2023-01-24T18:10:54.889 INFO:root:teuthology version: 0.0.1.dev22+g8e1bba1
2023-01-24T18:10:54.890 INFO:teuthology.results:subset: '111/120000'
2023-01-24T18:10:54.890 INFO:teuthology.results:seed: '1190'
2023-01-24T23:03:17.897 INFO:root:teuthology version: 0.0.1.dev22+g8e1bba1
2023-01-24T23:03:17.939 INFO:teuthology.results:Waiting up to 43200 seconds for tests to finish...
2023-01-25T01:03:28.861 INFO:teuthology.results:Tests finished! gathering results...
2023-01-25T01:03:51.462 INFO:scrape:Found 317 jobs
2023-01-25T01:04:37.628 INFO:scrape:Found 12 distinct failure reasons
2023-01-25T01:04:37.629 INFO:scrape:Failure: "2023-01-24T23:23:04.283514+0000 mon.a (mon.0) 95 : cluster [WRN] Health check failed: 1 mgr modules have recently crashed (RECENT_MGR_MODULE_CRASH)" in cluster log
2023-01-25T01:04:37.629 INFO:scrape:97 jobs
2023-01-25T01:04:37.630 INFO:scrape:suites intersection: []
2023-01-25T01:04:37.631 INFO:scrape:suites union: ['1-pg-log-overrides/normal_pg_log', '1-pg-log-overrides/short_pg_log', '2-inject-leak/mon', '2-inject-leak/none', '2-inject-leak/osd', '2-recovery-overrides/{default}', '2-recovery-overrides/{more-active-recovery}', '2-recovery-overrides/{more-async-partial-recovery}', '2-recovery-overrides/{more-async-recovery}', '2-recovery-overrides/{more-partial-recovery}', '3-scrub-overrides/{default}', '3-scrub-overrides/{max-simultaneous-scrubs-2}', '3-scrub-overrides/{max-simultaneous-scrubs-3}', 'backoff/normal', 'backoff/peering', 'backoff/peering_and_degraded', 'centos_latest}', 'ceph', 'cluster/{12-osds', 'clusters/3-mons', 'clusters/9-mons', 'clusters/{fixed-2', 'clusters/{fixed-4', 'crc-failures/bad_map_crc_failure', 'crc-failures/default', 'd-balancer/crush-compat', 'd-balancer/on', 'd-thrash/default/{default', 'd-thrash/none', 'debug/mgr', 'fast/fast', 'fast/normal', 'mgr_ttl_cache/disable', 'mon_election/classic', 'mon_election/connectivity', 'msgr-failures/fastclose', 'msgr-failures/few', 'msgr-failures/many', 'msgr-failures/mon-delay', 'msgr-failures/none', 'msgr-failures/osd-delay', 'msgr-failures/osd-dispatch-delay', 'msgr/async', 'msgr/async-v1only', 'msgr/async-v2only', 'no_pools', 'objectstore/bluestore-bitmap', 'objectstore/bluestore-comp-lz4', 'objectstore/bluestore-comp-snappy', 'objectstore/bluestore-comp-zlib', 'objectstore/bluestore-comp-zstd', 'objectstore/bluestore-hybrid', 'objectstore/bluestore-low-osd-mem-target', 'objectstore/bluestore-stupid', 'objectstore/filestore-xfs', 'openstack}', 'rados', 'rados/basic/{ceph', 'rados/mgr/{clusters/{2-node-mgr}', 'rados/monthrash/{ceph', 'rados/multimon/{clusters/6', 'rados/objectstore/{backends/objectcacher-stress', 'rados/singleton-nomsgr/{all/admin_socket_output', 'rados/singleton-nomsgr/{all/multi-backfill-reject', 'rados/singleton-nomsgr/{all/osd_stale_reads', 'rados/singleton/{all/ec-inconsistent-hinfo', 'rados/singleton/{all/ec-lost-unfound', 'rados/singleton/{all/lost-unfound', 'rados/singleton/{all/lost-unfound-delete', 'rados/singleton/{all/osd-backfill', 'rados/singleton/{all/osd-recovery-incomplete', 'rados/singleton/{all/thrash-eio', 'rados/singleton/{all/thrash_cache_writeback_proxy_none', 'rados/thrash-erasure-code-big/{ceph', 'rados/thrash-erasure-code-isa/{arch/x86_64', 'rados/thrash-erasure-code-overwrites/{bluestore-bitmap', 'rados/thrash-erasure-code-shec/{ceph', 'rados/thrash-erasure-code/{ceph', 'rados/thrash/{0-size-min-size-overrides/2-size-2-min-size', 'rados/thrash/{0-size-min-size-overrides/3-size-2-min-size', 'rados/valgrind-leaks/{1-start', 'rados/verify/{centos_latest', 'random-objectstore$/{bluestore-bitmap}', 'random-objectstore$/{bluestore-comp-lz4}', 'random-objectstore$/{bluestore-comp-snappy}', 'recovery-overrides/{default}', 'recovery-overrides/{more-active-recovery}', 'recovery-overrides/{more-async-partial-recovery}', 'recovery-overrides/{more-async-recovery}', 'recovery-overrides/{more-partial-recovery}', 'supported-random-distro$/{centos_8}', 'supported-random-distro$/{centos_8}}', 'supported-random-distro$/{rhel_8}', 'supported-random-distro$/{rhel_8}}', 'tasks/insights}', 'tasks/libcephsqlite}', 'tasks/mon_clock_with_skews}', 'tasks/mon_recovery', 'tasks/mon_recovery}', 'tasks/progress}', 'tasks/rados_api_tests', 'tasks/rados_cls_all', 'tasks/rados_python}', 'tasks/rados_striper}', 'tasks/rados_workunit_loadgen_big}', 'tasks/rados_workunit_loadgen_mix}', 'tasks/rados_workunit_loadgen_mostlyread}', 'tasks/readwrite}', 'tasks/workunits}', 'thrashers/careful', 'thrashers/default', 'thrashers/fastread', 'thrashers/force-sync-many', 'thrashers/many', 'thrashers/mapgap', 'thrashers/minsize_recovery', 'thrashers/morepggrow', 'thrashers/none', 'thrashers/one', 'thrashers/pggrow', 'thrashers/sync', 'thrashers/sync-many', 'thrashosds-health', 'thrashosds-health}', 'validater/lockdep}', 'validater/valgrind}', 'workloads/admin_socket_objecter_requests}', 'workloads/cache-agent-big}', 'workloads/cache-pool-snaps-readproxy}', 'workloads/cache-pool-snaps}', 'workloads/cache-snaps-balanced}', 'workloads/cache-snaps}', 'workloads/cache}', 'workloads/ec-pool-snaps-few-objects-overwrites}', 'workloads/ec-rados-plugin=clay-k=4-m=2}', 'workloads/ec-rados-plugin=isa-k=2-m=1}', 'workloads/ec-rados-plugin=jerasure-k=2-m=1}', 'workloads/ec-rados-plugin=jerasure-k=4-m=2}', 'workloads/ec-rados-plugin=shec-k=4-m=3-c=2}', 'workloads/ec-radosbench}', 'workloads/ec-small-objects-balanced}', 'workloads/ec-small-objects-fast-read-overwrites}', 'workloads/ec-small-objects-fast-read}', 'workloads/ec-small-objects-many-deletes}', 'workloads/ec-small-objects-overwrites}', 'workloads/ec-snaps-few-objects-overwrites}', 'workloads/pool-create-delete}', 'workloads/rados_api_tests}', 'workloads/rados_mon_osdmap_prune}', 'workloads/rados_mon_workunits}', 'workloads/radosbench-high-concurrency}', 'workloads/radosbench}', 'workloads/redirect_promote_tests}', 'workloads/redirect_set_object}', 'workloads/redirect}', 'workloads/small-objects-balanced}', 'workloads/small-objects-localized}', 'workloads/small-objects}', 'workloads/snaps-few-objects-balanced}', 'workloads/snaps-few-objects-localized}', 'workloads/snaps-few-objects}', 'workloads/write_fadvise_dontneed}']
2023-01-25T01:04:37.631 INFO:scrape:
2023-01-25T01:04:37.632 INFO:scrape:Failure: timeout expired in wait_until_healthy
2023-01-25T01:04:37.632 INFO:scrape:32 jobs
2023-01-25T01:04:37.632 INFO:scrape:suites intersection: []
2023-01-25T01:04:37.633 INFO:scrape:suites union: ['0-nvme-loop', '1-start', '2-ops/repave-all}', '2-ops/rm-zap-add}', '2-ops/rm-zap-flag}', '2-ops/rm-zap-wait}', '2-ops/rmdir-reactivate}', '2-services/basic', '2-services/rgw', '3-final}', 'agent/off', 'agent/on', 'fixed-2', 'mon_election/classic', 'mon_election/connectivity', 'msgr-failures/many', 'msgr/async-v1only', 'objectstore/bluestore-comp-zstd', 'rados', 'rados/cephadm/osds/{0-distro/centos_8.stream_container_tools', 'rados/cephadm/osds/{0-distro/centos_8.stream_container_tools_crun', 'rados/cephadm/osds/{0-distro/rhel_8.4_container_tools_3.0', 'rados/cephadm/osds/{0-distro/rhel_8.4_container_tools_rhel8', 'rados/cephadm/osds/{0-distro/ubuntu_20.04', 'rados/cephadm/smoke-singlehost/{0-random-distro$/{centos_8.stream_container_tools_crun}', 'rados/cephadm/smoke-singlehost/{0-random-distro$/{centos_8.stream_container_tools}', 'rados/cephadm/smoke-singlehost/{0-random-distro$/{ubuntu_20.04}', 'rados/cephadm/smoke/{0-distro/centos_8.stream_container_tools', 'rados/cephadm/smoke/{0-distro/centos_8.stream_container_tools_crun', 'rados/cephadm/smoke/{0-distro/rhel_8.4_container_tools_3.0', 'rados/cephadm/smoke/{0-distro/rhel_8.4_container_tools_rhel8', 'rados/cephadm/smoke/{0-distro/ubuntu_20.04', 'rados/singleton-nomsgr/{all/recovery-unfound-found', 'rados/singleton/{all/recovery-preemption', 'start}', 'supported-random-distro$/{centos_8}}', 'supported-random-distro$/{rhel_8}}']
2023-01-25T01:04:37.633 INFO:scrape:
2023-01-25T01:04:37.634 INFO:scrape:Failure: Command failed on smithi174 with status 127: 'sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:49f8fb05584886826e8eade75f7105fba754560c shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 08f9044c-9c22-11ed-9e56-001a4aab830c -- ceph mon dump -f json'
2023-01-25T01:04:37.634 INFO:scrape:1 jobs: ['7135701']
2023-01-25T01:04:37.635 INFO:scrape:suites: ['0-nvme-loop', '1-start', '2-ops/repave-all}', 'rados/cephadm/osds/{0-distro/rhel_8.4_container_tools_rhel8']
2023-01-25T01:04:37.635 INFO:scrape:
2023-01-25T01:04:37.635 INFO:scrape:Failure: Command failed on smithi173 with status 1: 'sudo systemctl enable --now kubelet && sudo kubeadm config images pull'
2023-01-25T01:04:37.636 INFO:scrape:4 jobs: ['7135635', '7135839', '7135760', '7135917']
2023-01-25T01:04:37.636 INFO:scrape:suites intersection: ['0-kubeadm', '0-nvme-loop', '1-rook', '3-final', 'k8s/1.21', 'rados/rook/smoke/{0-distro/ubuntu_20.04']
2023-01-25T01:04:37.636 INFO:scrape:suites union: ['0-kubeadm', '0-nvme-loop', '1-rook', '2-workload/none', '2-workload/radosbench', '3-final', 'cluster/1-node', 'cluster/3-node', 'k8s/1.21', 'net/calico', 'net/flannel', 'net/host', 'rados/rook/smoke/{0-distro/ubuntu_20.04', 'rook/1.7.2}', 'rook/master}']
2023-01-25T01:04:37.637 INFO:scrape:
2023-01-25T01:04:37.637 INFO:scrape:Failure: Command failed (workunit test cephadm/test_cephadm.sh) on smithi195 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=49f8fb05584886826e8eade75f7105fba754560c TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cephadm/test_cephadm.sh'
2023-01-25T01:04:37.638 INFO:scrape:2 jobs: ['7135897', '7135766']
2023-01-25T01:04:37.638 INFO:scrape:suites intersection: ['agent/on', 'mon_election/connectivity', 'task/test_cephadm}']
2023-01-25T01:04:37.638 INFO:scrape:suites union: ['agent/on', 'mon_election/connectivity', 'rados/cephadm/workunits/{0-distro/rhel_8.4_container_tools_3.0', 'rados/cephadm/workunits/{0-distro/rhel_8.4_container_tools_rhel8', 'task/test_cephadm}']
2023-01-25T01:04:37.639 INFO:scrape:
2023-01-25T01:04:37.639 INFO:scrape:Failure: Command failed (workunit test cephtool/test.sh) on smithi007 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=49f8fb05584886826e8eade75f7105fba754560c TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cephtool/test.sh'
2023-01-25T01:04:37.640 INFO:scrape:3 jobs: ['7135830', '7135629', '7135906']
2023-01-25T01:04:37.640 INFO:scrape:suites intersection: ['rados', 'rados/singleton-bluestore/{all/cephtool']
2023-01-25T01:04:37.640 INFO:scrape:suites union: ['mon_election/classic', 'mon_election/connectivity', 'msgr-failures/few', 'msgr-failures/many', 'msgr-failures/none', 'msgr/async-v1only', 'msgr/async-v2only', 'objectstore/bluestore-bitmap', 'objectstore/bluestore-comp-lz4', 'rados', 'rados/singleton-bluestore/{all/cephtool', 'supported-random-distro$/{centos_8}}', 'supported-random-distro$/{rhel_8}}']
2023-01-25T01:04:37.641 INFO:scrape:
2023-01-25T01:04:37.641 INFO:scrape:Failure: Command failed (workunit test cls/test_cls_rbd.sh) on smithi012 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=pacific TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cls/test_cls_rbd.sh'
2023-01-25T01:04:37.642 INFO:scrape:2 jobs: ['7135919', '7135763']
2023-01-25T01:04:37.642 INFO:scrape:suites intersection: ['0-start', '1-tasks', 'rados_api', 'rados_loadgenbig', 'rbd_import_export', 'test_rbd_api', 'test_rbd_python}}', 'upgrade-sequence', 'workload/{ec-rados-default']
2023-01-25T01:04:37.642 INFO:scrape:suites union: ['0-start', '1-tasks', 'mon_election/classic', 'mon_election/connectivity', 'rados/upgrade/parallel/{0-random-distro$/{centos_8.stream_container_tools}', 'rados/upgrade/parallel/{0-random-distro$/{ubuntu_20.04}', 'rados_api', 'rados_loadgenbig', 'rbd_import_export', 'test_rbd_api', 'test_rbd_python}}', 'upgrade-sequence', 'workload/{ec-rados-default']
2023-01-25T01:04:37.643 INFO:scrape:
2023-01-25T01:04:37.643 INFO:scrape:Failure: Test failure: test_remove_from_blocklist (tasks.mgr.dashboard.test_auth.AuthTest)
2023-01-25T01:04:37.643 INFO:scrape:1 jobs: ['7135916']
2023-01-25T01:04:37.644 INFO:scrape:suites: ['debug/mgr', 'mon_election/connectivity', 'rados/dashboard/{0-single-container-host', 'random-objectstore$/{bluestore-comp-zlib}', 'tasks/dashboard}']
2023-01-25T01:04:37.644 INFO:scrape:
2023-01-25T01:04:37.645 INFO:scrape:Crash: Command failed (workunit test rados/test_crash.sh) on smithi182 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=49f8fb05584886826e8eade75f7105fba754560c TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/rados/test_crash.sh'
2023-01-25T01:04:37.645 INFO:scrape:ceph version 17.2.5-727-g49f8fb05 (49f8fb05584886826e8eade75f7105fba754560c) quincy (stable)
 1: /lib64/libpthread.so.0(+0x12cf0) [0x7f1dd7ff7cf0]
 2: pthread_cond_wait()
 3: (std::condition_variable::wait(std::unique_lock<std::mutex>&)+0x10) [0x7f1dd763a8f0]
 4: (AsyncMessenger::wait()+0x147) [0x55e965f635b7]
 5: main()
 6: __libc_start_main()
 7: _start()
2023-01-25T01:04:37.645 INFO:scrape:1 jobs: ['7135808']
2023-01-25T01:04:37.646 INFO:scrape:suites: ['mon_election/connectivity', 'msgr-failures/few', 'msgr/async', 'objectstore/bluestore-low-osd-mem-target', 'rados', 'rados/singleton/{all/test-crash', 'supported-random-distro$/{centos_8}}']
2023-01-25T01:04:37.646 INFO:scrape:
2023-01-25T01:04:37.646 INFO:scrape:Failure: Test failure: test_prometheus (tasks.mgr.test_module_selftest.TestModuleSelftest)
2023-01-25T01:04:37.647 INFO:scrape:1 jobs: ['7135862']
2023-01-25T01:04:37.647 INFO:scrape:suites: ['debug/mgr', 'mgr_ttl_cache/enable', 'mon_election/connectivity', 'rados/mgr/{clusters/{2-node-mgr}', 'random-objectstore$/{bluestore-comp-lz4}', 'supported-random-distro$/{centos_8}', 'tasks/module_selftest}']
2023-01-25T01:04:37.648 INFO:scrape:
2023-01-25T01:04:37.648 INFO:scrape:Failure: Test failure: test_token_ttl (tasks.mgr.dashboard.test_auth.AuthTest)
2023-01-25T01:04:37.648 INFO:scrape:1 jobs: ['7135756']
2023-01-25T01:04:37.649 INFO:scrape:suites: ['debug/mgr', 'mon_election/classic', 'rados/dashboard/{0-single-container-host', 'random-objectstore$/{bluestore-comp-lz4}', 'tasks/dashboard}']
2023-01-25T01:04:37.649 INFO:scrape:
2023-01-25T01:04:37.649 INFO:scrape:Failure: {'smithi080.front.sepia.ceph.com': {'_ansible_no_log': False, 'changed': False, 'invocation': {'module_args': {'allow_unauthenticated': False, 'autoclean': False, 'autoremove': False, 'cache_valid_time': 0, 'deb': None, 'default_release': None, 'dpkg_options': 'force-confdef,force-confold', 'force': True, 'force_apt_get': False, 'install_recommends': None, 'name': ['ceph', 'ceph-common', 'libcephfs1', 'radosgw', 'python-ceph', 'python-rados', 'python-cephfs', 'python-rbd', 'librbd1', 'librados2', 'ceph-fs-common-dbg', 'ceph-fs-common', 'openmpi-common'], 'only_upgrade': False, 'package': ['ceph', 'ceph-common', 'libcephfs1', 'radosgw', 'python-ceph', 'python-rados', 'python-cephfs', 'python-rbd', 'librbd1', 'librados2', 'ceph-fs-common-dbg', 'ceph-fs-common', 'openmpi-common'], 'policy_rc_d': None, 'purge': False, 'state': 'absent', 'update_cache': None, 'update_cache_retries': 5, 'update_cache_retry_max_delay': 12, 'upgrade': None}}, 'msg': "'apt-get remove 'librbd1' 'librados2'' failed: W: --force-yes is deprecated, use one of the options starting with --allow instead.\nE: Could not get lock /var/lib/dpkg/lock-frontend. It is held by process 14185 (apt-get)\nE: Unable to acquire the dpkg frontend lock (/var/lib/dpkg/lock-frontend), is another process using it?\n", 'rc': 100, 'stderr': 'W: --force-yes is deprecated, use one of the options starting with --allow instead.\nE: Could not get lock /var/lib/dpkg/lock-frontend. It is held by process 14185 (apt-get)\nE: Unable to acquire the dpkg frontend lock (/var/lib/dpkg/lock-frontend), is another process using it?\n', 'stderr_lines': ['W: --force-yes is deprecated, use one of the options starting with --allow instead.', 'E: Could not get lock /var/lib/dpkg/lock-frontend. It is held by process 14185 (apt-get)', 'E: Unable to acquire the dpkg frontend lock (/var/lib/dpkg/lock-frontend), is another process using it?'], 'stdout': '', 'stdout_lines': []}}
2023-01-25T01:04:37.650 INFO:scrape:1 jobs: ['7135834']
2023-01-25T01:04:37.650 INFO:scrape:suites: ['cluster/{12-osds', 'mon_election/classic', 'msgr-failures/osd-delay', 'objectstore/bluestore-comp-snappy', 'openstack}', 'rados', 'rados/thrash-erasure-code-big/{ceph', 'recovery-overrides/{default}', 'supported-random-distro$/{ubuntu_latest}', 'thrashers/careful', 'thrashosds-health', 'workloads/ec-rados-plugin=lrc-k=4-m=2-l=3}']
2023-01-25T01:04:37.651 INFO:scrape:
2023-01-25T01:04:37.661 INFO:teuthology.results:Sending results to yweinstein@radhat.com: Test Run: yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi
=================================================================
info:    http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/
logs:    http://qa-proxy.ceph.com/teuthology/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/
failed:  145
dead:    1
running: 0
waiting: 0
queued:  0
passed:  171


Fail
=================================================================
[7135583]  rados/valgrind-leaks/{1-start 2-inject-leak/osd centos_latest}
-----------------------------------------------------------------
time:   00:21:45
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135583/
log:    http://qa-proxy.ceph.com/teuthology/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135583/

    "2023-01-24T18:40:20.262152+0000 mgr.x (mgr.4100) 1 : cluster [ERR] Failed
    to load ceph-mgr modules: prometheus" in cluster log

[7135589]  rados/thrash/{0-size-min-size-overrides/3-size-2-min-size 1-pg-log-overrides/short_pg_log 2-recovery-overrides/{default} 3-scrub-overrides/{default} backoff/peering ceph clusters/{fixed-2 openstack} crc-failures/default d-balancer/on mon_election/connectivity msgr-failures/osd-delay msgr/async-v2only objectstore/bluestore-comp-lz4 rados supported-random-distro$/{rhel_8} thrashers/default thrashosds-health workloads/cache-pool-snaps-readproxy}
-----------------------------------------------------------------
time:   00:27:22
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135589/
log:    http://qa-proxy.ceph.com/teuthology/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135589/

    "2023-01-24T18:38:31.306324+0000 mon.a (mon.0) 1181 : cluster [WRN] Health
    check failed: 1 mgr modules have recently crashed
    (RECENT_MGR_MODULE_CRASH)" in cluster log

[7135599]  rados/cephadm/osds/{0-distro/rhel_8.4_container_tools_3.0 0-nvme-loop 1-start 2-ops/rm-zap-wait}
-----------------------------------------------------------------
time:   00:19:09
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135599/
log:    http://qa-proxy.ceph.com/teuthology/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135599/
sentry: https://sentry.ceph.com/organizations/ceph/?query=ffb4a81af18a4df5a0610142cb7743dd

    timeout expired in wait_until_healthy

[7135607]  rados/thrash/{0-size-min-size-overrides/2-size-2-min-size 1-pg-log-overrides/normal_pg_log 2-recovery-overrides/{default} 3-scrub-overrides/{default} backoff/peering_and_degraded ceph clusters/{fixed-2 openstack} crc-failures/bad_map_crc_failure d-balancer/crush-compat mon_election/classic msgr-failures/osd-dispatch-delay msgr/async objectstore/bluestore-comp-snappy rados supported-random-distro$/{rhel_8} thrashers/mapgap thrashosds-health workloads/cache-pool-snaps}
-----------------------------------------------------------------
time:   00:29:04
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135607/
log:    http://qa-proxy.ceph.com/teuthology/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135607/

    "2023-01-24T18:52:17.887500+0000 mgr.y (mgr.4112) 1 : cluster [ERR] Failed
    to load ceph-mgr modules: prometheus" in cluster log

[7135611]  rados/cephadm/smoke/{0-distro/centos_8.stream_container_tools_crun 0-nvme-loop agent/on fixed-2 mon_election/classic start}
-----------------------------------------------------------------
time:   00:21:12
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135611/
log:    http://qa-proxy.ceph.com/teuthology/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135611/
sentry: https://sentry.ceph.com/organizations/ceph/?query=a2e78f6cb29b4c16b69a1b86a6f250f0

    timeout expired in wait_until_healthy

[7135614]  rados/verify/{centos_latest ceph clusters/{fixed-2 openstack} d-thrash/default/{default thrashosds-health} mon_election/connectivity msgr-failures/few msgr/async-v1only objectstore/bluestore-hybrid rados tasks/rados_api_tests validater/valgrind}
-----------------------------------------------------------------
time:   01:02:56
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135614/
log:    http://qa-proxy.ceph.com/teuthology/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135614/

    "2023-01-24T19:11:10.332591+0000 mgr.y (mgr.4107) 1 : cluster [ERR] Failed
    to load ceph-mgr modules: prometheus" in cluster log

[7135623]  rados/thrash-erasure-code-shec/{ceph clusters/{fixed-4 openstack} mon_election/classic msgr-failures/osd-dispatch-delay objectstore/bluestore-hybrid rados recovery-overrides/{more-async-partial-recovery} supported-random-distro$/{centos_8} thrashers/careful thrashosds-health workloads/ec-rados-plugin=shec-k=4-m=3-c=2}
-----------------------------------------------------------------
time:   00:14:46
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135623/
log:    http://qa-proxy.ceph.com/teuthology/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135623/

    "2023-01-24T19:04:53.706735+0000 mgr.x (mgr.4109) 1 : cluster [ERR] Failed
    to load ceph-mgr modules: prometheus" in cluster log

[7135629]  rados/singleton-bluestore/{all/cephtool mon_election/classic msgr-failures/many msgr/async-v2only objectstore/bluestore-comp-lz4 rados supported-random-distro$/{centos_8}}
-----------------------------------------------------------------
time:   00:28:09
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135629/
log:    http://qa-proxy.ceph.com/teuthology/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135629/
sentry: https://sentry.ceph.com/organizations/ceph/?query=0ab02816a55f4f2580c7a1151e5e70db

    Command failed (workunit test cephtool/test.sh) on smithi182 with status 1:
    'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd --
    /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1
    CEPH_REF=49f8fb05584886826e8eade75f7105fba754560c
    TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0"
    PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0
    CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0
    CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage
    /home/ubuntu/cephtest/archive/coverage timeout 3h
    /home/ubuntu/cephtest/clone.client.0/qa/workunits/cephtool/test.sh'

[7135635]  rados/rook/smoke/{0-distro/ubuntu_20.04 0-kubeadm 0-nvme-loop 1-rook 2-workload/radosbench 3-final cluster/1-node k8s/1.21 net/host rook/master}
-----------------------------------------------------------------
time:   00:05:34
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135635/
log:    http://qa-proxy.ceph.com/teuthology/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135635/
sentry: https://sentry.ceph.com/organizations/ceph/?query=457b3291dfb44a87b9b8400ec75fd62f

    Command failed on smithi173 with status 1: 'sudo systemctl enable --now
    kubelet && sudo kubeadm config images pull'

[7135639]  rados/thrash/{0-size-min-size-overrides/2-size-2-min-size 1-pg-log-overrides/normal_pg_log 2-recovery-overrides/{more-active-recovery} 3-scrub-overrides/{default} backoff/peering ceph clusters/{fixed-2 openstack} crc-failures/bad_map_crc_failure d-balancer/crush-compat mon_election/classic msgr-failures/few msgr/async-v2only objectstore/bluestore-comp-zstd rados supported-random-distro$/{centos_8} thrashers/none thrashosds-health workloads/cache-snaps}
-----------------------------------------------------------------
time:   00:21:25
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135639/
log:    http://qa-proxy.ceph.com/teuthology/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135639/

    "2023-01-24T19:16:03.240713+0000 mgr.y (mgr.4109) 1 : cluster [ERR] Failed
    to load ceph-mgr modules: prometheus" in cluster log

[7135641]  rados/thrash-erasure-code/{ceph clusters/{fixed-2 openstack} fast/fast mon_election/classic msgr-failures/fastclose objectstore/bluestore-comp-zlib rados recovery-overrides/{more-async-recovery} supported-random-distro$/{centos_8} thrashers/minsize_recovery thrashosds-health workloads/ec-small-objects-many-deletes}
-----------------------------------------------------------------
time:   00:26:31
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135641/
log:    http://qa-proxy.ceph.com/teuthology/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135641/

    "2023-01-24T19:16:00.621475+0000 mgr.x (mgr.4114) 1 : cluster [ERR] Failed
    to load ceph-mgr modules: prometheus" in cluster log

[7135645]  rados/cephadm/osds/{0-distro/rhel_8.4_container_tools_rhel8 0-nvme-loop 1-start 2-ops/rmdir-reactivate}
-----------------------------------------------------------------
time:   00:18:36
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135645/
log:    http://qa-proxy.ceph.com/teuthology/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135645/
sentry: https://sentry.ceph.com/organizations/ceph/?query=f21070e9126b46f3ae6d1d3a0f26169b

    timeout expired in wait_until_healthy

[7135653]  rados/thrash-erasure-code-overwrites/{bluestore-bitmap ceph clusters/{fixed-2 openstack} fast/fast mon_election/classic msgr-failures/fastclose rados recovery-overrides/{more-async-partial-recovery} supported-random-distro$/{centos_8} thrashers/fastread thrashosds-health workloads/ec-snaps-few-objects-overwrites}
-----------------------------------------------------------------
time:   00:30:02
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135653/
log:    http://qa-proxy.ceph.com/teuthology/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135653/

    "2023-01-24T19:28:10.770272+0000 mgr.y (mgr.4111) 1 : cluster [ERR] Failed
    to load ceph-mgr modules: prometheus" in cluster log

[7135655]  rados/thrash/{0-size-min-size-overrides/3-size-2-min-size 1-pg-log-overrides/short_pg_log 2-recovery-overrides/{default} 3-scrub-overrides/{max-simultaneous-scrubs-3} backoff/peering_and_degraded ceph clusters/{fixed-2 openstack} crc-failures/default d-balancer/on mon_election/connectivity msgr-failures/osd-delay msgr/async objectstore/bluestore-hybrid rados supported-random-distro$/{rhel_8} thrashers/pggrow thrashosds-health workloads/cache}
-----------------------------------------------------------------
time:   00:15:35
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135655/
log:    http://qa-proxy.ceph.com/teuthology/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135655/

    "2023-01-24T19:27:54.127733+0000 mgr.y (mgr.4107) 1 : cluster [ERR] Failed
    to load ceph-mgr modules: prometheus" in cluster log

[7135661]  rados/cephadm/smoke-singlehost/{0-random-distro$/{centos_8.stream_container_tools} 1-start 2-services/rgw 3-final}
-----------------------------------------------------------------
time:   00:18:03
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135661/
log:    http://qa-proxy.ceph.com/teuthology/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135661/
sentry: https://sentry.ceph.com/organizations/ceph/?query=7fa8899ea6c24942ad3137a9745f8b97

    timeout expired in wait_until_healthy

[7135667]  rados/multimon/{clusters/6 mon_election/connectivity msgr-failures/many msgr/async-v1only no_pools objectstore/bluestore-stupid rados supported-random-distro$/{centos_8} tasks/mon_clock_with_skews}
-----------------------------------------------------------------
time:   00:10:18
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135667/
log:    http://qa-proxy.ceph.com/teuthology/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135667/

    "2023-01-24T19:42:49.337618+0000 mgr.x (mgr.4105) 1 : cluster [ERR] Failed
    to load ceph-mgr modules: prometheus" in cluster log

[7135671]  rados/cephadm/smoke/{0-distro/rhel_8.4_container_tools_3.0 0-nvme-loop agent/off fixed-2 mon_election/connectivity start}
-----------------------------------------------------------------
time:   00:21:12
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135671/
log:    http://qa-proxy.ceph.com/teuthology/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135671/
sentry: https://sentry.ceph.com/organizations/ceph/?query=12a0340ca2c54f07bf104e9a6f516e54

    timeout expired in wait_until_healthy

[7135675]  rados/monthrash/{ceph clusters/9-mons mon_election/connectivity msgr-failures/few msgr/async objectstore/bluestore-low-osd-mem-target rados supported-random-distro$/{centos_8} thrashers/sync-many workloads/rados_api_tests}
-----------------------------------------------------------------
time:   00:29:25
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135675/
log:    http://qa-proxy.ceph.com/teuthology/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135675/

    "2023-01-24T19:54:08.017068+0000 mon.f (mon.1) 391 : cluster [WRN] Health
    check failed: 1 mgr modules have recently crashed
    (RECENT_MGR_MODULE_CRASH)" in cluster log

[7135677]  rados/verify/{centos_latest ceph clusters/{fixed-2 openstack} d-thrash/none mon_election/classic msgr-failures/few msgr/async-v2only objectstore/bluestore-low-osd-mem-target rados tasks/rados_cls_all validater/lockdep}
-----------------------------------------------------------------
time:   00:18:52
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135677/
log:    http://qa-proxy.ceph.com/teuthology/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135677/

    "2023-01-24T19:56:16.425060+0000 mon.a (mon.0) 518 : cluster [WRN] Health
    check failed: 1 mgr modules have recently crashed
    (RECENT_MGR_MODULE_CRASH)" in cluster log

[7135697]  rados/basic/{ceph clusters/{fixed-2 openstack} mon_election/classic msgr-failures/few msgr/async objectstore/bluestore-comp-snappy rados supported-random-distro$/{centos_8} tasks/libcephsqlite}
-----------------------------------------------------------------
time:   00:17:52
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135697/
log:    http://qa-proxy.ceph.com/teuthology/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135697/

    "2023-01-24T20:07:04.062261+0000 mgr.x (mgr.4114) 1 : cluster [ERR] Failed
    to load ceph-mgr modules: prometheus" in cluster log

[7135699]  rados/singleton/{all/osd-backfill mon_election/connectivity msgr-failures/few msgr/async objectstore/bluestore-comp-zlib rados supported-random-distro$/{centos_8}}
-----------------------------------------------------------------
time:   00:35:23
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135699/
log:    http://qa-proxy.ceph.com/teuthology/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135699/

    "2023-01-24T20:15:32.352968+0000 mon.a (mon.0) 228 : cluster [WRN] Health
    check failed: 1 mgr modules have recently crashed
    (RECENT_MGR_MODULE_CRASH)" in cluster log

[7135701]  rados/cephadm/osds/{0-distro/rhel_8.4_container_tools_rhel8 0-nvme-loop 1-start 2-ops/repave-all}
-----------------------------------------------------------------
time:   00:09:53
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135701/
log:    http://qa-proxy.ceph.com/teuthology/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135701/
sentry: https://sentry.ceph.com/organizations/ceph/?query=587f863e006b41a6810ecbc06deb8325

    Command failed on smithi174 with status 127: 'sudo
    /home/ubuntu/cephtest/cephadm --image quay-quay-
    quay.apps.os.sepia.ceph.com/ceph-
    ci/ceph:49f8fb05584886826e8eade75f7105fba754560c shell -c
    /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid
    08f9044c-9c22-11ed-9e56-001a4aab830c -- ceph mon dump -f json'

[7135707]  rados/singleton-nomsgr/{all/multi-backfill-reject mon_election/classic rados supported-random-distro$/{centos_8}}
-----------------------------------------------------------------
time:   00:23:16
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135707/
log:    http://qa-proxy.ceph.com/teuthology/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135707/

    "2023-01-24T20:21:32.998312+0000 mon.a (mon.0) 313 : cluster [WRN] Health
    check failed: 1 mgr modules have recently crashed
    (RECENT_MGR_MODULE_CRASH)" in cluster log

[7135709]  rados/singleton/{all/osd-recovery-incomplete mon_election/classic msgr-failures/many msgr/async-v1only objectstore/bluestore-comp-zstd rados supported-random-distro$/{rhel_8}}
-----------------------------------------------------------------
time:   00:47:12
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135709/
log:    http://qa-proxy.ceph.com/teuthology/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135709/

    "2023-01-24T20:19:13.447165+0000 mon.a (mon.0) 223 : cluster [WRN] Health
    check failed: 1 mgr modules have recently crashed
    (RECENT_MGR_MODULE_CRASH)" in cluster log

[7135711]  rados/thrash-erasure-code-isa/{arch/x86_64 ceph clusters/{fixed-2 openstack} mon_election/connectivity msgr-failures/osd-delay objectstore/bluestore-stupid rados recovery-overrides/{more-async-partial-recovery} supported-random-distro$/{centos_8} thrashers/morepggrow thrashosds-health workloads/ec-rados-plugin=isa-k=2-m=1}
-----------------------------------------------------------------
time:   00:29:16
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135711/
log:    http://qa-proxy.ceph.com/teuthology/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135711/

    "2023-01-24T20:17:22.480099+0000 mgr.x (mgr.4111) 1 : cluster [ERR] Failed
    to load ceph-mgr modules: prometheus" in cluster log

[7135713]  rados/cephadm/smoke/{0-distro/rhel_8.4_container_tools_rhel8 0-nvme-loop agent/on fixed-2 mon_election/classic start}
-----------------------------------------------------------------
time:   00:21:35
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135713/
log:    http://qa-proxy.ceph.com/teuthology/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135713/
sentry: https://sentry.ceph.com/organizations/ceph/?query=35b466ab87fb4ed98cc96988318183c5

    timeout expired in wait_until_healthy

[7135717]  rados/singleton-nomsgr/{all/osd_stale_reads mon_election/connectivity rados supported-random-distro$/{centos_8}}
-----------------------------------------------------------------
time:   00:20:49
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135717/
log:    http://qa-proxy.ceph.com/teuthology/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135717/

    "2023-01-24T20:30:50.284054+0000 mon.a (mon.0) 282 : cluster [WRN] Health
    check failed: 1 mgr modules have recently crashed
    (RECENT_MGR_MODULE_CRASH)" in cluster log

[7135723]  rados/mgr/{clusters/{2-node-mgr} debug/mgr mgr_ttl_cache/disable mon_election/connectivity random-objectstore$/{bluestore-bitmap} supported-random-distro$/{centos_8} tasks/workunits}
-----------------------------------------------------------------
time:   00:11:52
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135723/
log:    http://qa-proxy.ceph.com/teuthology/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135723/

    "2023-01-24T20:25:52.260278+0000 mgr.y (mgr.4114) 1 : cluster [ERR] Failed
    to load ceph-mgr modules: prometheus" in cluster log

[7135727]  rados/thrash/{0-size-min-size-overrides/2-size-2-min-size 1-pg-log-overrides/normal_pg_log 2-recovery-overrides/{more-async-recovery} 3-scrub-overrides/{default} backoff/peering ceph clusters/{fixed-2 openstack} crc-failures/bad_map_crc_failure d-balancer/crush-compat mon_election/classic msgr-failures/osd-dispatch-delay msgr/async-v2only objectstore/bluestore-comp-lz4 rados supported-random-distro$/{centos_8} thrashers/none thrashosds-health workloads/radosbench-high-concurrency}
-----------------------------------------------------------------
time:   00:25:44
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135727/
log:    http://qa-proxy.ceph.com/teuthology/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135727/

    "2023-01-24T20:30:15.114166+0000 mgr.x (mgr.4108) 1 : cluster [ERR] Failed
    to load ceph-mgr modules: prometheus" in cluster log

[7135729]  rados/monthrash/{ceph clusters/3-mons mon_election/classic msgr-failures/mon-delay msgr/async-v1only objectstore/bluestore-stupid rados supported-random-distro$/{rhel_8} thrashers/sync workloads/rados_mon_osdmap_prune}
-----------------------------------------------------------------
time:   00:44:25
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135729/
log:    http://qa-proxy.ceph.com/teuthology/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135729/

    "2023-01-24T20:36:51.528142+0000 mon.a (mon.0) 1239 : cluster [WRN] Health
    check failed: 1 mgr modules have recently crashed
    (RECENT_MGR_MODULE_CRASH)" in cluster log

[7135731]  rados/verify/{centos_latest ceph clusters/{fixed-2 openstack} d-thrash/default/{default thrashosds-health} mon_election/connectivity msgr-failures/few msgr/async objectstore/bluestore-stupid rados tasks/mon_recovery validater/valgrind}
-----------------------------------------------------------------
time:   00:31:17
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135731/
log:    http://qa-proxy.ceph.com/teuthology/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135731/

    "2023-01-24T20:44:26.698278+0000 mgr.x (mgr.4108) 1 : cluster [ERR] Failed
    to load ceph-mgr modules: prometheus" in cluster log

[7135732]  rados/thrash-erasure-code-overwrites/{bluestore-bitmap ceph clusters/{fixed-2 openstack} fast/normal mon_election/connectivity msgr-failures/few rados recovery-overrides/{more-partial-recovery} supported-random-distro$/{rhel_8} thrashers/minsize_recovery thrashosds-health workloads/ec-pool-snaps-few-objects-overwrites}
-----------------------------------------------------------------
time:   00:28:31
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135732/
log:    http://qa-proxy.ceph.com/teuthology/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135732/

    "2023-01-24T20:33:07.635056+0000 mgr.x (mgr.4111) 1 : cluster [ERR] Failed
    to load ceph-mgr modules: prometheus" in cluster log

[7135734]  rados/cephadm/osds/{0-distro/ubuntu_20.04 0-nvme-loop 1-start 2-ops/rm-zap-add}
-----------------------------------------------------------------
time:   00:19:37
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135734/
log:    http://qa-proxy.ceph.com/teuthology/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135734/
sentry: https://sentry.ceph.com/organizations/ceph/?query=d412f933c2b64a33bbe7024d0bf81f91

    timeout expired in wait_until_healthy

[7135736]  rados/thrash-erasure-code-shec/{ceph clusters/{fixed-4 openstack} mon_election/classic msgr-failures/few objectstore/bluestore-stupid rados recovery-overrides/{more-async-recovery} supported-random-distro$/{centos_8} thrashers/careful thrashosds-health workloads/ec-rados-plugin=shec-k=4-m=3-c=2}
-----------------------------------------------------------------
time:   00:14:16
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135736/
log:    http://qa-proxy.ceph.com/teuthology/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135736/

    "2023-01-24T20:39:51.547246+0000 mgr.y (mgr.4107) 1 : cluster [ERR] Failed
    to load ceph-mgr modules: prometheus" in cluster log

[7135738]  rados/thrash-erasure-code-big/{ceph cluster/{12-osds openstack} mon_election/connectivity msgr-failures/osd-dispatch-delay objectstore/filestore-xfs rados recovery-overrides/{more-async-partial-recovery} supported-random-distro$/{rhel_8} thrashers/morepggrow thrashosds-health workloads/ec-rados-plugin=jerasure-k=4-m=2}
-----------------------------------------------------------------
time:   00:30:27
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135738/
log:    http://qa-proxy.ceph.com/teuthology/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135738/

    "2023-01-24T20:45:20.501035+0000 mon.a (mon.0) 1314 : cluster [WRN] Health
    check failed: 1 mgr modules have recently crashed
    (RECENT_MGR_MODULE_CRASH)" in cluster log

[7135740]  rados/singleton-nomsgr/{all/recovery-unfound-found mon_election/connectivity rados supported-random-distro$/{rhel_8}}
-----------------------------------------------------------------
time:   00:38:15
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135740/
log:    http://qa-proxy.ceph.com/teuthology/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135740/
sentry: https://sentry.ceph.com/organizations/ceph/?query=8c020d8eaa714d258505a82b73d6072f

    timeout expired in wait_until_healthy

[7135742]  rados/thrash-erasure-code/{ceph clusters/{fixed-2 openstack} fast/fast mon_election/classic msgr-failures/osd-delay objectstore/bluestore-hybrid rados recovery-overrides/{default} supported-random-distro$/{rhel_8} thrashers/pggrow thrashosds-health workloads/ec-rados-plugin=clay-k=4-m=2}
-----------------------------------------------------------------
time:   00:32:37
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135742/
log:    http://qa-proxy.ceph.com/teuthology/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135742/

    "2023-01-24T20:46:33.414701+0000 mon.a (mon.0) 1161 : cluster [WRN] Health
    check failed: 1 mgr modules have recently crashed
    (RECENT_MGR_MODULE_CRASH)" in cluster log

[7135745]  rados/cephadm/smoke/{0-distro/ubuntu_20.04 0-nvme-loop agent/off fixed-2 mon_election/connectivity start}
-----------------------------------------------------------------
time:   00:27:41
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135745/
log:    http://qa-proxy.ceph.com/teuthology/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135745/
sentry: https://sentry.ceph.com/organizations/ceph/?query=3d17e73055184808891a198e7b766980

    timeout expired in wait_until_healthy

[7135751]  rados/thrash/{0-size-min-size-overrides/3-size-2-min-size 1-pg-log-overrides/short_pg_log 2-recovery-overrides/{default} 3-scrub-overrides/{max-simultaneous-scrubs-3} backoff/peering ceph clusters/{fixed-2 openstack} crc-failures/default d-balancer/on mon_election/connectivity msgr-failures/osd-delay msgr/async-v2only objectstore/bluestore-comp-zstd rados supported-random-distro$/{rhel_8} thrashers/default thrashosds-health workloads/redirect_promote_tests}
-----------------------------------------------------------------
time:   00:16:25
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135751/
log:    http://qa-proxy.ceph.com/teuthology/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135751/

    "2023-01-24T20:46:02.139498+0000 mgr.x (mgr.4099) 1 : cluster [ERR] Failed
    to load ceph-mgr modules: prometheus" in cluster log

[7135752]  rados/cephadm/osds/{0-distro/centos_8.stream_container_tools 0-nvme-loop 1-start 2-ops/rm-zap-flag}
-----------------------------------------------------------------
time:   00:19:03
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135752/
log:    http://qa-proxy.ceph.com/teuthology/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135752/
sentry: https://sentry.ceph.com/organizations/ceph/?query=e70b80aad9414333aa1663d779740ae4

    timeout expired in wait_until_healthy

[7135756]  rados/dashboard/{0-single-container-host debug/mgr mon_election/classic random-objectstore$/{bluestore-comp-lz4} tasks/dashboard}
-----------------------------------------------------------------
time:   00:20:14
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135756/
log:    http://qa-proxy.ceph.com/teuthology/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135756/
sentry: https://sentry.ceph.com/organizations/ceph/?query=cf1a23f1133d49c1b1f5ca248788bfe3

    Test failure: test_token_ttl (tasks.mgr.dashboard.test_auth.AuthTest)

[7135760]  rados/rook/smoke/{0-distro/ubuntu_20.04 0-kubeadm 0-nvme-loop 1-rook 2-workload/radosbench 3-final cluster/1-node k8s/1.21 net/calico rook/1.7.2}
-----------------------------------------------------------------
time:   00:05:30
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135760/
log:    http://qa-proxy.ceph.com/teuthology/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135760/
sentry: https://sentry.ceph.com/organizations/ceph/?query=356c97321b1c46d7a6d4f7b47f6202b9

    Command failed on smithi016 with status 1: 'sudo systemctl enable --now
    kubelet && sudo kubeadm config images pull'

[7135761]  rados/singleton-nomsgr/{all/admin_socket_output mon_election/classic rados supported-random-distro$/{centos_8}}
-----------------------------------------------------------------
time:   00:20:21
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135761/
log:    http://qa-proxy.ceph.com/teuthology/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135761/

    "2023-01-24T20:58:38.129584+0000 mon.a (mon.0) 205 : cluster [WRN] Health
    check failed: 1 mgr modules have recently crashed
    (RECENT_MGR_MODULE_CRASH)" in cluster log

[7135763]  rados/upgrade/parallel/{0-random-distro$/{centos_8.stream_container_tools} 0-start 1-tasks mon_election/classic upgrade-sequence workload/{ec-rados-default rados_api rados_loadgenbig rbd_import_export test_rbd_api test_rbd_python}}
-----------------------------------------------------------------
time:   00:36:51
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135763/
log:    http://qa-proxy.ceph.com/teuthology/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135763/
sentry: https://sentry.ceph.com/organizations/ceph/?query=d9678b992b5a4a8fbff8b7f91d6f781a

    Command failed (workunit test cls/test_cls_rbd.sh) on smithi062 with status
    1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd --
    /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1
    CEPH_REF=pacific TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph"
    CEPH_ID="0" PATH=$PATH:/usr/sbin
    CEPH_BASE=/home/ubuntu/cephtest/clone.client.0
    CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0
    CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage
    /home/ubuntu/cephtest/archive/coverage timeout 3h
    /home/ubuntu/cephtest/clone.client.0/qa/workunits/cls/test_cls_rbd.sh'

[7135764]  rados/valgrind-leaks/{1-start 2-inject-leak/mon centos_latest}
-----------------------------------------------------------------
time:   00:20:46
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135764/
log:    http://qa-proxy.ceph.com/teuthology/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135764/

    "2023-01-24T21:03:29.923370+0000 mgr.x (mgr.4100) 1 : cluster [ERR] Failed
    to load ceph-mgr modules: prometheus" in cluster log

[7135765]  rados/thrash/{0-size-min-size-overrides/2-size-2-min-size 1-pg-log-overrides/normal_pg_log 2-recovery-overrides/{more-async-recovery} 3-scrub-overrides/{max-simultaneous-scrubs-2} backoff/peering_and_degraded ceph clusters/{fixed-2 openstack} crc-failures/bad_map_crc_failure d-balancer/crush-compat mon_election/classic msgr-failures/osd-dispatch-delay msgr/async objectstore/bluestore-hybrid rados supported-random-distro$/{centos_8} thrashers/mapgap thrashosds-health workloads/redirect_set_object}
-----------------------------------------------------------------
time:   00:19:57
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135765/
log:    http://qa-proxy.ceph.com/teuthology/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135765/

    "2023-01-24T20:57:01.627667+0000 mgr.x (mgr.4120) 1 : cluster [ERR] Failed
    to load ceph-mgr modules: prometheus" in cluster log

[7135766]  rados/cephadm/workunits/{0-distro/rhel_8.4_container_tools_3.0 agent/on mon_election/connectivity task/test_cephadm}
-----------------------------------------------------------------
time:   00:12:30
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135766/
log:    http://qa-proxy.ceph.com/teuthology/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135766/
sentry: https://sentry.ceph.com/organizations/ceph/?query=cc744cbc556e411bb29f1322ce8715c3

    Command failed (workunit test cephadm/test_cephadm.sh) on smithi088 with
    status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd --
    /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1
    CEPH_REF=49f8fb05584886826e8eade75f7105fba754560c
    TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0"
    PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0
    CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0
    CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage
    /home/ubuntu/cephtest/archive/coverage timeout 3h
    /home/ubuntu/cephtest/clone.client.0/qa/workunits/cephadm/test_cephadm.sh'

[7135767]  rados/monthrash/{ceph clusters/9-mons mon_election/connectivity msgr-failures/few msgr/async-v2only objectstore/filestore-xfs rados supported-random-distro$/{centos_8} thrashers/force-sync-many workloads/rados_mon_workunits}
-----------------------------------------------------------------
time:   00:31:29
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135767/
log:    http://qa-proxy.ceph.com/teuthology/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135767/

    "2023-01-24T21:05:24.596613+0000 mon.d (mon.6) 1500 : cluster [WRN] Health
    check failed: 1 mgr modules have recently crashed
    (RECENT_MGR_MODULE_CRASH)" in cluster log

[7135768]  rados/verify/{centos_latest ceph clusters/{fixed-2 openstack} d-thrash/none mon_election/classic msgr-failures/few msgr/async-v1only objectstore/filestore-xfs rados tasks/rados_api_tests validater/lockdep}
-----------------------------------------------------------------
time:   00:26:04
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135768/
log:    http://qa-proxy.ceph.com/teuthology/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135768/

    "2023-01-24T20:59:15.317840+0000 mgr.x (mgr.4108) 1 : cluster [ERR] Failed
    to load ceph-mgr modules: prometheus" in cluster log

[7135773]  rados/cephadm/smoke/{0-distro/centos_8.stream_container_tools 0-nvme-loop agent/off fixed-2 mon_election/classic start}
-----------------------------------------------------------------
time:   00:21:46
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135773/
log:    http://qa-proxy.ceph.com/teuthology/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135773/
sentry: https://sentry.ceph.com/organizations/ceph/?query=c9a86528e7ca444b9992edf95653b29d

    timeout expired in wait_until_healthy

[7135777]  rados/thrash-erasure-code/{ceph clusters/{fixed-2 openstack} fast/normal mon_election/connectivity msgr-failures/osd-dispatch-delay objectstore/bluestore-low-osd-mem-target rados recovery-overrides/{default} supported-random-distro$/{rhel_8} thrashers/careful thrashosds-health workloads/ec-rados-plugin=jerasure-k=2-m=1}
-----------------------------------------------------------------
time:   00:28:55
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135777/
log:    http://qa-proxy.ceph.com/teuthology/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135777/

    "2023-01-24T21:08:00.044802+0000 mgr.y (mgr.4107) 1 : cluster [ERR] Failed
    to load ceph-mgr modules: prometheus" in cluster log

[7135778]  rados/cephadm/smoke-singlehost/{0-random-distro$/{centos_8.stream_container_tools_crun} 1-start 2-services/basic 3-final}
-----------------------------------------------------------------
time:   00:18:05
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135778/
log:    http://qa-proxy.ceph.com/teuthology/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135778/
sentry: https://sentry.ceph.com/organizations/ceph/?query=d180b2deeffe47e980bb89876edf5b09

    timeout expired in wait_until_healthy

[7135781]  rados/thrash/{0-size-min-size-overrides/2-size-2-min-size 1-pg-log-overrides/normal_pg_log 2-recovery-overrides/{more-partial-recovery} 3-scrub-overrides/{max-simultaneous-scrubs-2} backoff/peering ceph clusters/{fixed-2 openstack} crc-failures/bad_map_crc_failure d-balancer/crush-compat mon_election/classic msgr-failures/few msgr/async-v2only objectstore/bluestore-stupid rados supported-random-distro$/{centos_8} thrashers/none thrashosds-health workloads/small-objects-balanced}
-----------------------------------------------------------------
time:   00:21:44
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135781/
log:    http://qa-proxy.ceph.com/teuthology/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135781/

    "2023-01-24T21:16:56.004506+0000 mon.a (mon.0) 608 : cluster [WRN] Health
    check failed: 1 mgr modules have recently crashed
    (RECENT_MGR_MODULE_CRASH)" in cluster log

[7135782]  rados/basic/{ceph clusters/{fixed-2 openstack} mon_election/connectivity msgr-failures/many msgr/async objectstore/bluestore-hybrid rados supported-random-distro$/{rhel_8} tasks/rados_python}
-----------------------------------------------------------------
time:   00:16:16
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135782/
log:    http://qa-proxy.ceph.com/teuthology/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135782/

    "2023-01-24T21:06:06.611314+0000 mgr.y (mgr.4109) 1 : cluster [ERR] Failed
    to load ceph-mgr modules: prometheus" in cluster log

[7135785]  rados/cephadm/osds/{0-distro/centos_8.stream_container_tools_crun 0-nvme-loop 1-start 2-ops/rm-zap-wait}
-----------------------------------------------------------------
time:   00:18:41
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135785/
log:    http://qa-proxy.ceph.com/teuthology/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135785/
sentry: https://sentry.ceph.com/organizations/ceph/?query=1cbc3aa909004524802099b842bbb428

    timeout expired in wait_until_healthy

[7135787]  rados/thrash/{0-size-min-size-overrides/3-size-2-min-size 1-pg-log-overrides/short_pg_log 2-recovery-overrides/{more-async-partial-recovery} 3-scrub-overrides/{max-simultaneous-scrubs-2} backoff/peering_and_degraded ceph clusters/{fixed-2 openstack} crc-failures/default d-balancer/on mon_election/connectivity msgr-failures/osd-delay msgr/async objectstore/filestore-xfs rados supported-random-distro$/{centos_8} thrashers/pggrow thrashosds-health workloads/small-objects-localized}
-----------------------------------------------------------------
time:   00:22:27
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135787/
log:    http://qa-proxy.ceph.com/teuthology/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135787/

    "2023-01-24T21:12:40.906163+0000 mgr.y (mgr.4103) 1 : cluster [ERR] Failed
    to load ceph-mgr modules: prometheus" in cluster log

[7135790]  rados/singleton/{all/recovery-preemption mon_election/connectivity msgr-failures/many msgr/async-v1only objectstore/bluestore-comp-zstd rados supported-random-distro$/{rhel_8}}
-----------------------------------------------------------------
time:   01:07:53
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135790/
log:    http://qa-proxy.ceph.com/teuthology/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135790/
sentry: https://sentry.ceph.com/organizations/ceph/?query=d702a42e9c0b45239c8e0f6fa9ada139

    timeout expired in wait_until_healthy

[7135794]  rados/monthrash/{ceph clusters/3-mons mon_election/classic msgr-failures/mon-delay msgr/async objectstore/bluestore-bitmap rados supported-random-distro$/{rhel_8} thrashers/many workloads/rados_mon_workunits}
-----------------------------------------------------------------
time:   00:33:27
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135794/
log:    http://qa-proxy.ceph.com/teuthology/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135794/

    "2023-01-24T21:22:12.900152+0000 mon.a (mon.0) 306 : cluster [WRN] Health
    check failed: 1 mgr modules have recently crashed
    (RECENT_MGR_MODULE_CRASH)" in cluster log

[7135797]  rados/thrash/{0-size-min-size-overrides/2-size-2-min-size 1-pg-log-overrides/normal_pg_log 2-recovery-overrides/{more-async-partial-recovery} 3-scrub-overrides/{max-simultaneous-scrubs-2} backoff/normal ceph clusters/{fixed-2 openstack} crc-failures/bad_map_crc_failure d-balancer/crush-compat mon_election/classic msgr-failures/osd-dispatch-delay msgr/async-v1only objectstore/bluestore-bitmap rados supported-random-distro$/{centos_8} thrashers/careful thrashosds-health workloads/small-objects}
-----------------------------------------------------------------
time:   00:23:49
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135797/
log:    http://qa-proxy.ceph.com/teuthology/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135797/

    "2023-01-24T21:19:16.048225+0000 mgr.x (mgr.4106) 1 : cluster [ERR] Failed
    to load ceph-mgr modules: prometheus" in cluster log

[7135799]  rados/verify/{centos_latest ceph clusters/{fixed-2 openstack} d-thrash/default/{default thrashosds-health} mon_election/connectivity msgr-failures/few msgr/async-v2only objectstore/bluestore-bitmap rados tasks/rados_api_tests validater/valgrind}
-----------------------------------------------------------------
time:   00:55:47
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135799/
log:    http://qa-proxy.ceph.com/teuthology/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135799/

    "2023-01-24T21:30:05.145354+0000 mgr.y (mgr.4108) 1 : cluster [ERR] Failed
    to load ceph-mgr modules: prometheus" in cluster log

[7135801]  rados/cephadm/smoke/{0-distro/centos_8.stream_container_tools_crun 0-nvme-loop agent/on fixed-2 mon_election/connectivity start}
-----------------------------------------------------------------
time:   00:21:37
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135801/
log:    http://qa-proxy.ceph.com/teuthology/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135801/
sentry: https://sentry.ceph.com/organizations/ceph/?query=034a7341fce649e59f0b77532d487ce8

    timeout expired in wait_until_healthy

[7135803]  rados/thrash-erasure-code-big/{ceph cluster/{12-osds openstack} mon_election/connectivity msgr-failures/few objectstore/bluestore-comp-lz4 rados recovery-overrides/{more-async-partial-recovery} supported-random-distro$/{rhel_8} thrashers/pggrow thrashosds-health workloads/ec-rados-plugin=jerasure-k=4-m=2}
-----------------------------------------------------------------
time:   00:32:29
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135803/
log:    http://qa-proxy.ceph.com/teuthology/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135803/

    "2023-01-24T21:27:23.445107+0000 mon.a (mon.0) 1198 : cluster [WRN] Health
    check failed: 1 mgr modules have recently crashed
    (RECENT_MGR_MODULE_CRASH)" in cluster log

[7135804]  rados/thrash/{0-size-min-size-overrides/3-size-2-min-size 1-pg-log-overrides/short_pg_log 2-recovery-overrides/{more-async-partial-recovery} 3-scrub-overrides/{default} backoff/peering ceph clusters/{fixed-2 openstack} crc-failures/default d-balancer/on mon_election/connectivity msgr-failures/fastclose msgr/async-v2only objectstore/bluestore-comp-lz4 rados supported-random-distro$/{centos_8} thrashers/default thrashosds-health workloads/snaps-few-objects-balanced}
-----------------------------------------------------------------
time:   00:31:30
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135804/
log:    http://qa-proxy.ceph.com/teuthology/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135804/

    "2023-01-24T21:23:52.124431+0000 mgr.y (mgr.4109) 1 : cluster [ERR] Failed
    to load ceph-mgr modules: prometheus" in cluster log

[7135807]  rados/cephadm/osds/{0-distro/rhel_8.4_container_tools_3.0 0-nvme-loop 1-start 2-ops/rmdir-reactivate}
-----------------------------------------------------------------
time:   00:19:17
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135807/
log:    http://qa-proxy.ceph.com/teuthology/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135807/
sentry: https://sentry.ceph.com/organizations/ceph/?query=72241526cc604ccca61b680e9358e1fb

    timeout expired in wait_until_healthy

[7135808]  rados/singleton/{all/test-crash mon_election/connectivity msgr-failures/few msgr/async objectstore/bluestore-low-osd-mem-target rados supported-random-distro$/{centos_8}}
-----------------------------------------------------------------
time:   00:19:40
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135808/
log:    http://qa-proxy.ceph.com/teuthology/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135808/
sentry: https://sentry.ceph.com/organizations/ceph/?query=9d3317c47bf7456dbc38fe497eddbb74

    Command failed (workunit test rados/test_crash.sh) on smithi182 with status
    1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd --
    /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1
    CEPH_REF=49f8fb05584886826e8eade75f7105fba754560c
    TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0"
    PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0
    CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0
    CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage
    /home/ubuntu/cephtest/archive/coverage timeout 3h
    /home/ubuntu/cephtest/clone.client.0/qa/workunits/rados/test_crash.sh'

[7135810]  rados/thrash/{0-size-min-size-overrides/2-size-2-min-size 1-pg-log-overrides/normal_pg_log 2-recovery-overrides/{default} 3-scrub-overrides/{max-simultaneous-scrubs-2} backoff/peering_and_degraded ceph clusters/{fixed-2 openstack} crc-failures/bad_map_crc_failure d-balancer/crush-compat mon_election/classic msgr-failures/few msgr/async objectstore/bluestore-comp-snappy rados supported-random-distro$/{centos_8} thrashers/mapgap thrashosds-health workloads/snaps-few-objects-localized}
-----------------------------------------------------------------
time:   00:28:52
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135810/
log:    http://qa-proxy.ceph.com/teuthology/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135810/

    "2023-01-24T21:29:48.000575+0000 mgr.x (mgr.4108) 1 : cluster [ERR] Failed
    to load ceph-mgr modules: prometheus" in cluster log

[7135818]  rados/multimon/{clusters/6 mon_election/connectivity msgr-failures/many msgr/async-v2only no_pools objectstore/bluestore-comp-snappy rados supported-random-distro$/{centos_8} tasks/mon_clock_with_skews}
-----------------------------------------------------------------
time:   00:10:45
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135818/
log:    http://qa-proxy.ceph.com/teuthology/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135818/

    "2023-01-24T21:34:09.778010+0000 mgr.x (mgr.4108) 1 : cluster [ERR] Failed
    to load ceph-mgr modules: prometheus" in cluster log

[7135819]  rados/cephadm/smoke/{0-distro/rhel_8.4_container_tools_3.0 0-nvme-loop agent/off fixed-2 mon_election/classic start}
-----------------------------------------------------------------
time:   00:22:00
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135819/
log:    http://qa-proxy.ceph.com/teuthology/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135819/
sentry: https://sentry.ceph.com/organizations/ceph/?query=7a4113d96b2947dda93d1dc7395ed343

    timeout expired in wait_until_healthy

[7135823]  rados/verify/{centos_latest ceph clusters/{fixed-2 openstack} d-thrash/none mon_election/classic msgr-failures/few msgr/async objectstore/bluestore-comp-lz4 rados tasks/rados_cls_all validater/lockdep}
-----------------------------------------------------------------
time:   00:19:06
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135823/
log:    http://qa-proxy.ceph.com/teuthology/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135823/

    "2023-01-24T21:36:49.405373+0000 mgr.y (mgr.4101) 1 : cluster [ERR] Failed
    to load ceph-mgr modules: prometheus" in cluster log

[7135824]  rados/monthrash/{ceph clusters/9-mons mon_election/connectivity msgr-failures/few msgr/async-v1only objectstore/bluestore-comp-lz4 rados supported-random-distro$/{rhel_8} thrashers/one workloads/snaps-few-objects}
-----------------------------------------------------------------
time:   00:38:05
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135824/
log:    http://qa-proxy.ceph.com/teuthology/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135824/

    "2023-01-24T21:43:05.244221+0000 mon.b (mon.2) 898 : cluster [WRN] Health
    check failed: 1 mgr modules have recently crashed
    (RECENT_MGR_MODULE_CRASH)" in cluster log

[7135825]  rados/cephadm/osds/{0-distro/rhel_8.4_container_tools_3.0 0-nvme-loop 1-start 2-ops/repave-all}
-----------------------------------------------------------------
time:   00:19:07
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135825/
log:    http://qa-proxy.ceph.com/teuthology/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135825/
sentry: https://sentry.ceph.com/organizations/ceph/?query=d437fa2c903d47219ea8fc7888ef8e37

    timeout expired in wait_until_healthy

[7135826]  rados/thrash/{0-size-min-size-overrides/2-size-2-min-size 1-pg-log-overrides/normal_pg_log 2-recovery-overrides/{more-async-recovery} 3-scrub-overrides/{default} backoff/peering ceph clusters/{fixed-2 openstack} crc-failures/bad_map_crc_failure d-balancer/crush-compat mon_election/classic msgr-failures/osd-dispatch-delay msgr/async-v2only objectstore/bluestore-comp-zstd rados supported-random-distro$/{rhel_8} thrashers/none thrashosds-health workloads/write_fadvise_dontneed}
-----------------------------------------------------------------
time:   00:16:41
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135826/
log:    http://qa-proxy.ceph.com/teuthology/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135826/

    "2023-01-24T21:39:35.626582+0000 mgr.y (mgr.4116) 1 : cluster [ERR] Failed
    to load ceph-mgr modules: prometheus" in cluster log

[7135827]  rados/mgr/{clusters/{2-node-mgr} debug/mgr mgr_ttl_cache/disable mon_election/classic random-objectstore$/{bluestore-bitmap} supported-random-distro$/{centos_8} tasks/insights}
-----------------------------------------------------------------
time:   00:16:44
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135827/
log:    http://qa-proxy.ceph.com/teuthology/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135827/

    "2023-01-24T21:42:14.373170+0000 mgr.y (mgr.4114) 1 : cluster [ERR] Failed
    to load ceph-mgr modules: prometheus" in cluster log

[7135828]  rados/thrash-erasure-code-shec/{ceph clusters/{fixed-4 openstack} mon_election/connectivity msgr-failures/fastclose objectstore/bluestore-comp-lz4 rados recovery-overrides/{default} supported-random-distro$/{rhel_8} thrashers/default thrashosds-health workloads/ec-rados-plugin=shec-k=4-m=3-c=2}
-----------------------------------------------------------------
time:   00:15:21
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135828/
log:    http://qa-proxy.ceph.com/teuthology/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135828/

    "2023-01-24T21:41:50.907501+0000 mgr.x (mgr.4103) 1 : cluster [ERR] Failed
    to load ceph-mgr modules: prometheus" in cluster log

[7135829]  rados/basic/{ceph clusters/{fixed-2 openstack} mon_election/connectivity msgr-failures/many msgr/async-v2only objectstore/bluestore-stupid rados supported-random-distro$/{rhel_8} tasks/rados_striper}
-----------------------------------------------------------------
time:   00:10:38
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135829/
log:    http://qa-proxy.ceph.com/teuthology/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135829/

    "2023-01-24T21:38:26.444406+0000 mgr.y (mgr.4104) 1 : cluster [ERR] Failed
    to load ceph-mgr modules: prometheus" in cluster log

[7135830]  rados/singleton-bluestore/{all/cephtool mon_election/classic msgr-failures/none msgr/async-v1only objectstore/bluestore-bitmap rados supported-random-distro$/{rhel_8}}
-----------------------------------------------------------------
time:   00:29:19
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135830/
log:    http://qa-proxy.ceph.com/teuthology/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135830/
sentry: https://sentry.ceph.com/organizations/ceph/?query=4166787ca83247b081b5e489372477a2

    Command failed (workunit test cephtool/test.sh) on smithi007 with status 1:
    'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd --
    /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1
    CEPH_REF=49f8fb05584886826e8eade75f7105fba754560c
    TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0"
    PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0
    CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0
    CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage
    /home/ubuntu/cephtest/archive/coverage timeout 3h
    /home/ubuntu/cephtest/clone.client.0/qa/workunits/cephtool/test.sh'

[7135836]  rados/thrash/{0-size-min-size-overrides/3-size-2-min-size 1-pg-log-overrides/short_pg_log 2-recovery-overrides/{default} 3-scrub-overrides/{max-simultaneous-scrubs-3} backoff/peering_and_degraded ceph clusters/{fixed-2 openstack} crc-failures/default d-balancer/on mon_election/connectivity msgr-failures/fastclose msgr/async objectstore/bluestore-hybrid rados supported-random-distro$/{centos_8} thrashers/pggrow thrashosds-health workloads/admin_socket_objecter_requests}
-----------------------------------------------------------------
time:   00:45:43
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135836/
log:    http://qa-proxy.ceph.com/teuthology/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135836/

    "2023-01-24T21:44:19.011033+0000 mgr.x (mgr.4109) 1 : cluster [ERR] Failed
    to load ceph-mgr modules: prometheus" in cluster log

[7135839]  rados/rook/smoke/{0-distro/ubuntu_20.04 0-kubeadm 0-nvme-loop 1-rook 2-workload/none 3-final cluster/3-node k8s/1.21 net/flannel rook/master}
-----------------------------------------------------------------
time:   00:05:44
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135839/
log:    http://qa-proxy.ceph.com/teuthology/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135839/
sentry: https://sentry.ceph.com/organizations/ceph/?query=10947fab7c4f4d5aaf5676ac723903f4

    Command failed on smithi016 with status 1: 'sudo systemctl enable --now
    kubelet && sudo kubeadm config images pull'

[7135841]  rados/thrash-erasure-code/{ceph clusters/{fixed-2 openstack} fast/normal mon_election/connectivity msgr-failures/few objectstore/filestore-xfs rados recovery-overrides/{more-async-recovery} supported-random-distro$/{rhel_8} thrashers/fastread thrashosds-health workloads/ec-radosbench}
-----------------------------------------------------------------
time:   01:35:28
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135841/
log:    http://qa-proxy.ceph.com/teuthology/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135841/

    "2023-01-24T21:50:17.449437+0000 mgr.x (mgr.4108) 1 : cluster [ERR] Failed
    to load ceph-mgr modules: prometheus" in cluster log

[7135842]  rados/singleton/{all/thrash-eio mon_election/connectivity msgr-failures/many msgr/async-v1only objectstore/bluestore-comp-lz4 rados supported-random-distro$/{rhel_8}}
-----------------------------------------------------------------
time:   00:42:32
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135842/
log:    http://qa-proxy.ceph.com/teuthology/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135842/

    "2023-01-24T21:56:06.135126+0000 mon.a (mon.0) 797 : cluster [WRN] Health
    check failed: 1 mgr modules have recently crashed
    (RECENT_MGR_MODULE_CRASH)" in cluster log

[7135844]  rados/cephadm/smoke/{0-distro/rhel_8.4_container_tools_rhel8 0-nvme-loop agent/on fixed-2 mon_election/connectivity start}
-----------------------------------------------------------------
time:   00:21:32
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135844/
log:    http://qa-proxy.ceph.com/teuthology/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135844/
sentry: https://sentry.ceph.com/organizations/ceph/?query=5d14bbf6206f46da9dc55db3d3836f94

    timeout expired in wait_until_healthy

[7135845]  rados/thrash/{0-size-min-size-overrides/2-size-2-min-size 1-pg-log-overrides/normal_pg_log 2-recovery-overrides/{more-async-partial-recovery} 3-scrub-overrides/{max-simultaneous-scrubs-2} backoff/normal ceph clusters/{fixed-2 openstack} crc-failures/bad_map_crc_failure d-balancer/crush-compat mon_election/classic msgr-failures/few msgr/async-v1only objectstore/bluestore-low-osd-mem-target rados supported-random-distro$/{centos_8} thrashers/careful thrashosds-health workloads/cache-agent-big}
-----------------------------------------------------------------
time:   00:25:10
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135845/
log:    http://qa-proxy.ceph.com/teuthology/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135845/

    "2023-01-24T21:54:02.565148+0000 mgr.x (mgr.4109) 1 : cluster [ERR] Failed
    to load ceph-mgr modules: prometheus" in cluster log

[7135846]  rados/thrash-erasure-code-isa/{arch/x86_64 ceph clusters/{fixed-2 openstack} mon_election/connectivity msgr-failures/osd-delay objectstore/bluestore-comp-snappy rados recovery-overrides/{more-async-recovery} supported-random-distro$/{centos_8} thrashers/careful thrashosds-health workloads/ec-rados-plugin=isa-k=2-m=1}
-----------------------------------------------------------------
time:   00:28:34
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135846/
log:    http://qa-proxy.ceph.com/teuthology/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135846/

    "2023-01-24T21:55:05.812247+0000 mgr.x (mgr.4104) 1 : cluster [ERR] Failed
    to load ceph-mgr modules: prometheus" in cluster log

[7135850]  rados/cephadm/smoke-singlehost/{0-random-distro$/{ubuntu_20.04} 1-start 2-services/rgw 3-final}
-----------------------------------------------------------------
time:   00:19:26
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135850/
log:    http://qa-proxy.ceph.com/teuthology/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135850/
sentry: https://sentry.ceph.com/organizations/ceph/?query=82a5c245facc47bc9fa407001da5a399

    timeout expired in wait_until_healthy

[7135853]  rados/basic/{ceph clusters/{fixed-2 openstack} mon_election/classic msgr-failures/few msgr/async objectstore/filestore-xfs rados supported-random-distro$/{rhel_8} tasks/rados_workunit_loadgen_big}
-----------------------------------------------------------------
time:   00:33:49
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135853/
log:    http://qa-proxy.ceph.com/teuthology/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135853/

    "2023-01-24T21:55:40.261872+0000 mgr.x (mgr.4104) 1 : cluster [ERR] Failed
    to load ceph-mgr modules: prometheus" in cluster log

[7135854]  rados/singleton/{all/thrash_cache_writeback_proxy_none mon_election/connectivity msgr-failures/few msgr/async objectstore/bluestore-comp-zlib rados supported-random-distro$/{centos_8}}
-----------------------------------------------------------------
time:   00:23:23
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135854/
log:    http://qa-proxy.ceph.com/teuthology/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135854/

    "2023-01-24T22:09:03.400342+0000 mon.a (mon.0) 209 : cluster [WRN] Health
    check failed: 1 mgr modules have recently crashed
    (RECENT_MGR_MODULE_CRASH)" in cluster log

[7135857]  rados/verify/{centos_latest ceph clusters/{fixed-2 openstack} d-thrash/default/{default thrashosds-health} mon_election/connectivity msgr-failures/few msgr/async-v1only objectstore/bluestore-comp-snappy rados tasks/mon_recovery validater/valgrind}
-----------------------------------------------------------------
time:   00:31:50
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135857/
log:    http://qa-proxy.ceph.com/teuthology/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135857/

    "2023-01-24T22:18:27.748946+0000 mgr.y (mgr.4107) 1 : cluster [ERR] Failed
    to load ceph-mgr modules: prometheus" in cluster log

[7135859]  rados/monthrash/{ceph clusters/3-mons mon_election/classic msgr-failures/mon-delay msgr/async-v2only objectstore/bluestore-comp-snappy rados supported-random-distro$/{centos_8} thrashers/sync-many workloads/pool-create-delete}
-----------------------------------------------------------------
time:   00:20:25
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135859/
log:    http://qa-proxy.ceph.com/teuthology/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135859/

    "2023-01-24T22:13:13.454322+0000 mon.a (mon.0) 14 : cluster [WRN] Health
    check failed: 1 mgr modules have recently crashed
    (RECENT_MGR_MODULE_CRASH)" in cluster log

[7135860]  rados/thrash-erasure-code-shec/{ceph clusters/{fixed-4 openstack} mon_election/classic msgr-failures/few objectstore/bluestore-comp-snappy rados recovery-overrides/{more-async-recovery} supported-random-distro$/{rhel_8} thrashers/careful thrashosds-health workloads/ec-rados-plugin=shec-k=4-m=3-c=2}
-----------------------------------------------------------------
time:   00:15:45
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135860/
log:    http://qa-proxy.ceph.com/teuthology/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135860/

    "2023-01-24T22:04:19.475653+0000 mgr.y (mgr.4107) 1 : cluster [ERR] Failed
    to load ceph-mgr modules: prometheus" in cluster log

[7135862]  rados/mgr/{clusters/{2-node-mgr} debug/mgr mgr_ttl_cache/enable mon_election/connectivity random-objectstore$/{bluestore-comp-lz4} supported-random-distro$/{centos_8} tasks/module_selftest}
-----------------------------------------------------------------
time:   00:20:41
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135862/
log:    http://qa-proxy.ceph.com/teuthology/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135862/
sentry: https://sentry.ceph.com/organizations/ceph/?query=1c0adc1608f34c3e8d65ad79c2a6d6fa

    Test failure: test_prometheus
    (tasks.mgr.test_module_selftest.TestModuleSelftest)

[7135863]  rados/cephadm/osds/{0-distro/rhel_8.4_container_tools_rhel8 0-nvme-loop 1-start 2-ops/rm-zap-add}
-----------------------------------------------------------------
time:   00:18:57
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135863/
log:    http://qa-proxy.ceph.com/teuthology/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135863/
sentry: https://sentry.ceph.com/organizations/ceph/?query=417d74f6e65c480e956589709a25f9ce

    timeout expired in wait_until_healthy

[7135865]  rados/thrash-erasure-code-big/{ceph cluster/{12-osds openstack} mon_election/connectivity msgr-failures/osd-dispatch-delay objectstore/bluestore-comp-zlib rados recovery-overrides/{default} supported-random-distro$/{rhel_8} thrashers/default thrashosds-health workloads/ec-rados-plugin=jerasure-k=4-m=2}
-----------------------------------------------------------------
time:   00:33:21
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135865/
log:    http://qa-proxy.ceph.com/teuthology/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135865/

    "2023-01-24T22:12:43.692085+0000 mon.a (mon.0) 1240 : cluster [WRN] Health
    check failed: 1 mgr modules have recently crashed
    (RECENT_MGR_MODULE_CRASH)" in cluster log

[7135868]  rados/valgrind-leaks/{1-start 2-inject-leak/none centos_latest}
-----------------------------------------------------------------
time:   00:22:56
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135868/
log:    http://qa-proxy.ceph.com/teuthology/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135868/

    "2023-01-24T22:17:45.248035+0000 mgr.y (mgr.4100) 1 : cluster [ERR] Failed
    to load ceph-mgr modules: prometheus" in cluster log

[7135869]  rados/thrash-erasure-code-overwrites/{bluestore-bitmap ceph clusters/{fixed-2 openstack} fast/fast mon_election/classic msgr-failures/osd-dispatch-delay rados recovery-overrides/{more-partial-recovery} supported-random-distro$/{centos_8} thrashers/careful thrashosds-health workloads/ec-pool-snaps-few-objects-overwrites}
-----------------------------------------------------------------
time:   00:33:31
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135869/
log:    http://qa-proxy.ceph.com/teuthology/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135869/

    "2023-01-24T22:08:50.181975+0000 mgr.x (mgr.4108) 1 : cluster [ERR] Failed
    to load ceph-mgr modules: prometheus" in cluster log

[7135870]  rados/thrash/{0-size-min-size-overrides/3-size-2-min-size 1-pg-log-overrides/short_pg_log 2-recovery-overrides/{more-partial-recovery} 3-scrub-overrides/{default} backoff/normal ceph clusters/{fixed-2 openstack} crc-failures/default d-balancer/on mon_election/connectivity msgr-failures/fastclose msgr/async-v1only objectstore/bluestore-bitmap rados supported-random-distro$/{rhel_8} thrashers/morepggrow thrashosds-health workloads/cache-pool-snaps}
-----------------------------------------------------------------
time:   00:25:37
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135870/
log:    http://qa-proxy.ceph.com/teuthology/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135870/

    "2023-01-24T22:08:18.967039+0000 mgr.y (mgr.4109) 1 : cluster [ERR] Failed
    to load ceph-mgr modules: prometheus" in cluster log

[7135871]  rados/cephadm/smoke/{0-distro/ubuntu_20.04 0-nvme-loop agent/off fixed-2 mon_election/classic start}
-----------------------------------------------------------------
time:   00:26:56
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135871/
log:    http://qa-proxy.ceph.com/teuthology/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135871/
sentry: https://sentry.ceph.com/organizations/ceph/?query=ba07b5b13cb9427db82c18bb133d4362

    timeout expired in wait_until_healthy

[7135872]  rados/thrash-erasure-code/{ceph clusters/{fixed-2 openstack} fast/fast mon_election/classic msgr-failures/osd-delay objectstore/bluestore-bitmap rados recovery-overrides/{more-active-recovery} supported-random-distro$/{centos_8} thrashers/minsize_recovery thrashosds-health workloads/ec-small-objects-balanced}
-----------------------------------------------------------------
time:   00:23:03
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135872/
log:    http://qa-proxy.ceph.com/teuthology/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135872/

    "2023-01-24T22:11:18.015749+0000 mgr.x (mgr.4111) 1 : cluster [ERR] Failed
    to load ceph-mgr modules: prometheus" in cluster log

[7135876]  rados/thrash/{0-size-min-size-overrides/2-size-2-min-size 1-pg-log-overrides/normal_pg_log 2-recovery-overrides/{more-async-partial-recovery} 3-scrub-overrides/{default} backoff/peering ceph clusters/{fixed-2 openstack} crc-failures/bad_map_crc_failure d-balancer/crush-compat mon_election/classic msgr-failures/few msgr/async-v2only objectstore/bluestore-comp-lz4 rados supported-random-distro$/{rhel_8} thrashers/none thrashosds-health workloads/cache-snaps-balanced}
-----------------------------------------------------------------
time:   00:23:56
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135876/
log:    http://qa-proxy.ceph.com/teuthology/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135876/

    "2023-01-24T22:15:24.421825+0000 mgr.x (mgr.4120) 1 : cluster [ERR] Failed
    to load ceph-mgr modules: prometheus" in cluster log

[7135877]  rados/thrash-erasure-code-isa/{arch/x86_64 ceph clusters/{fixed-2 openstack} mon_election/classic msgr-failures/osd-dispatch-delay objectstore/bluestore-comp-zlib rados recovery-overrides/{more-async-partial-recovery} supported-random-distro$/{rhel_8} thrashers/default thrashosds-health workloads/ec-rados-plugin=isa-k=2-m=1}
-----------------------------------------------------------------
time:   00:33:21
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135877/
log:    http://qa-proxy.ceph.com/teuthology/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135877/

    "2023-01-24T22:18:22.533010+0000 mgr.x (mgr.4108) 1 : cluster [ERR] Failed
    to load ceph-mgr modules: prometheus" in cluster log

[7135878]  rados/basic/{ceph clusters/{fixed-2 openstack} mon_election/connectivity msgr-failures/many msgr/async-v1only objectstore/bluestore-bitmap rados supported-random-distro$/{rhel_8} tasks/rados_workunit_loadgen_mix}
-----------------------------------------------------------------
time:   00:21:46
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135878/
log:    http://qa-proxy.ceph.com/teuthology/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135878/

    "2023-01-24T22:18:45.567578+0000 mgr.y (mgr.4107) 1 : cluster [ERR] Failed
    to load ceph-mgr modules: prometheus" in cluster log

[7135885]  rados/cephadm/osds/{0-distro/ubuntu_20.04 0-nvme-loop 1-start 2-ops/rm-zap-flag}
-----------------------------------------------------------------
time:   00:20:09
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135885/
log:    http://qa-proxy.ceph.com/teuthology/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135885/
sentry: https://sentry.ceph.com/organizations/ceph/?query=97a8e00dc4c44146a1c082133fe734ad

    timeout expired in wait_until_healthy

[7135888]  rados/verify/{centos_latest ceph clusters/{fixed-2 openstack} d-thrash/none mon_election/classic msgr-failures/few msgr/async-v2only objectstore/bluestore-comp-zlib rados tasks/rados_api_tests validater/lockdep}
-----------------------------------------------------------------
time:   00:26:51
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135888/
log:    http://qa-proxy.ceph.com/teuthology/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135888/

    "2023-01-24T22:27:49.278369+0000 mgr.x (mgr.4105) 1 : cluster [ERR] Failed
    to load ceph-mgr modules: prometheus" in cluster log

[7135892]  rados/cephadm/smoke/{0-distro/centos_8.stream_container_tools 0-nvme-loop agent/on fixed-2 mon_election/connectivity start}
-----------------------------------------------------------------
time:   00:21:45
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135892/
log:    http://qa-proxy.ceph.com/teuthology/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135892/
sentry: https://sentry.ceph.com/organizations/ceph/?query=17ea42e85c97428d9783fbd429a8e6a1

    timeout expired in wait_until_healthy

[7135893]  rados/thrash/{0-size-min-size-overrides/2-size-2-min-size 1-pg-log-overrides/normal_pg_log 2-recovery-overrides/{more-active-recovery} 3-scrub-overrides/{default} backoff/normal ceph clusters/{fixed-2 openstack} crc-failures/bad_map_crc_failure d-balancer/crush-compat mon_election/classic msgr-failures/osd-dispatch-delay msgr/async-v1only objectstore/bluestore-comp-zlib rados supported-random-distro$/{centos_8} thrashers/careful thrashosds-health workloads/cache}
-----------------------------------------------------------------
time:   00:17:34
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135893/
log:    http://qa-proxy.ceph.com/teuthology/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135893/

    "2023-01-24T22:34:13.964462+0000 mgr.y (mgr.4107) 1 : cluster [ERR] Failed
    to load ceph-mgr modules: prometheus" in cluster log

[7135894]  rados/mgr/{clusters/{2-node-mgr} debug/mgr mgr_ttl_cache/disable mon_election/classic random-objectstore$/{bluestore-comp-lz4} supported-random-distro$/{rhel_8} tasks/progress}
-----------------------------------------------------------------
time:   00:29:44
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135894/
log:    http://qa-proxy.ceph.com/teuthology/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135894/

    "2023-01-24T22:31:59.424548+0000 mgr.z (mgr.4113) 1 : cluster [ERR] Failed
    to load ceph-mgr modules: prometheus" in cluster log

[7135897]  rados/cephadm/workunits/{0-distro/rhel_8.4_container_tools_rhel8 agent/on mon_election/connectivity task/test_cephadm}
-----------------------------------------------------------------
time:   00:12:22
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135897/
log:    http://qa-proxy.ceph.com/teuthology/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135897/
sentry: https://sentry.ceph.com/organizations/ceph/?query=1cb10ae5d7764307bf1314cc98fe98d2

    Command failed (workunit test cephadm/test_cephadm.sh) on smithi195 with
    status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd --
    /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1
    CEPH_REF=49f8fb05584886826e8eade75f7105fba754560c
    TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0"
    PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0
    CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0
    CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage
    /home/ubuntu/cephtest/archive/coverage timeout 3h
    /home/ubuntu/cephtest/clone.client.0/qa/workunits/cephadm/test_cephadm.sh'

[7135898]  rados/singleton-nomsgr/{all/recovery-unfound-found mon_election/classic rados supported-random-distro$/{centos_8}}
-----------------------------------------------------------------
time:   00:36:58
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135898/
log:    http://qa-proxy.ceph.com/teuthology/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135898/
sentry: https://sentry.ceph.com/organizations/ceph/?query=633627aa97f4447d8c2c49558f40a9e2

    timeout expired in wait_until_healthy

[7135901]  rados/basic/{ceph clusters/{fixed-2 openstack} mon_election/classic msgr-failures/few msgr/async-v2only objectstore/bluestore-comp-lz4 rados supported-random-distro$/{centos_8} tasks/rados_workunit_loadgen_mostlyread}
-----------------------------------------------------------------
time:   00:27:44
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135901/
log:    http://qa-proxy.ceph.com/teuthology/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135901/

    "2023-01-24T22:40:16.203457+0000 mgr.y (mgr.4099) 1 : cluster [ERR] Failed
    to load ceph-mgr modules: prometheus" in cluster log

[7135903]  rados/thrash-erasure-code/{ceph clusters/{fixed-2 openstack} fast/normal mon_election/connectivity msgr-failures/osd-dispatch-delay objectstore/bluestore-comp-lz4 rados recovery-overrides/{more-async-recovery} supported-random-distro$/{centos_8} thrashers/morepggrow thrashosds-health workloads/ec-small-objects-fast-read}
-----------------------------------------------------------------
time:   00:24:19
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135903/
log:    http://qa-proxy.ceph.com/teuthology/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135903/

    "2023-01-24T22:41:36.098358+0000 mgr.x (mgr.4109) 1 : cluster [ERR] Failed
    to load ceph-mgr modules: prometheus" in cluster log

[7135906]  rados/singleton-bluestore/{all/cephtool mon_election/connectivity msgr-failures/few msgr/async-v2only objectstore/bluestore-comp-lz4 rados supported-random-distro$/{rhel_8}}
-----------------------------------------------------------------
time:   00:28:18
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135906/
log:    http://qa-proxy.ceph.com/teuthology/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135906/
sentry: https://sentry.ceph.com/organizations/ceph/?query=c0b7631b6fc04ea690a74f42bf447d42

    Command failed (workunit test cephtool/test.sh) on smithi093 with status 1:
    'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd --
    /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1
    CEPH_REF=49f8fb05584886826e8eade75f7105fba754560c
    TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0"
    PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0
    CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0
    CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage
    /home/ubuntu/cephtest/archive/coverage timeout 3h
    /home/ubuntu/cephtest/clone.client.0/qa/workunits/cephtool/test.sh'

[7135907]  rados/cephadm/osds/{0-distro/centos_8.stream_container_tools 0-nvme-loop 1-start 2-ops/rm-zap-wait}
-----------------------------------------------------------------
time:   00:19:02
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135907/
log:    http://qa-proxy.ceph.com/teuthology/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135907/
sentry: https://sentry.ceph.com/organizations/ceph/?query=be2bb3c304744718af3fbf0474e24e28

    timeout expired in wait_until_healthy

[7135911]  rados/thrash-erasure-code-overwrites/{bluestore-bitmap ceph clusters/{fixed-2 openstack} fast/normal mon_election/connectivity msgr-failures/fastclose rados recovery-overrides/{default} supported-random-distro$/{centos_8} thrashers/default thrashosds-health workloads/ec-small-objects-fast-read-overwrites}
-----------------------------------------------------------------
time:   00:28:13
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135911/
log:    http://qa-proxy.ceph.com/teuthology/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135911/

    "2023-01-24T22:44:38.408869+0000 mgr.y (mgr.4113) 1 : cluster [ERR] Failed
    to load ceph-mgr modules: prometheus" in cluster log

[7135913]  rados/cephadm/smoke/{0-distro/centos_8.stream_container_tools_crun 0-nvme-loop agent/off fixed-2 mon_election/classic start}
-----------------------------------------------------------------
time:   00:21:20
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135913/
log:    http://qa-proxy.ceph.com/teuthology/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135913/
sentry: https://sentry.ceph.com/organizations/ceph/?query=093f8897851241249543057c9a381b62

    timeout expired in wait_until_healthy

[7135916]  rados/dashboard/{0-single-container-host debug/mgr mon_election/connectivity random-objectstore$/{bluestore-comp-zlib} tasks/dashboard}
-----------------------------------------------------------------
time:   00:20:14
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135916/
log:    http://qa-proxy.ceph.com/teuthology/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135916/
sentry: https://sentry.ceph.com/organizations/ceph/?query=6dbe129f161d40449e300b35a1e33ebe

    Test failure: test_remove_from_blocklist
    (tasks.mgr.dashboard.test_auth.AuthTest)

[7135917]  rados/rook/smoke/{0-distro/ubuntu_20.04 0-kubeadm 0-nvme-loop 1-rook 2-workload/radosbench 3-final cluster/1-node k8s/1.21 net/host rook/1.7.2}
-----------------------------------------------------------------
time:   00:05:36
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135917/
log:    http://qa-proxy.ceph.com/teuthology/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135917/
sentry: https://sentry.ceph.com/organizations/ceph/?query=b008e3b1e8aa4d168fbc5dcd48208de3

    Command failed on smithi026 with status 1: 'sudo systemctl enable --now
    kubelet && sudo kubeadm config images pull'

[7135918]  rados/singleton-nomsgr/{all/admin_socket_output mon_election/connectivity rados supported-random-distro$/{centos_8}}
-----------------------------------------------------------------
time:   00:21:31
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135918/
log:    http://qa-proxy.ceph.com/teuthology/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135918/

    "2023-01-24T22:57:27.629931+0000 mon.a (mon.0) 206 : cluster [WRN] Health
    check failed: 1 mgr modules have recently crashed
    (RECENT_MGR_MODULE_CRASH)" in cluster log

[7135919]  rados/upgrade/parallel/{0-random-distro$/{ubuntu_20.04} 0-start 1-tasks mon_election/connectivity upgrade-sequence workload/{ec-rados-default rados_api rados_loadgenbig rbd_import_export test_rbd_api test_rbd_python}}
-----------------------------------------------------------------
time:   00:39:32
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135919/
log:    http://qa-proxy.ceph.com/teuthology/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135919/
sentry: https://sentry.ceph.com/organizations/ceph/?query=966718d8d408459b9d5257a3ef60916b

    Command failed (workunit test cls/test_cls_rbd.sh) on smithi012 with status
    1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd --
    /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1
    CEPH_REF=pacific TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph"
    CEPH_ID="0" PATH=$PATH:/usr/sbin
    CEPH_BASE=/home/ubuntu/cephtest/clone.client.0
    CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0
    CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage
    /home/ubuntu/cephtest/archive/coverage timeout 3h
    /home/ubuntu/cephtest/clone.client.0/qa/workunits/cls/test_cls_rbd.sh'

[7135920]  rados/singleton/{all/ec-inconsistent-hinfo mon_election/connectivity msgr-failures/none msgr/async-v2only objectstore/bluestore-comp-snappy rados supported-random-distro$/{rhel_8}}
-----------------------------------------------------------------
time:   01:04:02
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135920/
log:    http://qa-proxy.ceph.com/teuthology/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135920/

    "2023-01-24T22:53:16.179749+0000 mon.a (mon.0) 416 : cluster [WRN] Health
    check failed: 1 mgr modules have recently crashed
    (RECENT_MGR_MODULE_CRASH)" in cluster log

[7135921]  rados/verify/{centos_latest ceph clusters/{fixed-2 openstack} d-thrash/default/{default thrashosds-health} mon_election/connectivity msgr-failures/few msgr/async objectstore/bluestore-comp-zstd rados tasks/rados_cls_all validater/valgrind}
-----------------------------------------------------------------
time:   02:23:39
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135921/
log:    http://qa-proxy.ceph.com/teuthology/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135921/

    "2023-01-24T23:00:24.449184+0000 mgr.y (mgr.4108) 1 : cluster [ERR] Failed
    to load ceph-mgr modules: prometheus" in cluster log

[7135922]  rados/thrash-erasure-code-shec/{ceph clusters/{fixed-4 openstack} mon_election/classic msgr-failures/osd-dispatch-delay objectstore/bluestore-comp-zstd rados recovery-overrides/{more-async-partial-recovery} supported-random-distro$/{rhel_8} thrashers/careful thrashosds-health workloads/ec-rados-plugin=shec-k=4-m=3-c=2}
-----------------------------------------------------------------
time:   00:15:35
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135922/
log:    http://qa-proxy.ceph.com/teuthology/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135922/

    "2023-01-24T22:50:17.450068+0000 mgr.y (mgr.4106) 1 : cluster [ERR] Failed
    to load ceph-mgr modules: prometheus" in cluster log

[7135926]  rados/thrash/{0-size-min-size-overrides/2-size-2-min-size 1-pg-log-overrides/normal_pg_log 2-recovery-overrides/{more-partial-recovery} 3-scrub-overrides/{max-simultaneous-scrubs-2} backoff/peering ceph clusters/{fixed-2 openstack} crc-failures/bad_map_crc_failure d-balancer/crush-compat mon_election/classic msgr-failures/osd-dispatch-delay msgr/async-v2only objectstore/bluestore-stupid rados supported-random-distro$/{rhel_8} thrashers/none thrashosds-health workloads/rados_api_tests}
-----------------------------------------------------------------
time:   00:37:19
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135926/
log:    http://qa-proxy.ceph.com/teuthology/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135926/

    "2023-01-24T22:50:15.773071+0000 mgr.y (mgr.4106) 1 : cluster [ERR] Failed
    to load ceph-mgr modules: prometheus" in cluster log

[7135927]  rados/basic/{ceph clusters/{fixed-2 openstack} mon_election/connectivity msgr-failures/many msgr/async objectstore/bluestore-comp-snappy rados supported-random-distro$/{centos_8} tasks/readwrite}
-----------------------------------------------------------------
time:   00:14:09
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135927/
log:    http://qa-proxy.ceph.com/teuthology/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135927/

    "2023-01-24T22:51:07.260641+0000 mgr.y (mgr.4106) 1 : cluster [ERR] Failed
    to load ceph-mgr modules: prometheus" in cluster log

[7135929]  rados/thrash-erasure-code-big/{ceph cluster/{12-osds openstack} mon_election/connectivity msgr-failures/few objectstore/bluestore-hybrid rados recovery-overrides/{more-async-partial-recovery} supported-random-distro$/{rhel_8} thrashers/mapgap thrashosds-health workloads/ec-rados-plugin=jerasure-k=4-m=2}
-----------------------------------------------------------------
time:   00:30:16
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135929/
log:    http://qa-proxy.ceph.com/teuthology/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135929/

    "2023-01-24T23:00:00.742841+0000 mon.a (mon.0) 1155 : cluster [WRN] Health
    check failed: 1 mgr modules have recently crashed
    (RECENT_MGR_MODULE_CRASH)" in cluster log

[7135930]  rados/singleton/{all/ec-lost-unfound mon_election/classic msgr-failures/few msgr/async objectstore/bluestore-comp-zlib rados supported-random-distro$/{centos_8}}
-----------------------------------------------------------------
time:   01:09:50
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135930/
log:    http://qa-proxy.ceph.com/teuthology/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135930/

    "2023-01-24T23:02:49.484523+0000 mon.a (mon.0) 301 : cluster [WRN] Health
    check failed: 1 mgr modules have recently crashed
    (RECENT_MGR_MODULE_CRASH)" in cluster log

[7135931]  rados/cephadm/smoke-singlehost/{0-random-distro$/{centos_8.stream_container_tools_crun} 1-start 2-services/basic 3-final}
-----------------------------------------------------------------
time:   00:18:04
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135931/
log:    http://qa-proxy.ceph.com/teuthology/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135931/
sentry: https://sentry.ceph.com/organizations/ceph/?query=1b787fc81b7d4c5388e434a69c5c8719

    timeout expired in wait_until_healthy

[7135935]  rados/thrash/{0-size-min-size-overrides/3-size-2-min-size 1-pg-log-overrides/short_pg_log 2-recovery-overrides/{more-async-partial-recovery} 3-scrub-overrides/{max-simultaneous-scrubs-2} backoff/peering_and_degraded ceph clusters/{fixed-2 openstack} crc-failures/default d-balancer/on mon_election/connectivity msgr-failures/fastclose msgr/async objectstore/filestore-xfs rados supported-random-distro$/{rhel_8} thrashers/pggrow thrashosds-health workloads/radosbench-high-concurrency}
-----------------------------------------------------------------
time:   00:36:17
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135935/
log:    http://qa-proxy.ceph.com/teuthology/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135935/

    "2023-01-24T22:56:58.075346+0000 mgr.y (mgr.4113) 1 : cluster [ERR] Failed
    to load ceph-mgr modules: prometheus" in cluster log

[7135936]  rados/thrash-erasure-code/{ceph clusters/{fixed-2 openstack} fast/fast mon_election/classic msgr-failures/fastclose objectstore/bluestore-comp-snappy rados recovery-overrides/{more-active-recovery} supported-random-distro$/{rhel_8} thrashers/pggrow thrashosds-health workloads/ec-small-objects-many-deletes}
-----------------------------------------------------------------
time:   00:27:00
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135936/
log:    http://qa-proxy.ceph.com/teuthology/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135936/

    "2023-01-24T22:57:20.476910+0000 mgr.x (mgr.4103) 1 : cluster [ERR] Failed
    to load ceph-mgr modules: prometheus" in cluster log

[7135937]  rados/cephadm/osds/{0-distro/centos_8.stream_container_tools_crun 0-nvme-loop 1-start 2-ops/rmdir-reactivate}
-----------------------------------------------------------------
time:   00:18:43
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135937/
log:    http://qa-proxy.ceph.com/teuthology/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135937/
sentry: https://sentry.ceph.com/organizations/ceph/?query=99e1586010634f59bcc9b3cd43990872

    timeout expired in wait_until_healthy

[7135941]  rados/thrash-erasure-code-isa/{arch/x86_64 ceph clusters/{fixed-2 openstack} mon_election/classic msgr-failures/few objectstore/bluestore-hybrid rados recovery-overrides/{more-async-partial-recovery} supported-random-distro$/{centos_8} thrashers/morepggrow thrashosds-health workloads/ec-rados-plugin=isa-k=2-m=1}
-----------------------------------------------------------------
time:   00:31:18
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135941/
log:    http://qa-proxy.ceph.com/teuthology/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135941/

    "2023-01-24T23:01:03.822073+0000 mgr.y (mgr.4115) 1 : cluster [ERR] Failed
    to load ceph-mgr modules: prometheus" in cluster log

[7135942]  rados/thrash/{0-size-min-size-overrides/2-size-2-min-size 1-pg-log-overrides/normal_pg_log 2-recovery-overrides/{more-async-recovery} 3-scrub-overrides/{max-simultaneous-scrubs-2} backoff/normal ceph clusters/{fixed-2 openstack} crc-failures/bad_map_crc_failure d-balancer/crush-compat mon_election/classic msgr-failures/few msgr/async-v1only objectstore/bluestore-bitmap rados supported-random-distro$/{rhel_8} thrashers/careful thrashosds-health workloads/radosbench}
-----------------------------------------------------------------
time:   01:22:06
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135942/
log:    http://qa-proxy.ceph.com/teuthology/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135942/

    "2023-01-24T23:01:31.929176+0000 mgr.y (mgr.4113) 1 : cluster [ERR] Failed
    to load ceph-mgr modules: prometheus" in cluster log

[7135943]  rados/cephadm/smoke/{0-distro/rhel_8.4_container_tools_3.0 0-nvme-loop agent/on fixed-2 mon_election/connectivity start}
-----------------------------------------------------------------
time:   00:22:11
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135943/
log:    http://qa-proxy.ceph.com/teuthology/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135943/
sentry: https://sentry.ceph.com/organizations/ceph/?query=ebce554917214219b37ec8426679e291

    timeout expired in wait_until_healthy

[7135944]  rados/singleton/{all/lost-unfound-delete mon_election/classic msgr-failures/none msgr/async-v2only objectstore/bluestore-hybrid rados supported-random-distro$/{rhel_8}}
-----------------------------------------------------------------
time:   01:23:49
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135944/
log:    http://qa-proxy.ceph.com/teuthology/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135944/

    "2023-01-24T23:06:59.915957+0000 mon.a (mon.0) 290 : cluster [WRN] Health
    check failed: 1 mgr modules have recently crashed
    (RECENT_MGR_MODULE_CRASH)" in cluster log

[7135947]  rados/multimon/{clusters/6 mon_election/connectivity msgr-failures/many msgr/async no_pools objectstore/bluestore-low-osd-mem-target rados supported-random-distro$/{centos_8} tasks/mon_recovery}
-----------------------------------------------------------------
time:   00:15:08
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135947/
log:    http://qa-proxy.ceph.com/teuthology/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135947/

    "2023-01-24T23:04:41.518891+0000 mgr.y (mgr.4111) 1 : cluster [ERR] Failed
    to load ceph-mgr modules: prometheus" in cluster log

[7135949]  rados/thrash/{0-size-min-size-overrides/3-size-2-min-size 1-pg-log-overrides/short_pg_log 2-recovery-overrides/{more-async-partial-recovery} 3-scrub-overrides/{max-simultaneous-scrubs-3} backoff/peering ceph clusters/{fixed-2 openstack} crc-failures/default d-balancer/on mon_election/connectivity msgr-failures/osd-delay msgr/async-v2only objectstore/bluestore-comp-lz4 rados supported-random-distro$/{centos_8} thrashers/default thrashosds-health workloads/redirect}
-----------------------------------------------------------------
time:   00:16:02
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135949/
log:    http://qa-proxy.ceph.com/teuthology/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135949/

    "2023-01-24T23:11:21.139232+0000 mgr.y (mgr.4105) 1 : cluster [ERR] Failed
    to load ceph-mgr modules: prometheus" in cluster log

[7135951]  rados/thrash-erasure-code-shec/{ceph clusters/{fixed-4 openstack} mon_election/connectivity msgr-failures/fastclose objectstore/bluestore-hybrid rados recovery-overrides/{more-async-recovery} supported-random-distro$/{rhel_8} thrashers/default thrashosds-health workloads/ec-rados-plugin=shec-k=4-m=3-c=2}
-----------------------------------------------------------------
time:   00:15:17
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135951/
log:    http://qa-proxy.ceph.com/teuthology/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135951/

    "2023-01-24T23:10:06.398570+0000 mgr.y (mgr.4107) 1 : cluster [ERR] Failed
    to load ceph-mgr modules: prometheus" in cluster log

[7135952]  rados/verify/{centos_latest ceph clusters/{fixed-2 openstack} d-thrash/none mon_election/classic msgr-failures/few msgr/async-v1only objectstore/bluestore-hybrid rados tasks/mon_recovery validater/lockdep}
-----------------------------------------------------------------
time:   00:13:30
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135952/
log:    http://qa-proxy.ceph.com/teuthology/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135952/

    "2023-01-24T23:10:52.723208+0000 mgr.x (mgr.4108) 1 : cluster [ERR] Failed
    to load ceph-mgr modules: prometheus" in cluster log

[7135953]  rados/singleton/{all/lost-unfound mon_election/connectivity msgr-failures/few msgr/async objectstore/bluestore-low-osd-mem-target rados supported-random-distro$/{rhel_8}}
-----------------------------------------------------------------
time:   01:17:16
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135953/
log:    http://qa-proxy.ceph.com/teuthology/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135953/

    "2023-01-24T23:16:38.508116+0000 mon.a (mon.0) 324 : cluster [WRN] Health
    check failed: 1 mgr modules have recently crashed
    (RECENT_MGR_MODULE_CRASH)" in cluster log

[7135954]  rados/monthrash/{ceph clusters/9-mons mon_election/connectivity msgr-failures/few msgr/async-v2only objectstore/bluestore-hybrid rados supported-random-distro$/{centos_8} thrashers/many workloads/rados_mon_osdmap_prune}
-----------------------------------------------------------------
time:   01:01:02
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135954/
log:    http://qa-proxy.ceph.com/teuthology/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135954/

    "2023-01-24T23:18:54.474959+0000 mon.h (mon.5) 1159 : cluster [WRN] Health
    check failed: 1 mgr modules have recently crashed
    (RECENT_MGR_MODULE_CRASH)" in cluster log

[7135956]  rados/thrash-erasure-code-overwrites/{bluestore-bitmap ceph clusters/{fixed-2 openstack} fast/fast mon_election/classic msgr-failures/few rados recovery-overrides/{default} supported-random-distro$/{centos_8} thrashers/fastread thrashosds-health workloads/ec-small-objects-overwrites}
-----------------------------------------------------------------
time:   00:26:34
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135956/
log:    http://qa-proxy.ceph.com/teuthology/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135956/

    "2023-01-24T23:14:50.551849+0000 mgr.x (mgr.4105) 1 : cluster [ERR] Failed
    to load ceph-mgr modules: prometheus" in cluster log

[7135957]  rados/cephadm/osds/{0-distro/centos_8.stream_container_tools_crun 0-nvme-loop 1-start 2-ops/repave-all}
-----------------------------------------------------------------
time:   00:18:46
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135957/
log:    http://qa-proxy.ceph.com/teuthology/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135957/
sentry: https://sentry.ceph.com/organizations/ceph/?query=b275d63d7ae945e49a860ecf28b2de51

    timeout expired in wait_until_healthy

[7135958]  rados/thrash/{0-size-min-size-overrides/2-size-2-min-size 1-pg-log-overrides/normal_pg_log 2-recovery-overrides/{more-active-recovery} 3-scrub-overrides/{default} backoff/peering_and_degraded ceph clusters/{fixed-2 openstack} crc-failures/bad_map_crc_failure d-balancer/crush-compat mon_election/classic msgr-failures/osd-dispatch-delay msgr/async objectstore/bluestore-comp-snappy rados supported-random-distro$/{rhel_8} thrashers/mapgap thrashosds-health workloads/redirect_promote_tests}
-----------------------------------------------------------------
time:   00:20:06
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135958/
log:    http://qa-proxy.ceph.com/teuthology/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135958/

    "2023-01-24T23:14:05.264604+0000 mgr.y (mgr.4108) 1 : cluster [ERR] Failed
    to load ceph-mgr modules: prometheus" in cluster log

[7135961]  rados/objectstore/{backends/objectcacher-stress supported-random-distro$/{centos_8}}
-----------------------------------------------------------------
time:   00:26:25
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135961/
log:    http://qa-proxy.ceph.com/teuthology/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135961/

    "2023-01-24T23:23:04.283514+0000 mon.a (mon.0) 95 : cluster [WRN] Health
    check failed: 1 mgr modules have recently crashed
    (RECENT_MGR_MODULE_CRASH)" in cluster log

[7135963]  rados/mgr/{clusters/{2-node-mgr} debug/mgr mgr_ttl_cache/disable mon_election/classic random-objectstore$/{bluestore-comp-snappy} supported-random-distro$/{rhel_8} tasks/workunits}
-----------------------------------------------------------------
time:   00:12:59
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135963/
log:    http://qa-proxy.ceph.com/teuthology/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135963/

    "2023-01-24T23:16:48.616471+0000 mgr.x (mgr.4103) 1 : cluster [ERR] Failed
    to load ceph-mgr modules: prometheus" in cluster log

[7135964]  rados/cephadm/smoke/{0-distro/rhel_8.4_container_tools_rhel8 0-nvme-loop agent/off fixed-2 mon_election/classic start}
-----------------------------------------------------------------
time:   00:21:13
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135964/
log:    http://qa-proxy.ceph.com/teuthology/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135964/
sentry: https://sentry.ceph.com/organizations/ceph/?query=22fe4a8790164395ab2733f0c57924c9

    timeout expired in wait_until_healthy

[7135966]  rados/thrash/{0-size-min-size-overrides/3-size-2-min-size 1-pg-log-overrides/short_pg_log 2-recovery-overrides/{more-partial-recovery} 3-scrub-overrides/{max-simultaneous-scrubs-2} backoff/normal ceph clusters/{fixed-2 openstack} crc-failures/default d-balancer/on mon_election/connectivity msgr-failures/fastclose msgr/async-v1only objectstore/bluestore-comp-zlib rados supported-random-distro$/{centos_8} thrashers/morepggrow thrashosds-health workloads/redirect_set_object}
-----------------------------------------------------------------
time:   00:15:22
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135966/
log:    http://qa-proxy.ceph.com/teuthology/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135966/

    "2023-01-24T23:21:34.943948+0000 mgr.x (mgr.4104) 1 : cluster [ERR] Failed
    to load ceph-mgr modules: prometheus" in cluster log



Dead
=================================================================
[7135834]  rados/thrash-erasure-code-big/{ceph cluster/{12-osds openstack} mon_election/classic msgr-failures/osd-delay objectstore/bluestore-comp-snappy rados recovery-overrides/{default} supported-random-distro$/{ubuntu_latest} thrashers/careful thrashosds-health workloads/ec-rados-plugin=lrc-k=4-m=2-l=3}
-----------------------------------------------------------------
time:   00:04:45
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135834/
log:    http://qa-proxy.ceph.com/teuthology/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135834/
sentry: https://sentry.ceph.com/organizations/ceph/?query=c375012e60a448a39df76bd8569c6a60

    {'smithi080.front.sepia.ceph.com': {'_ansible_no_log': False, 'changed':
    False, 'invocation': {'module_args': {'allow_unauthenticated': False,
    'autoclean': False, 'autoremove': False, 'cache_valid_time': 0, 'deb':
    None, 'default_release': None, 'dpkg_options': 'force-confdef,force-
    confold', 'force': True, 'force_apt_get': False, 'install_recommends':
    None, 'name': ['ceph', 'ceph-common', 'libcephfs1', 'radosgw', 'python-
    ceph', 'python-rados', 'python-cephfs', 'python-rbd', 'librbd1',
    'librados2', 'ceph-fs-common-dbg', 'ceph-fs-common', 'openmpi-common'],
    'only_upgrade': False, 'package': ['ceph', 'ceph-common', 'libcephfs1',
    'radosgw', 'python-ceph', 'python-rados', 'python-cephfs', 'python-rbd',
    'librbd1', 'librados2', 'ceph-fs-common-dbg', 'ceph-fs-common', 'openmpi-
    common'], 'policy_rc_d': None, 'purge': False, 'state': 'absent',
    'update_cache': None, 'update_cache_retries': 5,
    'update_cache_retry_max_delay': 12, 'upgrade': None}}, 'msg': "'apt-get
    remove 'librbd1' 'librados2'' failed: W: --force-yes is deprecated, use one
    of the options starting with --allow instead.\nE: Could not get lock
    /var/lib/dpkg/lock-frontend. It is held by process 14185 (apt-get)\nE:
    Unable to acquire the dpkg frontend lock (/var/lib/dpkg/lock-frontend), is
    another process using it?\n", 'rc': 100, 'stderr': 'W: --force-yes is
    deprecated, use one of the options starting with --allow instead.\nE: Could
    not get lock /var/lib/dpkg/lock-frontend. It is held by process 14185 (apt-
    get)\nE: Unable to acquire the dpkg frontend lock (/var/lib/dpkg/lock-
    frontend), is another process using it?\n', 'stderr_lines': ['W: --force-
    yes is deprecated, use one of the options starting with --allow instead.',
    'E: Could not get lock /var/lib/dpkg/lock-frontend. It is held by process
    14185 (apt-get)', 'E: Unable to acquire the dpkg frontend lock
    (/var/lib/dpkg/lock-frontend), is another process using it?'], 'stdout':
    '', 'stdout_lines': []}}



Pass
=================================================================
[7135579] rados/singleton-nomsgr/{all/export-after-evict mon_election/connectivity rados supported-random-distro$/{rhel_8}}
time:   00:11:56
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135579/

[7135581] rados/standalone/{supported-random-distro$/{ubuntu_latest} workloads/osd-backfill}
time:   03:26:26
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135581/

[7135585] rados/cephadm/workunits/{0-distro/rhel_8.4_container_tools_3.0 agent/off mon_election/classic task/test_cephadm_repos}
time:   00:09:25
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135585/

[7135587] rados/thrash-erasure-code-isa/{arch/x86_64 ceph clusters/{fixed-2 openstack} mon_election/connectivity msgr-failures/fastclose objectstore/bluestore-hybrid rados recovery-overrides/{more-partial-recovery} supported-random-distro$/{ubuntu_latest} thrashers/default thrashosds-health workloads/ec-rados-plugin=isa-k=2-m=1}
time:   00:29:22
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135587/

[7135591] rados/mgr/{clusters/{2-node-mgr} debug/mgr mgr_ttl_cache/disable mon_election/connectivity random-objectstore$/{filestore-xfs} supported-random-distro$/{ubuntu_latest} tasks/progress}
time:   00:17:59
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135591/

[7135594] rados/basic/{ceph clusters/{fixed-2 openstack} mon_election/classic msgr-failures/few msgr/async-v1only objectstore/bluestore-bitmap rados supported-random-distro$/{ubuntu_latest} tasks/repair_test}
time:   00:16:09
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135594/

[7135596] rados/thrash-old-clients/{0-distro$/{centos_8.stream_container_tools} 0-size-min-size-overrides/3-size-2-min-size 1-install/nautilus backoff/peering_and_degraded ceph clusters/{openstack three-plus-one} d-balancer/on mon_election/connectivity msgr-failures/few rados thrashers/morepggrow thrashosds-health workloads/rbd_cls}
time:   00:22:58
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135596/

[7135598] rados/perf/{ceph mon_election/connectivity objectstore/bluestore-stupid openstack scheduler/dmclock_1Shard_16Threads settings/optimized ubuntu_latest workloads/fio_4K_rand_read}
time:   00:09:57
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135598/

[7135601] rados/singleton/{all/max-pg-per-osd.from-primary mon_election/connectivity msgr-failures/few msgr/async objectstore/bluestore-low-osd-mem-target rados supported-random-distro$/{rhel_8}}
time:   00:15:52
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135601/

[7135603] rados/multimon/{clusters/3 mon_election/classic msgr-failures/few msgr/async no_pools objectstore/bluestore-low-osd-mem-target rados supported-random-distro$/{centos_8} tasks/mon_clock_no_skews}
time:   00:10:47
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135603/

[7135605] rados/singleton-nomsgr/{all/full-tiering mon_election/classic rados supported-random-distro$/{centos_8}}
time:   00:11:30
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135605/

[7135609] rados/monthrash/{ceph clusters/3-mons mon_election/classic msgr-failures/mon-delay msgr/async-v2only objectstore/bluestore-hybrid rados supported-random-distro$/{ubuntu_latest} thrashers/one workloads/rados_5925}
time:   00:09:22
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135609/

[7135615] rados/singleton/{all/max-pg-per-osd.from-replica mon_election/classic msgr-failures/many msgr/async-v1only objectstore/bluestore-stupid rados supported-random-distro$/{centos_8}}
time:   00:15:23
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135615/

[7135617] rados/singleton-nomsgr/{all/health-warnings mon_election/connectivity rados supported-random-distro$/{rhel_8}}
time:   00:13:48
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135617/

[7135619] rados/thrash/{0-size-min-size-overrides/3-size-2-min-size 1-pg-log-overrides/short_pg_log 2-recovery-overrides/{more-active-recovery} 3-scrub-overrides/{max-simultaneous-scrubs-2} backoff/normal ceph clusters/{fixed-2 openstack} crc-failures/default d-balancer/on mon_election/connectivity msgr-failures/fastclose msgr/async-v1only objectstore/bluestore-comp-zlib rados supported-random-distro$/{ubuntu_latest} thrashers/morepggrow thrashosds-health workloads/cache-snaps-balanced}
time:   00:21:20
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135619/

[7135621] rados/objectstore/{backends/objectstore-bluestore-a supported-random-distro$/{centos_8}}
time:   02:17:40
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135621/

[7135625] rados/thrash-erasure-code-big/{ceph cluster/{12-osds openstack} mon_election/connectivity msgr-failures/few objectstore/bluestore-low-osd-mem-target rados recovery-overrides/{more-partial-recovery} supported-random-distro$/{ubuntu_latest} thrashers/fastread thrashosds-health workloads/ec-rados-plugin=jerasure-k=4-m=2}
time:   00:28:03
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135625/

[7135627] rados/cephadm/workunits/{0-distro/rhel_8.4_container_tools_rhel8 agent/on mon_election/connectivity task/test_nfs}
time:   00:30:12
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135627/

[7135631] rados/singleton/{all/mon-auth-caps mon_election/connectivity msgr-failures/none msgr/async-v2only objectstore/filestore-xfs rados supported-random-distro$/{ubuntu_latest}}
time:   00:10:37
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135631/

[7135633] rados/dashboard/{0-single-container-host debug/mgr mon_election/classic random-objectstore$/{bluestore-comp-zlib} tasks/e2e}
time:   00:24:11
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135633/

[7135637] rados/singleton-nomsgr/{all/large-omap-object-warnings mon_election/classic rados supported-random-distro$/{rhel_8}}
time:   00:13:12
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135637/

[7135643] rados/perf/{ceph mon_election/classic objectstore/bluestore-basic-min-osd-mem-target openstack scheduler/dmclock_default_shards settings/optimized ubuntu_latest workloads/fio_4K_rand_rw}
time:   00:10:15
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135643/

[7135647] rados/basic/{ceph clusters/{fixed-2 openstack} mon_election/connectivity msgr-failures/many msgr/async-v2only objectstore/bluestore-comp-lz4 rados supported-random-distro$/{ubuntu_latest} tasks/scrub_test}
time:   00:11:07
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135647/

[7135649] rados/singleton/{all/mon-config-key-caps mon_election/classic msgr-failures/few msgr/async objectstore/bluestore-bitmap rados supported-random-distro$/{ubuntu_latest}}
time:   00:10:08
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135649/

[7135651] rados/thrash-erasure-code-isa/{arch/x86_64 ceph clusters/{fixed-2 openstack} mon_election/classic msgr-failures/few objectstore/bluestore-low-osd-mem-target rados recovery-overrides/{more-active-recovery} supported-random-distro$/{ubuntu_latest} thrashers/mapgap thrashosds-health workloads/ec-rados-plugin=isa-k=2-m=1}
time:   00:24:30
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135651/

[7135657] rados/singleton-nomsgr/{all/lazy_omap_stats_output mon_election/connectivity rados supported-random-distro$/{rhel_8}}
time:   00:13:05
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135657/

[7135659] rados/standalone/{supported-random-distro$/{centos_8} workloads/osd}
time:   02:52:46
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135659/

[7135663] rados/mgr/{clusters/{2-node-mgr} debug/mgr mgr_ttl_cache/enable mon_election/classic random-objectstore$/{bluestore-comp-zlib} supported-random-distro$/{ubuntu_latest} tasks/prometheus}
time:   00:11:50
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135663/

[7135665] rados/singleton/{all/mon-config-keys mon_election/connectivity msgr-failures/many msgr/async-v1only objectstore/bluestore-comp-lz4 rados supported-random-distro$/{centos_8}}
time:   00:16:49
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135665/

[7135669] rados/thrash/{0-size-min-size-overrides/2-size-2-min-size 1-pg-log-overrides/normal_pg_log 2-recovery-overrides/{more-partial-recovery} 3-scrub-overrides/{max-simultaneous-scrubs-2} backoff/normal ceph clusters/{fixed-2 openstack} crc-failures/bad_map_crc_failure d-balancer/crush-compat mon_election/classic msgr-failures/osd-dispatch-delay msgr/async-v1only objectstore/bluestore-low-osd-mem-target rados supported-random-distro$/{ubuntu_latest} thrashers/careful thrashosds-health workloads/dedup-io-mixed}
time:   00:14:13
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135669/

[7135673] rados/singleton-nomsgr/{all/librados_hello_world mon_election/classic rados supported-random-distro$/{centos_8}}
time:   00:14:31
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135673/

[7135679] rados/singleton/{all/mon-config mon_election/classic msgr-failures/none msgr/async-v2only objectstore/bluestore-comp-snappy rados supported-random-distro$/{centos_8}}
time:   00:12:44
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135679/

[7135681] rados/objectstore/{backends/objectstore-bluestore-b supported-random-distro$/{rhel_8}}
time:   02:14:11
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135681/

[7135683] rados/perf/{ceph mon_election/connectivity objectstore/bluestore-bitmap openstack scheduler/wpq_default_shards settings/optimized ubuntu_latest workloads/fio_4M_rand_read}
time:   00:09:49
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135683/

[7135685] rados/thrash-erasure-code-shec/{ceph clusters/{fixed-4 openstack} mon_election/connectivity msgr-failures/fastclose objectstore/bluestore-low-osd-mem-target rados recovery-overrides/{default} supported-random-distro$/{ubuntu_latest} thrashers/default thrashosds-health workloads/ec-rados-plugin=shec-k=4-m=3-c=2}
time:   00:11:26
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135685/

[7135687] rados/thrash-old-clients/{0-distro$/{centos_8.stream_container_tools} 0-size-min-size-overrides/2-size-2-min-size 1-install/octopus backoff/normal ceph clusters/{openstack three-plus-one} d-balancer/crush-compat mon_election/classic msgr-failures/osd-delay rados thrashers/none thrashosds-health workloads/snaps-few-objects}
time:   00:34:55
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135687/

[7135689] rados/cephadm/workunits/{0-distro/ubuntu_20.04 agent/off mon_election/classic task/test_orch_cli}
time:   00:17:53
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135689/

[7135691] rados/thrash/{0-size-min-size-overrides/3-size-2-min-size 1-pg-log-overrides/short_pg_log 2-recovery-overrides/{more-active-recovery} 3-scrub-overrides/{max-simultaneous-scrubs-2} backoff/peering ceph clusters/{fixed-2 openstack} crc-failures/default d-balancer/on mon_election/connectivity msgr-failures/fastclose msgr/async-v2only objectstore/bluestore-stupid rados supported-random-distro$/{ubuntu_latest} thrashers/default thrashosds-health workloads/dedup-io-snaps}
time:   00:14:28
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135691/

[7135693] rados/thrash-erasure-code-big/{ceph cluster/{12-osds openstack} mon_election/classic msgr-failures/osd-delay objectstore/bluestore-stupid rados recovery-overrides/{more-partial-recovery} supported-random-distro$/{rhel_8} thrashers/mapgap thrashosds-health workloads/ec-rados-plugin=lrc-k=4-m=2-l=3}
time:   00:14:48
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135693/

[7135695] rados/singleton-nomsgr/{all/msgr mon_election/connectivity rados supported-random-distro$/{ubuntu_latest}}
time:   00:16:54
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135695/

[7135703] rados/thrash-erasure-code/{ceph clusters/{fixed-2 openstack} fast/normal mon_election/connectivity msgr-failures/few objectstore/bluestore-comp-zstd rados recovery-overrides/{more-partial-recovery} supported-random-distro$/{ubuntu_latest} thrashers/morepggrow thrashosds-health workloads/ec-small-objects}
time:   00:21:49
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135703/

[7135705] rados/thrash/{0-size-min-size-overrides/2-size-2-min-size 1-pg-log-overrides/normal_pg_log 2-recovery-overrides/{more-active-recovery} 3-scrub-overrides/{max-simultaneous-scrubs-3} backoff/peering_and_degraded ceph clusters/{fixed-2 openstack} crc-failures/bad_map_crc_failure d-balancer/crush-compat mon_election/classic msgr-failures/few msgr/async objectstore/filestore-xfs rados supported-random-distro$/{ubuntu_latest} thrashers/mapgap thrashosds-health workloads/pool-snaps-few-objects}
time:   00:25:02
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135705/

[7135715] rados/thrash/{0-size-min-size-overrides/3-size-2-min-size 1-pg-log-overrides/short_pg_log 2-recovery-overrides/{more-async-partial-recovery} 3-scrub-overrides/{max-simultaneous-scrubs-3} backoff/normal ceph clusters/{fixed-2 openstack} crc-failures/default d-balancer/on mon_election/connectivity msgr-failures/osd-delay msgr/async-v1only objectstore/bluestore-bitmap rados supported-random-distro$/{ubuntu_latest} thrashers/morepggrow thrashosds-health workloads/rados_api_tests}
time:   00:27:35
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135715/

[7135719] rados/standalone/{supported-random-distro$/{ubuntu_latest} workloads/scrub}
time:   02:31:17
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135719/

[7135721] rados/multimon/{clusters/9 mon_election/classic msgr-failures/few msgr/async-v2only no_pools objectstore/filestore-xfs rados supported-random-distro$/{centos_8} tasks/mon_recovery}
time:   00:17:14
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135721/

[7135724] rados/perf/{ceph mon_election/classic objectstore/bluestore-comp openstack scheduler/dmclock_1Shard_16Threads settings/optimized ubuntu_latest workloads/fio_4M_rand_rw}
time:   00:10:09
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135724/

[7135725] rados/singleton/{all/osd-recovery mon_election/connectivity msgr-failures/none msgr/async-v2only objectstore/bluestore-hybrid rados supported-random-distro$/{ubuntu_latest}}
time:   00:17:53
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135725/

[7135726] rados/cephadm/workunits/{0-distro/centos_8.stream_container_tools agent/on mon_election/connectivity task/test_orch_cli_mon}
time:   00:27:02
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135726/

[7135728] rados/objectstore/{backends/objectstore-filestore-memstore supported-random-distro$/{centos_8}}
time:   00:31:27
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135728/

[7135730] rados/basic/{ceph clusters/{fixed-2 openstack} mon_election/connectivity msgr-failures/many msgr/async-v1only objectstore/bluestore-comp-zlib rados supported-random-distro$/{ubuntu_latest} tasks/rados_api_tests}
time:   00:27:20
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135730/

[7135733] rados/singleton-nomsgr/{all/pool-access mon_election/classic rados supported-random-distro$/{rhel_8}}
time:   00:12:51
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135733/

[7135735] rados/singleton/{all/peer mon_election/classic msgr-failures/few msgr/async objectstore/bluestore-low-osd-mem-target rados supported-random-distro$/{centos_8}}
time:   00:11:06
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135735/

[7135737] rados/thrash/{0-size-min-size-overrides/3-size-2-min-size 1-pg-log-overrides/short_pg_log 2-recovery-overrides/{more-async-partial-recovery} 3-scrub-overrides/{max-simultaneous-scrubs-3} backoff/peering_and_degraded ceph clusters/{fixed-2 openstack} crc-failures/default d-balancer/on mon_election/connectivity msgr-failures/fastclose msgr/async objectstore/bluestore-comp-snappy rados supported-random-distro$/{ubuntu_latest} thrashers/pggrow thrashosds-health workloads/radosbench}
time:   01:09:35
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135737/

[7135739] rados/cephadm/workunits/{0-distro/centos_8.stream_container_tools_crun agent/off mon_election/classic task/test_adoption}
time:   00:13:23
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135739/

[7135741] rados/singleton/{all/pg-autoscaler-progress-off mon_election/connectivity msgr-failures/many msgr/async-v1only objectstore/bluestore-stupid rados supported-random-distro$/{ubuntu_latest}}
time:   00:14:11
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135741/

[7135743] rados/thrash/{0-size-min-size-overrides/2-size-2-min-size 1-pg-log-overrides/normal_pg_log 2-recovery-overrides/{default} 3-scrub-overrides/{default} backoff/normal ceph clusters/{fixed-2 openstack} crc-failures/bad_map_crc_failure d-balancer/crush-compat mon_election/classic msgr-failures/few msgr/async-v1only objectstore/bluestore-comp-zlib rados supported-random-distro$/{ubuntu_latest} thrashers/careful thrashosds-health workloads/redirect}
time:   00:12:44
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135743/

[7135744] rados/perf/{ceph mon_election/connectivity objectstore/bluestore-low-osd-mem-target openstack scheduler/dmclock_default_shards settings/optimized ubuntu_latest workloads/fio_4M_rand_write}
time:   00:10:15
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135744/

[7135746] rados/thrash-old-clients/{0-distro$/{centos_8.stream_container_tools} 0-size-min-size-overrides/3-size-2-min-size 1-install/pacific backoff/peering ceph clusters/{openstack three-plus-one} d-balancer/on mon_election/connectivity msgr-failures/fastclose rados thrashers/pggrow thrashosds-health workloads/test_rbd_api}
time:   00:22:38
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135746/

[7135747] rados/singleton/{all/pg-autoscaler mon_election/classic msgr-failures/none msgr/async-v2only objectstore/filestore-xfs rados supported-random-distro$/{centos_8}}
time:   00:16:12
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135747/

[7135748] rados/thrash-erasure-code-isa/{arch/x86_64 ceph clusters/{fixed-2 openstack} mon_election/classic msgr-failures/osd-dispatch-delay objectstore/filestore-xfs rados recovery-overrides/{default} supported-random-distro$/{ubuntu_latest} thrashers/none thrashosds-health workloads/ec-rados-plugin=isa-k=2-m=1}
time:   00:21:48
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135748/

[7135749] rados/singleton-nomsgr/{all/version-number-sanity mon_election/classic rados supported-random-distro$/{rhel_8}}
time:   00:14:47
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135749/

[7135750] rados/singleton-bluestore/{all/cephtool mon_election/connectivity msgr-failures/none msgr/async objectstore/bluestore-comp-snappy rados supported-random-distro$/{ubuntu_latest}}
time:   00:30:59
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135750/

[7135753] rados/basic/{ceph clusters/{fixed-2 openstack} mon_election/classic msgr-failures/few msgr/async-v2only objectstore/bluestore-comp-zstd rados supported-random-distro$/{ubuntu_latest} tasks/rados_cls_all}
time:   00:18:48
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135753/

[7135754] rados/multimon/{clusters/21 mon_election/connectivity msgr-failures/many msgr/async no_pools objectstore/bluestore-bitmap rados supported-random-distro$/{ubuntu_latest} tasks/mon_recovery}
time:   00:23:45
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135754/

[7135755] rados/singleton/{all/pg-removal-interruption mon_election/connectivity msgr-failures/few msgr/async objectstore/bluestore-bitmap rados supported-random-distro$/{rhel_8}}
time:   00:12:19
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135755/

[7135757] rados/mgr/{clusters/{2-node-mgr} debug/mgr mgr_ttl_cache/disable mon_election/classic random-objectstore$/{bluestore-stupid} supported-random-distro$/{ubuntu_latest} tasks/crash}
time:   00:11:27
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135757/

[7135758] rados/objectstore/{backends/alloc-hint supported-random-distro$/{ubuntu_latest}}
time:   00:08:34
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135758/

[7135759] rados/rest/{mgr-restful supported-random-distro$/{ubuntu_latest}}
time:   00:12:03
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135759/

[7135762] rados/standalone/{supported-random-distro$/{ubuntu_latest} workloads/c2c}
time:   00:12:41
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135762/

[7135769] rados/thrash-erasure-code-shec/{ceph clusters/{fixed-4 openstack} mon_election/connectivity msgr-failures/osd-delay objectstore/filestore-xfs rados recovery-overrides/{more-active-recovery} supported-random-distro$/{ubuntu_latest} thrashers/default thrashosds-health workloads/ec-rados-plugin=shec-k=4-m=3-c=2}
time:   00:10:38
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135769/

[7135770] rados/singleton/{all/radostool mon_election/classic msgr-failures/many msgr/async-v1only objectstore/bluestore-comp-lz4 rados supported-random-distro$/{ubuntu_latest}}
time:   00:21:09
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135770/

[7135771] rados/singleton-nomsgr/{all/balancer mon_election/connectivity rados supported-random-distro$/{rhel_8}}
time:   00:13:14
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135771/

[7135772] rados/thrash-erasure-code-big/{ceph cluster/{12-osds openstack} mon_election/classic msgr-failures/fastclose objectstore/bluestore-bitmap rados recovery-overrides/{more-active-recovery} supported-random-distro$/{ubuntu_latest} thrashers/morepggrow thrashosds-health workloads/ec-rados-plugin=lrc-k=4-m=2-l=3}
time:   00:11:19
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135772/

[7135774] rados/perf/{ceph mon_election/classic objectstore/bluestore-stupid openstack scheduler/wpq_default_shards settings/optimized ubuntu_latest workloads/radosbench_4K_rand_read}
time:   00:08:18
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135774/

[7135775] rados/thrash/{0-size-min-size-overrides/3-size-2-min-size 1-pg-log-overrides/short_pg_log 2-recovery-overrides/{more-async-recovery} 3-scrub-overrides/{max-simultaneous-scrubs-3} backoff/normal ceph clusters/{fixed-2 openstack} crc-failures/default d-balancer/on mon_election/connectivity msgr-failures/fastclose msgr/async-v1only objectstore/bluestore-low-osd-mem-target rados supported-random-distro$/{ubuntu_latest} thrashers/morepggrow thrashosds-health workloads/set-chunks-read}
time:   00:14:36
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135775/

[7135776] rados/singleton/{all/random-eio mon_election/connectivity msgr-failures/none msgr/async-v2only objectstore/bluestore-comp-snappy rados supported-random-distro$/{ubuntu_latest}}
time:   00:19:35
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135776/

[7135779] rados/thrash-erasure-code-overwrites/{bluestore-bitmap ceph clusters/{fixed-2 openstack} fast/fast mon_election/classic msgr-failures/osd-delay rados recovery-overrides/{more-async-partial-recovery} supported-random-distro$/{ubuntu_latest} thrashers/morepggrow thrashosds-health workloads/ec-small-objects-fast-read-overwrites}
time:   00:28:46
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135779/

[7135780] rados/singleton-nomsgr/{all/cache-fs-trunc mon_election/classic rados supported-random-distro$/{centos_8}}
time:   00:15:15
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135780/

[7135783] rados/thrash-erasure-code-isa/{arch/x86_64 ceph clusters/{fixed-2 openstack} mon_election/connectivity msgr-failures/fastclose objectstore/bluestore-bitmap rados recovery-overrides/{more-partial-recovery} supported-random-distro$/{ubuntu_latest} thrashers/none thrashosds-health workloads/ec-rados-plugin=isa-k=2-m=1}
time:   00:24:05
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135783/

[7135784] rados/singleton/{all/rebuild-mondb mon_election/classic msgr-failures/few msgr/async objectstore/bluestore-comp-zlib rados supported-random-distro$/{ubuntu_latest}}
time:   00:14:54
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135784/

[7135786] rados/singleton-nomsgr/{all/ceph-kvstore-tool mon_election/connectivity rados supported-random-distro$/{ubuntu_latest}}
time:   00:09:01
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135786/

[7135788] rados/objectstore/{backends/ceph_objectstore_tool supported-random-distro$/{centos_8}}
time:   00:15:10
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135788/

[7135789] rados/multimon/{clusters/3 mon_election/classic msgr-failures/few msgr/async-v1only no_pools objectstore/bluestore-comp-lz4 rados supported-random-distro$/{rhel_8} tasks/mon_clock_no_skews}
time:   00:11:48
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135789/

[7135791] rados/cephadm/workunits/{0-distro/rhel_8.4_container_tools_rhel8 agent/off mon_election/classic task/test_cephadm_repos}
time:   00:09:04
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135791/

[7135792] rados/perf/{ceph mon_election/connectivity objectstore/bluestore-basic-min-osd-mem-target openstack scheduler/dmclock_1Shard_16Threads settings/optimized ubuntu_latest workloads/radosbench_4K_seq_read}
time:   00:08:23
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135792/

[7135793] rados/mgr/{clusters/{2-node-mgr} debug/mgr mgr_ttl_cache/enable mon_election/connectivity random-objectstore$/{bluestore-comp-zstd} supported-random-distro$/{ubuntu_latest} tasks/failover}
time:   00:13:11
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135793/

[7135795] rados/singleton-nomsgr/{all/ceph-post-file mon_election/classic rados supported-random-distro$/{ubuntu_latest}}
time:   00:07:29
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135795/

[7135796] rados/standalone/{supported-random-distro$/{centos_8} workloads/crush}
time:   00:23:00
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135796/

[7135798] rados/thrash-old-clients/{0-distro$/{centos_8.stream_container_tools} 0-size-min-size-overrides/2-size-2-min-size 1-install/pacific backoff/peering_and_degraded ceph clusters/{openstack three-plus-one} d-balancer/crush-compat mon_election/classic msgr-failures/few rados thrashers/careful thrashosds-health workloads/cache-snaps}
time:   00:35:07
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135798/

[7135800] rados/thrash-erasure-code-shec/{ceph clusters/{fixed-4 openstack} mon_election/classic msgr-failures/osd-dispatch-delay objectstore/bluestore-bitmap rados recovery-overrides/{more-active-recovery} supported-random-distro$/{ubuntu_latest} thrashers/careful thrashosds-health workloads/ec-rados-plugin=shec-k=4-m=3-c=2}
time:   00:11:59
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135800/

[7135802] rados/singleton/{all/resolve_stuck_peering mon_election/classic msgr-failures/none msgr/async-v2only objectstore/bluestore-hybrid rados supported-random-distro$/{ubuntu_latest}}
time:   00:08:59
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135802/

[7135805] rados/singleton-nomsgr/{all/crushdiff mon_election/connectivity rados supported-random-distro$/{rhel_8}}
time:   00:13:27
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135805/

[7135806] rados/basic/{ceph clusters/{fixed-2 openstack} mon_election/classic msgr-failures/few msgr/async-v1only objectstore/bluestore-low-osd-mem-target rados supported-random-distro$/{ubuntu_latest} tasks/rados_stress_watch}
time:   00:18:21
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135806/

[7135809] rados/thrash-erasure-code/{ceph clusters/{fixed-2 openstack} fast/fast mon_election/classic msgr-failures/fastclose objectstore/bluestore-stupid rados recovery-overrides/{more-active-recovery} supported-random-distro$/{ubuntu_latest} thrashers/default thrashosds-health workloads/ec-rados-plugin=jerasure-k=3-m=1}
time:   00:25:50
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135809/

[7135811] rados/singleton-nomsgr/{all/export-after-evict mon_election/classic rados supported-random-distro$/{centos_8}}
time:   00:11:15
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135811/

[7135812] rados/cephadm/workunits/{0-distro/ubuntu_20.04 agent/on mon_election/connectivity task/test_nfs}
time:   00:36:25
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135812/

[7135813] rados/perf/{ceph mon_election/classic objectstore/bluestore-bitmap openstack scheduler/dmclock_default_shards settings/optimized ubuntu_latest workloads/radosbench_4M_rand_read}
time:   00:08:35
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135813/

[7135814] rados/thrash-erasure-code-isa/{arch/x86_64 ceph clusters/{fixed-2 openstack} mon_election/classic msgr-failures/few objectstore/bluestore-comp-lz4 rados recovery-overrides/{default} supported-random-distro$/{ubuntu_latest} thrashers/pggrow thrashosds-health workloads/ec-rados-plugin=isa-k=2-m=1}
time:   00:26:05
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135814/

[7135815] rados/singleton/{all/test-noautoscale-flag mon_election/classic msgr-failures/many msgr/async-v1only objectstore/bluestore-stupid rados supported-random-distro$/{rhel_8}}
time:   00:12:22
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135815/

[7135816] rados/objectstore/{backends/filejournal supported-random-distro$/{centos_8}}
time:   00:10:45
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135816/

[7135817] rados/thrash/{0-size-min-size-overrides/3-size-2-min-size 1-pg-log-overrides/short_pg_log 2-recovery-overrides/{more-partial-recovery} 3-scrub-overrides/{default} backoff/normal ceph clusters/{fixed-2 openstack} crc-failures/default d-balancer/on mon_election/connectivity msgr-failures/osd-delay msgr/async-v1only objectstore/bluestore-comp-zlib rados supported-random-distro$/{ubuntu_latest} thrashers/morepggrow thrashosds-health workloads/snaps-few-objects}
time:   00:25:50
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135817/

[7135820] rados/singleton-nomsgr/{all/full-tiering mon_election/connectivity rados supported-random-distro$/{rhel_8}}
time:   00:12:14
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135820/

[7135821] rados/thrash-erasure-code-overwrites/{bluestore-bitmap ceph clusters/{fixed-2 openstack} fast/normal mon_election/connectivity msgr-failures/osd-dispatch-delay rados recovery-overrides/{more-async-recovery} supported-random-distro$/{ubuntu_latest} thrashers/pggrow thrashosds-health workloads/ec-small-objects-overwrites}
time:   00:22:21
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135821/

[7135822] rados/singleton/{all/test_envlibrados_for_rocksdb mon_election/connectivity msgr-failures/none msgr/async-v2only objectstore/filestore-xfs rados supported-random-distro$/{ubuntu_latest}}
time:   00:15:58
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135822/

[7135831] rados/singleton-nomsgr/{all/health-warnings mon_election/classic rados supported-random-distro$/{centos_8}}
time:   00:12:19
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135831/

[7135832] rados/standalone/{supported-random-distro$/{rhel_8} workloads/erasure-code}
time:   01:29:41
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135832/

[7135833] rados/singleton/{all/thrash-backfill-full mon_election/classic msgr-failures/few msgr/async objectstore/bluestore-bitmap rados supported-random-distro$/{ubuntu_latest}}
time:   01:16:01
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135833/

[7135835] rados/cephadm/workunits/{0-distro/centos_8.stream_container_tools agent/off mon_election/classic task/test_orch_cli}
time:   00:17:17
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135835/

[7135837] rados/perf/{ceph mon_election/connectivity objectstore/bluestore-comp openstack scheduler/wpq_default_shards settings/optimized ubuntu_latest workloads/radosbench_4M_seq_read}
time:   00:08:21
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135837/

[7135838] rados/dashboard/{0-single-container-host debug/mgr mon_election/connectivity random-objectstore$/{bluestore-hybrid} tasks/e2e}
time:   00:24:03
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135838/

[7135840] rados/singleton-nomsgr/{all/large-omap-object-warnings mon_election/connectivity rados supported-random-distro$/{centos_8}}
time:   00:11:52
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135840/

[7135843] rados/thrash-old-clients/{0-distro$/{centos_8.stream_container_tools} 0-size-min-size-overrides/3-size-2-min-size 1-install/nautilus-v1only backoff/normal ceph clusters/{openstack three-plus-one} d-balancer/on mon_election/connectivity msgr-failures/osd-delay rados thrashers/default thrashosds-health workloads/radosbench}
time:   01:35:09
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135843/

[7135847] rados/objectstore/{backends/filestore-idempotent-aio-journal supported-random-distro$/{ubuntu_latest}}
time:   02:29:32
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135847/

[7135848] rados/singleton/{all/thrash-rados/{thrash-rados thrashosds-health} mon_election/classic msgr-failures/none msgr/async-v2only objectstore/bluestore-comp-snappy rados supported-random-distro$/{ubuntu_latest}}
time:   00:19:28
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135848/

[7135849] rados/singleton-nomsgr/{all/lazy_omap_stats_output mon_election/classic rados supported-random-distro$/{rhel_8}}
time:   00:13:35
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135849/

[7135851] rados/multimon/{clusters/9 mon_election/classic msgr-failures/few msgr/async no_pools objectstore/bluestore-comp-zlib rados supported-random-distro$/{ubuntu_latest} tasks/mon_recovery}
time:   00:14:22
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135851/

[7135852] rados/thrash/{0-size-min-size-overrides/3-size-2-min-size 1-pg-log-overrides/short_pg_log 2-recovery-overrides/{default} 3-scrub-overrides/{default} backoff/peering ceph clusters/{fixed-2 openstack} crc-failures/default d-balancer/on mon_election/connectivity msgr-failures/osd-delay msgr/async-v2only objectstore/bluestore-stupid rados supported-random-distro$/{ubuntu_latest} thrashers/default thrashosds-health workloads/cache-agent-small}
time:   00:13:56
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135852/

[7135855] rados/cephadm/workunits/{0-distro/centos_8.stream_container_tools_crun agent/on mon_election/connectivity task/test_orch_cli_mon}
time:   00:27:13
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135855/

[7135856] rados/singleton-nomsgr/{all/librados_hello_world mon_election/connectivity rados supported-random-distro$/{centos_8}}
time:   00:14:34
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135856/

[7135858] rados/perf/{ceph mon_election/classic objectstore/bluestore-low-osd-mem-target openstack scheduler/dmclock_1Shard_16Threads settings/optimized ubuntu_latest workloads/radosbench_4M_write}
time:   00:08:44
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135858/

[7135861] rados/thrash/{0-size-min-size-overrides/2-size-2-min-size 1-pg-log-overrides/normal_pg_log 2-recovery-overrides/{more-async-partial-recovery} 3-scrub-overrides/{default} backoff/peering_and_degraded ceph clusters/{fixed-2 openstack} crc-failures/bad_map_crc_failure d-balancer/crush-compat mon_election/classic msgr-failures/osd-dispatch-delay msgr/async objectstore/filestore-xfs rados supported-random-distro$/{ubuntu_latest} thrashers/mapgap thrashosds-health workloads/cache-pool-snaps-readproxy}
time:   00:27:05
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135861/

[7135864] rados/singleton/{all/watch-notify-same-primary mon_election/classic msgr-failures/many msgr/async-v1only objectstore/bluestore-comp-zstd rados supported-random-distro$/{centos_8}}
time:   00:11:10
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135864/

[7135866] rados/singleton-nomsgr/{all/msgr mon_election/classic rados supported-random-distro$/{rhel_8}}
time:   00:18:02
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135866/

[7135867] rados/standalone/{supported-random-distro$/{centos_8} workloads/mgr}
time:   00:17:36
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135867/

[7135873] rados/singleton/{all/admin-socket mon_election/connectivity msgr-failures/none msgr/async-v2only objectstore/bluestore-hybrid rados supported-random-distro$/{rhel_8}}
time:   00:12:05
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135873/

[7135874] rados/singleton-nomsgr/{all/multi-backfill-reject mon_election/connectivity rados supported-random-distro$/{ubuntu_latest}}
time:   00:25:05
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135874/

[7135875] rados/objectstore/{backends/filestore-idempotent supported-random-distro$/{ubuntu_latest}}
time:   02:31:36
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135875/

[7135879] rados/cephadm/workunits/{0-distro/rhel_8.4_container_tools_3.0 agent/off mon_election/classic task/test_adoption}
time:   00:13:21
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135879/

[7135880] rados/perf/{ceph mon_election/connectivity objectstore/bluestore-stupid openstack scheduler/dmclock_default_shards settings/optimized ubuntu_latest workloads/radosbench_omap_write}
time:   00:23:32
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135880/

[7135881] rados/singleton/{all/backfill-toofull mon_election/classic msgr-failures/few msgr/async objectstore/bluestore-low-osd-mem-target rados supported-random-distro$/{ubuntu_latest}}
time:   00:22:24
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135881/

[7135882] rados/singleton-nomsgr/{all/osd_stale_reads mon_election/classic rados supported-random-distro$/{ubuntu_latest}}
time:   00:18:12
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135882/

[7135883] rados/multimon/{clusters/21 mon_election/connectivity msgr-failures/many msgr/async-v1only no_pools objectstore/bluestore-comp-zstd rados supported-random-distro$/{centos_8} tasks/mon_clock_no_skews}
time:   00:10:43
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135883/

[7135884] rados/thrash/{0-size-min-size-overrides/3-size-2-min-size 1-pg-log-overrides/short_pg_log 2-recovery-overrides/{default} 3-scrub-overrides/{max-simultaneous-scrubs-2} backoff/peering_and_degraded ceph clusters/{fixed-2 openstack} crc-failures/default d-balancer/on mon_election/connectivity msgr-failures/osd-delay msgr/async objectstore/bluestore-comp-snappy rados supported-random-distro$/{ubuntu_latest} thrashers/pggrow thrashosds-health workloads/cache-snaps}
time:   00:20:30
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135884/

[7135886] rados/thrash-old-clients/{0-distro$/{centos_8.stream_container_tools} 0-size-min-size-overrides/2-size-2-min-size 1-install/nautilus-v2only backoff/peering ceph clusters/{openstack three-plus-one} d-balancer/crush-compat mon_election/classic msgr-failures/fastclose rados thrashers/mapgap thrashosds-health workloads/rbd_cls}
time:   00:23:36
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135886/

[7135887] rados/singleton/{all/deduptool mon_election/connectivity msgr-failures/many msgr/async-v1only objectstore/bluestore-stupid rados supported-random-distro$/{ubuntu_latest}}
time:   00:09:28
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135887/

[7135889] rados/thrash-erasure-code-shec/{ceph clusters/{fixed-4 openstack} mon_election/connectivity msgr-failures/osd-delay objectstore/bluestore-comp-zlib rados recovery-overrides/{more-active-recovery} supported-random-distro$/{rhel_8} thrashers/default thrashosds-health workloads/ec-rados-plugin=shec-k=4-m=3-c=2}
time:   00:15:37
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135889/

[7135890] rados/monthrash/{ceph clusters/9-mons mon_election/connectivity msgr-failures/few msgr/async objectstore/bluestore-comp-zlib rados supported-random-distro$/{ubuntu_latest} thrashers/sync workloads/rados_5925}
time:   00:10:45
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135890/

[7135891] rados/singleton-nomsgr/{all/pool-access mon_election/connectivity rados supported-random-distro$/{ubuntu_latest}}
time:   00:08:23
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135891/

[7135895] rados/thrash-erasure-code-big/{ceph cluster/{12-osds openstack} mon_election/classic msgr-failures/fastclose objectstore/bluestore-comp-zstd rados recovery-overrides/{more-async-partial-recovery} supported-random-distro$/{rhel_8} thrashers/fastread thrashosds-health workloads/ec-rados-plugin=lrc-k=4-m=2-l=3}
time:   00:14:29
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135895/

[7135896] rados/singleton/{all/divergent_priors mon_election/classic msgr-failures/none msgr/async-v2only objectstore/filestore-xfs rados supported-random-distro$/{centos_8}}
time:   00:11:51
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135896/

[7135899] rados/standalone/{supported-random-distro$/{rhel_8} workloads/misc}
time:   01:08:02
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135899/

[7135900] rados/thrash/{0-size-min-size-overrides/3-size-2-min-size 1-pg-log-overrides/short_pg_log 2-recovery-overrides/{more-partial-recovery} 3-scrub-overrides/{max-simultaneous-scrubs-3} backoff/peering ceph clusters/{fixed-2 openstack} crc-failures/default d-balancer/on mon_election/connectivity msgr-failures/fastclose msgr/async-v2only objectstore/bluestore-comp-zstd rados supported-random-distro$/{ubuntu_latest} thrashers/default thrashosds-health workloads/dedup-io-mixed}
time:   00:11:29
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135900/

[7135902] rados/perf/{ceph mon_election/classic objectstore/bluestore-basic-min-osd-mem-target openstack scheduler/wpq_default_shards settings/optimized ubuntu_latest workloads/sample_fio}
time:   00:09:57
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135902/

[7135904] rados/objectstore/{backends/fusestore supported-random-distro$/{ubuntu_latest}}
time:   00:07:33
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135904/

[7135905] rados/singleton/{all/divergent_priors2 mon_election/connectivity msgr-failures/few msgr/async objectstore/bluestore-bitmap rados supported-random-distro$/{rhel_8}}
time:   00:13:10
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135905/

[7135908] rados/thrash-erasure-code-isa/{arch/x86_64 ceph clusters/{fixed-2 openstack} mon_election/connectivity msgr-failures/fastclose objectstore/bluestore-comp-zstd rados recovery-overrides/{default} supported-random-distro$/{ubuntu_latest} thrashers/mapgap thrashosds-health workloads/ec-rados-plugin=isa-k=2-m=1}
time:   00:24:08
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135908/

[7135909] rados/thrash/{0-size-min-size-overrides/2-size-2-min-size 1-pg-log-overrides/normal_pg_log 2-recovery-overrides/{more-async-partial-recovery} 3-scrub-overrides/{max-simultaneous-scrubs-3} backoff/peering_and_degraded ceph clusters/{fixed-2 openstack} crc-failures/bad_map_crc_failure d-balancer/crush-compat mon_election/classic msgr-failures/few msgr/async objectstore/bluestore-hybrid rados supported-random-distro$/{ubuntu_latest} thrashers/mapgap thrashosds-health workloads/dedup-io-snaps}
time:   00:14:58
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135909/

[7135910] rados/singleton-nomsgr/{all/version-number-sanity mon_election/connectivity rados supported-random-distro$/{centos_8}}
time:   00:13:31
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135910/

[7135912] rados/singleton/{all/dump-stuck mon_election/classic msgr-failures/many msgr/async-v1only objectstore/bluestore-comp-lz4 rados supported-random-distro$/{rhel_8}}
time:   00:13:48
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135912/

[7135914] rados/multimon/{clusters/3 mon_election/classic msgr-failures/few msgr/async-v2only no_pools objectstore/bluestore-hybrid rados supported-random-distro$/{ubuntu_latest} tasks/mon_clock_with_skews}
time:   00:07:38
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135914/

[7135915] rados/thrash/{0-size-min-size-overrides/3-size-2-min-size 1-pg-log-overrides/short_pg_log 2-recovery-overrides/{more-async-recovery} 3-scrub-overrides/{default} backoff/normal ceph clusters/{fixed-2 openstack} crc-failures/default d-balancer/on mon_election/connectivity msgr-failures/osd-delay msgr/async-v1only objectstore/bluestore-low-osd-mem-target rados supported-random-distro$/{ubuntu_latest} thrashers/morepggrow thrashosds-health workloads/pool-snaps-few-objects}
time:   00:27:18
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135915/

[7135923] rados/monthrash/{ceph clusters/3-mons mon_election/classic msgr-failures/mon-delay msgr/async-v1only objectstore/bluestore-comp-zstd rados supported-random-distro$/{ubuntu_latest} thrashers/force-sync-many workloads/rados_api_tests}
time:   00:28:26
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135923/

[7135924] rados/cephadm/workunits/{0-distro/ubuntu_20.04 agent/off mon_election/classic task/test_cephadm_repos}
time:   00:06:17
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135924/

[7135925] rados/perf/{ceph mon_election/connectivity objectstore/bluestore-bitmap openstack scheduler/dmclock_1Shard_16Threads settings/optimized ubuntu_latest workloads/sample_radosbench}
time:   00:08:32
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135925/

[7135928] rados/singleton-nomsgr/{all/balancer mon_election/classic rados supported-random-distro$/{centos_8}}
time:   00:12:33
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135928/

[7135932] rados/mgr/{clusters/{2-node-mgr} debug/mgr mgr_ttl_cache/enable mon_election/connectivity random-objectstore$/{bluestore-stupid} supported-random-distro$/{ubuntu_latest} tasks/prometheus}
time:   00:12:00
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135932/

[7135933] rados/thrash-old-clients/{0-distro$/{centos_8.stream_container_tools} 0-size-min-size-overrides/3-size-2-min-size 1-install/nautilus backoff/peering_and_degraded ceph clusters/{openstack three-plus-one} d-balancer/on mon_election/connectivity msgr-failures/few rados thrashers/morepggrow thrashosds-health workloads/snaps-few-objects}
time:   00:37:21
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135933/

[7135934] rados/objectstore/{backends/keyvaluedb supported-random-distro$/{ubuntu_latest}}
time:   00:14:55
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135934/

[7135938] rados/singleton-nomsgr/{all/cache-fs-trunc mon_election/connectivity rados supported-random-distro$/{ubuntu_latest}}
time:   00:13:35
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135938/

[7135939] rados/standalone/{supported-random-distro$/{rhel_8} workloads/mon}
time:   01:07:46
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135939/

[7135940] rados/singleton/{all/erasure-code-nonregression mon_election/connectivity msgr-failures/many msgr/async-v1only objectstore/bluestore-comp-zstd rados supported-random-distro$/{rhel_8}}
time:   00:11:45
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135940/

[7135945] rados/singleton-nomsgr/{all/ceph-kvstore-tool mon_election/classic rados supported-random-distro$/{ubuntu_latest}}
time:   00:08:33
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135945/

[7135946] rados/perf/{ceph mon_election/classic objectstore/bluestore-comp openstack scheduler/dmclock_default_shards settings/optimized ubuntu_latest workloads/fio_4K_rand_read}
time:   00:10:08
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135946/

[7135948] rados/basic/{ceph clusters/{fixed-2 openstack} mon_election/classic msgr-failures/few msgr/async-v1only objectstore/bluestore-comp-zlib rados supported-random-distro$/{ubuntu_latest} tasks/repair_test}
time:   00:16:15
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135948/

[7135950] rados/cephadm/workunits/{0-distro/centos_8.stream_container_tools agent/on mon_election/connectivity task/test_nfs}
time:   00:29:42
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135950/

[7135955] rados/singleton-nomsgr/{all/ceph-post-file mon_election/connectivity rados supported-random-distro$/{centos_8}}
time:   00:10:30
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135955/

[7135959] rados/thrash-erasure-code-big/{ceph cluster/{12-osds openstack} mon_election/classic msgr-failures/osd-delay objectstore/bluestore-low-osd-mem-target rados recovery-overrides/{more-partial-recovery} supported-random-distro$/{centos_8} thrashers/morepggrow thrashosds-health workloads/ec-rados-plugin=lrc-k=4-m=2-l=3}
time:   00:13:36
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135959/

[7135960] rados/singleton/{all/max-pg-per-osd.from-mon mon_election/classic msgr-failures/many msgr/async-v1only objectstore/bluestore-stupid rados supported-random-distro$/{centos_8}}
time:   00:11:05
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135960/

[7135962] rados/singleton-nomsgr/{all/crushdiff mon_election/classic rados supported-random-distro$/{centos_8}}
time:   00:11:28
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135962/

[7135965] rados/thrash-erasure-code/{ceph clusters/{fixed-2 openstack} fast/normal mon_election/connectivity msgr-failures/few objectstore/bluestore-comp-zlib rados recovery-overrides/{more-active-recovery} supported-random-distro$/{ubuntu_latest} thrashers/careful thrashosds-health workloads/ec-small-objects}
time:   00:20:40
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135965/

[7135967] rados/perf/{ceph mon_election/connectivity objectstore/bluestore-low-osd-mem-target openstack scheduler/wpq_default_shards settings/optimized ubuntu_latest workloads/fio_4K_rand_rw}
time:   00:09:57
info:   http://pulpito.ceph.com/yuriw-2023-01-24_16:05:13-rados-wip-yuri7-testing-2023-01-23-1532-quincy-distro-default-smithi/7135967/