Found 374 jobs Found 6 distinct failure reasons Failure: "2021-02-17T07:39:08.611748+0000 mgr.x (mgr.4101) 1 : cluster [ERR] Unhandled exception from module 'devicehealth' while running on mgr.x: name 'cast' is not defined" in cluster log 169 jobs suites intersection: [] suites union: ['1-pg-log-overrides/normal_pg_log', '1-pg-log-overrides/short_pg_log', '2-inject-leak/mon', '2-inject-leak/none', '2-inject-leak/osd', '2-recovery-overrides/{default}', '2-recovery-overrides/{more-active-recovery}', '2-recovery-overrides/{more-async-partial-recovery}', '2-recovery-overrides/{more-async-recovery}', '2-recovery-overrides/{more-partial-recovery}', 'backoff/normal', 'backoff/peering', 'backoff/peering_and_degraded', 'centos_latest}', 'ceph', 'cluster/{12-osds', 'clusters/3-mons', 'clusters/9-mons', 'clusters/{fixed-2', 'clusters/{fixed-4', 'crc-failures/bad_map_crc_failure', 'crc-failures/default', 'd-balancer/crush-compat', 'd-balancer/on', 'd-thrash/default/{default', 'd-thrash/none', 'debug/mgr', 'fast/fast', 'fast/normal', 'mon_election/classic', 'mon_election/connectivity', 'msgr-failures/fastclose', 'msgr-failures/few', 'msgr-failures/many', 'msgr-failures/mon-delay', 'msgr-failures/osd-delay', 'msgr/async', 'msgr/async-v1only', 'msgr/async-v2only', 'objectstore/bluestore-basic-min-osd-mem-target', 'objectstore/bluestore-bitmap', 'objectstore/bluestore-comp', 'objectstore/bluestore-comp-lz4', 'objectstore/bluestore-comp-snappy', 'objectstore/bluestore-comp-zlib', 'objectstore/bluestore-comp-zstd', 'objectstore/bluestore-hybrid', 'objectstore/bluestore-low-osd-mem-target', 'objectstore/bluestore-stupid', 'objectstore/filestore-xfs', 'openstack', 'openstack}', 'orchestrator_cli', 'rados', 'rados/basic/{ceph', 'rados/cephadm/orchestrator_cli/{2-node-mgr', 'rados/dashboard/{clusters/{2-node-mgr}', 'rados/mgr/{clusters/{2-node-mgr}', 'rados/monthrash/{ceph', 'rados/objectstore/{backends/alloc-hint', 'rados/objectstore/{backends/ceph_objectstore_tool', 'rados/objectstore/{backends/filejournal', 'rados/objectstore/{backends/filestore-idempotent', 'rados/objectstore/{backends/filestore-idempotent-aio-journal', 'rados/objectstore/{backends/objectcacher-stress', 'rados/perf/{ceph', 'rados/rest/{mgr-restful', 'rados/singleton-bluestore/{all/cephtool', 'rados/singleton/{all/admin-socket', 'rados/singleton/{all/rebuild-mondb', 'rados/singleton/{all/thrash-rados/{thrash-rados', 'rados/thrash-erasure-code-big/{ceph', 'rados/thrash-erasure-code-isa/{arch/x86_64', 'rados/thrash-erasure-code-overwrites/{bluestore-bitmap', 'rados/thrash-erasure-code-shec/{ceph', 'rados/thrash-erasure-code/{ceph', 'rados/thrash/{0-size-min-size-overrides/2-size-2-min-size', 'rados/thrash/{0-size-min-size-overrides/3-size-2-min-size', 'rados/valgrind-leaks/{1-start', 'rados/verify/{centos_latest', 'recovery-overrides/{default}', 'recovery-overrides/{more-active-recovery}', 'recovery-overrides/{more-async-partial-recovery}', 'recovery-overrides/{more-async-recovery}', 'recovery-overrides/{more-partial-recovery}', 'scheduler/dmclock_1Shard_16Threads', 'scheduler/dmclock_default_shards', 'scheduler/wpq_default_shards', 'settings/optimized', 'supported-random-distro$/{centos_8}', 'supported-random-distro$/{centos_8}}', 'supported-random-distro$/{rhel_8}', 'supported-random-distro$/{rhel_8}}', 'supported-random-distro$/{ubuntu_latest}', 'supported-random-distro$/{ubuntu_latest}}', 'tasks/crash}', 'tasks/dashboard}', 'tasks/failover}', 'tasks/insights}', 'tasks/module_selftest}', 'tasks/mon_recovery', 'tasks/progress}', 'tasks/prometheus}', 'tasks/rados_api_tests', 'tasks/rados_api_tests}', 'tasks/rados_cls_all', 'tasks/rados_cls_all}', 'tasks/rados_python}', 'tasks/rados_stress_watch}', 'tasks/rados_striper}', 'tasks/rados_workunit_loadgen_big}', 'tasks/rados_workunit_loadgen_mix}', 'tasks/rados_workunit_loadgen_mostlyread}', 'tasks/readwrite}', 'tasks/repair_test}', 'tasks/scrub_test}', 'tasks/workunits}', 'thrashers/careful', 'thrashers/default', 'thrashers/fastread', 'thrashers/force-sync-many', 'thrashers/many', 'thrashers/mapgap', 'thrashers/morepggrow', 'thrashers/none', 'thrashers/one', 'thrashers/pggrow', 'thrashers/sync', 'thrashers/sync-many', 'thrashosds-health', 'thrashosds-health}', 'ubuntu_18.04', 'validater/lockdep}', 'validater/valgrind}', 'workloads/admin_socket_objecter_requests}', 'workloads/cache-agent-big}', 'workloads/cache-agent-small}', 'workloads/cache-pool-snaps-readproxy}', 'workloads/cache-pool-snaps}', 'workloads/cache-snaps-balanced}', 'workloads/cache-snaps}', 'workloads/cache}', 'workloads/cosbench_64K_read_write}', 'workloads/cosbench_64K_write}', 'workloads/ec-pool-snaps-few-objects-overwrites}', 'workloads/ec-rados-plugin=clay-k=4-m=2}', 'workloads/ec-rados-plugin=isa-k=2-m=1}', 'workloads/ec-rados-plugin=jerasure-k=2-m=1}', 'workloads/ec-rados-plugin=jerasure-k=3-m=1}', 'workloads/ec-rados-plugin=jerasure-k=4-m=2}', 'workloads/ec-rados-plugin=lrc-k=4-m=2-l=3}', 'workloads/ec-rados-plugin=shec-k=4-m=3-c=2}', 'workloads/ec-radosbench}', 'workloads/ec-small-objects-fast-read-overwrites}', 'workloads/ec-small-objects-fast-read}', 'workloads/ec-small-objects-many-deletes}', 'workloads/ec-small-objects}', 'workloads/ec-snaps-few-objects-overwrites}', 'workloads/fio_4K_rand_read}', 'workloads/fio_4K_rand_rw}', 'workloads/fio_4M_rand_read}', 'workloads/fio_4M_rand_rw}', 'workloads/fio_4M_rand_write}', 'workloads/pool-create-delete}', 'workloads/pool-snaps-few-objects}', 'workloads/rados_5925}', 'workloads/rados_api_tests}', 'workloads/rados_mon_osdmap_prune}', 'workloads/rados_mon_workunits}', 'workloads/radosbench-high-concurrency}', 'workloads/radosbench_4K_rand_read}', 'workloads/radosbench_4K_seq_read}', 'workloads/radosbench_4M_rand_read}', 'workloads/radosbench_4M_seq_read}', 'workloads/radosbench_4M_write}', 'workloads/radosbench_omap_write}', 'workloads/radosbench}', 'workloads/redirect_promote_tests}', 'workloads/redirect_set_object}', 'workloads/redirect}', 'workloads/sample_fio}', 'workloads/sample_radosbench}', 'workloads/set-chunks-read}', 'workloads/small-objects-balanced}', 'workloads/small-objects-localized}', 'workloads/small-objects}', 'workloads/snaps-few-objects-balanced}', 'workloads/snaps-few-objects-localized}', 'workloads/snaps-few-objects}', 'workloads/write_fadvise_dontneed}'] Failure: timeout expired in wait_until_healthy 90 jobs suites intersection: [] suites union: ['1-install/luminous', '1-install/luminous-v1only', '1-install/mimic', '1-install/mimic-v1only', '1-install/nautilus', '1-install/nautilus-v1only', '1-install/nautilus-v2only', '1-install/octopus', '1-start', '2-partial-upgrade/firsthalf', '2-thrash', '3-tasks/radosbench', '3-tasks/small-objects', '3-tasks/snaps-few-objects', '3-thrash/default', '4-workload/{rbd-cls', '5-workload/{radosbench', '6-finish-upgrade', '7-octopus', '8-workload/{rbd-python', 'backoff/normal', 'backoff/peering', 'backoff/peering_and_degraded', 'bluestore-bitmap', 'ceph', 'clusters/{openstack', 'd-balancer/crush-compat', 'd-balancer/on', 'distro$/{ubuntu_18.04}', 'fixed-2', 'mode/packaged', 'mode/root', 'mon_election/classic', 'mon_election/connectivity', 'msgr-failures/fastclose', 'msgr-failures/few', 'msgr-failures/osd-delay', 'msgr/async', 'msgr/async-v1only', 'msgr/async-v2only', 'rados', 'rados/cephadm/dashboard/{distro/centos_latest', 'rados/cephadm/smoke-roleless/{distro/centos_8.0', 'rados/cephadm/smoke-roleless/{distro/centos_latest', 'rados/cephadm/smoke-roleless/{distro/rhel_8.0', 'rados/cephadm/smoke-roleless/{distro/rhel_latest', 'rados/cephadm/smoke-roleless/{distro/ubuntu_18.04', 'rados/cephadm/smoke-roleless/{distro/ubuntu_20.04', 'rados/cephadm/smoke/{distro/centos_8.0', 'rados/cephadm/smoke/{distro/centos_latest', 'rados/cephadm/smoke/{distro/rhel_8.0', 'rados/cephadm/smoke/{distro/rhel_latest', 'rados/cephadm/smoke/{distro/ubuntu_18.04', 'rados/cephadm/smoke/{distro/ubuntu_20.04', 'rados/cephadm/thrash-old-clients/{0-size-min-size-overrides/2-size-2-min-size', 'rados/cephadm/thrash-old-clients/{0-size-min-size-overrides/3-size-2-min-size', 'rados/cephadm/thrash/{0-distro/centos_8.0', 'rados/cephadm/thrash/{0-distro/centos_latest', 'rados/cephadm/thrash/{0-distro/rhel_latest', 'rados/cephadm/thrash/{0-distro/ubuntu_18.04', 'rados/cephadm/thrash/{0-distro/ubuntu_20.04', 'rados/cephadm/with-work/{distro/centos_8.0', 'rados/cephadm/with-work/{distro/centos_latest', 'rados/cephadm/with-work/{distro/rhel_8.0', 'rados/cephadm/with-work/{distro/rhel_latest', 'rados/cephadm/with-work/{distro/ubuntu_18.04', 'rados/cephadm/with-work/{distro/ubuntu_20.04', 'rados/cephadm/workunits/{distro/centos_latest', 'rados/upgrade/nautilus-x-singleton/{0-cluster/{openstack', 'rbd-import-export', 'rbd_api}', 'readwrite', 'root}', 'snaps-few-objects}', 'snaps-many-objects}', 'start', 'start}', 'task/test_e2e}', 'task/test_orch_cli}', 'tasks/rados_api_tests}', 'tasks/rados_python}', 'thrashers/careful', 'thrashers/default', 'thrashers/mapgap', 'thrashers/morepggrow', 'thrashers/none', 'thrashers/pggrow', 'thrashosds-health', 'three-plus-one}', 'ubuntu_18.04}', 'workloads/cache-snaps}', 'workloads/radosbench}', 'workloads/rbd_cls}', 'workloads/snaps-few-objects}', 'workloads/test_rbd_api}'] Crash: "2021-02-17T08:56:12.720916+0000 mon.a (mon.0) 147 : cluster [WRN] Health check failed: 1 osds down (OSD_DOWN)" in cluster log ceph version 16.1.0-309-g150664a2 (150664a24fd2908651bf7acb7cdc7c82a897e183) pacific (rc) 1: /lib64/libpthread.so.0(+0x12b20) [0x7fa155ee8b20] 2: gsignal() 3: abort() 4: /lib64/libstdc++.so.6(+0x9009b) [0x7fa15550609b] 5: /lib64/libstdc++.so.6(+0x9653c) [0x7fa15550c53c] 6: /lib64/libstdc++.so.6(+0x96597) [0x7fa15550c597] 7: /lib64/libstdc++.so.6(+0x967f8) [0x7fa15550c7f8] 8: ceph-osd(+0x587fa6) [0x55fee4ccefa6] 9: (ceph::buffer::v15_2_0::create_aligned(unsigned int, unsigned int)+0x26) [0x55fee55e2dc6] 10: (ceph::buffer::v15_2_0::create(unsigned int)+0x26) [0x55fee55e2e16] 11: (ceph::buffer::v15_2_0::list::iterator_impl::copy_shallow(unsigned int, ceph::buffer::v15_2_0::ptr&)+0x5a) [0x55fee55e7b8a] 12: (FileStore::_do_transaction(ceph::os::Transaction&, unsigned long, int, ThreadPool::TPHandle*, char const*)+0x312e) [0x55fee51f8a1e] 13: (FileStore::_do_transactions(std::vector >&, unsigned long, ThreadPool::TPHandle*, char const*)+0x50) [0x55fee51fc220] 14: (FileStore::_do_op(FileStore::OpSequencer*, ThreadPool::TPHandle&)+0x5a7) [0x55fee51fc807] 15: (ThreadPool::worker(ThreadPool::WorkThread*)+0xe9a) [0x55fee543b59a] 16: (ThreadPool::WorkThread::entry()+0x15) [0x55fee543bdf5] 17: /lib64/libpthread.so.0(+0x814a) [0x7fa155ede14a] 18: clone() 1 jobs: ['5885953'] suites: ['mon_election/connectivity', 'msgr-failures/many', 'msgr/async', 'objectstore/filestore-xfs', 'rados', 'rados/singleton/{all/radostool', 'supported-random-distro$/{rhel_8}}'] Dead: b'2021-02-17T17:14:33.649 DEBUG:teuthology.exit:Finished running handlers' 3 jobs: ['5886112', '5886077', '5885914'] suites intersection: [] suites union: ['2-repo_digest/repo_digest', '3-start-upgrade', '4-wait', 'clusters/{fixed-2', 'distro$/{ubuntu_20.04}', 'fast/fast', 'fixed-2', 'mon_election/classic', 'mon_election/classic}', 'msgr-failures/few', 'objectstore/bluestore-bitmap', 'openstack}', 'rados', 'rados/cephadm/upgrade/{1-start', 'rados/objectstore/{backends/objectstore', 'rados/thrash-erasure-code/{ceph', 'recovery-overrides/{more-partial-recovery}', 'supported-random-distro$/{centos_8}', 'supported-random-distro$/{centos_8}}', 'thrashers/minsize_recovery', 'thrashosds-health', 'workloads/ec-small-objects-balanced}'] Failure: Command failed (workunit test osd/osd-recovery-space.sh) on smithi023 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=150664a24fd2908651bf7acb7cdc7c82a897e183 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/standalone/osd/osd-recovery-space.sh' 1 jobs: ['5886102'] suites: ['rados/standalone/{mon_election/classic', 'supported-random-distro$/{centos_8}', 'workloads/osd}'] Crash: Command crashed: 'sudo TESTDIR=/home/ubuntu/cephtest bash -c ceph_test_msgr' ceph version 16.1.0-309-g150664a2 (150664a24fd2908651bf7acb7cdc7c82a897e183) pacific (rc) 1: /lib64/libpthread.so.0(+0x12dc0) [0x7fda9050ddc0] 2: gsignal() 3: abort() 4: (ceph::__ceph_abort(char const*, int, char const*, std::__cxx11::basic_string, std::allocator > const&)+0x1b6) [0x7fda863672eb] 5: (SyntheticWorkload::wait_for_done()+0xdc) [0x559e67aa305c] 6: (MessengerTest_SyntheticStressTest1_Test::TestBody()+0x528) [0x559e67a89308] 7: (void testing::internal::HandleExceptionsInMethodIfSupported(testing::Test*, void (testing::Test::*)(), char const*)+0x4e) [0x559e67ae5e4e] 8: (testing::Test::Run()+0xcb) [0x559e67ad8a7b] 9: (testing::TestInfo::Run()+0x135) [0x559e67ad8be5] 10: (testing::TestSuite::Run()+0xc1) [0x559e67ad9141] 11: (testing::internal::UnitTestImpl::RunAllTests()+0x445) [0x559e67adad55] 12: (bool testing::internal::HandleExceptionsInMethodIfSupported(testing::internal::UnitTestImpl*, bool (testing::internal::UnitTestImpl::*)(), char const*)+0x4e) [0x559e67ae63be] 13: (testing::UnitTest::Run()+0xa0) [0x559e67ad8cb0] 14: main() 15: __libc_start_main() 16: _start() 1 jobs: ['5885903'] suites: ['mon_election/connectivity', 'rados', 'rados/singleton-nomsgr/{all/msgr', 'supported-random-distro$/{centos_8}}']