Found 64 jobs Found 8 distinct failure reasons Failure: Command failed (workunit test rbd/test_librbd_python.sh) on smithi169 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=mimic TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/rbd/test_librbd_python.sh' 2 jobs: ['5905632', '5905633'] suites intersection: ['1-ceph-install/mimic', 'objectstore/bluestore-bitmap', 'rgw_ragweed_prepare', 'start}'] suites union: ['1-ceph-install/mimic', '1.1-pg-log-overrides/normal_pg_log', '1.1-pg-log-overrides/short_pg_log', '2-partial-upgrade/firsthalf', '2-workload/{blogbench', '3-thrash/default', '3-upgrade-sequence/upgrade-mon-osd-mds', '4-msgr2', '4-nautilus', '4-workload/{radosbench', '5-final-workload/{blogbench', '5-finish-upgrade', '6-nautilus', '6.1-msgr2', '7-final-workload/{rbd-python', 'ec-rados-default', 'objectstore/bluestore-bitmap', 'rados-snaps-few-objects', 'rados_api', 'rados_loadgenbig', 'rados_loadgenmix', 'rados_mon_thrash', 'rbd-cls', 'rbd-import-export', 'rbd_api', 'rbd_cls', 'rbd_import_export', 'readwrite', 'rgw', 'rgw-swift-ragweed_check', 'rgw_ragweed_check', 'rgw_ragweed_prepare', 'rgw_swift}', 'snaps-few-objects}', 'snaps-many-objects}', 'start}', 'supported-all-distro/rhel_7', 'supported-all-distro/ubuntu_16.04}', 'test_rbd_api', 'test_rbd_python}', 'thrashosds-health}', 'upgrade:mimic-x/parallel/{0-cluster/{openstack', 'upgrade:mimic-x/stress-split/{0-cluster/{openstack'] Dead: b'2021-02-23T15:19:23.640 DEBUG:teuthology.exit:Finished running handlers' 8 jobs: ['5905605', '5905601', '5905635', '5905613', '5905589', '5905597', '5905617', '5905625'] suites intersection: ['1-ceph-install/mimic', 'rgw_ragweed_prepare', 'start}'] suites union: ['1-ceph-install/mimic', '1.1-pg-log-overrides/normal_pg_log', '1.1-pg-log-overrides/short_pg_log', '2-partial-upgrade/firsthalf', '2-workload/{blogbench', '3-thrash/default', '3-upgrade-sequence/upgrade-all', '4-msgr2', '4-nautilus', '4-workload/{radosbench', '5-final-workload/{blogbench', '5-finish-upgrade', '6-nautilus', '6.1-msgr2', '7-final-workload/{rbd-python', 'ec-rados-default', 'objectstore/bluestore-bitmap', 'objectstore/filestore-xfs', 'rados-snaps-few-objects', 'rados_api', 'rados_loadgenbig', 'rados_loadgenmix', 'rados_mon_thrash', 'rbd-cls', 'rbd-import-export', 'rbd_api', 'rbd_cls', 'rbd_import_export', 'readwrite', 'rgw', 'rgw-swift-ragweed_check', 'rgw_ragweed_check', 'rgw_ragweed_prepare', 'rgw_swift}', 'snaps-few-objects}', 'snaps-many-objects}', 'start}', 'supported-all-distro/centos_latest', 'supported-all-distro/rhel_7', 'supported-all-distro/ubuntu_16.04', 'supported-all-distro/ubuntu_latest', 'supported-all-distro/ubuntu_latest}', 'test_rbd_api', 'test_rbd_python}', 'thrashosds-health}', 'upgrade:mimic-x/parallel/{0-cluster/{openstack', 'upgrade:mimic-x/stress-split/{0-cluster/{openstack'] Failure: failed to recover before timeout expired 3 jobs: ['5905637', '5905641', '5905621'] suites intersection: ['1-ceph-install/mimic', '2-partial-upgrade/firsthalf', '3-thrash/default', '4-workload/{radosbench', '5-finish-upgrade', '6-nautilus', '6.1-msgr2', '7-final-workload/{rbd-python', 'rbd-cls', 'rbd-import-export', 'rbd_api', 'readwrite', 'rgw-swift-ragweed_check', 'rgw_ragweed_prepare', 'snaps-few-objects}', 'snaps-many-objects}', 'start}', 'thrashosds-health}', 'upgrade:mimic-x/stress-split/{0-cluster/{openstack'] suites union: ['1-ceph-install/mimic', '1.1-pg-log-overrides/normal_pg_log', '1.1-pg-log-overrides/short_pg_log', '2-partial-upgrade/firsthalf', '3-thrash/default', '4-workload/{radosbench', '5-finish-upgrade', '6-nautilus', '6.1-msgr2', '7-final-workload/{rbd-python', 'objectstore/bluestore-bitmap', 'objectstore/filestore-xfs', 'rbd-cls', 'rbd-import-export', 'rbd_api', 'readwrite', 'rgw-swift-ragweed_check', 'rgw_ragweed_prepare', 'snaps-few-objects}', 'snaps-many-objects}', 'start}', 'supported-all-distro/ubuntu_16.04', 'supported-all-distro/ubuntu_latest', 'thrashosds-health}', 'upgrade:mimic-x/stress-split/{0-cluster/{openstack'] Dead: b'2021-02-23T14:44:29.554 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read"' 1 jobs: ['5905585'] suites: ['1-ceph-install/mimic', '1.1-pg-log-overrides/short_pg_log', '2-partial-upgrade/firsthalf', '3-thrash/default', '4-workload/{radosbench', '5-finish-upgrade', '6-nautilus', '6.1-msgr2', '7-final-workload/{rbd-python', 'objectstore/filestore-xfs', 'rbd-cls', 'rbd-import-export', 'rbd_api', 'readwrite', 'rgw-swift-ragweed_check', 'rgw_ragweed_prepare', 'snaps-few-objects}', 'snaps-many-objects}', 'start}', 'supported-all-distro/rhel_7', 'thrashosds-health}', 'upgrade:mimic-x/stress-split/{0-cluster/{openstack'] Dead: b'2021-02-23T14:40:51.624 INFO:teuthology.orchestra.run.smithi095.stderr:2021-02-23 14:40:49.246 7f5e44887700 0 --1- 172.21.15.95:0/2858363573 >> v1:172.21.15.95:6825/330505 conn(0x7f5e20016e50 0x7f5e2001ba60 :-1 s=CONNECTING_SEND_CONNECT_MSG pgs=0 cs=0 l=1).handle_connect_reply_2 connect got BADAUTHORIZER' 3 jobs: ['5905581', '5905629', '5905609'] suites intersection: ['1-ceph-install/mimic', '2-partial-upgrade/firsthalf', '3-thrash/default', '4-workload/{radosbench', '5-finish-upgrade', '6-nautilus', '6.1-msgr2', '7-final-workload/{rbd-python', 'rbd-cls', 'rbd-import-export', 'rbd_api', 'readwrite', 'rgw-swift-ragweed_check', 'rgw_ragweed_prepare', 'snaps-few-objects}', 'snaps-many-objects}', 'start}', 'thrashosds-health}', 'upgrade:mimic-x/stress-split/{0-cluster/{openstack'] suites union: ['1-ceph-install/mimic', '1.1-pg-log-overrides/normal_pg_log', '1.1-pg-log-overrides/short_pg_log', '2-partial-upgrade/firsthalf', '3-thrash/default', '4-workload/{radosbench', '5-finish-upgrade', '6-nautilus', '6.1-msgr2', '7-final-workload/{rbd-python', 'objectstore/bluestore-bitmap', 'objectstore/filestore-xfs', 'rbd-cls', 'rbd-import-export', 'rbd_api', 'readwrite', 'rgw-swift-ragweed_check', 'rgw_ragweed_prepare', 'snaps-few-objects}', 'snaps-many-objects}', 'start}', 'supported-all-distro/centos_latest', 'supported-all-distro/ubuntu_latest', 'thrashosds-health}', 'upgrade:mimic-x/stress-split/{0-cluster/{openstack'] Failure: "2021-02-23 06:37:56.158332 mon.a (mon.0) 10 : cluster [WRN] daemon mds.b is not responding, replacing it as rank 0 with standby daemon mds.a" in cluster log 1 jobs: ['5905603'] suites: ['1-ceph-install/mimic', '1.1-pg-log-overrides/short_pg_log', '2-workload/{blogbench', '3-upgrade-sequence/upgrade-all', '4-msgr2', '4-nautilus', '5-final-workload/{blogbench', 'ec-rados-default', 'objectstore/filestore-xfs', 'rados-snaps-few-objects', 'rados_api', 'rados_loadgenbig', 'rados_loadgenmix', 'rados_mon_thrash', 'rbd_cls', 'rbd_import_export', 'rgw', 'rgw_ragweed_check', 'rgw_ragweed_prepare', 'rgw_swift}', 'start}', 'supported-all-distro/ubuntu_latest}', 'test_rbd_api', 'test_rbd_python}', 'upgrade:mimic-x/parallel/{0-cluster/{openstack'] Failure: Command failed (workunit test rados/test.sh) on smithi141 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.1/client.1/tmp && cd -- /home/ubuntu/cephtest/mnt.1/client.1/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=7dbba1bc150916539993edfbe467e84a697568a3 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="1" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.1 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.1 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.1/qa/workunits/rados/test.sh' 1 jobs: ['5905595'] suites: ['1-ceph-install/mimic', '1.1-pg-log-overrides/short_pg_log', '2-workload/{blogbench', '3-upgrade-sequence/upgrade-mon-osd-mds', '4-msgr2', '4-nautilus', '5-final-workload/{blogbench', 'ec-rados-default', 'objectstore/bluestore-bitmap', 'rados-snaps-few-objects', 'rados_api', 'rados_loadgenbig', 'rados_loadgenmix', 'rados_mon_thrash', 'rbd_cls', 'rbd_import_export', 'rgw', 'rgw_ragweed_check', 'rgw_ragweed_prepare', 'rgw_swift}', 'start}', 'supported-all-distro/ubuntu_latest}', 'test_rbd_api', 'test_rbd_python}', 'upgrade:mimic-x/parallel/{0-cluster/{openstack'] Dead: b'2021-02-23T14:55:34.811 INFO:tasks.ceph.osd.4.smithi093.stderr:2021-02-23 14:55:34.804 7f3037230700 -1 received signal: Hangup from /usr/bin/python3 /usr/bin/daemon-helper kill ceph-osd -f --cluster ceph -i 4 (PID: 13861) UID: 0' 1 jobs: ['5905593'] suites: ['1-ceph-install/mimic', '1.1-pg-log-overrides/short_pg_log', '2-partial-upgrade/firsthalf', '3-thrash/default', '4-workload/{radosbench', '5-finish-upgrade', '6-nautilus', '6.1-msgr2', '7-final-workload/{rbd-python', 'objectstore/filestore-xfs', 'rbd-cls', 'rbd-import-export', 'rbd_api', 'readwrite', 'rgw-swift-ragweed_check', 'rgw_ragweed_prepare', 'snaps-few-objects}', 'snaps-many-objects}', 'start}', 'supported-all-distro/ubuntu_latest', 'thrashosds-health}', 'upgrade:mimic-x/stress-split/{0-cluster/{openstack']