Found 308 jobs Found 9 distinct failure reasons Failure: Command failed on smithi097 with status 128: 'rm -rf /home/ubuntu/cephtest/clone.client.0 && git clone git://git.ceph.com/ceph.git /home/ubuntu/cephtest/clone.client.0 && cd /home/ubuntu/cephtest/clone.client.0 && git checkout baa1ea6a9656c3db06c66032fa80b476721947ba' 81 jobs suites intersection: [] suites union: ['1-install/nautilus', '1-pg-log-overrides/normal_pg_log', '2-partial-upgrade/firsthalf', '2-recovery-overrides/{more-active-recovery}', '3-thrash/default', '4-workload/{rbd-cls', '5-workload/{radosbench', '6-finish-upgrade', '7-octopus', '8-workload/{rbd-python', 'backoff/normal', 'bluestore-bitmap', 'ceph', 'clusters/3-mons', 'clusters/9-mons', 'clusters/{fixed-2', 'd-balancer/crush-compat', 'd-thrash/default/{default', 'd-thrash/none', 'debug/mgr', 'fixed-2', 'mode/packaged', 'mode/root', 'msgr-failures/fastclose', 'msgr-failures/few', 'msgr-failures/many', 'msgr-failures/mon-delay', 'msgr/async', 'msgr/async-v1only', 'msgr/async-v2only', 'objectstore/bluestore-bitmap', 'objectstore/bluestore-comp-lz4', 'objectstore/bluestore-comp-snappy', 'objectstore/bluestore-comp-zlib', 'objectstore/bluestore-comp-zstd', 'objectstore/bluestore-hybrid', 'objectstore/bluestore-low-osd-mem-target', 'objectstore/bluestore-stupid', 'objectstore/filestore-xfs', 'openstack}', 'rados', 'rados/basic/{ceph', 'rados/cephadm/with-work/{distro/centos_8.0', 'rados/cephadm/with-work/{distro/centos_latest', 'rados/cephadm/with-work/{distro/rhel_8.0', 'rados/cephadm/with-work/{distro/rhel_latest', 'rados/cephadm/with-work/{distro/ubuntu_18.04', 'rados/cephadm/with-work/{distro/ubuntu_18.04_podman', 'rados/cephadm/workunits/{distro/centos_latest', 'rados/cephadm/workunits/{distro/ubuntu_18.04_podman', 'rados/mgr/{clusters/{2-node-mgr}', 'rados/monthrash/{ceph', 'rados/objectstore/{backends/alloc-hint', 'rados/objectstore/{backends/fusestore', 'rados/objectstore/{backends/objectcacher-stress', 'rados/rest/{mgr-restful', 'rados/singleton-bluestore/{all/cephtool', 'rados/singleton-nomsgr/{all/balancer', 'rados/singleton-nomsgr/{all/ceph-kvstore-tool', 'rados/singleton-nomsgr/{all/ceph-post-file', 'rados/singleton-nomsgr/{all/health-warnings', 'rados/singleton-nomsgr/{all/large-omap-object-warnings', 'rados/singleton-nomsgr/{all/librados_hello_world', 'rados/singleton-nomsgr/{all/pool-access', 'rados/singleton-nomsgr/{all/version-number-sanity', 'rados/singleton/{all/deduptool', 'rados/singleton/{all/erasure-code-nonregression', 'rados/singleton/{all/mon-auth-caps', 'rados/singleton/{all/mon-config', 'rados/singleton/{all/mon-config-key-caps', 'rados/singleton/{all/mon-config-keys', 'rados/singleton/{all/pg-autoscaler', 'rados/singleton/{all/radostool', 'rados/singleton/{all/test-crash', 'rados/singleton/{all/test_envlibrados_for_rocksdb', 'rados/singleton/{all/thrash-rados/{thrash-rados', 'rados/standalone/{supported-random-distro$/{centos_8}', 'rados/standalone/{supported-random-distro$/{rhel_8}', 'rados/standalone/{supported-random-distro$/{ubuntu_latest}', 'rados/thrash/{0-size-min-size-overrides/2-size-2-min-size', 'rados/upgrade/nautilus-x-singleton/{0-cluster/{openstack', 'rados/verify/{centos_latest', 'rbd-import-export', 'rbd_api}', 'readwrite', 'snaps-few-objects}', 'snaps-many-objects}', 'start', 'start}', 'supported-random-distro$/{centos_8}', 'supported-random-distro$/{centos_8}}', 'supported-random-distro$/{rhel_8}', 'supported-random-distro$/{rhel_8}}', 'supported-random-distro$/{ubuntu_latest}', 'supported-random-distro$/{ubuntu_latest}}', 'task/test_adoption}', 'task/test_cephadm_repos}', 'task/test_cephadm}', 'tasks/rados_api_tests', 'tasks/rados_api_tests}', 'tasks/rados_cls_all', 'tasks/rados_cls_all}', 'tasks/rados_python}', 'tasks/rados_stress_watch}', 'tasks/rados_workunit_loadgen_big}', 'tasks/rados_workunit_loadgen_mix}', 'tasks/rados_workunit_loadgen_mostlyread}', 'tasks/workunits}', 'thrashers/force-sync-many', 'thrashers/one', 'thrashers/pggrow', 'thrashers/sync', 'thrashers/sync-many', 'thrashosds-health', 'thrashosds-health}', 'ubuntu_latest}', 'validater/lockdep}', 'validater/valgrind}', 'workloads/crush}', 'workloads/erasure-code}', 'workloads/mgr}', 'workloads/misc}', 'workloads/mon}', 'workloads/osd}', 'workloads/rados_api_tests}', 'workloads/rados_mon_osdmap_prune}', 'workloads/rados_mon_workunits}', 'workloads/scrub}'] Crash: Command crashed: "sudo TESTDIR=/home/ubuntu/cephtest bash -c 'mkdir $TESTDIR/kvtest && cd $TESTDIR/kvtest && ceph_test_keyvaluedb'" ceph version 16.0.0-2962-g5ccfc2d5d3 (5ccfc2d5d3a03d700cf9c4a48b194d7836328981) pacific (dev) 1: (()+0x12890) [0x7f53ee27e890] 2: (gsignal()+0xc7) [0x7f53e35aae97] 3: (abort()+0x141) [0x7f53e35ac801] 4: (()+0x3039a) [0x7f53e359c39a] 5: (()+0x30412) [0x7f53e359c412] 6: (rocksdb::ColumnFamilySet::~ColumnFamilySet()+0x91) [0x555a5f6dbfcf] 7: (std::default_delete::operator()(rocksdb::ColumnFamilySet*) const+0x22) [0x555a5f52074c] 8: (std::unique_ptr >::reset(rocksdb::ColumnFamilySet*)+0x55) [0x555a5f51ac73] 9: (rocksdb::VersionSet::~VersionSet()+0x5e) [0x555a5f4ffb00] 10: (rocksdb::VersionSet::~VersionSet()+0x18) [0x555a5f4ffc88] 11: (std::default_delete::operator()(rocksdb::VersionSet*) const+0x2e) [0x555a5f3a5f22] 12: (std::unique_ptr >::reset(rocksdb::VersionSet*)+0x55) [0x555a5f3985c1] 13: (rocksdb::DBImpl::CloseHelper()+0xaaa) [0x555a5f361b82] 14: (rocksdb::DBImpl::~DBImpl()+0x5d) [0x555a5f36200b] 15: (rocksdb::DBImpl::~DBImpl()+0x18) [0x555a5f3623ce] 16: (RocksDBStore::close()+0x31a) [0x555a5f333faa] 17: (RocksDBStore::reshard(std::__cxx11::basic_string, std::allocator > const&, RocksDBStore::resharding_ctrl const*)+0x31e) [0x555a5f33abfe] 18: (RocksDBResharding_change_reshard_Test::TestBody()+0x15a) [0x555a5f2acc6a] 19: (void testing::internal::HandleExceptionsInMethodIfSupported(testing::Test*, void (testing::Test::*)(), char const*)+0x4a) [0x555a5f2f5bca] 20: (testing::Test::Run()+0xc3) [0x555a5f2eb843] 21: (testing::TestInfo::Run()+0x135) [0x555a5f2eb9a5] 22: (testing::TestSuite::Run()+0xb5) [0x555a5f2eba85] 23: (testing::internal::UnitTestImpl::RunAllTests()+0x478) [0x555a5f2ebf98] 24: (bool testing::internal::HandleExceptionsInMethodIfSupported(testing::internal::UnitTestImpl*, bool (testing::internal::UnitTestImpl::*)(), char const*)+0x4a) [0x555a5f2f60da] 25: (testing::UnitTest::Run()+0x98) [0x555a5f2ec1a8] 26: (main()+0x152) [0x555a5f2969e2] 27: (__libc_start_main()+0xe7) [0x7f53e358db97] 28: (_start()+0x2a) [0x555a5f2994ca] 1 jobs: ['5199781'] suites: ['rados/objectstore/{backends/keyvaluedb', 'supported-random-distro$/{ubuntu_latest}}'] Failure: reached maximum tries (180) after waiting for 180 seconds 8 jobs: ['5199720', '5199847', '5199603', '5199751', '5199740', '5199818', '5199593', '5199655'] suites intersection: [] suites union: ['1-install/luminous', '1-install/luminous-v1only', '1-install/mimic', '1-install/nautilus', '1-install/nautilus-v2only', 'backoff/normal', 'backoff/peering', 'backoff/peering_and_degraded', 'ceph', 'clusters/{openstack', 'd-balancer/crush-compat', 'd-balancer/on', 'distro$/{centos_7.6}', 'fixed-2', 'mode/root', 'msgr-failures/fastclose', 'msgr-failures/few', 'msgr-failures/osd-delay', 'msgr/async-v2only', 'rados', 'rados/cephadm/with-work/{distro/ubuntu_18.04_podman', 'rados/thrash-old-clients/{0-size-min-size-overrides/2-size-2-min-size', 'rados/thrash-old-clients/{0-size-min-size-overrides/3-size-2-min-size', 'start', 'tasks/rados_api_tests}', 'thrashers/careful', 'thrashers/default', 'thrashers/morepggrow', 'thrashers/none', 'thrashers/pggrow', 'thrashosds-health', 'three-plus-one}', 'workloads/cache-snaps}', 'workloads/radosbench}', 'workloads/rbd_cls}', 'workloads/test_rbd_api}'] Failure: wait_for_recovery: failed before timeout expired 1 jobs: ['5199624'] suites: ['1-install/mimic-v1only', 'backoff/normal', 'ceph', 'clusters/{openstack', 'd-balancer/crush-compat', 'distro$/{centos_7.6}', 'msgr-failures/osd-delay', 'rados', 'rados/thrash-old-clients/{0-size-min-size-overrides/2-size-2-min-size', 'thrashers/careful', 'thrashosds-health', 'three-plus-one}', 'workloads/snaps-few-objects}'] Failure: Command failed on smithi187 with status 100: "sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo apt -y install podman'" 1 jobs: ['5199761'] suites: ['rados/cephadm/workunits/{distro/ubuntu_18.04_podman', 'task/test_cephadm}'] Failure: Scrubbing terminated -- not all pgs were active and clean. 4 jobs: ['5199725', '5199806', '5199793', '5199796'] suites intersection: ['rados'] suites union: ['1-pg-log-overrides/short_pg_log', '2-recovery-overrides/{more-async-partial-recovery}', 'backoff/peering_and_degraded', 'ceph', 'clusters/3-mons', 'clusters/{fixed-2', 'd-balancer/on', 'd-thrash/default/{default', 'msgr-failures/few', 'msgr-failures/osd-delay', 'msgr/async', 'msgr/async-v2only', 'objectstore/bluestore-comp-snappy', 'objectstore/bluestore-low-osd-mem-target', 'objectstore/filestore-xfs', 'openstack}', 'rados', 'rados/monthrash/{ceph', 'rados/singleton-nomsgr/{all/multi-backfill-reject', 'rados/thrash/{0-size-min-size-overrides/3-size-2-min-size', 'rados/verify/{centos_latest', 'supported-random-distro$/{centos_8}', 'supported-random-distro$/{ubuntu_latest}', 'supported-random-distro$/{ubuntu_latest}}', 'tasks/rados_api_tests', 'thrashers/many', 'thrashers/none', 'thrashosds-health', 'thrashosds-health}', 'validater/valgrind}', 'workloads/pool-create-delete}', 'workloads/pool-snaps-few-objects}'] Failure: Test failure: test_create_and_delete_cluster (tasks.cephfs.test_nfs.TestNFS) 4 jobs: ['5199726', '5199650', '5199802', '5199567'] suites intersection: ['task/test_orch_cli}'] suites union: ['rados/cephadm/workunits/{distro/centos_latest', 'rados/cephadm/workunits/{distro/ubuntu_18.04_podman', 'task/test_orch_cli}'] Failure: '/home/ubuntu/cephtest/archive/syslog/misc.log:2020-07-05T11:27:33.235365+00:00 smithi133 bash[12421]: debug 2020-07-05T11:27:33.228+0000 7f18cf821700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr:INFO:cephadm:/usr/bin/podman:stderr Error: container_linux.go:349: starting container process caused "exec: \"/usr/sbin/ceph-volume\": stat /usr/sbin/ceph-volume: no such file or directory": OCI runtime command not found error ' in syslog 1 jobs: ['5199855'] suites: ['rados/cephadm/smoke-roleless/{distro/ubuntu_18.04_podman', 'start}'] Failure: "2020-07-05T11:13:18.546056+0000 mon.a (mon.0) 127 : cluster [WRN] Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY)" in cluster log 1 jobs: ['5199828'] suites: ['ceph', 'clusters/{fixed-2', 'd-thrash/none', 'msgr-failures/few', 'msgr/async', 'objectstore/bluestore-comp-zlib', 'openstack}', 'rados', 'rados/verify/{centos_latest', 'tasks/rados_cls_all', 'validater/lockdep}']