Found 308 jobs Found 7 distinct failure reasons Failure: Command failed on smithi149 with status 128: 'rm -rf /home/ubuntu/cephtest/clone.client.0 && git clone git://git.ceph.com/ceph.git /home/ubuntu/cephtest/clone.client.0 && cd /home/ubuntu/cephtest/clone.client.0 && git checkout baa1ea6a9656c3db06c66032fa80b476721947ba' 80 jobs suites intersection: [] suites union: ['1-install/nautilus', '1-pg-log-overrides/short_pg_log', '2-partial-upgrade/firsthalf', '2-recovery-overrides/{more-active-recovery}', '3-thrash/default', '4-workload/{rbd-cls', '5-workload/{radosbench', '6-finish-upgrade', '7-octopus', '8-workload/{rbd-python', 'backoff/peering_and_degraded', 'bluestore-bitmap', 'ceph', 'clusters/3-mons', 'clusters/9-mons', 'clusters/{fixed-2', 'd-balancer/on', 'd-thrash/default/{default', 'd-thrash/none', 'debug/mgr', 'fixed-2', 'mode/packaged', 'mode/root', 'msgr-failures/few', 'msgr-failures/many', 'msgr-failures/mon-delay', 'msgr-failures/osd-delay', 'msgr/async', 'msgr/async-v1only', 'msgr/async-v2only', 'objectstore/bluestore-bitmap', 'objectstore/bluestore-comp-lz4', 'objectstore/bluestore-comp-snappy', 'objectstore/bluestore-comp-zlib', 'objectstore/bluestore-comp-zstd', 'objectstore/bluestore-hybrid', 'objectstore/bluestore-low-osd-mem-target', 'objectstore/bluestore-stupid', 'objectstore/filestore-xfs', 'openstack}', 'rados', 'rados/basic/{ceph', 'rados/cephadm/with-work/{distro/centos_8.0', 'rados/cephadm/with-work/{distro/centos_latest', 'rados/cephadm/with-work/{distro/rhel_8.0', 'rados/cephadm/with-work/{distro/rhel_latest', 'rados/cephadm/with-work/{distro/ubuntu_18.04', 'rados/cephadm/with-work/{distro/ubuntu_18.04_podman', 'rados/cephadm/workunits/{distro/centos_latest', 'rados/cephadm/workunits/{distro/ubuntu_18.04_podman', 'rados/mgr/{clusters/{2-node-mgr}', 'rados/monthrash/{ceph', 'rados/objectstore/{backends/alloc-hint', 'rados/objectstore/{backends/fusestore', 'rados/objectstore/{backends/objectcacher-stress', 'rados/rest/{mgr-restful', 'rados/singleton-bluestore/{all/cephtool', 'rados/singleton-nomsgr/{all/balancer', 'rados/singleton-nomsgr/{all/ceph-kvstore-tool', 'rados/singleton-nomsgr/{all/ceph-post-file', 'rados/singleton-nomsgr/{all/large-omap-object-warnings', 'rados/singleton-nomsgr/{all/librados_hello_world', 'rados/singleton-nomsgr/{all/pool-access', 'rados/singleton-nomsgr/{all/version-number-sanity', 'rados/singleton/{all/deduptool', 'rados/singleton/{all/erasure-code-nonregression', 'rados/singleton/{all/mon-auth-caps', 'rados/singleton/{all/mon-config', 'rados/singleton/{all/mon-config-key-caps', 'rados/singleton/{all/mon-config-keys', 'rados/singleton/{all/pg-autoscaler', 'rados/singleton/{all/radostool', 'rados/singleton/{all/test-crash', 'rados/singleton/{all/test_envlibrados_for_rocksdb', 'rados/singleton/{all/thrash-rados/{thrash-rados', 'rados/standalone/{supported-random-distro$/{centos_8}', 'rados/standalone/{supported-random-distro$/{rhel_8}', 'rados/standalone/{supported-random-distro$/{ubuntu_latest}', 'rados/thrash/{0-size-min-size-overrides/3-size-2-min-size', 'rados/upgrade/nautilus-x-singleton/{0-cluster/{openstack', 'rados/verify/{centos_latest', 'rbd-import-export', 'rbd_api}', 'readwrite', 'snaps-few-objects}', 'snaps-many-objects}', 'start', 'start}', 'supported-random-distro$/{centos_8}', 'supported-random-distro$/{centos_8}}', 'supported-random-distro$/{rhel_8}', 'supported-random-distro$/{rhel_8}}', 'supported-random-distro$/{ubuntu_latest}', 'supported-random-distro$/{ubuntu_latest}}', 'task/test_adoption}', 'task/test_cephadm_repos}', 'task/test_cephadm}', 'tasks/rados_api_tests', 'tasks/rados_api_tests}', 'tasks/rados_cls_all', 'tasks/rados_cls_all}', 'tasks/rados_python}', 'tasks/rados_stress_watch}', 'tasks/rados_workunit_loadgen_big}', 'tasks/rados_workunit_loadgen_mix}', 'tasks/rados_workunit_loadgen_mostlyread}', 'tasks/workunits}', 'thrashers/force-sync-many', 'thrashers/many', 'thrashers/none', 'thrashers/sync', 'thrashosds-health', 'thrashosds-health}', 'ubuntu_latest}', 'validater/lockdep}', 'validater/valgrind}', 'workloads/crush}', 'workloads/erasure-code}', 'workloads/mgr}', 'workloads/misc}', 'workloads/mon}', 'workloads/osd}', 'workloads/rados_api_tests}', 'workloads/rados_mon_osdmap_prune}', 'workloads/rados_mon_workunits}', 'workloads/scrub}'] Failure: Test failure: test_create_and_delete_cluster (tasks.cephfs.test_nfs.TestNFS) 4 jobs: ['5193917', '5193833', '5193990', '5194068'] suites intersection: ['task/test_orch_cli}'] suites union: ['rados/cephadm/workunits/{distro/centos_latest', 'rados/cephadm/workunits/{distro/ubuntu_18.04_podman', 'task/test_orch_cli}'] Failure: Scrubbing terminated -- not all pgs were active and clean. 4 jobs: ['5193987', '5194100', '5193865', '5193968'] suites intersection: ['rados'] suites union: ['1-pg-log-overrides/short_pg_log', '2-recovery-overrides/{more-async-partial-recovery}', 'backoff/peering_and_degraded', 'ceph', 'clusters/{fixed-2', 'd-balancer/on', 'd-thrash/default/{default', 'msgr-failures/few', 'msgr-failures/osd-delay', 'msgr/async', 'msgr/async-v2only', 'objectstore/bluestore-comp-lz4', 'objectstore/bluestore-hybrid', 'openstack}', 'rados', 'rados/singleton-nomsgr/{all/health-warnings', 'rados/thrash/{0-size-min-size-overrides/3-size-2-min-size', 'rados/verify/{centos_latest', 'supported-random-distro$/{rhel_8}', 'supported-random-distro$/{ubuntu_latest}}', 'tasks/rados_api_tests', 'tasks/rados_cls_all', 'thrashers/none', 'thrashosds-health', 'thrashosds-health}', 'validater/valgrind}', 'workloads/set-chunk-promote-flush}'] Failure: reached maximum tries (180) after waiting for 180 seconds 4 jobs: ['5194067', '5193937', '5193906', '5193969'] suites intersection: ['ceph', 'clusters/{openstack', 'distro$/{centos_7.6}', 'rados', 'thrashosds-health', 'three-plus-one}'] suites union: ['1-install/mimic', '1-install/mimic-v1only', '1-install/nautilus-v1only', '1-install/octopus', 'backoff/normal', 'backoff/peering', 'backoff/peering_and_degraded', 'ceph', 'clusters/{openstack', 'd-balancer/crush-compat', 'd-balancer/on', 'distro$/{centos_7.6}', 'msgr-failures/fastclose', 'msgr-failures/few', 'msgr-failures/osd-delay', 'rados', 'rados/thrash-old-clients/{0-size-min-size-overrides/2-size-2-min-size', 'rados/thrash-old-clients/{0-size-min-size-overrides/3-size-2-min-size', 'thrashers/careful', 'thrashers/morepggrow', 'thrashers/none', 'thrashers/pggrow', 'thrashosds-health', 'three-plus-one}', 'workloads/cache-snaps}', 'workloads/radosbench}', 'workloads/test_rbd_api}'] Failure: {'Failure object was': {'smithi142.front.sepia.ceph.com': {'msg': 'Failed to download packages: Cannot download Packages/boost-random-1.66.0-6.el8.x86_64.rpm: All mirrors were tried', 'results': [], 'invocation': {'module_args': {'name': ['dnf-utils', 'sysstat', 'libedit', 'boost-thread', 'xfsprogs', 'gdisk', 'parted', 'libgcrypt', 'fuse-libs', 'openssl', 'libuuid', 'podman', 'podman-docker', 'attr', 'ant', 'lsof', 'gettext', 'bc', 'xfsdump', 'blktrace', 'usbredir', 'libev-devel', 'valgrind', 'nfs-utils', 'ncurses-devel', 'gcc', 'git', 'python3-nose', 'python3-virtualenv', 'genisoimage', 'qemu-img', 'qemu-kvm-core', 'qemu-kvm-block-rbd', 'libacl-devel', 'dbench', 'autoconf', 'gdb', 'iozone'], 'state': 'present', 'allow_downgrade': False, 'autoremove': False, 'bugfix': False, 'disable_gpg_check': False, 'disable_plugin': [], 'disablerepo': [], 'download_only': False, 'enable_plugin': [], 'enablerepo': [], 'exclude': [], 'installroot': '/', 'install_repoquery': True, 'install_weak_deps': True, 'security': False, 'skip_broken': False, 'update_cache': False, 'update_only': False, 'validate_certs': True, 'lock_timeout': 30, 'conf_file': 'None', 'disable_excludes': 'None', 'download_dir': 'None', 'list': 'None', 'releasever': 'None'}}, '_ansible_no_log': False, 'changed': False}}, 'Traceback (most recent call last)': 'File "/home/teuthworker/src/git.ceph.com_git_ceph-cm-ansible_master/callback_plugins/failure_log.py", line 44, in log_failure log.error(yaml.safe_dump(failure)) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/lib/python3.6/site-packages/yaml/__init__.py", line 306, in safe_dump return dump_all([data], stream, Dumper=SafeDumper, **kwds) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/lib/python3.6/site-packages/yaml/__init__.py", line 278, in dump_all dumper.represent(data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/lib/python3.6/site-packages/yaml/representer.py", line 27, in represent node = self.represent_data(data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/lib/python3.6/site-packages/yaml/representer.py", line 48, in represent_data node = self.yaml_representers[data_types[0]](self, data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/lib/python3.6/site-packages/yaml/representer.py", line 207, in represent_dict return self.represent_mapping(\'tag:yaml.org,2002:map\', data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/lib/python3.6/site-packages/yaml/representer.py", line 118, in represent_mapping node_value = self.represent_data(item_value) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/lib/python3.6/site-packages/yaml/representer.py", line 48, in represent_data node = self.yaml_representers[data_types[0]](self, data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/lib/python3.6/site-packages/yaml/representer.py", line 207, in represent_dict return self.represent_mapping(\'tag:yaml.org,2002:map\', data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/lib/python3.6/site-packages/yaml/representer.py", line 118, in represent_mapping node_value = self.represent_data(item_value) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/lib/python3.6/site-packages/yaml/representer.py", line 48, in represent_data node = self.yaml_representers[data_types[0]](self, data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/lib/python3.6/site-packages/yaml/representer.py", line 207, in represent_dict return self.represent_mapping(\'tag:yaml.org,2002:map\', data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/lib/python3.6/site-packages/yaml/representer.py", line 118, in represent_mapping node_value = self.represent_data(item_value) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/lib/python3.6/site-packages/yaml/representer.py", line 48, in represent_data node = self.yaml_representers[data_types[0]](self, data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/lib/python3.6/site-packages/yaml/representer.py", line 207, in represent_dict return self.represent_mapping(\'tag:yaml.org,2002:map\', data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/lib/python3.6/site-packages/yaml/representer.py", line 118, in represent_mapping node_value = self.represent_data(item_value) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/lib/python3.6/site-packages/yaml/representer.py", line 58, in represent_data node = self.yaml_representers[None](self, data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/lib/python3.6/site-packages/yaml/representer.py", line 231, in represent_undefined raise RepresenterError("cannot represent an object", data)', 'yaml.representer.RepresenterError': "('cannot represent an object', '/')"} 1 jobs: ['5193870'] suites: ['debug/mgr', 'objectstore/bluestore-stupid', 'rados/mgr/{clusters/{2-node-mgr}', 'supported-random-distro$/{centos_8}', 'tasks/failover}'] Crash: Command crashed: "sudo TESTDIR=/home/ubuntu/cephtest bash -c 'mkdir $TESTDIR/kvtest && cd $TESTDIR/kvtest && ceph_test_keyvaluedb'" ceph version 16.0.0-2962-g5ccfc2d5d3 (5ccfc2d5d3a03d700cf9c4a48b194d7836328981) pacific (dev) 1: (()+0x12890) [0x7f395a0bf890] 2: (gsignal()+0xc7) [0x7f394f3ebe97] 3: (abort()+0x141) [0x7f394f3ed801] 4: (()+0x3039a) [0x7f394f3dd39a] 5: (()+0x30412) [0x7f394f3dd412] 6: (rocksdb::ColumnFamilySet::~ColumnFamilySet()+0x91) [0x5617f932cfcf] 7: (std::default_delete::operator()(rocksdb::ColumnFamilySet*) const+0x22) [0x5617f917174c] 8: (std::unique_ptr >::reset(rocksdb::ColumnFamilySet*)+0x55) [0x5617f916bc73] 9: (rocksdb::VersionSet::~VersionSet()+0x5e) [0x5617f9150b00] 10: (rocksdb::VersionSet::~VersionSet()+0x18) [0x5617f9150c88] 11: (std::default_delete::operator()(rocksdb::VersionSet*) const+0x2e) [0x5617f8ff6f22] 12: (std::unique_ptr >::reset(rocksdb::VersionSet*)+0x55) [0x5617f8fe95c1] 13: (rocksdb::DBImpl::CloseHelper()+0xaaa) [0x5617f8fb2b82] 14: (rocksdb::DBImpl::~DBImpl()+0x5d) [0x5617f8fb300b] 15: (rocksdb::DBImpl::~DBImpl()+0x18) [0x5617f8fb33ce] 16: (RocksDBStore::close()+0x31a) [0x5617f8f84faa] 17: (RocksDBStore::reshard(std::__cxx11::basic_string, std::allocator > const&, RocksDBStore::resharding_ctrl const*)+0x31e) [0x5617f8f8bbfe] 18: (RocksDBResharding_change_reshard_Test::TestBody()+0x15a) [0x5617f8efdc6a] 19: (void testing::internal::HandleExceptionsInMethodIfSupported(testing::Test*, void (testing::Test::*)(), char const*)+0x4a) [0x5617f8f46bca] 20: (testing::Test::Run()+0xc3) [0x5617f8f3c843] 21: (testing::TestInfo::Run()+0x135) [0x5617f8f3c9a5] 22: (testing::TestSuite::Run()+0xb5) [0x5617f8f3ca85] 23: (testing::internal::UnitTestImpl::RunAllTests()+0x478) [0x5617f8f3cf98] 24: (bool testing::internal::HandleExceptionsInMethodIfSupported(testing::internal::UnitTestImpl*, bool (testing::internal::UnitTestImpl::*)(), char const*)+0x4a) [0x5617f8f470da] 25: (testing::UnitTest::Run()+0x98) [0x5617f8f3d1a8] 26: (main()+0x152) [0x5617f8ee79e2] 27: (__libc_start_main()+0xe7) [0x7f394f3ceb97] 28: (_start()+0x2a) [0x5617f8eea4ca] 1 jobs: ['5194051'] suites: ['rados/objectstore/{backends/keyvaluedb', 'supported-random-distro$/{ubuntu_latest}}'] Failure: "2020-07-02T06:05:59.895940+0000 mon.a (mon.0) 124 : cluster [WRN] Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY)" in cluster log 1 jobs: ['5193896'] suites: ['ceph', 'clusters/{fixed-2', 'd-thrash/none', 'msgr-failures/few', 'msgr/async', 'objectstore/bluestore-comp-snappy', 'openstack}', 'rados', 'rados/verify/{centos_latest', 'tasks/rados_cls_all', 'validater/lockdep}']