Found 160 jobs
Found 8 distinct failure reasons
Crash: 'wait_until_healthy' reached maximum tries (150) after waiting for 900 seconds
ceph version 14.2.21-248-g712a56ab7ae (712a56ab7aeaed280c06d871319a38308d8c834d) nautilus (stable)
 1: (()+0x12980) [0x7fcfeb0a7980]
 2: (gsignal()+0xc7) [0x7fcfea19ffb7]
 3: (abort()+0x141) [0x7fcfea1a1921]
 4: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x1a3) [0x7fcfebd05543]
 5: (ceph::__ceph_assertf_fail(char const*, char const*, int, char const*, char const*, ...)+0) [0x7fcfebd056cd]
 6: (Mutex::lock(bool)+0xa8) [0x7fcfebd85b38]
 7: (PyModuleRunner::log(int, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&)+0x100) [0x559965026720]
 8: (()+0x156931) [0x559964f84931]
 9: (PyEval_EvalFrameEx()+0x8010) [0x7fcfeb5c6780]
 10: (PyEval_EvalFrameEx()+0x8b5b) [0x7fcfeb5c72cb]
 11: (PyEval_EvalFrameEx()+0x8b5b) [0x7fcfeb5c72cb]
 12: (PyEval_EvalFrameEx()+0x8b5b) [0x7fcfeb5c72cb]
 13: (PyEval_EvalFrameEx()+0x8b5b) [0x7fcfeb5c72cb]
 14: (PyEval_EvalCodeEx()+0x7d8) [0x7fcfeb703908]
 15: (()+0x17a72d) [0x7fcfeb69e72d]
 16: (PyObject_Call()+0x43) [0x7fcfeb698913]
 17: (PyEval_EvalFrameEx()+0x5314) [0x7fcfeb5c3a84]
 18: (PyEval_EvalCodeEx()+0x7d8) [0x7fcfeb703908]
 19: (PyEval_EvalFrameEx()+0x5bf6) [0x7fcfeb5c4366]
 20: (PyEval_EvalCodeEx()+0x7d8) [0x7fcfeb703908]
 21: (()+0x17a72d) [0x7fcfeb69e72d]
 22: (PyObject_Call()+0x43) [0x7fcfeb698913]
 23: (PyEval_EvalFrameEx()+0x5314) [0x7fcfeb5c3a84]
 24: (PyEval_EvalFrameEx()+0x8b5b) [0x7fcfeb5c72cb]
 25: (PyEval_EvalFrameEx()+0x8b5b) [0x7fcfeb5c72cb]
 26: (PyEval_EvalCodeEx()+0x7d8) [0x7fcfeb703908]
 27: (()+0x17a649) [0x7fcfeb69e649]
 28: (PyObject_Call()+0x43) [0x7fcfeb698913]
 29: (()+0x1a1eec) [0x7fcfeb6c5eec]
 30: (PyObject_Call()+0x43) [0x7fcfeb698913]
 31: (PyEval_CallObjectWithKeywords()+0x47) [0x7fcfeb702e37]
 32: (()+0xeb462) [0x7fcfeb60f462]
 33: (()+0x1b2ae7) [0x7fcfeb6d6ae7]
 34: (()+0x76db) [0x7fcfeb09c6db]
 35: (clone()+0x3f) [0x7fcfea28271f]
18 jobs: ['6123732', '6123827', '6123805', '6123912', '6123615', '6123922', '6123783', '6123628', '6123689', '6123857', '6123791', '6123697', '6123677', '6123930', '6123635', '6123819', '6123833', '6123760']
suites intersection: ['1-client', '2-upgrade', 'conf/{client', 'mds', 'mon', 'osd}', 'overrides/{frag_enable', 'whitelist_health', 'whitelist_wrongly_marked_down}']
suites union: ['1-client', '2-upgrade', '3-client-upgrade', '3-compat_client/mimic}}', '3-compat_client/no}}', '3-sanity', '3-verify}', '4-client-upgrade/no', '4-client-upgrade/yes', '4-compat_client', '5-client-sanity', '5-client-sanity}}', '6-snap-upgrade', '7-client-sanity}}', 'clusters/1-mds-2-client-micro', 'conf/{client', 'fs/upgrade/featureful_client/old_client/{bluestore-bitmap', 'fs/upgrade/featureful_client/upgraded_client/{bluestore-bitmap', 'fs/upgrade/snaps/{clusters/3-mds', 'fs/upgrade/volumes/import-legacy/{bluestore-bitmap', 'mds', 'mon', 'multimds/no', 'multimds/yes', 'objectstore-ec/bluestore-bitmap', 'objectstore-ec/bluestore-comp', 'objectstore-ec/bluestore-comp-ec-root', 'objectstore-ec/bluestore-ec-root', 'objectstore-ec/filestore-xfs', 'osd}', 'overrides/{frag_enable', 'pg-warn', 'tasks/{0-luminous', 'tasks/{0-mimic', 'ubuntu_18.04}', 'whitelist_health', 'whitelist_rstat', 'whitelist_wrongly_marked_down}']

Failure: "2021-05-20 06:17:28.843714 mon.a (mon.0) 116 : cluster [ERR] Health check failed: Module 'volumes' has failed: Not found or unloadable (MGR_MODULE_ERROR)" in cluster log
2 jobs: ['6123711', '6123671']
suites intersection: ['1-client', '2-upgrade', '3-sanity', '4-client-upgrade/no', '5-client-sanity', '6-snap-upgrade', '7-client-sanity}}', 'conf/{client', 'fs/upgrade/snaps/{clusters/3-mds', 'mds', 'mon', 'multimds/no', 'osd}', 'overrides/{frag_enable', 'tasks/{0-luminous', 'whitelist_health', 'whitelist_rstat', 'whitelist_wrongly_marked_down}']
suites union: ['1-client', '2-upgrade', '3-sanity', '4-client-upgrade/no', '5-client-sanity', '6-snap-upgrade', '7-client-sanity}}', 'conf/{client', 'fs/upgrade/snaps/{clusters/3-mds', 'mds', 'mon', 'multimds/no', 'objectstore-ec/bluestore-bitmap', 'objectstore-ec/bluestore-comp', 'osd}', 'overrides/{frag_enable', 'tasks/{0-luminous', 'whitelist_health', 'whitelist_rstat', 'whitelist_wrongly_marked_down}']

Failure: 'wait_until_healthy' reached maximum tries (150) after waiting for 900 seconds
6 jobs: ['6123727', '6123768', '6123661', '6123865', '6123898', '6123885']
suites intersection: ['1-client', '2-upgrade', 'conf/{client', 'mds', 'mon', 'osd}', 'overrides/{frag_enable', 'tasks/{0-luminous', 'whitelist_health', 'whitelist_wrongly_marked_down}']
suites union: ['1-client', '2-upgrade', '3-client-upgrade', '3-compat_client/mimic}}', '3-compat_client/no}}', '3-sanity', '4-client-upgrade/no', '4-client-upgrade/yes', '4-compat_client', '5-client-sanity', '5-client-sanity}}', '6-snap-upgrade', '7-client-sanity}}', 'clusters/1-mds-2-client-micro', 'conf/{client', 'fs/upgrade/featureful_client/old_client/{bluestore-bitmap', 'fs/upgrade/featureful_client/upgraded_client/{bluestore-bitmap', 'fs/upgrade/snaps/{clusters/3-mds', 'mds', 'mon', 'multimds/no', 'multimds/yes', 'objectstore-ec/bluestore-ec-root', 'objectstore-ec/filestore-xfs', 'osd}', 'overrides/{frag_enable', 'tasks/{0-luminous', 'whitelist_health', 'whitelist_rstat', 'whitelist_wrongly_marked_down}']

Failure: Test failure: test_async_subvolume_rm (tasks.cephfs.test_volumes.TestVolumes)
2 jobs: ['6123641', '6123809']
suites intersection: ['clusters/1-mds-4-client-coloc', 'conf/{client', 'fs/basic_functional/{begin', 'mds', 'mon', 'mount/fuse', 'no_client_pidfile', 'osd}', 'overrides/{frag_enable', 'supported-random-distros$/{ubuntu_16.04}', 'tasks/volumes}', 'whitelist_health', 'whitelist_wrongly_marked_down}']
suites union: ['clusters/1-mds-4-client-coloc', 'conf/{client', 'fs/basic_functional/{begin', 'mds', 'mon', 'mount/fuse', 'no_client_pidfile', 'objectstore/bluestore-bitmap', 'objectstore/bluestore-ec-root', 'osd}', 'overrides/{frag_enable', 'supported-random-distros$/{ubuntu_16.04}', 'tasks/volumes}', 'whitelist_health', 'whitelist_wrongly_marked_down}']

Failure: 
3 jobs: ['6123893', '6123801', '6123703']
suites intersection: ['centos_latest', 'clusters/fixed-2-ucephfs', 'conf/{client', 'fs/verify/{begin', 'mds', 'mon', 'mon-debug', 'mount/fuse', 'osd}', 'overrides/{frag_enable', 'session_timeout', 'tasks/cfuse_workunit_suites_fsstress', 'validater/valgrind}', 'whitelist_health', 'whitelist_wrongly_marked_down}']
suites union: ['centos_latest', 'clusters/fixed-2-ucephfs', 'conf/{client', 'fs/verify/{begin', 'mds', 'mon', 'mon-debug', 'mount/fuse', 'objectstore-ec/bluestore-bitmap', 'objectstore-ec/bluestore-comp-ec-root', 'objectstore-ec/bluestore-ec-root', 'osd}', 'overrides/{frag_enable', 'session_timeout', 'tasks/cfuse_workunit_suites_fsstress', 'validater/valgrind}', 'whitelist_health', 'whitelist_wrongly_marked_down}']

Failure: {'Failure object was': {'smithi096.front.sepia.ceph.com': {'msg': 'stdout: Collecting git+https://github.com/sebastian-philipp/registries-conf-ctl\\n  Cloning https://github.com/sebastian-philipp/registries-conf-ctl to ./pip-FNf5Gy-build\\nCollecting docopt (from registries-conf-ctl==1.0)\\n  Downloading https://files.pythonhosted.org/packages/a2/55/8f8cab2afd404cf578136ef2cc5dfb50baa1761b68c9da1fb1e4eed343c9/docopt-0.6.2.tar.gz\\nCollecting toml (from registries-conf-ctl==1.0)\\n  Downloading https://files.pythonhosted.org/packages/44/6f/7120676b6d73228c96e17f1f794d8ab046fc910d781c8d151120c3f1569e/toml-0.10.2-py2.py3-none-any.whl\\nCollecting typing (from registries-conf-ctl==1.0)\\n\\n:stderr: Exception:\\nTraceback (most recent call last):\\n  File "/usr/lib/python2.7/dist-packages/pip/basecommand.py", line 215, in main\\n    status = self.run(options, args)\\n  File "/usr/lib/python2.7/dist-packages/pip/commands/install.py", line 353, in run\\n    wb.build(autobuilding=True)\\n  File "/usr/lib/python2.7/dist-packages/pip/wheel.py", line 749, in build\\n    self.requirement_set.prepare_files(self.finder)\\n  File "/usr/lib/python2.7/dist-packages/pip/req/req_set.py", line 380, in prepare_files\\n    ignore_dependencies=self.ignore_dependencies))\\n  File "/usr/lib/python2.7/dist-packages/pip/req/req_set.py", line 554, in _prepare_file\\n    require_hashes\\n  File "/usr/lib/python2.7/dist-packages/pip/req/req_install.py", line 278, in populate_link\\n    self.link = finder.find_requirement(self, upgrade)\\n  File "/usr/lib/python2.7/dist-packages/pip/index.py", line 465, in find_requirement\\n    all_candidates = self.find_all_candidates(req.name)\\n  File "/usr/lib/python2.7/dist-packages/pip/index.py", line 423, in find_all_candidates\\n    for page in self._get_pages(url_locations, project_name):\\n  File "/usr/lib/python2.7/dist-packages/pip/index.py", line 568, in _get_pages\\n    page = self._get_page(location)\\n  File "/usr/lib/python2.7/dist-packages/pip/index.py", line 683, in _get_page\\n    return HTMLPage.get_page(link, session=self.session)\\n  File "/usr/lib/python2.7/dist-packages/pip/index.py", line 795, in get_page\\n    resp.raise_for_status()\\n  File "/usr/share/python-wheels/requests-2.18.4-py2.py3-none-any.whl/requests/models.py", line 935, in raise_for_status\\n    raise HTTPError(http_error_msg, response=self)\\nHTTPError: 502 Server Error: Gateway Error for url: https://pypi.python.org/simple/typing/\\n', 'cmd': ['/usr/bin/pip2', 'install', '-U', 'git+https://github.com/sebastian-philipp/registries-conf-ctl'], 'invocation': {'module_args': {'virtualenv': 'None', 'extra_args': 'None', 'virtualenv_command': 'virtualenv', 'chdir': 'None', 'requirements': 'None', 'name': ['git+https://github.com/sebastian-philipp/registries-conf-ctl'], 'virtualenv_python': 'None', 'umask': 'None', 'editable': False, 'executable': 'None', 'virtualenv_site_packages': False, 'state': 'latest', 'version': 'None'}}, '_ansible_no_log': False, 'changed': False}}, 'Traceback (most recent call last)': 'File "/home/teuthworker/src/git.ceph.com_git_ceph-cm-ansible_master/callback_plugins/failure_log.py", line 44, in log_failure log.error(yaml.safe_dump(failure)) File "/home/teuthworker/src/git.ceph.com_git_teuthology_471a2f1efab907807e62c17be86bf6fb1a2f4522/virtualenv/lib/python3.6/site-packages/yaml/__init__.py", line 306, in safe_dump return dump_all([data], stream, Dumper=SafeDumper, **kwds) File "/home/teuthworker/src/git.ceph.com_git_teuthology_471a2f1efab907807e62c17be86bf6fb1a2f4522/virtualenv/lib/python3.6/site-packages/yaml/__init__.py", line 278, in dump_all dumper.represent(data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_471a2f1efab907807e62c17be86bf6fb1a2f4522/virtualenv/lib/python3.6/site-packages/yaml/representer.py", line 27, in represent node = self.represent_data(data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_471a2f1efab907807e62c17be86bf6fb1a2f4522/virtualenv/lib/python3.6/site-packages/yaml/representer.py", line 48, in represent_data node = self.yaml_representers[data_types[0]](self, data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_471a2f1efab907807e62c17be86bf6fb1a2f4522/virtualenv/lib/python3.6/site-packages/yaml/representer.py", line 207, in represent_dict return self.represent_mapping(\'tag:yaml.org,2002:map\', data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_471a2f1efab907807e62c17be86bf6fb1a2f4522/virtualenv/lib/python3.6/site-packages/yaml/representer.py", line 118, in represent_mapping node_value = self.represent_data(item_value) File "/home/teuthworker/src/git.ceph.com_git_teuthology_471a2f1efab907807e62c17be86bf6fb1a2f4522/virtualenv/lib/python3.6/site-packages/yaml/representer.py", line 48, in represent_data node = self.yaml_representers[data_types[0]](self, data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_471a2f1efab907807e62c17be86bf6fb1a2f4522/virtualenv/lib/python3.6/site-packages/yaml/representer.py", line 207, in represent_dict return self.represent_mapping(\'tag:yaml.org,2002:map\', data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_471a2f1efab907807e62c17be86bf6fb1a2f4522/virtualenv/lib/python3.6/site-packages/yaml/representer.py", line 117, in represent_mapping node_key = self.represent_data(item_key) File "/home/teuthworker/src/git.ceph.com_git_teuthology_471a2f1efab907807e62c17be86bf6fb1a2f4522/virtualenv/lib/python3.6/site-packages/yaml/representer.py", line 58, in represent_data node = self.yaml_representers[None](self, data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_471a2f1efab907807e62c17be86bf6fb1a2f4522/virtualenv/lib/python3.6/site-packages/yaml/representer.py", line 231, in represent_undefined raise RepresenterError("cannot represent an object", data)', 'yaml.representer.RepresenterError': "('cannot represent an object', 'cmd')"}
1 jobs: ['6123932']
suites: ['clusters/1-mds-4-client-coloc', 'conf/{client', 'fs/basic_functional/{begin', 'mds', 'mon', 'mount/fuse', 'no_client_pidfile', 'objectstore/bluestore-ec-root', 'osd}', 'overrides/{frag_enable', 'supported-random-distros$/{ubuntu_latest}', 'tasks/pool-perm}', 'whitelist_health', 'whitelist_wrongly_marked_down}']

Crash: 'wait_until_healthy' reached maximum tries (150) after waiting for 900 seconds
ceph version 14.2.21-248-g712a56ab7ae (712a56ab7aeaed280c06d871319a38308d8c834d) nautilus (stable)
 1: (()+0x12980) [0x7f1b2335d980]
 2: (gsignal()+0xc7) [0x7f1b22455fb7]
 3: (abort()+0x141) [0x7f1b22457921]
 4: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x1a3) [0x7f1b23fbb543]
 5: (ceph::__ceph_assertf_fail(char const*, char const*, int, char const*, char const*, ...)+0) [0x7f1b23fbb6cd]
 6: (Mutex::lock(bool)+0xa8) [0x7f1b2403bb38]
 7: (PyModuleRunner::log(int, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&)+0x100) [0x55b7f080e720]
 8: (()+0x156931) [0x55b7f076c931]
 9: (PyEval_EvalFrameEx()+0x8010) [0x7f1b2387c780]
 10: (PyEval_EvalFrameEx()+0x8b5b) [0x7f1b2387d2cb]
 11: (PyEval_EvalFrameEx()+0x8b5b) [0x7f1b2387d2cb]
 12: (PyEval_EvalFrameEx()+0x8b5b) [0x7f1b2387d2cb]
 13: (PyEval_EvalFrameEx()+0x8b5b) [0x7f1b2387d2cb]
 14: (PyEval_EvalCodeEx()+0x7d8) [0x7f1b239b9908]
 15: (()+0x17a72d) [0x7f1b2395472d]
 16: (PyObject_Call()+0x43) [0x7f1b2394e913]
 17: (PyEval_EvalFrameEx()+0x5314) [0x7f1b23879a84]
 18: (PyEval_EvalCodeEx()+0x7d8) [0x7f1b239b9908]
 19: (PyEval_EvalFrameEx()+0x5bf6) [0x7f1b2387a366]
 20: (PyEval_EvalCodeEx()+0x7d8) [0x7f1b239b9908]
 21: (()+0x17a72d) [0x7f1b2395472d]
 22: (PyObject_Call()+0x43) [0x7f1b2394e913]
 23: (PyEval_EvalFrameEx()+0x5314) [0x7f1b23879a84]
 24: (PyEval_EvalFrameEx()+0x8b5b) [0x7f1b2387d2cb]
 25: (PyEval_EvalFrameEx()+0x8b5b) [0x7f1b2387d2cb]
 26: (PyEval_EvalCodeEx()+0x7d8) [0x7f1b239b9908]
 27: (()+0x17a649) [0x7f1b23954649]
 28: (PyObject_Call()+0x43) [0x7f1b2394e913]
 29: (()+0x1a1eec) [0x7f1b2397beec]
 30: (PyObject_Call()+0x43) [0x7f1b2394e913]
 31: (PyEval_CallObjectWithKeywords()+0x47) [0x7f1b239b8e37]
 32: (()+0xeb462) [0x7f1b238c5462]
 33: (()+0x1b2ae7) [0x7f1b2398cae7]
 34: (()+0x76db) [0x7f1b233526db]
 35: (clone()+0x3f) [0x7f1b2253871f]
1 jobs: ['6123873']
suites: ['1-client', '2-upgrade', '3-compat_client/no}}', 'clusters/1-mds-2-client-micro', 'conf/{client', 'fs/upgrade/featureful_client/old_client/{bluestore-bitmap', 'mds', 'mon', 'multimds/yes', 'osd}', 'overrides/{frag_enable', 'tasks/{0-luminous', 'whitelist_health', 'whitelist_wrongly_marked_down}']

Crash: 'wait_until_healthy' reached maximum tries (150) after waiting for 900 seconds
ceph version 14.2.21-248-g712a56ab7ae (712a56ab7aeaed280c06d871319a38308d8c834d) nautilus (stable)
 1: (()+0x12980) [0x7ffbc19cf980]
 2: (gsignal()+0xc7) [0x7ffbc0ac7fb7]
 3: (abort()+0x141) [0x7ffbc0ac9921]
 4: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x1a3) [0x7ffbc262d543]
 5: (ceph::__ceph_assertf_fail(char const*, char const*, int, char const*, char const*, ...)+0) [0x7ffbc262d6cd]
 6: (Mutex::lock(bool)+0xa8) [0x7ffbc26adb38]
 7: (PyModuleRunner::log(int, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&)+0x100) [0x5632e2db0720]
 8: (()+0x156931) [0x5632e2d0e931]
 9: (PyEval_EvalFrameEx()+0x8010) [0x7ffbc1eee780]
 10: (PyEval_EvalFrameEx()+0x8b5b) [0x7ffbc1eef2cb]
 11: (PyEval_EvalFrameEx()+0x8b5b) [0x7ffbc1eef2cb]
 12: (PyEval_EvalFrameEx()+0x8b5b) [0x7ffbc1eef2cb]
 13: (PyEval_EvalFrameEx()+0x8b5b) [0x7ffbc1eef2cb]
 14: (PyEval_EvalCodeEx()+0x7d8) [0x7ffbc202b908]
 15: (()+0x17a72d) [0x7ffbc1fc672d]
 16: (PyObject_Call()+0x43) [0x7ffbc1fc0913]
 17: (PyEval_EvalFrameEx()+0x5314) [0x7ffbc1eeba84]
 18: (PyEval_EvalCodeEx()+0x7d8) [0x7ffbc202b908]
 19: (PyEval_EvalFrameEx()+0x5bf6) [0x7ffbc1eec366]
 20: (PyEval_EvalCodeEx()+0x7d8) [0x7ffbc202b908]
 21: (()+0x17a72d) [0x7ffbc1fc672d]
 22: (PyObject_Call()+0x43) [0x7ffbc1fc0913]
 23: (PyEval_EvalFrameEx()+0x5314) [0x7ffbc1eeba84]
 24: (PyEval_EvalFrameEx()+0x8b5b) [0x7ffbc1eef2cb]
 25: (PyEval_EvalFrameEx()+0x8b5b) [0x7ffbc1eef2cb]
 26: (PyEval_EvalCodeEx()+0x7d8) [0x7ffbc202b908]
 27: (()+0x17a649) [0x7ffbc1fc6649]
 28: (PyObject_Call()+0x43) [0x7ffbc1fc0913]
 29: (()+0x1a1eec) [0x7ffbc1fedeec]
 30: (PyObject_Call()+0x43) [0x7ffbc1fc0913]
 31: (PyEval_CallObjectWithKeywords()+0x47) [0x7ffbc202ae37]
 32: (()+0xeb462) [0x7ffbc1f37462]
 33: (()+0x1b2ae7) [0x7ffbc1ffeae7]
 34: (()+0x76db) [0x7ffbc19c46db]
 35: (clone()+0x3f) [0x7ffbc0baa71f]
1 jobs: ['6123740']
suites: ['1-client', '2-upgrade', '3-compat_client/mimic}}', 'clusters/1-mds-2-client-micro', 'conf/{client', 'fs/upgrade/featureful_client/old_client/{bluestore-bitmap', 'mds', 'mon', 'multimds/yes', 'osd}', 'overrides/{frag_enable', 'tasks/{0-luminous', 'whitelist_health', 'whitelist_wrongly_marked_down}']