Found 25 jobs Missing teuthology log /home/teuthworker/archive/ideepika-2021-03-12_19:31:07-rados-wip-yuri2-testing-2021-03-09-1006-octopus-distro-basic-gibba/5958930/teuthology.log Found 8 distinct failure reasons Failure: Command failed on gibba022 with status 5: 'sudo systemctl stop ceph-1c9a3fce-8370-11eb-908a-001a4aab830c@mon.a' 8 jobs: ['5958937', '5958934', '5958941', '5958923', '5958931', '5958932', '5958935', '5958922'] suites intersection: [] suites union: ['fixed-2', 'mode/root', 'msgr/async', 'msgr/async-v1only', 'rados/cephadm/smoke-roleless/{distro/ubuntu_18.04_podman', 'rados/cephadm/smoke/{distro/ubuntu_18.04_podman', 'rados/cephadm/with-work/{distro/ubuntu_18.04_podman', 'rados/cephadm/workunits/{distro/ubuntu_18.04_podman', 'start', 'start}', 'task/test_orch_cli}', 'tasks/rados_api_tests}', 'tasks/rados_python}'] Failure: Command failed (workunit test cephadm/test_cephadm.sh) on gibba024 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=dbb79e0547db3abf076b9bc9b6ad97ede0519a0e TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cephadm/test_cephadm.sh' 2 jobs: ['5958936', '5958928'] suites intersection: ['rados/cephadm/workunits/{distro/ubuntu_18.04_podman', 'task/test_cephadm}'] suites union: ['rados/cephadm/workunits/{distro/ubuntu_18.04_podman', 'task/test_cephadm}'] Failure: {'Failure object was': {'gibba040.front.sepia.ceph.com': {'msg': 'Failed to checkout branch master', 'stdout': '', 'stderr': "fatal: Unable to create '/home/teuthworker/.cache/src/keys/.git/index.lock': File exists.\n\nAnother git process seems to be running in this repository, e.g.\nan editor opened by 'git commit'. Please make sure all processes\nare terminated then try again. If it still fails, a git process\nmay have crashed in this repository earlier:\nremove the file manually to continue.\n", 'rc': 128, 'invocation': {'module_args': {'repo': 'https://github.com/ceph/keys', 'version': 'master', 'force': True, 'dest': '/home/teuthworker/.cache/src/keys', 'remote': 'origin', 'clone': True, 'update': True, 'verify_commit': False, 'gpg_whitelist': [], 'accept_hostkey': False, 'bare': False, 'recursive': True, 'track_submodules': False, 'refspec': 'None', 'reference': 'None', 'depth': 'None', 'key_file': 'None', 'ssh_opts': 'None', 'executable': 'None', 'umask': 'None', 'archive': 'None', 'separate_git_dir': 'None'}}, 'stdout_lines': [], 'stderr_lines': ["fatal: Unable to create '/home/teuthworker/.cache/src/keys/.git/index.lock': File exists.", '', 'Another git process seems to be running in this repository, e.g.', "an editor opened by 'git commit'. Please make sure all processes", 'are terminated then try again. If it still fails, a git process', 'may have crashed in this repository earlier:', 'remove the file manually to continue.'], '_ansible_no_log': False, 'attempts': 5, 'changed': False, '_ansible_delegated_vars': {}}}, 'Traceback (most recent call last)': 'File "/home/teuthworker/src/git.ceph.com_git_ceph-cm-ansible_master/callback_plugins/failure_log.py", line 44, in log_failure log.error(yaml.safe_dump(failure)) File "/home/teuthworker/src/git.ceph.com_git_teuthology_e8438aef42f98e8ef3e8cdc260580ac63cdc5a1f/virtualenv/lib/python3.6/site-packages/yaml/__init__.py", line 306, in safe_dump return dump_all([data], stream, Dumper=SafeDumper, **kwds) File "/home/teuthworker/src/git.ceph.com_git_teuthology_e8438aef42f98e8ef3e8cdc260580ac63cdc5a1f/virtualenv/lib/python3.6/site-packages/yaml/__init__.py", line 278, in dump_all dumper.represent(data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_e8438aef42f98e8ef3e8cdc260580ac63cdc5a1f/virtualenv/lib/python3.6/site-packages/yaml/representer.py", line 27, in represent node = self.represent_data(data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_e8438aef42f98e8ef3e8cdc260580ac63cdc5a1f/virtualenv/lib/python3.6/site-packages/yaml/representer.py", line 48, in represent_data node = self.yaml_representers[data_types[0]](self, data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_e8438aef42f98e8ef3e8cdc260580ac63cdc5a1f/virtualenv/lib/python3.6/site-packages/yaml/representer.py", line 207, in represent_dict return self.represent_mapping(\'tag:yaml.org,2002:map\', data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_e8438aef42f98e8ef3e8cdc260580ac63cdc5a1f/virtualenv/lib/python3.6/site-packages/yaml/representer.py", line 118, in represent_mapping node_value = self.represent_data(item_value) File "/home/teuthworker/src/git.ceph.com_git_teuthology_e8438aef42f98e8ef3e8cdc260580ac63cdc5a1f/virtualenv/lib/python3.6/site-packages/yaml/representer.py", line 48, in represent_data node = self.yaml_representers[data_types[0]](self, data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_e8438aef42f98e8ef3e8cdc260580ac63cdc5a1f/virtualenv/lib/python3.6/site-packages/yaml/representer.py", line 207, in represent_dict return self.represent_mapping(\'tag:yaml.org,2002:map\', data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_e8438aef42f98e8ef3e8cdc260580ac63cdc5a1f/virtualenv/lib/python3.6/site-packages/yaml/representer.py", line 117, in represent_mapping node_key = self.represent_data(item_key) File "/home/teuthworker/src/git.ceph.com_git_teuthology_e8438aef42f98e8ef3e8cdc260580ac63cdc5a1f/virtualenv/lib/python3.6/site-packages/yaml/representer.py", line 58, in represent_data node = self.yaml_representers[None](self, data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_e8438aef42f98e8ef3e8cdc260580ac63cdc5a1f/virtualenv/lib/python3.6/site-packages/yaml/representer.py", line 231, in represent_undefined raise RepresenterError("cannot represent an object", data)', 'yaml.representer.RepresenterError': "('cannot represent an object', '_ansible_no_log')"} 1 jobs: ['5958925'] suites: ['1-install/mimic', '2-partial-upgrade/firsthalf', '3-thrash/default', '4-workload/{rbd-cls', '5-workload/{radosbench', '6-finish-upgrade', '7-octopus', '8-workload/{rbd-python', 'bluestore-bitmap', 'rados/upgrade/mimic-x-singleton/{0-cluster/{openstack', 'rbd-import-export', 'rbd_api}', 'readwrite', 'snaps-few-objects}', 'snaps-many-objects}', 'start}', 'thrashosds-health', 'ubuntu_latest}'] Dead: None 4 jobs: ['5958930', '5958939', '5958929', '5958938'] Failure: Test failure: test_osd_cannot_recover (tasks.mgr.test_progress.TestProgress) 2 jobs: ['5958926', '5958940'] suites intersection: ['debug/mgr', 'rados/mgr/{clusters/{2-node-mgr}', 'tasks/progress}'] suites union: ['debug/mgr', 'objectstore/bluestore-comp-zlib', 'objectstore/bluestore-hybrid', 'rados/mgr/{clusters/{2-node-mgr}', 'supported-random-distro$/{rhel_latest}', 'supported-random-distro$/{ubuntu_latest}', 'tasks/progress}'] Failure: Command failed (workunit test cephadm/test_adoption.sh) on gibba036 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=dbb79e0547db3abf076b9bc9b6ad97ede0519a0e TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cephadm/test_adoption.sh' 2 jobs: ['5958933', '5958943'] suites intersection: ['rados/cephadm/workunits/{distro/ubuntu_18.04_podman', 'task/test_adoption}'] suites union: ['rados/cephadm/workunits/{distro/ubuntu_18.04_podman', 'task/test_adoption}'] Failure: Command failed (workunit test rados/test.sh) on gibba014 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=dbb79e0547db3abf076b9bc9b6ad97ede0519a0e TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/rados/test.sh' 1 jobs: ['5958924'] suites: ['1-pg-log-overrides/short_pg_log', '2-recovery-overrides/{more-partial-recovery}', 'backoff/peering', 'ceph', 'clusters/{fixed-2', 'crc-failures/default', 'd-balancer/off', 'msgr-failures/osd-delay', 'msgr/async-v2only', 'objectstore/bluestore-comp-zstd', 'openstack}', 'rados', 'rados/thrash/{0-size-min-size-overrides/3-size-2-min-size', 'supported-random-distro$/{ubuntu_latest}', 'thrashers/careful', 'thrashosds-health', 'workloads/rados_api_tests}'] Failure: reached maximum tries (180) after waiting for 180 seconds 1 jobs: ['5958927'] suites: ['2-repo_digest/defaut', '3-start-upgrade', '4-wait', 'distro$/{rhel_latest}', 'fixed-2}', 'rados/cephadm/upgrade/{1-start']