Found 307 jobs Missing teuthology log /home/teuthworker/archive/yuriw-2023-06-26_18:56:05-rados-wip-yuri7-testing-2023-06-23-1022-distro-default-smithi/7317133/teuthology.log Found 15 distinct failure reasons Dead: None 40 jobs Failure: Command crashed: 'sudo TESTDIR=/home/ubuntu/cephtest bash -c ceph_test_lazy_omap_stats' 1 jobs: ['7316894'] suites: ['mon_election/classic', 'rados', 'rados/singleton-nomsgr/{all/lazy_omap_stats_output', 'supported-random-distro$/{ubuntu_latest}}'] Failure: Command failed (workunit test rados/test_envlibrados_for_rocksdb.sh) on smithi067 with status 2: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=e4919e7f3fd451cf8205569752e8c3a1018b941c TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/rados/test_envlibrados_for_rocksdb.sh' 2 jobs: ['7317124', '7317030'] suites intersection: ['mon_election/connectivity', 'msgr-failures/few', 'msgr/async-v2only', 'rados', 'test_envlibrados_for_rocksdb}'] suites union: ['mon_election/connectivity', 'msgr-failures/few', 'msgr/async-v2only', 'objectstore/bluestore-comp-zstd', 'objectstore/bluestore-low-osd-mem-target', 'rados', 'rados/singleton/{all/test_envlibrados_for_rocksdb/{supported/centos_latest', 'rados/singleton/{all/test_envlibrados_for_rocksdb/{supported/rhel_latest', 'supported-random-distro$/{centos_8}}', 'supported-random-distro$/{ubuntu_20.04}}', 'test_envlibrados_for_rocksdb}'] Failure: Command failed on smithi160 with status 100: 'sudo DEBIAN_FRONTEND=noninteractive apt-get -y --force-yes -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install ceph=18.0.0-4596-ge4919e7f-1jammy cephadm=18.0.0-4596-ge4919e7f-1jammy ceph-mds=18.0.0-4596-ge4919e7f-1jammy ceph-mgr=18.0.0-4596-ge4919e7f-1jammy ceph-common=18.0.0-4596-ge4919e7f-1jammy ceph-fuse=18.0.0-4596-ge4919e7f-1jammy ceph-test=18.0.0-4596-ge4919e7f-1jammy ceph-volume=18.0.0-4596-ge4919e7f-1jammy radosgw=18.0.0-4596-ge4919e7f-1jammy python3-rados=18.0.0-4596-ge4919e7f-1jammy python3-rgw=18.0.0-4596-ge4919e7f-1jammy python3-cephfs=18.0.0-4596-ge4919e7f-1jammy python3-rbd=18.0.0-4596-ge4919e7f-1jammy libcephfs2=18.0.0-4596-ge4919e7f-1jammy libcephfs-dev=18.0.0-4596-ge4919e7f-1jammy librados2=18.0.0-4596-ge4919e7f-1jammy librbd1=18.0.0-4596-ge4919e7f-1jammy rbd-fuse=18.0.0-4596-ge4919e7f-1jammy' 1 jobs: ['7316955'] suites: ['rados/standalone/{supported-random-distro$/{ubuntu_latest}', 'workloads/misc}'] Failure: {'smithi179.front.sepia.ceph.com': {'changed': False, 'msg': 'All items completed', 'results': [{'_ansible_item_label': 'http://download.ceph.com/keys/autobuild.asc', '_ansible_no_log': False, 'ansible_loop_var': 'item', 'changed': False, 'failed': True, 'invocation': {'module_args': {'data': None, 'file': None, 'id': None, 'key': None, 'keyring': None, 'keyserver': None, 'state': 'present', 'url': 'http://download.ceph.com/keys/autobuild.asc', 'validate_certs': True}}, 'item': 'http://download.ceph.com/keys/autobuild.asc', 'msg': 'Failed to download key at http://download.ceph.com/keys/autobuild.asc: Request failed: '}, {'_ansible_item_label': 'http://download.ceph.com/keys/release.asc', '_ansible_no_log': False, 'ansible_loop_var': 'item', 'changed': False, 'failed': True, 'invocation': {'module_args': {'data': None, 'file': None, 'id': None, 'key': None, 'keyring': None, 'keyserver': None, 'state': 'present', 'url': 'http://download.ceph.com/keys/release.asc', 'validate_certs': True}}, 'item': 'http://download.ceph.com/keys/release.asc', 'msg': 'Failed to download key at http://download.ceph.com/keys/release.asc: Request failed: '}]}} 3 jobs: ['7317045', '7316943', '7316862'] suites intersection: ['rados'] suites union: ['1-pg-log-overrides/short_pg_log', '2-recovery-overrides/{default}', '3-scrub-overrides/{max-simultaneous-scrubs-3}', 'backoff/normal', 'ceph', 'clusters/{fixed-2', 'crc-failures/default', 'd-balancer/on', 'mon_election/classic', 'mon_election/connectivity', 'msgr-failures/fastclose', 'msgr-failures/few', 'msgr-failures/many', 'msgr/async', 'msgr/async-v1only', 'no_pools', 'objectstore/bluestore-bitmap', 'objectstore/bluestore-comp-zlib', 'objectstore/bluestore-stupid', 'openstack}', 'rados', 'rados/multimon/{clusters/6', 'rados/singleton/{all/osd-backfill', 'rados/thrash/{0-size-min-size-overrides/3-size-2-min-size', 'supported-random-distro$/{ubuntu_20.04}}', 'supported-random-distro$/{ubuntu_latest}', 'tasks/mon_clock_with_skews}', 'thrashers/morepggrow', 'thrashosds-health', 'workloads/cache-snaps-balanced}'] Failure: failed to install new kernel version within timeout 1 jobs: ['7316809'] suites: ['mon_election/classic', 'rados', 'rados/singleton-nomsgr/{all/admin_socket_output', 'supported-random-distro$/{rhel_8}}'] Failure: {'smithi149.front.sepia.ceph.com': {'changed': False, 'msg': 'All items completed', 'results': [{'_ansible_item_label': {'key': 'vg_nvme', 'value': {'pvs': '/dev/nvme0n1'}}, '_ansible_no_log': False, 'ansible_loop_var': 'item', 'changed': False, 'err': " /dev/vg_nvme: already exists in filesystem\n Run `vgcreate --help' for more information.\n", 'failed': True, 'invocation': {'module_args': {'force': False, 'pesize': '4', 'pv_options': '', 'pvresize': False, 'pvs': ['/dev/nvme0n1'], 'state': 'present', 'vg': 'vg_nvme', 'vg_options': ''}}, 'item': {'key': 'vg_nvme', 'value': {'pvs': '/dev/nvme0n1'}}, 'msg': "Creating volume group 'vg_nvme' failed", 'rc': 3}]}, 'smithi169.front.sepia.ceph.com': {'changed': False, 'msg': 'All items completed', 'results': [{'_ansible_item_label': {'key': 'vg_nvme', 'value': {'pvs': '/dev/nvme0n1'}}, '_ansible_no_log': False, 'ansible_loop_var': 'item', 'changed': False, 'err': " /dev/vg_nvme: already exists in filesystem\n Run `vgcreate --help' for more information.\n", 'failed': True, 'invocation': {'module_args': {'force': False, 'pesize': '4', 'pv_options': '', 'pvresize': False, 'pvs': ['/dev/nvme0n1'], 'state': 'present', 'vg': 'vg_nvme', 'vg_options': ''}}, 'item': {'key': 'vg_nvme', 'value': {'pvs': '/dev/nvme0n1'}}, 'msg': "Creating volume group 'vg_nvme' failed", 'rc': 3}]}} 4 jobs: ['7317098', '7316813', '7316812', '7317138'] suites intersection: [] suites union: ['1-pg-log-overrides/normal_pg_log', '2-inject-leak/mon', '2-recovery-overrides/{default}', '3-scrub-overrides/{max-simultaneous-scrubs-2}', 'backoff/normal', 'centos_latest}', 'ceph', 'clusters/{fixed-2', 'crc-failures/bad_map_crc_failure', 'd-balancer/crush-compat', 'd-thrash/default/{default', 'mon_election/classic', 'mon_election/connectivity', 'msgr-failures/few', 'msgr-failures/osd-dispatch-delay', 'msgr/async-v1only', 'msgr/async-v2only', 'objectstore/bluestore-comp-snappy', 'objectstore/bluestore-stupid', 'openstack}', 'rados', 'rados/singleton-nomsgr/{all/full-tiering', 'rados/thrash/{0-size-min-size-overrides/2-size-2-min-size', 'rados/valgrind-leaks/{1-start', 'rados/verify/{centos_latest', 'supported-random-distro$/{centos_8}', 'supported-random-distro$/{centos_8}}', 'tasks/mon_recovery', 'thrashers/careful', 'thrashosds-health', 'thrashosds-health}', 'validater/lockdep}', 'workloads/dedup-io-mixed}'] Failure: {'smithi158.front.sepia.ceph.com': {'_ansible_no_log': False, 'changed': False, 'cmd': '/usr/bin/cpanm Amazon::S3', 'invocation': {'module_args': {'executable': None, 'from_path': None, 'installdeps': False, 'locallib': None, 'mirror': None, 'mirror_only': False, 'name': 'Amazon::S3', 'notest': False, 'system_lib': False, 'version': None}}, 'msg': "! Finding Amazon::S3 on cpanmetadb failed.\n! cannot open file '/root/.cpanm/sources/http%www.cpan.org/02packages.details.txt.gz': No such file or directory opening compressed index\n! Couldn't find module or a distribution Amazon::S3\n"}, 'smithi052.front.sepia.ceph.com': {'_ansible_no_log': False, 'changed': False, 'cmd': '/usr/bin/cpanm Amazon::S3', 'invocation': {'module_args': {'executable': None, 'from_path': None, 'installdeps': False, 'locallib': None, 'mirror': None, 'mirror_only': False, 'name': 'Amazon::S3', 'notest': False, 'system_lib': False, 'version': None}}, 'msg': "! Finding Amazon::S3 on cpanmetadb failed.\n! cannot open file '/root/.cpanm/sources/http%www.cpan.org/02packages.details.txt.gz': No such file or directory opening compressed index\n! Couldn't find module or a distribution Amazon::S3\n"}} 2 jobs: ['7317003', '7317019'] suites intersection: [] suites union: ['clusters/3-mons', 'mon_election/classic', 'msgr-failures/mon-delay', 'msgr/async-v1only', 'objectstore/bluestore-low-osd-mem-target', 'rados', 'rados/monthrash/{ceph', 'rados/standalone/{supported-random-distro$/{ubuntu_latest}', 'supported-random-distro$/{ubuntu_latest}', 'thrashers/many', 'workloads/mon-stretch}', 'workloads/rados_mon_osdmap_prune}'] Failure: HTTPSConnectionPool(host='shaman.ceph.com', port=443): Max retries exceeded with url: /api/search?status=ready&project=ceph&flavor=default&distros=ubuntu%2F22.04%2Fx86_64&sha1=e4919e7f3fd451cf8205569752e8c3a1018b941c (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 110] Connection timed out')) 2 jobs: ['7317158', '7317178'] suites: ['mon_election/connectivity', 'rados', 'rados/singleton-nomsgr/{all/health-warnings', 'supported-random-distro$/{ubuntu_latest}}'] Failure: SELinux denials found on ubuntu@smithi156.front.sepia.ceph.com: ['type=AVC msg=audit(1687824353.578:20637): avc: denied { ioctl } for pid=111979 comm="iptables" path="/var/lib/containers/storage/overlay/e304f4e00b0d7b74643d9f0bd78245c061f0f2cd8410fa1dca48fe046905ec47/merged" dev="overlay" ino=2753542 scontext=system_u:system_r:iptables_t:s0 tcontext=system_u:object_r:container_file_t:s0:c1022,c1023 tclass=dir permissive=1', 'type=AVC msg=audit(1687824353.513:20636): avc: denied { ioctl } for pid=111976 comm="iptables" path="/var/lib/containers/storage/overlay/e304f4e00b0d7b74643d9f0bd78245c061f0f2cd8410fa1dca48fe046905ec47/merged" dev="overlay" ino=2753542 scontext=system_u:system_r:iptables_t:s0 tcontext=system_u:object_r:container_file_t:s0:c1022,c1023 tclass=dir permissive=1'] 1 jobs: ['7317126'] suites: ['agent/on', 'mon_election/connectivity', 'rados/cephadm/workunits/{0-distro/rhel_8.6_container_tools_rhel8', 'task/test_cephadm}'] Failure: Command failed (workunit test cephadm/test_dashboard_e2e.sh) on smithi023 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=e4919e7f3fd451cf8205569752e8c3a1018b941c TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cephadm/test_dashboard_e2e.sh' 2 jobs: ['7316884', '7317174'] suites intersection: ['debug/mgr', 'rados/dashboard/{0-single-container-host', 'tasks/e2e}'] suites union: ['debug/mgr', 'mon_election/classic', 'mon_election/connectivity', 'rados/dashboard/{0-single-container-host', 'random-objectstore$/{bluestore-hybrid}', 'random-objectstore$/{bluestore-low-osd-mem-target}', 'tasks/e2e}'] Failure: {'smithi067.front.sepia.ceph.com': {'changed': True, 'msg': 'All items completed', 'results': [{'_ansible_item_label': 'ceph/python3-asyncssh', '_ansible_no_log': False, 'ansible_loop_var': 'item', 'changed': True, 'cmd': ['dnf', '-y', 'copr', 'enable', 'ceph/python3-asyncssh'], 'delta': '0:02:11.718928', 'end': '2023-06-26 22:27:19.774156', 'failed': True, 'invocation': {'module_args': {'_raw_params': 'dnf -y copr enable ceph/python3-asyncssh', '_uses_shell': False, 'argv': None, 'chdir': None, 'creates': None, 'executable': None, 'removes': None, 'stdin': None, 'stdin_add_newline': True, 'strip_empty_ends': True, 'warn': True}}, 'item': 'ceph/python3-asyncssh', 'msg': 'non-zero return code', 'rc': 1, 'start': '2023-06-26 22:25:08.055228', 'stderr': 'Enabling a Copr repository. Please note that this repository is not part\nof the main distribution, and quality may vary.\n\nThe Fedora Project does not exercise any power over the contents of\nthis repository beyond the rules outlined in the Copr FAQ at\n,\nand packages are not held to any quality or security level.\n\nPlease do not file bug reports about these packages in Fedora\nBugzilla. In case of problems, contact the owner of this repository.\nError: Failed to connect to https://copr.fedorainfracloud.org/coprs/ceph/python3-asyncssh/repo/epel-8/dnf.repo?arch=x86_64: Network is unreachable', 'stderr_lines': ['Enabling a Copr repository. Please note that this repository is not part', 'of the main distribution, and quality may vary.', '', 'The Fedora Project does not exercise any power over the contents of', 'this repository beyond the rules outlined in the Copr FAQ at', ',', 'and packages are not held to any quality or security level.', '', 'Please do not file bug reports about these packages in Fedora', 'Bugzilla. In case of problems, contact the owner of this repository.', 'Error: Failed to connect to https://copr.fedorainfracloud.org/coprs/ceph/python3-asyncssh/repo/epel-8/dnf.repo?arch=x86_64: Network is unreachable'], 'stdout': '', 'stdout_lines': []}], 'warnings': ["Consider using the dnf module rather than running 'dnf'. If you need to use command because dnf is insufficient you can add 'warn: false' to this command task or set 'command_warnings=False' in ansible.cfg to get rid of this message."]}} 1 jobs: ['7317004'] suites: ['1-pg-log-overrides/short_pg_log', '2-recovery-overrides/{more-partial-recovery}', '3-scrub-overrides/{default}', 'backoff/peering', 'ceph', 'clusters/{fixed-2', 'crc-failures/default', 'd-balancer/on', 'mon_election/connectivity', 'msgr-failures/osd-delay', 'msgr/async-v2only', 'objectstore/bluestore-hybrid', 'openstack}', 'rados', 'rados/thrash/{0-size-min-size-overrides/3-size-2-min-size', 'supported-random-distro$/{centos_8}', 'thrashers/default', 'thrashosds-health', 'workloads/cache-pool-snaps-readproxy}'] Failure: {'smithi145.front.sepia.ceph.com': {'_ansible_no_log': False, 'changed': False, 'cmd': '/usr/bin/cpanm Amazon::S3', 'invocation': {'module_args': {'executable': None, 'from_path': None, 'installdeps': False, 'locallib': None, 'mirror': None, 'mirror_only': False, 'name': 'Amazon::S3', 'notest': False, 'system_lib': False, 'version': None}}, 'msg': '! Finding Digest::HMAC_SHA1 on cpanmetadb failed.\n! Finding Class::Accessor::Fast on cpanmetadb failed.\nread timeout at /usr/share/perl5/Net/HTTP/Methods.pm line 249. at /usr/share/perl5/LWP/UserAgent.pm line 1017.\n'}, 'smithi040.front.sepia.ceph.com': {'_ansible_no_log': False, 'changed': False, 'cmd': '/usr/bin/cpanm Amazon::S3', 'invocation': {'module_args': {'executable': None, 'from_path': None, 'installdeps': False, 'locallib': None, 'mirror': None, 'mirror_only': False, 'name': 'Amazon::S3', 'notest': False, 'system_lib': False, 'version': None}}, 'msg': '! Finding XML::Simple on cpanmetadb failed.\n! Finding Digest::HMAC_SHA1 on cpanmetadb failed.\nread timeout at /usr/share/perl5/Net/HTTP/Methods.pm line 249. at /usr/share/perl5/LWP/UserAgent.pm line 1017.\n'}} 1 jobs: ['7317009'] suites: ['mon_election/connectivity', 'msgr-failures/many', 'msgr/async-v2only', 'no_pools', 'objectstore/bluestore-comp-zstd', 'rados', 'rados/multimon/{clusters/9', 'supported-random-distro$/{ubuntu_latest}', 'tasks/mon_recovery}'] Dead: b'2023-06-27T10:20:25.625 DEBUG:teuthology.exit:Finished running handlers' 1 jobs: ['7317017'] suites: ['mon_election/connectivity', 'rados', 'rados/singleton-nomsgr/{all/admin_socket_output', 'supported-random-distro$/{centos_8}}'] Failure: [Errno None] Unable to connect to port 22 on 172.21.15.39 1 jobs: ['7316842'] suites: ['mon_election/classic', 'rados', 'rados/singleton-nomsgr/{all/ceph-post-file', 'supported-random-distro$/{ubuntu_latest}}']