Found 57 jobs Found 7 distinct failure reasons Failure: Command failed on smithi079 with status 1: 'cd /home/ubuntu/cephtest/tempest && /home/ubuntu/cephtest/tox-venv/bin/tox -e venv --notest' 1 jobs: ['7327656'] suites: ['frontend/beast', 'overrides', 'rgw/tempest/{clusters/fixed-1', 'tasks/rgw_tempest', 'ubuntu_latest}'] Failure: Command failed (s3 tests against rgw) on smithi102 with status 1: "S3TEST_CONF=/home/ubuntu/cephtest/archive/s3-tests.client.0.conf BOTO_CONFIG=/home/ubuntu/cephtest/boto-client.0.cfg REQUESTS_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt /home/ubuntu/cephtest/s3-tests-client.0/virtualenv/bin/python -m nose -w /home/ubuntu/cephtest/s3-tests-client.0 -v -a '!fails_on_rgw,!lifecycle_expiration,!fails_strict_rfc2616,!test_of_sts,!webidentity_test,!fails_with_subdomain,!sse-s3'" 11 jobs: ['7327669', '7327694', '7327671', '7327648', '7327698', '7327660', '7327679', '7327665', '7327696', '7327690', '7327673'] suites intersection: [] suites union: ['1-ceph-install/install', '2-kms/barbican', '2-kms/kmip', '2-kms/testing', '2-kms/vault_kv', '2-kms/vault_old', '2-kms/vault_transit', '3-rgw/rgw', '4-tests/{s3tests}', 'frontend/beast', 'http', 'ignore-pg-availability', 'install', 'objectstore/bluestore-bitmap', 'objectstore/filestore-xfs', 'overrides', 'rgw/crypt/{0-cluster/fixed-1', 'rgw/multifs/{clusters/fixed-2', 'rgw/thrash/{clusters/fixed-2', 'rgw/website/{clusters/fixed-2', 'rgw_pool_type/ec', 'rgw_pool_type/ec-profile', 'supported-random-distro$/{centos_8}}', 'supported-random-distro$/{rhel_8}}', 'supported-random-distro$/{ubuntu_latest}}', 'tasks/rgw_s3tests', 'tasks/s3tests-website', 'thrasher/default', 'thrashosds-health', 'ubuntu_latest', 'ubuntu_latest}', 'workload/rgw_s3tests}'] Crash: Command failed (s3 tests against rgw) on smithi072 with status 1: "S3TEST_CONF=/home/ubuntu/cephtest/archive/s3-tests.client.0.conf BOTO_CONFIG=/home/ubuntu/cephtest/boto-client.0.cfg REQUESTS_CA_BUNDLE=/etc/pki/tls/certs/ca-bundle.crt /home/ubuntu/cephtest/s3-tests-client.0/virtualenv/bin/python -m nose -w /home/ubuntu/cephtest/s3-tests-client.0 -v -a '!fails_on_rgw,!lifecycle_expiration,!fails_strict_rfc2616,!test_of_sts,!webidentity_test,!fails_with_subdomain,!sse-s3'" ceph version 17.2.6-676-g151650f1 (151650f184a67af9ebce490cf526c615e14997b1) quincy (stable) 1: /lib64/libpthread.so.0(+0x12cf0) [0x7fda0fedecf0] 2: gsignal() 3: abort() 4: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x18f) [0x7fda113754fd] 5: /usr/lib64/ceph/libceph-common.so.2(+0x26a669) [0x7fda11375669] 6: (std::__detail::__variant::__gen_vtable_impl (*)(FaultInjector > >::check(std::basic_string_view > const&) const::visitor&&, std::variant const&)>, std::integer_sequence >::__visit_invoke(FaultInjector > >::check(std::basic_string_view > const&) const::visitor&&, std::variant const&)+0x41) [0x55966301c7f1] 7: (RGWBucketReshard::execute(int, FaultInjector > >&, int, DoutPrefixProvider const*, bool, std::ostream*, ceph::Formatter*, RGWReshard*)+0x16b) [0x55966301a38b] 8: main() 9: __libc_start_main() 10: _start() 16 jobs: ['7327664', '7327692', '7327689', '7327661', '7327676', '7327672', '7327697', '7327701', '7327667', '7327686', '7327659', '7327685', '7327695', '7327678', '7327703', '7327670'] suites intersection: ['centos_latest', 'clusters/fixed-2', 'frontend/beast', 'ignore-pg-availability', 'msgr-failures/few', 'overrides', 'ragweed', 'reshard', 'rgw/verify/{0-install', 's3tests-java', 's3tests}', 'tasks/{cls'] suites union: ['centos_latest', 'clusters/fixed-2', 'datacache/no_datacache', 'datacache/rgw-datacache', 'frontend/beast', 'ignore-pg-availability', 'msgr-failures/few', 'objectstore/bluestore-bitmap', 'objectstore/filestore-xfs', 'overrides', 'proto/http', 'proto/https', 'ragweed', 'reshard', 'rgw/verify/{0-install', 'rgw_pool_type/ec', 'rgw_pool_type/ec-profile', 'rgw_pool_type/replicated', 's3tests-java', 's3tests}', 'sharding$/{default}', 'sharding$/{single}', 'striping$/{stripe-equals-chunk}', 'striping$/{stripe-greater-than-chunk}', 'tasks/{cls', 'validater/lockdep}', 'validater/valgrind}'] Failure: {'smithi003.front.sepia.ceph.com': {'_ansible_no_log': False, 'cache_update_time': 1688619005, 'cache_updated': False, 'changed': False, 'invocation': {'module_args': {'allow_unauthenticated': False, 'autoclean': False, 'autoremove': False, 'cache_valid_time': 0, 'deb': None, 'default_release': None, 'dpkg_options': 'force-confdef,force-confold', 'force': True, 'force_apt_get': False, 'install_recommends': None, 'name': ['mpich', 'qemu-system-x86', 'lttng-tools', 'libtool-bin', 'docker.io', 'python3-nose', 'python3-venv', 'python3-virtualenv', 'python3-configobj', 'python3-gevent', 'python3-numpy', 'python3-matplotlib', 'python3-setuptools', 'python-dev', 'libfcgi0ldbl', 'libev-dev', 'perl', 'libwww-perl', 'lsb-release', 'build-essential', 'sysstat', 'gdb', 'libedit2', 'cryptsetup-bin', 'xfsprogs', 'gdisk', 'parted', 'libuuid1', 'libatomic-ops-dev', 'git-core', 'attr', 'dbench', 'bonnie++', 'valgrind', 'ant', 'libtool', 'automake', 'gettext', 'uuid-dev', 'libacl1-dev', 'bc', 'xfsdump', 'xfslibs-dev', 'libattr1-dev', 'quota', 'libcap2-bin', 'libncurses5-dev', 'lvm2', 'vim', 'pdsh', 'blktrace', 'genisoimage', 'libjson-xs-perl', 'xml-twig-tools', 'default-jdk', 'junit4', 'tgt', 'open-iscsi', 'cifs-utils', 'ipcalc', 'nfs-common', 'nfs-kernel-server', 'software-properties-common', 'libgnutls30'], 'only_upgrade': False, 'package': ['mpich', 'qemu-system-x86', 'lttng-tools', 'libtool-bin', 'docker.io', 'python3-nose', 'python3-venv', 'python3-virtualenv', 'python3-configobj', 'python3-gevent', 'python3-numpy', 'python3-matplotlib', 'python3-setuptools', 'python-dev', 'libfcgi0ldbl', 'libev-dev', 'perl', 'libwww-perl', 'lsb-release', 'build-essential', 'sysstat', 'gdb', 'libedit2', 'cryptsetup-bin', 'xfsprogs', 'gdisk', 'parted', 'libuuid1', 'libatomic-ops-dev', 'git-core', 'attr', 'dbench', 'bonnie++', 'valgrind', 'ant', 'libtool', 'automake', 'gettext', 'uuid-dev', 'libacl1-dev', 'bc', 'xfsdump', 'xfslibs-dev', 'libattr1-dev', 'quota', 'libcap2-bin', 'libncurses5-dev', 'lvm2', 'vim', 'pdsh', 'blktrace', 'genisoimage', 'libjson-xs-perl', 'xml-twig-tools', 'default-jdk', 'junit4', 'tgt', 'open-iscsi', 'cifs-utils', 'ipcalc', 'nfs-common', 'nfs-kernel-server', 'software-properties-common', 'libgnutls30'], 'policy_rc_d': None, 'purge': False, 'state': 'present', 'update_cache': None, 'update_cache_retries': 5, 'update_cache_retry_max_delay': 12, 'upgrade': None}}, 'msg': '\'/usr/bin/apt-get -y -o "Dpkg::Options::=--force-confdef" -o "Dpkg::Options::=--force-confold" --force-yes install \'qemu-system-x86\' \'git-core\'\' failed: W: --force-yes is deprecated, use one of the options starting with --allow instead.\nE: Failed to fetch http://archive.ubuntu.com/ubuntu/pool/main/c/ceph/librados2_15.2.17-0ubuntu0.20.04.4_amd64.deb Unable to connect to archive.ubuntu.com:http:\nE: Failed to fetch http://archive.ubuntu.com/ubuntu/pool/main/c/ceph/librbd1_15.2.17-0ubuntu0.20.04.4_amd64.deb Unable to connect to archive.ubuntu.com:http:\nE: Unable to fetch some archives, maybe run apt-get update or try with --fix-missing?\n', 'rc': 100, 'stderr': 'W: --force-yes is deprecated, use one of the options starting with --allow instead.\nE: Failed to fetch http://archive.ubuntu.com/ubuntu/pool/main/c/ceph/librados2_15.2.17-0ubuntu0.20.04.4_amd64.deb Unable to connect to archive.ubuntu.com:http:\nE: Failed to fetch http://archive.ubuntu.com/ubuntu/pool/main/c/ceph/librbd1_15.2.17-0ubuntu0.20.04.4_amd64.deb Unable to connect to archive.ubuntu.com:http:\nE: Unable to fetch some archives, maybe run apt-get update or try with --fix-missing?\n', 'stderr_lines': ['W: --force-yes is deprecated, use one of the options starting with --allow instead.', 'E: Failed to fetch http://archive.ubuntu.com/ubuntu/pool/main/c/ceph/librados2_15.2.17-0ubuntu0.20.04.4_amd64.deb Unable to connect to archive.ubuntu.com:http:', 'E: Failed to fetch http://archive.ubuntu.com/ubuntu/pool/main/c/ceph/librbd1_15.2.17-0ubuntu0.20.04.4_amd64.deb Unable to connect to archive.ubuntu.com:http:', 'E: Unable to fetch some archives, maybe run apt-get update or try with --fix-missing?'], 'stdout': 'Reading package lists...\nBuilding dependency tree...\nReading state information...\nThe following additional packages will be installed:\n librados2 librbd1 qemu-block-extra qemu-system-common qemu-system-data\n qemu-system-gui qemu-utils\nSuggested packages:\n git-daemon-run | git-daemon-sysvinit git-doc git-el git-email git-gui gitk\n gitweb git-cvs git-mediawiki git-svn samba vde2 debootstrap\nThe following NEW packages will be installed:\n librados2 librbd1 qemu-block-extra qemu-system-common qemu-system-x86\n qemu-utils\nThe following packages will be upgraded:\n git qemu-system-data qemu-system-gui\n3 upgraded, 6 newly installed, 0 to remove and 212 not upgraded.\nNeed to get 18.9 MB of archives.\nAfter this operation, 63.1 MB of additional disk space will be used.\nIgn:1 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 git amd64 1:2.25.1-1ubuntu3.11\nErr:2 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 librados2 amd64 15.2.17-0ubuntu0.20.04.4\n Unable to connect to archive.ubuntu.com:http:\nErr:3 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 librbd1 amd64 15.2.17-0ubuntu0.20.04.4\n Unable to connect to archive.ubuntu.com:http:\nIgn:4 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 qemu-block-extra amd64 1:4.2-3ubuntu6.27\nIgn:5 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 qemu-system-common amd64 1:4.2-3ubuntu6.27\nIgn:6 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 qemu-system-data all 1:4.2-3ubuntu6.27\nIgn:7 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 qemu-system-gui amd64 1:4.2-3ubuntu6.27\nIgn:8 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 qemu-system-x86 amd64 1:4.2-3ubuntu6.27\nIgn:9 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 qemu-utils amd64 1:4.2-3ubuntu6.27\nIgn:6 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 qemu-system-data all 1:4.2-3ubuntu6.27\nGet:1 http://security.ubuntu.com/ubuntu focal-updates/main amd64 git amd64 1:2.25.1-1ubuntu3.11 [4605 kB]\nGet:4 http://security.ubuntu.com/ubuntu focal-updates/main amd64 qemu-block-extra amd64 1:4.2-3ubuntu6.27 [53.4 kB]\nGet:5 http://security.ubuntu.com/ubuntu focal-updates/main amd64 qemu-system-common amd64 1:4.2-3ubuntu6.27 [1057 kB]\nGet:7 http://security.ubuntu.com/ubuntu focal-updates/main amd64 qemu-system-gui amd64 1:4.2-3ubuntu6.27 [39.2 kB]\nGet:8 http://security.ubuntu.com/ubuntu focal-updates/main amd64 qemu-system-x86 amd64 1:4.2-3ubuntu6.27 [6756 kB]\nGet:9 http://security.ubuntu.com/ubuntu focal-updates/main amd64 qemu-utils amd64 1:4.2-3ubuntu6.27 [969 kB]\nGet:6 http://security.ubuntu.com/ubuntu focal-updates/main amd64 qemu-system-data all 1:4.2-3ubuntu6.27 [562 kB]\nFetched 14.0 MB in 42s (337 kB/s)\n', 'stdout_lines': ['Reading package lists...', 'Building dependency tree...', 'Reading state information...', 'The following additional packages will be installed:', ' librados2 librbd1 qemu-block-extra qemu-system-common qemu-system-data', ' qemu-system-gui qemu-utils', 'Suggested packages:', ' git-daemon-run | git-daemon-sysvinit git-doc git-el git-email git-gui gitk', ' gitweb git-cvs git-mediawiki git-svn samba vde2 debootstrap', 'The following NEW packages will be installed:', ' librados2 librbd1 qemu-block-extra qemu-system-common qemu-system-x86', ' qemu-utils', 'The following packages will be upgraded:', ' git qemu-system-data qemu-system-gui', '3 upgraded, 6 newly installed, 0 to remove and 212 not upgraded.', 'Need to get 18.9 MB of archives.', 'After this operation, 63.1 MB of additional disk space will be used.', 'Ign:1 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 git amd64 1:2.25.1-1ubuntu3.11', 'Err:2 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 librados2 amd64 15.2.17-0ubuntu0.20.04.4', ' Unable to connect to archive.ubuntu.com:http:', 'Err:3 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 librbd1 amd64 15.2.17-0ubuntu0.20.04.4', ' Unable to connect to archive.ubuntu.com:http:', 'Ign:4 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 qemu-block-extra amd64 1:4.2-3ubuntu6.27', 'Ign:5 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 qemu-system-common amd64 1:4.2-3ubuntu6.27', 'Ign:6 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 qemu-system-data all 1:4.2-3ubuntu6.27', 'Ign:7 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 qemu-system-gui amd64 1:4.2-3ubuntu6.27', 'Ign:8 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 qemu-system-x86 amd64 1:4.2-3ubuntu6.27', 'Ign:9 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 qemu-utils amd64 1:4.2-3ubuntu6.27', 'Ign:6 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 qemu-system-data all 1:4.2-3ubuntu6.27', 'Get:1 http://security.ubuntu.com/ubuntu focal-updates/main amd64 git amd64 1:2.25.1-1ubuntu3.11 [4605 kB]', 'Get:4 http://security.ubuntu.com/ubuntu focal-updates/main amd64 qemu-block-extra amd64 1:4.2-3ubuntu6.27 [53.4 kB]', 'Get:5 http://security.ubuntu.com/ubuntu focal-updates/main amd64 qemu-system-common amd64 1:4.2-3ubuntu6.27 [1057 kB]', 'Get:7 http://security.ubuntu.com/ubuntu focal-updates/main amd64 qemu-system-gui amd64 1:4.2-3ubuntu6.27 [39.2 kB]', 'Get:8 http://security.ubuntu.com/ubuntu focal-updates/main amd64 qemu-system-x86 amd64 1:4.2-3ubuntu6.27 [6756 kB]', 'Get:9 http://security.ubuntu.com/ubuntu focal-updates/main amd64 qemu-utils amd64 1:4.2-3ubuntu6.27 [969 kB]', 'Get:6 http://security.ubuntu.com/ubuntu focal-updates/main amd64 qemu-system-data all 1:4.2-3ubuntu6.27 [562 kB]', 'Fetched 14.0 MB in 42s (337 kB/s)']}} 1 jobs: ['7327666'] suites: ['frontend/beast', 'ignore-pg-availability', 'objectstore/filestore-xfs', 'overrides', 'rgw/singleton/{all/radosgw-admin', 'rgw_pool_type/ec', 'supported-random-distro$/{ubuntu_latest}}'] Dead: b'2023-07-06T16:40:06.816 DEBUG:teuthology.exit:Finished running handlers' 1 jobs: ['7327653'] suites: ['bluestore-bitmap', 'fixed-2', 'overrides', 'rgw/notifications/{beast', 'supported-all-distro$/{centos_8}', 'tasks/{0-install', 'test_amqp', 'test_kafka', 'test_others}}'] Failure: rgw multisite test failures 2 jobs: ['7327682', '7327652'] suites intersection: ['frontend/beast', 'ignore-pg-availability', 'notify', 'omap_limits', 'overrides', 'rgw/multisite/{clusters', 'tasks/test_multi}'] suites union: ['frontend/beast', 'ignore-pg-availability', 'notify', 'omap_limits', 'overrides', 'realms/three-zones', 'realms/two-zonegroup', 'rgw/multisite/{clusters', 'supported-random-distro$/{centos_8}', 'supported-random-distro$/{ubuntu_latest}', 'tasks/test_multi}'] Failure: Command failed (s3 tests against rgw) on smithi082 with status 1: 'S3TEST_CONF=/home/ubuntu/cephtest/archive/s3-tests.client.0.conf BOTO_CONFIG=/home/ubuntu/cephtest/boto-client.0.cfg REQUESTS_CA_BUNDLE=/etc/pki/tls/certs/ca-bundle.crt /home/ubuntu/cephtest/s3-tests-client.0/virtualenv/bin/python -m nose -w /home/ubuntu/cephtest/s3-tests-client.0 -v -a lifecycle' 1 jobs: ['7327650'] suites: ['overrides', 'rgw/lifecycle/{cluster', 'supported-random-distro$/{centos_8}', 'tasks/rgw_s3tests}']