Found 199 jobs Found 15 distinct failure reasons Failure: Command failed on smithi191 with status 5: 'rm -f /tmp/linux-image.deb && echo linux-image-6.6.0-gcd68252d7b4d_6.6.0-gcd68252d7b4d-1_amd64.deb | wget -nv -O /tmp/linux-image.deb --base=https://2.chacra.ceph.com/r/kernel/testing/cd68252d7b4d7802f3cae7f5945b38e9b20dad4c/ubuntu/jammy/flavors/default/pool/main/l/linux-6.6.0-gcd68252d7b4d/ --input-file=-' 11 jobs: ['7490620', '7490608', '7490690', '7490619', '7490651', '7490662', '7490639', '7490691', '7490670', '7490678', '7490638'] suites intersection: ['1-ceph', '2-logrotate}', 'conf/{client', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'osd}'] suites union: ['1-ceph', '1-tests/fscrypt-dbench}}', '1-tests/fscrypt-iozone}}', '2-logrotate}', '2-workunit/cfuse_workunit_snaptests}}', '2-workunit/suites/ffsb}}', 'bluestore-bitmap', 'clusters/1-mds-1-client', 'clusters/1a3s-mds-2c-client', 'clusters/1a3s-mds-4c-client', 'clusters/1a5s-mds-1c-client', 'conf/{client', 'distro/{ubuntu/{overrides', 'distro/{ubuntu_latest}', 'frag', 'fs/fscrypt/{begin/{0-install', 'fs/functional/{begin/{0-install', 'fs/mixed-clients/{begin/{0-install', 'fs/thrash/multifs/{begin/{0-install', 'fs/thrash/workloads/{begin/{0-install', 'fs/volumes/{begin/{0-install', 'ignorelist_health', 'ignorelist_health_more', 'ignorelist_wrongly_marked_down', 'kclient-overrides/{distro/testing/k-testing', 'mds', 'mon', 'mount', 'mount/kclient/{mount-syntax/{v1}', 'mount/kclient/{mount-syntax/{v2}', 'ms-die-on-skipped}', 'ms-die-on-skipped}}', 'msgr-failures/osd-mds-delay', 'multifs', 'no_client_pidfile}', 'objectstore-ec/bluestore-comp-ec-root', 'objectstore/bluestore-bitmap', 'objectstore/bluestore-ec-root', 'osd-asserts}', 'osd}', 'overrides/{client-shutdown', 'overrides/{distro/testing/k-testing', 'overrides/{ignorelist_health', 'pg-warn}', 'prefetch_dirfrags/no', 'prefetch_entire_dirfrags/no', 'races', 'ranks/3', 'session_timeout', 'tasks/journal-repair}', 'tasks/kernel_cfuse_workunits_untarbuild_blogbench}', 'tasks/mds_creation_retry}', 'tasks/openfiletable}', 'tasks/recovery-fs}', 'tasks/volumes/{overrides', 'tasks/{0-client', 'tasks/{1-thrash/mds', 'test/clone}}', 'test/misc}}', 'thrashosds-health}', 'ubuntu_latest}}'] Failure: smithi160.front.sepia.ceph.com: _ansible_no_log: false changed: false invocation: module_args: allow_downgrade: false allowerasing: false autoremove: false bugfix: false conf_file: null disable_excludes: null disable_gpg_check: false disable_plugin: [] disablerepo: [] download_dir: null download_only: false enable_plugin: [] enablerepo: [] exclude: [] install_repoquery: true install_weak_deps: true installroot: / list: null lock_timeout: 30 name: - krb5-workstation releasever: null security: false skip_broken: false state: present update_cache: false update_only: false validate_certs: true msg: 'Failed to download metadata for repo ''codeready-builder-for-rhel-8-x86_64-rpms'': Cannot download repomd.xml: Cannot download repodata/repomd.xml: All mirrors were tried' rc: 1 results: []smithi148.front.sepia.ceph.com: _ansible_no_log: false changed: false invocation: module_args: allow_downgrade: false allowerasing: false autoremove: false bugfix: false conf_file: null disable_excludes: null disable_gpg_check: false disable_plugin: [] disablerepo: [] download_dir: null download_only: false enable_plugin: [] enablerepo: [] exclude: [] install_repoquery: true install_weak_deps: true installroot: / list: null lock_timeout: 30 name: - krb5-workstation releasever: null security: false skip_broken: false state: present update_cache: false update_only: false validate_certs: true msg: 'Failed to download metadata for repo ''codeready-builder-for-rhel-8-x86_64-rpms'': Cannot download repomd.xml: Cannot download repodata/repomd.xml: All mirrors were tried' rc: 1 results: [] 46 jobs suites intersection: ['2-logrotate}', 'conf/{client', 'mds', 'mon', 'osd}'] suites union: ['1-ceph', '1-cephadm', '1-check-counter', '2-logrotate}', '2-scrub/no', '2-scrub/yes', '2-workunit/cfuse_workunit_suites_fsstress}}', '2-workunit/cfuse_workunit_suites_pjd}}', '2-workunit/ffsb}}', '2-workunit/fs/snaps}}', '2-workunit/fs/trivial_sync}}', '2-workunit/iozone}}', '3-snaps/no', '3-snaps/yes', '4-flush/no', '4-flush/yes', '5-workunit/fs/misc}}', '5-workunit/fs/norstats}}', '5-workunit/postgres}}', '5-workunit/suites/blogbench}}', '5-workunit/suites/dbench}}', '5-workunit/suites/ffsb}}', '5-workunit/suites/fsstress}}', '5-workunit/suites/fsx}}', '5-workunit/suites/fsync-tester}}', '5-workunit/suites/iogen}}', '5-workunit/suites/pjd}}', 'begin/{0-install', 'clusters/1a11s-mds-1c-client-3node', 'clusters/1a3s-mds-1c-client', 'clusters/1a3s-mds-2c-client', 'clusters/1a3s-mds-4c-client', 'clusters/1a5s-mds-1c-client', 'conf/{client', 'distro/{centos_latest}', 'distro/{ubuntu/{overrides', 'distro/{ubuntu_latest}', 'frag', 'fs/functional/{begin/{0-install', 'fs/mixed-clients/{begin/{0-install', 'fs/multifs/{begin/{0-install', 'fs/snaps/{begin/{0-install', 'fs/thrash/multifs/{begin/{0-install', 'fs/thrash/workloads/{begin/{0-install', 'fs/volumes/{begin/{0-install', 'fs/workload/{0-rhel_8', 'ignorelist_health', 'ignorelist_wrongly_marked_down', 'ignorelist_wrongly_marked_down}', 'kclient-overrides/{distro/stock/{k-stock', 'mds', 'mon', 'mon-debug}', 'mount', 'mount/fuse', 'mount/kclient/{base/{mount-syntax/{v1}', 'mount/kclient/{base/{mount-syntax/{v2}', 'mount/kclient/{mount-syntax/{v1}', 'mount/kclient/{mount-syntax/{v2}', 'ms-die-on-skipped}', 'ms-die-on-skipped}}', 'ms_mode/crc', 'ms_mode/legacy', 'ms_mode/secure', 'msgr-failures/none', 'msgr-failures/osd-mds-delay', 'multifs', 'n/3', 'n/5', 'no_client_pidfile}', 'objectstore-ec/bluestore-bitmap', 'objectstore-ec/bluestore-comp', 'objectstore-ec/bluestore-comp-ec-root', 'objectstore-ec/bluestore-ec-root', 'objectstore/bluestore-bitmap', 'objectstore/bluestore-ec-root', 'omap_limit/10', 'omap_limit/10000', 'osd-asserts', 'osd-asserts}', 'osd}', 'overrides/{cephsqlite-timeout', 'overrides/{client-shutdown', 'overrides/{distro/stock/{k-stock', 'overrides/{distro/testing/k-testing', 'overrides/{ignorelist_health', 'prefetch_dirfrags/no', 'prefetch_dirfrags/yes', 'prefetch_entire_dirfrags/no', 'prefetch_entire_dirfrags/yes', 'races', 'ranks/1', 'ranks/multi/{export-check', 'replication/always}', 'replication/default}', 'rhel_8}', 'session_timeout', 'session_timeout}', 'standby-replay', 'tasks/alternate-pool}', 'tasks/backtrace}', 'tasks/client-readahead}', 'tasks/data-scan}', 'tasks/failover}', 'tasks/fragment}', 'tasks/kernel_cfuse_workunits_dbench_iozone}', 'tasks/mds-full}', 'tasks/multifs-auth}', 'tasks/multimds_misc}', 'tasks/quota}', 'tasks/scrub}', 'tasks/sessionmap}', 'tasks/snap_schedule_snapdir}', 'tasks/snapshots}', 'tasks/test_journal_migration}', 'tasks/volumes/{overrides', 'tasks/workunit/snaps}', 'tasks/xfstests-dev}', 'tasks/{0-subvolume/{no-subvolume}', 'tasks/{0-subvolume/{with-namespace-isolated}', 'tasks/{0-subvolume/{with-no-extra-options}', 'tasks/{0-subvolume/{with-quota}', 'tasks/{1-thrash/mds', 'tasks/{1-thrash/mon', 'tasks/{1-thrash/osd', 'test/basic}}', 'test/finisher_per_module}}', 'thrashosds-health}', 'ubuntu_latest}}', 'wsync/no}', 'wsync/yes}'] Failure: HTTPSConnectionPool(host='chacra.ceph.com', port=443): Max retries exceeded with url: /repos/ceph/octopus/8a82819d84cf884bd39c17e3236e0632ac146dc4/centos/8/flavors/default/repo (Caused by SSLError(SSLCertVerificationError(1, '[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: certificate has expired (_ssl.c:1131)'))) 15 jobs: ['7490642', '7490592', '7490585', '7490606', '7490685', '7490658', '7490689', '7490671', '7490618', '7490663', '7490615', '7490624', '7490696', '7490637', '7490706'] suites intersection: [] suites union: ['1-client', '1-client-sanity}}}', '1-mount', '1-mount/mount/fuse', '1-rename', '1-tests/dbench}}}', '1-tests/fsstress}}}', '1-tests/kernel_untar_build}}}', '1-upgrade}}', '2-upgrade', '2-workload/dbench', '2-workload/new_ops/{0-clients/fuse-upgrade', '2-workload/stress_tests/{0-client-upgrade', '3-client-upgrade', '3-compat_client/no}}', '3-compat_client/quincy}}', '4-compat_client', '5-client-sanity}}', 'centos_8.stream', 'clusters/1-mds-2-client-micro', 'clusters/{1-mds-1-client-micro}', 'conf/{client', 'distro/single-container-host', 'distro/single-container-host}', 'fs/cephadm/multivolume/{0-start', 'fs/cephadm/renamevolume/{0-start', 'fs/upgrade/featureful_client/old_client/{bluestore-bitmap', 'fs/upgrade/featureful_client/upgraded_client/{bluestore-bitmap', 'fs/upgrade/nofs/{bluestore-bitmap', 'fs/upgrade/upgraded_client/{bluestore-bitmap', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'multimds/no', 'multimds/yes', 'no-mds-cluster', 'osd}', 'overrides/ignorelist_health}', 'overrides/{ignorelist_health', 'pg-warn}', 'tasks/{0-from/nautilus', 'tasks/{0-from/pacific', 'tasks/{0-octopus', 'tasks/{0-pacific'] Timeout 120 running ceph --cluster ceph osd dump --format=json 1 jobs: ['7490511'] suites: ['1-upgrade}}', 'centos_8.stream', 'conf/{client', 'fs/upgrade/nofs/{bluestore-bitmap', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'no-mds-cluster', 'osd}', 'overrides/{ignorelist_health', 'pg-warn}', 'tasks/{0-pacific'] Failure: valgrind error: Leak_StillReachable calloc calloc _dl_check_map_versions 2 jobs: ['7490529', '7490683'] suites intersection: ['1-ceph', '2-logrotate}', 'clusters/1a5s-mds-1c-client', 'conf/{client', 'fs/verify/{begin/{0-install', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mon-debug', 'mount/fuse', 'objectstore-ec/bluestore-ec-root', 'osd}', 'overrides/{ignorelist_health', 'session_timeout}', 'tasks/dbench', 'validater/valgrind}'] suites union: ['1-ceph', '2-logrotate}', 'clusters/1a5s-mds-1c-client', 'conf/{client', 'distro/{centos_latest}', 'distro/{ubuntu/{overrides', 'fs/verify/{begin/{0-install', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mon-debug', 'mount/fuse', 'objectstore-ec/bluestore-ec-root', 'osd}', 'overrides/{ignorelist_health', 'ranks/3', 'ranks/5', 'session_timeout}', 'tasks/dbench', 'ubuntu_latest}}', 'validater/valgrind}'] Failure: Test failure: test_cd_with_args (tasks.cephfs.test_cephfs_shell.TestCD) 1 jobs: ['7490701'] suites: ['1-ceph', '2-logrotate}', 'clusters/1-mds-1-client-coloc', 'conf/{client', 'distro/ubuntu_latest', 'fs/shell/{begin/{0-install', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mount/fuse', 'no_client_pidfile}', 'objectstore/bluestore-bitmap', 'osd}', 'overrides/{ignorelist_health', 'tasks/cephfs-shell}'] Failure: Command failed on smithi017 with status 3: 'sudo logrotate /etc/logrotate.d/ceph-test.conf' 4 jobs: ['7490603', '7490598', '7490536', '7490611'] suites intersection: ['1-ceph', '2-logrotate}', 'conf/{client', 'mds', 'mon', 'osd}', 'overrides/{ignorelist_health'] suites union: ['1-ceph', '2-logrotate}', 'clusters/1-mds-3-client', 'clusters/1a3s-mds-1c-client', 'clusters/1a3s-mds-4c-client', 'clusters/1a5s-mds-1c-client', 'conf/{client', 'distro/{ubuntu/{overrides', 'distro/{ubuntu_latest}', 'distros/ubuntu_latest', 'fs/multiclient/{begin/{0-install', 'fs/snaps/{begin/{0-install', 'fs/verify/{begin/{0-install', 'fs/volumes/{begin/{0-install', 'ignorelist_wrongly_marked_down', 'ignorelist_wrongly_marked_down}', 'mds', 'mon', 'mon-debug', 'mount', 'mount/fuse', 'mount/kclient/{mount-syntax/{v2}', 'ms-die-on-skipped}}', 'no_client_pidfile}', 'objectstore-ec/bluestore-comp', 'objectstore-ec/bluestore-comp-ec-root', 'objectstore/bluestore-bitmap', 'osd}', 'overrides/{distro/testing/k-testing', 'overrides/{ignorelist_health', 'ranks/1', 'session_timeout}', 'tasks/cephfs_misc_tests}', 'tasks/dbench', 'tasks/volumes/{overrides', 'tasks/workunit/snaps}', 'test/basic}}', 'ubuntu_latest}}', 'validater/valgrind}'] Failure: Test failure: test_r_with_fsname_and_no_path_in_cap (tasks.cephfs.test_multifs_auth.TestMDSCaps) 1 jobs: ['7490666'] suites: ['1-ceph', '2-logrotate}', 'clusters/1a3s-mds-2c-client', 'conf/{client', 'distro/{centos_latest}', 'fs/multifs/{begin/{0-install', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mon-debug}', 'mount', 'mount/kclient/{mount-syntax/{v2}', 'ms-die-on-skipped}}', 'objectstore-ec/bluestore-bitmap', 'osd}', 'overrides/{distro/testing/k-testing', 'overrides/{ignorelist_health', 'tasks/multifs-auth}'] Dead: b'2023-12-14T11:05:06.396 DEBUG:teuthology.exit:Finished running handlers' 1 jobs: ['7490607'] suites: ['1-ceph', '2-logrotate}', 'centos_latest', 'clients/mirror', 'cluster/1-node', 'debug', 'fs/valgrind/{begin/{0-install', 'mirror/{cephfs-mirror/one-per-cluster', 'mount/fuse', 'overrides/ignorelist_health', 'tasks/mirror}}'] Failure: Command failed on smithi115 with status 5: 'rm -f /tmp/kernel.x86_64.rpm && echo kernel-6.6.0_gcd68252d7b4d-1.x86_64.rpm | wget -nv -O /tmp/kernel.x86_64.rpm --base=https://1.chacra.ceph.com/r/kernel/testing/cd68252d7b4d7802f3cae7f5945b38e9b20dad4c/centos/8/flavors/default/x86_64/ --input-file=-' 5 jobs: ['7490705', '7490656', '7490613', '7490631', '7490679'] suites intersection: ['1-cephadm', '1-check-counter', '2-logrotate}', '2-scrub/no', '3-snaps/no', '4-flush/yes', 'begin/{0-install', 'clusters/1a11s-mds-1c-client-3node', 'conf/{client', 'frag', 'fs/workload/{0-rhel_8', 'ignorelist_health', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mount', 'ms-die-on-skipped}}', 'omap_limit/10000', 'osd-asserts', 'osd}', 'overrides/{cephsqlite-timeout', 'overrides/{distro/testing/k-testing', 'session_timeout}', 'standby-replay', 'wsync/yes}'] suites union: ['1-cephadm', '1-check-counter', '2-logrotate}', '2-scrub/no', '3-snaps/no', '4-flush/yes', '5-workunit/direct_io}}', '5-workunit/fs/test_o_trunc}}', '5-workunit/kernel_untar_build}}', '5-workunit/suites/blogbench}}', '5-workunit/suites/iozone}}', 'begin/{0-install', 'clusters/1a11s-mds-1c-client-3node', 'conf/{client', 'frag', 'fs/workload/{0-rhel_8', 'ignorelist_health', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mount', 'mount/kclient/{base/{mount-syntax/{v1}', 'mount/kclient/{base/{mount-syntax/{v2}', 'ms-die-on-skipped}}', 'ms_mode/crc', 'ms_mode/secure', 'n/3', 'n/5', 'objectstore-ec/bluestore-bitmap', 'objectstore-ec/bluestore-comp', 'omap_limit/10000', 'osd-asserts', 'osd}', 'overrides/{cephsqlite-timeout', 'overrides/{distro/testing/k-testing', 'ranks/1', 'ranks/multi/{export-check', 'replication/always}', 'replication/default}', 'session_timeout}', 'standby-replay', 'tasks/{0-subvolume/{with-namespace-isolated-and-quota}', 'tasks/{0-subvolume/{with-namespace-isolated}', 'tasks/{0-subvolume/{with-no-extra-options}', 'tasks/{0-subvolume/{with-quota}', 'wsync/yes}'] Failure: Test failure: test_fscrypt_encrypt (tasks.cephfs.test_fscrypt.TestFscrypt) 1 jobs: ['7490532'] suites: ['1-ceph', '1-tests/fscrypt-common}}', '2-logrotate}', 'bluestore-bitmap', 'clusters/1-mds-1-client', 'conf/{client', 'distro/{ubuntu_latest}', 'fs/fscrypt/{begin/{0-install', 'ignorelist_health_more', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mount', 'mount/kclient/{mount-syntax/{v2}', 'ms-die-on-skipped}}', 'osd}', 'overrides/{distro/testing/k-testing', 'overrides/{ignorelist_health', 'pg-warn}', 'tasks/{0-client'] Failure: Test failure: test_snapshot_remove (tasks.cephfs.test_strays.TestStrays) 1 jobs: ['7490707'] suites: ['1-ceph', '2-logrotate}', 'clusters/1a3s-mds-4c-client', 'conf/{client', 'distro/{centos_latest}', 'fs/functional/{begin/{0-install', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mount', 'mount/kclient/{mount-syntax/{v2}', 'ms-die-on-skipped}}', 'no_client_pidfile}', 'objectstore/bluestore-bitmap', 'osd}', 'overrides/{distro/testing/k-testing', 'overrides/{ignorelist_health', 'tasks/strays}'] Timeout 3h running clone.client.1/qa/workunits/suites/fsstress.sh 1 jobs: ['7490680'] suites: ['1-ranks/2', '1-volume/{0-create', '2-allow_standby_replay/no', '2-client', '3-inline/no', '3-upgrade-mgr-staggered', '4-config-upgrade/{fail_fs}', '4-verify}', '5-upgrade-with-workload', '6-verify}}', 'centos_8.stream_container_tools', 'conf/{client', 'fail_fs/no', 'fs/upgrade/mds_upgrade_sequence/{bluestore-bitmap', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'osd}', 'overrides/{ignorelist_health', 'pg-warn', 'roles', 'syntax}', 'tasks/{0-from/quincy'] Failure: Test failure: test_pool_perm (tasks.cephfs.test_pool_perm.TestPoolPerm) 1 jobs: ['7490669'] suites: ['1-ceph', '2-logrotate}', 'clusters/1a3s-mds-4c-client', 'conf/{client', 'distro/{centos_latest}', 'fs/functional/{begin/{0-install', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mount/fuse', 'no_client_pidfile}', 'objectstore/bluestore-bitmap', 'osd}', 'overrides/{ignorelist_health', 'tasks/pool-perm}'] Failure: Test failure: test_reading_conf (tasks.cephfs.test_cephfs_shell.TestShellOpts) 1 jobs: ['7490602'] suites: ['1-ceph', '2-logrotate}', 'clusters/1-mds-1-client-coloc', 'conf/{client', 'distro/centos_latest', 'fs/shell/{begin/{0-install', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mount/fuse', 'no_client_pidfile}', 'objectstore/bluestore-bitmap', 'osd}', 'overrides/{ignorelist_health', 'tasks/cephfs-shell}']