Found 104 jobs Found 7 distinct failure reasons Failure: smithi124.front.sepia.ceph.com: _ansible_no_log: false changed: false invocation: module_args: allow_downgrade: false allowerasing: false autoremove: false bugfix: false conf_file: null disable_excludes: null disable_gpg_check: false disable_plugin: [] disablerepo: [] download_dir: null download_only: false enable_plugin: [] enablerepo: [] exclude: [] install_repoquery: true install_weak_deps: true installroot: / list: null lock_timeout: 30 name: - krb5-workstation releasever: null security: false skip_broken: false state: present update_cache: false update_only: false validate_certs: true msg: 'Failed to download metadata for repo ''codeready-builder-for-rhel-8-x86_64-rpms'': Cannot download repomd.xml: Cannot download repodata/repomd.xml: All mirrors were tried' rc: 1 results: [] 54 jobs suites intersection: ['1-ceph', '2-logrotate}'] suites union: ['1-ceph', '2-logrotate}', '2-three-per-cluster}', '2-workunit/cfuse_workunit_snaptests}}', '2-workunit/cfuse_workunit_suites_pjd}}', '2-workunit/iozone}}', '2-workunit/suites/ffsb}}', '2-workunit/suites/fsstress}}', '2-workunit/suites/iozone}}', '2-workunit/suites/pjd}}', 'cephfs-mirror/one-per-cluster', 'cephfs-mirror/{1-volume-create-rm', 'clients/{mirror}', 'cluster/{1-node}', 'clusters/1-mds-1-client-coloc', 'clusters/1a3s-mds-1c-client', 'clusters/1a3s-mds-2c-client', 'clusters/1a3s-mds-4c-client', 'clusters/1a5s-mds-1c-client', 'clusters/fixed-2-ucephfs', 'conf/{client', 'distro/{centos_8}', 'distro/{rhel_8}', 'distro/{ubuntu_latest}', 'fs/32bits/{begin/{0-install', 'fs/functional/{begin/{0-install', 'fs/libcephfs/{begin/{0-install', 'fs/mirror-ha/{begin/{0-install', 'fs/mirror/{begin/{0-install', 'fs/mixed-clients/{begin/{0-install', 'fs/multifs/{begin/{0-install', 'fs/permission/{begin/{0-install', 'fs/snaps/{begin/{0-install', 'fs/thrash/multifs/{begin/{0-install', 'fs/thrash/workloads/{begin/{0-install', 'fs/traceless/{begin/{0-install', 'fs/verify/{begin/{0-install', 'fs/volumes/{begin/{0-install', 'ignorelist_health', 'ignorelist_wrongly_marked_down', 'ignorelist_wrongly_marked_down}', 'kclient-overrides/{distro/stock/{k-stock', 'mds', 'mon', 'mon-debug', 'mon-debug}', 'mount', 'mount/fuse', 'mount/kclient/{mount-syntax/{v1}', 'mount/kclient/{mount-syntax/{v2}', 'ms-die-on-skipped}', 'ms-die-on-skipped}}', 'msgr-failures/none', 'msgr-failures/osd-mds-delay', 'multifs', 'no_client_pidfile}', 'objectstore-ec/bluestore-bitmap', 'objectstore-ec/bluestore-comp', 'objectstore-ec/bluestore-comp-ec-root', 'objectstore-ec/bluestore-ec-root', 'objectstore/bluestore-bitmap', 'objectstore/bluestore-ec-root', 'osd-asserts}', 'osd}', 'overrides/{distro/stock/{k-stock', 'overrides/{faked-ino', 'overrides/{frag', 'overrides/{ignorelist_health', 'overrides/{ignorelist_health}', 'prefetch_dirfrags/no', 'prefetch_dirfrags/yes', 'races', 'ranks/3', 'ranks/5', 'rhel_8}', 'session_timeout', 'session_timeout}', 'supported-random-distro$/{rhel_8}', 'supported-random-distros$/{rhel_8}', 'tasks/acls}', 'tasks/alternate-pool}', 'tasks/asok_dump_tree}', 'tasks/cap-flush}', 'tasks/cfuse_workunit_misc}', 'tasks/cfuse_workunit_suites_blogbench', 'tasks/cfuse_workunit_suites_fsstress', 'tasks/cfuse_workunit_suites_fsstress}', 'tasks/cfuse_workunit_suites_pjd}', 'tasks/client-recovery}', 'tasks/dbench', 'tasks/exports}', 'tasks/failover}', 'tasks/journal-repair}', 'tasks/kernel_cfuse_workunits_untarbuild_blogbench}', 'tasks/libcephfs_python}', 'tasks/mds-full}', 'tasks/mds_creation_retry}', 'tasks/mirror}', 'tasks/multifs-auth}', 'tasks/openfiletable}', 'tasks/pool-perm}', 'tasks/quota}', 'tasks/recovery-fs}', 'tasks/sessionmap}', 'tasks/snapshots}', 'tasks/test_journal_migration}', 'tasks/truncate_delay}', 'tasks/volumes/{overrides', 'tasks/workunit/quota}', 'tasks/workunit/snaps}', 'tasks/{1-thrash/mds', 'tasks/{1-thrash/mon', 'tasks/{1-thrash/osd', 'test/clone}}', 'test/misc}}', 'test/snapshot}}', 'thrashosds-health}', 'traceless/50pc}', 'validater/lockdep}', 'workloads/cephfs-mirror-ha-workunit}'] Failure: smithi079.front.sepia.ceph.com: _ansible_no_log: false changed: false invocation: module_args: allow_downgrade: false allowerasing: false autoremove: false bugfix: false conf_file: null disable_excludes: null disable_gpg_check: false disable_plugin: [] disablerepo: [] download_dir: null download_only: false enable_plugin: [] enablerepo: [] exclude: [] install_repoquery: true install_weak_deps: true installroot: / list: null lock_timeout: 30 name: - krb5-workstation releasever: null security: false skip_broken: false state: present update_cache: false update_only: false validate_certs: true msg: 'Failed to download metadata for repo ''codeready-builder-for-rhel-8-x86_64-rpms'': Cannot download repomd.xml: Cannot download repodata/repomd.xml: All mirrors were tried' rc: 1 results: []smithi045.front.sepia.ceph.com: _ansible_no_log: false changed: false invocation: module_args: allow_downgrade: false allowerasing: false autoremove: false bugfix: false conf_file: null disable_excludes: null disable_gpg_check: false disable_plugin: [] disablerepo: [] download_dir: null download_only: false enable_plugin: [] enablerepo: [] exclude: [] install_repoquery: true install_weak_deps: true installroot: / list: null lock_timeout: 30 name: - krb5-workstation releasever: null security: false skip_broken: false state: present update_cache: false update_only: false validate_certs: true msg: 'Failed to download metadata for repo ''codeready-builder-for-rhel-8-x86_64-rpms'': Cannot download repomd.xml: Cannot download repodata/repomd.xml: All mirrors were tried' rc: 1 results: []smithi012.front.sepia.ceph.com: _ansible_no_log: false changed: false invocation: module_args: allow_downgrade: false allowerasing: false autoremove: false bugfix: false conf_file: null disable_excludes: null disable_gpg_check: false disable_plugin: [] disablerepo: [] download_dir: null download_only: false enable_plugin: [] enablerepo: [] exclude: [] install_repoquery: true install_weak_deps: true installroot: / list: null lock_timeout: 30 name: - krb5-workstation releasever: null security: false skip_broken: false state: present update_cache: false update_only: false validate_certs: true msg: 'Failed to download metadata for repo ''codeready-builder-for-rhel-8-x86_64-rpms'': Cannot download repomd.xml: Cannot download repodata/repomd.xml: All mirrors were tried' rc: 1 results: [] 11 jobs: ['7491366', '7491348', '7491356', '7491332', '7491310', '7491413', '7491370', '7491381', '7491392', '7491404', '7491326'] suites intersection: ['1-cephadm', '2-logrotate}', 'begin/{0-install', 'clusters/1a11s-mds-1c-client-3node', 'conf/{client', 'fs/workload/{0-rhel_8', 'ignorelist_health', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'osd-asserts', 'osd}', 'overrides/{frag', 'session_timeout}', 'standby-replay', 'tasks/{0-check-counter'] suites union: ['1-cephadm', '2-logrotate}', 'begin/{0-install', 'clusters/1a11s-mds-1c-client-3node', 'conf/{client', 'fs/workload/{0-rhel_8', 'ignorelist_health', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mount', 'mount/fuse', 'mount/kclient/{mount-syntax/{v1}', 'mount/kclient/{mount-syntax/{v2}', 'ms-die-on-skipped}}', 'ms_mode/{crc}', 'ms_mode/{legacy}', 'ms_mode/{secure}', 'objectstore-ec/bluestore-bitmap', 'objectstore-ec/bluestore-comp', 'objectstore-ec/bluestore-comp-ec-root', 'objectstore-ec/bluestore-ec-root', 'omap_limit/10', 'omap_limit/10000', 'osd-asserts', 'osd}', 'overrides/{distro/stock/{k-stock', 'overrides/{frag', 'ranks/3', 'ranks/5', 'rhel_8}', 'scrub/no', 'scrub/yes', 'session_timeout}', 'standby-replay', 'tasks/{0-check-counter', 'workunit/direct_io}', 'workunit/fs/misc}', 'workunit/fs/norstats}', 'workunit/suites/blogbench}', 'workunit/suites/dbench}', 'workunit/suites/fsstress}', 'workunit/suites/fsx}', 'workunit/suites/fsync-tester}', 'workunit/suites/iogen}', 'workunit/suites/iozone}', 'wsync/{no}}', 'wsync/{yes}}'] Failure: HTTPSConnectionPool(host='2.chacra.ceph.com', port=443): Max retries exceeded with url: /repos/ceph/pacific/a28aed6037399e2b2b3d0ba5ff408f99da20207c/centos/8/flavors/default/repo (Caused by SSLError(SSLCertVerificationError(1, '[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: certificate has expired (_ssl.c:1131)'))) 13 jobs: ['7491330', '7491365', '7491389', '7491312', '7491401', '7491340', '7491380', '7491335', '7491352', '7491328', '7491317', '7491386', '7491355'] suites intersection: ['conf/{client', 'mds', 'mon', 'osd}', 'overrides/{ignorelist_health'] suites union: ['1-ceph', '1-client', '1-ranks/1', '1-ranks/2', '1-volume/{0-create', '2-allow_standby_replay/no', '2-allow_standby_replay/yes', '2-client', '2-logrotate}', '2-upgrade', '3-client-upgrade', '3-compat_client/no}}', '3-compat_client/pacific}}', '3-inline/no', '3-inline/yes', '3-upgrade-with-workload', '4-compat_client', '4-verify}', '4-verify}}', '5-client-sanity}}', 'centos_8.stream_container_tools', 'centos_latest', 'clusters/1-mds-2-client', 'clusters/1-mds-2-client-micro', 'clusters/1-mds-3-client', 'clusters/1a3s-mds-4c-client', 'conf/{client', 'distro/{ubuntu_latest}', 'distros/ubuntu_latest', 'fs/functional/{begin/{0-install', 'fs/multiclient/{begin/{0-install', 'fs/upgrade/featureful_client/old_client/{bluestore-bitmap', 'fs/upgrade/featureful_client/upgraded_client/{bluestore-bitmap', 'fs/upgrade/mds_upgrade_sequence/{bluestore-bitmap', 'ignorelist_wrongly_marked_down', 'ignorelist_wrongly_marked_down}', 'mds', 'mon', 'mount', 'mount/fuse', 'mount/kclient/{mount-syntax/{v2}', 'ms-die-on-skipped}}', 'multimds/no', 'multimds/yes', 'no_client_pidfile}', 'objectstore-ec/bluestore-bitmap', 'objectstore-ec/bluestore-comp-ec-root', 'objectstore/bluestore-bitmap', 'objectstore/bluestore-ec-root', 'osd}', 'overrides/{distro/testing/k-testing', 'overrides/{ignorelist_health', 'pg-warn', 'pg-warn}', 'roles', 'syntax}', 'tasks/admin}', 'tasks/ior-shared-file}', 'tasks/snap-schedule}', 'tasks/{0-from/pacific', 'tasks/{0-from/v16.2.4', 'tasks/{0-octopus'] Failure: Command failed on smithi178 with status 5: 'rm -f /tmp/kernel.x86_64.rpm && echo kernel-6.6.0_gcd68252d7b4d-1.x86_64.rpm | wget -nv -O /tmp/kernel.x86_64.rpm --base=https://1.chacra.ceph.com/r/kernel/testing/cd68252d7b4d7802f3cae7f5945b38e9b20dad4c/centos/8/flavors/default/x86_64/ --input-file=-' 19 jobs: ['7491371', '7491337', '7491324', '7491360', '7491311', '7491345', '7491405', '7491351', '7491396', '7491343', '7491361', '7491385', '7491338', '7491378', '7491402', '7491383', '7491399', '7491325', '7491415'] suites intersection: ['2-logrotate}', 'conf/{client', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mount', 'osd}'] suites union: ['1-ceph', '1-cephadm', '2-logrotate}', '2-workunit/cfuse_workunit_snaptests}}', '2-workunit/cfuse_workunit_trivial_sync}}', '2-workunit/fs/snaps}}', '2-workunit/fs/trivial_sync}}', 'begin/{0-install', 'clusters/1a11s-mds-1c-client-3node', 'clusters/1a3s-mds-2c-client', 'clusters/1a3s-mds-4c-client', 'clusters/1a5s-mds-1c-client', 'conf/{client', 'distro/{rhel_8}', 'fs/functional/{begin/{0-install', 'fs/multifs/{begin/{0-install', 'fs/thrash/multifs/{begin/{0-install', 'fs/thrash/workloads/{begin/{0-install', 'fs/verify/{begin/{0-install', 'fs/volumes/{begin/{0-install', 'fs/workload/{0-rhel_8', 'ignorelist_health', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mon-debug', 'mon-debug}', 'mount', 'mount/kclient/{k-testing', 'mount/kclient/{mount-syntax/{v1}', 'mount/kclient/{mount-syntax/{v2}', 'ms-die-on-skipped}', 'ms-die-on-skipped}}', 'ms_mode/{crc}', 'ms_mode/{legacy}', 'ms_mode/{secure}', 'msgr-failures/none', 'msgr-failures/osd-mds-delay', 'multifs', 'no_client_pidfile}', 'objectstore-ec/bluestore-bitmap', 'objectstore-ec/bluestore-comp', 'objectstore-ec/bluestore-comp-ec-root', 'objectstore-ec/bluestore-ec-root', 'objectstore/bluestore-bitmap', 'objectstore/bluestore-ec-root', 'omap_limit/10', 'omap_limit/10000', 'osd-asserts', 'osd}', 'overrides/{distro/testing/k-testing', 'overrides/{frag', 'overrides/{ignorelist_health', 'prefetch_dirfrags/no', 'prefetch_dirfrags/yes', 'races', 'ranks/1', 'ranks/3', 'scrub/no', 'scrub/yes', 'session_timeout', 'session_timeout}', 'standby-replay', 'tasks/damage}', 'tasks/fsstress', 'tasks/mds-flush}', 'tasks/metrics}', 'tasks/multifs-auth}', 'tasks/openfiletable}', 'tasks/pool-perm}', 'tasks/recovery-fs}', 'tasks/strays}', 'tasks/volumes/{overrides', 'tasks/{0-check-counter', 'tasks/{1-thrash/mon', 'tasks/{1-thrash/osd', 'test/finisher_per_module}}', 'thrashosds-health}', 'validater/valgrind}', 'workunit/fs/test_o_trunc}', 'workunit/kernel_untar_build}', 'workunit/suites/ffsb}', 'workunit/suites/fsstress}', 'workunit/suites/pjd}', 'wsync/{no}}', 'wsync/{yes}}'] Crash: Command failed (workunit test fs/full/subvolume_snapshot_rm.sh) on smithi086 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=2167f1c9167e5cb12f58a96ff6fa1bd01d4c8350 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/fs/full/subvolume_snapshot_rm.sh' ceph version 17.2.7-169-g2167f1c9 (2167f1c9167e5cb12f58a96ff6fa1bd01d4c8350) quincy (stable) 1: (ceph::__ceph_abort(char const*, int, char const*, std::__cxx11::basic_string, std::allocator > const&)+0xd7) [0x560736bc0fad] 2: (BlueStore::_txc_add_transaction(BlueStore::TransContext*, ceph::os::Transaction*)+0x9fb) [0x5607372c170b] 3: (BlueStore::queue_transactions(boost::intrusive_ptr&, std::vector >&, boost::intrusive_ptr, ThreadPool::TPHandle*)+0x2df) [0x5607372d28af] 4: (non-virtual thunk to PrimaryLogPG::queue_transactions(std::vector >&, boost::intrusive_ptr)+0x53) [0x560736e8cab3] 5: (ReplicatedBackend::submit_transaction(hobject_t const&, object_stat_sum_t const&, eversion_t const&, std::unique_ptr >&&, eversion_t const&, eversion_t const&, std::vector >&&, std::optional&, Context*, unsigned long, osd_reqid_t, boost::intrusive_ptr)+0x7c2) [0x5607370e3b32] 6: (PrimaryLogPG::issue_repop(PrimaryLogPG::RepGather*, PrimaryLogPG::OpContext*)+0x50d) [0x560736e048ed] 7: (PrimaryLogPG::execute_ctx(PrimaryLogPG::OpContext*)+0xd35) [0x560736e62ef5] 8: (PrimaryLogPG::do_op(boost::intrusive_ptr&)+0x288d) [0x560736e6adad] 9: (OSD::dequeue_op(boost::intrusive_ptr, boost::intrusive_ptr, ThreadPool::TPHandle&)+0x1c0) [0x560736cc2510] 10: (ceph::osd::scheduler::PGOpItem::run(OSD*, OSDShard*, boost::intrusive_ptr&, ThreadPool::TPHandle&)+0x6d) [0x560736fcd29d] 11: (OSD::ShardedOpWQ::_process(unsigned int, ceph::heartbeat_handle_d*)+0x112f) [0x560736cfef9f] 12: (ShardedThreadPool::shardedthreadpool_worker(unsigned int)+0x435) [0x560737443325] 13: (ShardedThreadPool::WorkThreadSharded::entry()+0x14) [0x560737445a44] 14: /lib64/libpthread.so.0(+0x81ca) [0x7f53685b61ca] 1 jobs: ['7491320'] suites: ['1-ceph', '2-logrotate}', 'clusters/1-node-1-mds-1-osd', 'conf/{client', 'distro/{centos_8}', 'fs/full/{begin/{0-install', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mount/fuse', 'no_client_pidfile}', 'objectstore/bluestore-bitmap', 'osd}', 'overrides', 'overrides/{ignorelist_health', 'tasks/mgr-osd-full}'] Failure: Command failed on smithi119 with status 5: 'rm -f /tmp/linux-image.deb && echo linux-image-6.6.0-gcd68252d7b4d_6.6.0-gcd68252d7b4d-1_amd64.deb | wget -nv -O /tmp/linux-image.deb --base=https://2.chacra.ceph.com/r/kernel/testing/cd68252d7b4d7802f3cae7f5945b38e9b20dad4c/ubuntu/jammy/flavors/default/pool/main/l/linux-6.6.0-gcd68252d7b4d/ --input-file=-' 3 jobs: ['7491395', '7491339', '7491373'] suites intersection: ['1-ceph', '2-logrotate}', 'clusters/1a3s-mds-2c-client', 'conf/{client', 'fs/mixed-clients/{begin/{0-install', 'ignorelist_wrongly_marked_down', 'kclient-overrides/{distro/testing/k-testing', 'mds', 'mon', 'ms-die-on-skipped}', 'osd-asserts}', 'osd}', 'overrides/{ignorelist_health', 'tasks/kernel_cfuse_workunits_dbench_iozone}'] suites union: ['1-ceph', '2-logrotate}', 'clusters/1a3s-mds-2c-client', 'conf/{client', 'fs/mixed-clients/{begin/{0-install', 'ignorelist_wrongly_marked_down', 'kclient-overrides/{distro/testing/k-testing', 'mds', 'mon', 'ms-die-on-skipped}', 'objectstore-ec/bluestore-bitmap', 'objectstore-ec/bluestore-comp-ec-root', 'objectstore-ec/bluestore-ec-root', 'osd-asserts}', 'osd}', 'overrides/{ignorelist_health', 'tasks/kernel_cfuse_workunits_dbench_iozone}'] Dead: b'2023-12-15T03:47:11.575 DEBUG:teuthology.exit:Finished running handlers' 1 jobs: ['7491323'] suites: ['1-ceph', '2-logrotate}', 'centos_latest', 'clients/mirror', 'cluster/1-node', 'debug', 'fs/valgrind/{begin/{0-install', 'mirror/{cephfs-mirror/one-per-cluster', 'mount/fuse', 'overrides/ignorelist_health', 'tasks/mirror}}']