Found 215 jobs Missing teuthology log /home/teuthworker/archive/vshankar-2024-02-17_05:57:56-fs-wip-vshankar-testing-20240217.015652-testing-default-smithi/7564299/teuthology.log Found 17 distinct failure reasons Failure: "2024-02-18T11:49:56.429153+0000 mon.a (mon.0) 190 : cluster 3 [WRN] MON_DOWN: 1/3 mons down, quorum a,c" in cluster log 7 jobs: ['7564389', '7564185', '7564342', '7564261', '7564362', '7564287', '7564318'] suites intersection: ['1-cephadm', '1-check-counter', '2-logrotate}', '4-flush/no', 'begin/{0-install', 'clusters/1a11s-mds-1c-client-3node', 'conf/{client', 'frag', 'fs/workload/{0-centos_9.stream', 'ignorelist_health', 'ignorelist_wrongly_marked_down', 'k-stock}', 'mds', 'mon', 'mount', 'ms-die-on-skipped}}', 'ms_mode/legacy', 'objectstore-ec/bluestore-bitmap', 'omap_limit/10000', 'osd-asserts', 'osd}', 'overrides/{cephsqlite-timeout', 'overrides/{distro/stock/{centos_9.stream', 'session_timeout}', 'standby-replay', 'wsync/yes}'] suites union: ['1-cephadm', '1-check-counter', '2-logrotate}', '2-scrub/no', '2-scrub/yes', '3-snaps/no', '3-snaps/yes', '4-flush/no', '5-workunit/direct_io}}', '5-workunit/fs/misc}}', '5-workunit/fs/test_o_trunc}}', '5-workunit/suites/blogbench}}', '5-workunit/suites/fsstress}}', '5-workunit/suites/iogen}}', '5-workunit/suites/pjd}}', 'begin/{0-install', 'clusters/1a11s-mds-1c-client-3node', 'conf/{client', 'export-check', 'frag', 'fs/workload/{0-centos_9.stream', 'ignorelist_health', 'ignorelist_wrongly_marked_down', 'k-stock}', 'mds', 'mon', 'mount', 'mount/kclient/{base/{mount-syntax/{v1}', 'mount/kclient/{base/{mount-syntax/{v2}', 'ms-die-on-skipped}}', 'ms_mode/legacy', 'n/3', 'n/5', 'objectstore-ec/bluestore-bitmap', 'omap_limit/10000', 'osd-asserts', 'osd}', 'overrides/{cephsqlite-timeout', 'overrides/{distro/stock/{centos_9.stream', 'ranks/1', 'ranks/multi/{balancer/automatic', 'ranks/multi/{balancer/random', 'replication/always}', 'replication/default}', 'session_timeout}', 'standby-replay', 'tasks/{0-subvolume/{with-namespace-isolated-and-quota}', 'tasks/{0-subvolume/{with-namespace-isolated}', 'tasks/{0-subvolume/{with-no-extra-options}', 'tasks/{0-subvolume/{with-quota}', 'wsync/yes}'] Failure: Command failed on smithi082 with status 1: 'sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy pull' 38 jobs suites intersection: [] suites union: ['1-cephadm', '1-check-counter', '1-client', '1-mount', '1-ranks/1', '1-ranks/2', '1-rename', '1-volume/{0-create', '2-allow_standby_replay/no', '2-allow_standby_replay/yes', '2-client', '2-logrotate}', '2-scrub/no', '2-scrub/yes', '2-upgrade', '2-workload/dbench', '3-compat_client/quincy}}', '3-inline/no', '3-inline/yes', '3-snaps/no', '3-snaps/yes', '3-upgrade-mgr-staggered', '4-config-upgrade/{fail_fs}', '4-flush/no', '4-flush/yes', '4-verify}', '5-upgrade-with-workload', '5-workunit/direct_io}}', '5-workunit/fs/misc}}', '5-workunit/fs/norstats}}', '5-workunit/fs/test_o_trunc}}', '5-workunit/kernel_untar_build}}', '5-workunit/suites/blogbench}}', '5-workunit/suites/dbench}}', '5-workunit/suites/ffsb}}', '5-workunit/suites/fsstress}}', '5-workunit/suites/fsx}}', '5-workunit/suites/fsync-tester}}', '5-workunit/suites/iogen}}', '5-workunit/suites/iozone}}', '5-workunit/suites/pjd}}', '6-verify}}', 'begin/{0-install', 'centos_8.stream', 'centos_9.stream', 'clusters/1-mds-2-client-micro', 'clusters/1a11s-mds-1c-client-3node', 'conf/{client', 'distro/single-container-host', 'distro/single-container-host}', 'export-check', 'fail_fs/no', 'fail_fs/yes', 'frag', 'fs/cephadm/multivolume/{0-start', 'fs/cephadm/renamevolume/{0-start', 'fs/nfs/{cluster/{1-node}', 'fs/upgrade/featureful_client/old_client/{bluestore-bitmap', 'fs/upgrade/mds_upgrade_sequence/{bluestore-bitmap', 'fs/workload/{0-centos_9.stream', 'ignorelist_health', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mount', 'mount/fuse', 'mount/kclient/{base/{mount-syntax/{v1}', 'mount/kclient/{base/{mount-syntax/{v2}', 'ms-die-on-skipped}}', 'ms_mode/crc', 'ms_mode/secure', 'multimds/yes', 'n/3', 'n/5', 'objectstore-ec/bluestore-comp', 'objectstore-ec/bluestore-comp-ec-root', 'objectstore-ec/bluestore-ec-root', 'omap_limit/10', 'omap_limit/10000', 'osd-asserts', 'osd}', 'overrides/ignorelist_health', 'overrides/ignorelist_health}', 'overrides/{cephsqlite-timeout', 'overrides/{distro/testing/k-testing', 'overrides/{ignorelist_health', 'pg-warn', 'pg-warn}', 'ranks/1', 'ranks/multi/{balancer/automatic', 'ranks/multi/{balancer/random', 'replication/always}', 'replication/default}', 'roles', 'session_timeout}', 'standby-replay', 'supported-random-distros$/{ubuntu_latest}', 'syntax}', 'tasks/nfs}', 'tasks/{0-from/quincy', 'tasks/{0-octopus', 'tasks/{0-subvolume/{with-namespace-isolated-and-quota}', 'tasks/{0-subvolume/{with-namespace-isolated}', 'tasks/{0-subvolume/{with-no-extra-options}', 'tasks/{0-subvolume/{with-quota}', 'wsync/no}'] Failure: SELinux denials found on ubuntu@smithi112.front.sepia.ceph.com: ['type=AVC msg=audit(1708248525.867:191): avc: denied { checkpoint_restore } for pid=1190 comm="agetty" capability=40 scontext=system_u:system_r:getty_t:s0-s0:c0.c1023 tcontext=system_u:system_r:getty_t:s0-s0:c0.c1023 tclass=capability2 permissive=1', 'type=AVC msg=audit(1708248525.867:192): avc: denied { checkpoint_restore } for pid=1189 comm="agetty" capability=40 scontext=system_u:system_r:getty_t:s0-s0:c0.c1023 tcontext=system_u:system_r:getty_t:s0-s0:c0.c1023 tclass=capability2 permissive=1'] 8 jobs: ['7564220', '7564372', '7564315', '7564268', '7564204', '7564356', '7564327', '7564303'] suites intersection: ['1-ceph', '2-logrotate}'] suites union: ['1-ceph', '1-tests/fscrypt-iozone}}', '2-logrotate}', '2-workunit/cfuse_workunit_trivial_sync}}', '2-workunit/ffsb}}', 'bluestore-bitmap', 'centos_latest', 'cluster/{1-node}', 'clusters/1-mds-1-client', 'clusters/1a3s-mds-1c-client', 'clusters/1a3s-mds-2c-client', 'clusters/1a3s-mds-4c-client', 'clusters/small-cluster', 'conf/{client', 'distro/{centos_latest}', 'frag', 'fs/bugs/client_trim_caps/{begin/{0-install', 'fs/fscrypt/{begin/{0-install', 'fs/functional/{begin/{0-install', 'fs/snaps/{begin/{0-install', 'fs/thrash/multifs/{begin/{0-install', 'fs/top/{begin/{0-install', 'fs/volumes/{begin/{0-install', 'ignorelist_health', 'ignorelist_health_more', 'ignorelist_wrongly_marked_down', 'ignorelist_wrongly_marked_down}', 'mds', 'mon', 'mount', 'mount/fuse', 'mount/kclient/{mount-syntax/{v1}', 'mount/kclient/{mount-syntax/{v2}', 'ms-die-on-skipped}}', 'msgr-failures/none', 'multifs', 'no_client_pidfile}', 'objectstore-ec/bluestore-ec-root', 'objectstore/bluestore-bitmap', 'objectstore/bluestore-ec-root', 'osd', 'osd}', 'overrides/ignorelist_health', 'overrides/{client-shutdown', 'overrides/{distro/testing/k-testing', 'overrides/{ignorelist_health', 'pg-warn}', 'session_timeout', 'subvol_versions/create_subvol_version_v2', 'supported-random-distros$/{centos_latest}', 'tasks/exports}', 'tasks/fstop}', 'tasks/trim-i24137}', 'tasks/volumes/{overrides', 'tasks/workunit/snaps}', 'tasks/{0-client', 'tasks/{1-thrash/mds', 'tasks/{1-thrash/mon', 'test/snapshot}}', 'thrashosds-health}'] Dead: None 4 jobs: ['7564299', '7564202', '7564322', '7564353'] Crash: Test failure: test_generic (tasks.cephfs.tests_from_xfstests_dev.TestXFSTestsDev), test_generic (tasks.cephfs.tests_from_xfstests_dev.TestXFSTestsDev) ceph version 19.0.0-1555-g1c920afa (1c920afa8316ea21877f3186850254947531ea22) squid (dev) 1: /lib64/libc.so.6(+0x54db0) [0x7fa78ea54db0] 2: /lib64/libc.so.6(+0xa154c) [0x7fa78eaa154c] 3: raise() 4: abort() 5: (ceph::__ceph_abort(char const*, int, char const*, std::__cxx11::basic_string, std::allocator > const&)+0x187) [0x5582a40cd687] 6: (KernelDevice::_aio_thread()+0x11e0) [0x5582a4a955e0] 7: ceph-osd(+0xdbb751) [0x5582a4a95751] 8: /lib64/libc.so.6(+0x9f802) [0x7fa78ea9f802] 9: /lib64/libc.so.6(+0x3f450) [0x7fa78ea3f450] 1 jobs: ['7564191'] suites: ['1-ceph', '2-logrotate}', 'clusters/1a3s-mds-4c-client', 'conf/{client', 'distro/{centos_latest}', 'fs/functional/{begin/{0-install', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mount', 'mount/kclient/{mount-syntax/{v1}', 'ms-die-on-skipped}}', 'no_client_pidfile}', 'objectstore/bluestore-ec-root', 'osd}', 'overrides/{distro/testing/k-testing', 'overrides/{ignorelist_health', 'subvol_versions/create_subvol_version_v1', 'tasks/xfstests-dev}'] Failure: SSH connection to smithi132 was lost: 'rm -f /tmp/kernel.x86_64.rpm && echo kernel-6.8.0_rc4_g8d59ab0bd93c-1.x86_64.rpm | wget -nv -O /tmp/kernel.x86_64.rpm --base=https://1.chacra.ceph.com/r/kernel/testing/8d59ab0bd93c806231018a6729893bf309624262/centos/9/flavors/default/x86_64/ --input-file=-' 1 jobs: ['7564339'] suites: ['1-ceph', '2-logrotate}', 'clusters/1a3s-mds-4c-client', 'conf/{client', 'distro/{centos_latest}', 'fs/functional/{begin/{0-install', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mount/fuse', 'no_client_pidfile}', 'objectstore/bluestore-ec-root', 'osd}', 'overrides/{ignorelist_health', 'subvol_versions/create_subvol_version_v2', 'tasks/recovery-fs}'] Failure: Test failure: test_subvolume_snapshot_info_if_clone_pending_for_no_group (tasks.cephfs.test_volumes.TestSubvolumeSnapshotClones) 1 jobs: ['7564265'] suites: ['1-ceph', '2-logrotate}', 'clusters/1a3s-mds-4c-client', 'conf/{client', 'distro/{ubuntu_latest}', 'fs/volumes/{begin/{0-install', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mount/fuse', 'no_client_pidfile}', 'objectstore/bluestore-bitmap', 'osd}', 'overrides/{ignorelist_health', 'tasks/volumes/{overrides', 'test/clone}}'] Assertion: 5: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x188) [0x7fb33d380d4a] ceph version 19.0.0-1555-g1c920afa (1c920afa8316ea21877f3186850254947531ea22) squid (dev) 1: /lib64/libc.so.6(+0x54db0) [0x7fb33ca54db0] 2: /lib64/libc.so.6(+0xa154c) [0x7fb33caa154c] 3: raise() 4: abort() 5: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x188) [0x7fb33d380d4a] 6: /usr/lib64/ceph/libceph-common.so.2(+0x180eae) [0x7fb33d380eae] 7: (MDSTableClient::handle_request(boost::intrusive_ptr const&)+0x119f) [0x560fba04d78f] 8: (MDSRank::handle_message(boost::intrusive_ptr const&)+0xd55) [0x560fb9de0e35] 9: (MDSRank::_dispatch(boost::intrusive_ptr const&, bool)+0x4eb) [0x560fb9de25ab] 10: (MDSRankDispatcher::ms_dispatch(boost::intrusive_ptr const&)+0x5c) [0x560fb9de2a9c] 11: (MDSDaemon::ms_dispatch2(boost::intrusive_ptr const&)+0x195) [0x560fb9dca0a5] 12: (DispatchQueue::entry()+0x53a) [0x7fb33d5766da] 13: /usr/lib64/ceph/libceph-common.so.2(+0x408aa1) [0x7fb33d608aa1] 14: /lib64/libc.so.6(+0x9f802) [0x7fb33ca9f802] 15: /lib64/libc.so.6(+0x3f450) [0x7fb33ca3f450] 1 jobs: ['7564361'] suites: ['1-ceph', '2-logrotate}', 'clusters/1a3s-mds-4c-client', 'conf/{client', 'distro/{centos_latest}', 'fs/functional/{begin/{0-install', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mount', 'mount/kclient/{mount-syntax/{v1}', 'ms-die-on-skipped}}', 'no_client_pidfile}', 'objectstore/bluestore-bitmap', 'osd}', 'overrides/{distro/testing/k-testing', 'overrides/{ignorelist_health', 'subvol_versions/create_subvol_version_v1', 'tasks/snapshots}'] Failure: Test failure: test_idem_unaffected_root_squash (tasks.cephfs.test_admin.TestFsAuthorizeUpdate) 1 jobs: ['7564215'] suites: ['1-ceph', '2-logrotate}', 'clusters/1a3s-mds-4c-client', 'conf/{client', 'distro/{centos_latest}', 'fs/functional/{begin/{0-install', 'ignorelist_wrongly_marked_down', 'k-stock}', 'mds', 'mon', 'mount', 'mount/kclient/{mount-syntax/{v1}', 'ms-die-on-skipped}}', 'no_client_pidfile}', 'objectstore/bluestore-bitmap', 'osd}', 'overrides/{distro/stock/{centos_9.stream', 'overrides/{ignorelist_health', 'subvol_versions/create_subvol_version_v1', 'tasks/admin}'] Failure: valgrind error: Leak_StillReachable calloc calloc _dl_check_map_versions 3 jobs: ['7564334', '7564316', '7564244'] suites intersection: ['1-ceph', '2-logrotate}', 'mount/fuse'] suites union: ['1-ceph', '2-logrotate}', 'centos_latest', 'clients/mirror', 'cluster/1-node', 'clusters/1a5s-mds-1c-client', 'conf/{client', 'debug', 'distro/{ubuntu/{overrides', 'fs/valgrind/{begin/{0-install', 'fs/verify/{begin/{0-install', 'ignorelist_wrongly_marked_down', 'mds', 'mirror/{cephfs-mirror/one-per-cluster', 'mon', 'mon-debug', 'mount/fuse', 'objectstore-ec/bluestore-comp-ec-root', 'objectstore-ec/bluestore-ec-root', 'osd}', 'overrides/ignorelist_health', 'overrides/{ignorelist_health', 'ranks/3', 'ranks/5', 'session_timeout}', 'tasks/dbench', 'tasks/mirror}}', 'ubuntu_latest}}', 'validater/valgrind}'] Failure: Failed to reconnect to smithi132 1 jobs: ['7564338'] suites: ['1-ceph', '2-logrotate}', '2-workunit/suites/iozone}}', 'clusters/1a5s-mds-1c-client', 'conf/{client', 'distro/{ubuntu_latest}', 'frag', 'fs/thrash/workloads/{begin/{0-install', 'ignorelist_health', 'ignorelist_wrongly_marked_down', 'k-stock}', 'mds', 'mon', 'mount', 'mount/kclient/{mount-syntax/{v2}', 'ms-die-on-skipped}}', 'msgr-failures/none', 'objectstore-ec/bluestore-comp-ec-root', 'osd}', 'overrides/{client-shutdown', 'overrides/{distro/stock/{centos_9.stream', 'prefetch_dirfrags/no', 'prefetch_entire_dirfrags/no', 'races', 'ranks/1', 'session_timeout', 'tasks/{1-thrash/mon', 'thrashosds-health}'] Failure: Test failure: test_r_with_fsname_and_no_path_in_cap (tasks.cephfs.test_multifs_auth.TestMDSCaps) 1 jobs: ['7564272'] suites: ['1-ceph', '2-logrotate}', 'clusters/1a3s-mds-2c-client', 'conf/{client', 'distro/{ubuntu_latest}', 'fs/multifs/{begin/{0-install', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mon-debug}', 'mount', 'mount/kclient/{mount-syntax/{v1}', 'ms-die-on-skipped}}', 'objectstore-ec/bluestore-comp-ec-root', 'osd}', 'overrides/{distro/testing/k-testing', 'overrides/{ignorelist_health', 'tasks/multifs-auth}'] Failure: No module named 'tasks.fs' 2 jobs: ['7564391', '7564398'] suites intersection: ['conf/{client', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'osd}'] suites union: ['1-ceph', '1-client', '2-logrotate}', '2-upgrade', '2-workunit/iozone}}', '3-client-upgrade', '4-compat_client', '5-client-sanity}}', 'centos_8.stream', 'clusters/1-mds-2-client-micro', 'clusters/1a3s-mds-2c-client', 'conf/{client', 'distro/{ubuntu_latest}', 'frag', 'fs/thrash/multifs/{begin/{0-install', 'fs/upgrade/featureful_client/upgraded_client/{bluestore-bitmap', 'ignorelist_health', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mount', 'mount/kclient/{mount-syntax/{v1}', 'ms-die-on-skipped}}', 'msgr-failures/none', 'multifs', 'multimds/yes', 'objectstore/bluestore-bitmap', 'osd}', 'overrides/{client-shutdown', 'overrides/{distro/testing/k-testing', 'overrides/{ignorelist_health', 'pg-warn}', 'session_timeout', 'tasks/{0-octopus', 'tasks/{1-thrash/mds', 'thrashosds-health}'] Failure: Test failure: test_cd_with_args (tasks.cephfs.test_cephfs_shell.TestCD) 1 jobs: ['7564203'] suites: ['1-ceph', '2-logrotate}', 'clusters/1-mds-1-client-coloc', 'conf/{client', 'distro/ubuntu_latest', 'fs/shell/{begin/{0-install', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mount/fuse', 'no_client_pidfile}', 'objectstore/bluestore-bitmap', 'osd}', 'overrides/{ignorelist_health', 'tasks/cephfs-shell}'] Dead: b'2024-02-18T23:38:37.481 DEBUG:teuthology.exit:Finished running handlers' ceph version 19.0.0-1555-g1c920afa (1c920afa8316ea21877f3186850254947531ea22) squid (dev) 1: /lib64/libc.so.6(+0x54db0) [0x7f5c56654db0] 2: /lib64/libc.so.6(+0xa154c) [0x7f5c566a154c] 3: raise() 4: abort() 5: (ceph::__ceph_abort(char const*, int, char const*, std::__cxx11::basic_string, std::allocator > const&)+0x187) [0x56535064a687] 6: (KernelDevice::_aio_thread()+0x11e0) [0x5653510125e0] 7: ceph-osd(+0xdbb751) [0x565351012751] 8: /lib64/libc.so.6(+0x9f802) [0x7f5c5669f802] 9: /lib64/libc.so.6(+0x3f450) [0x7f5c5663f450] 1 jobs: ['7564390'] suites: ['1-ceph', '2-logrotate}', 'clusters/1a3s-mds-4c-client', 'conf/{client', 'distro/{centos_latest}', 'fs/functional/{begin/{0-install', 'ignorelist_wrongly_marked_down', 'k-stock}', 'mds', 'mon', 'mount', 'mount/kclient/{mount-syntax/{v1}', 'ms-die-on-skipped}}', 'no_client_pidfile}', 'objectstore/bluestore-ec-root', 'osd}', 'overrides/{distro/stock/{centos_9.stream', 'overrides/{ignorelist_health', 'subvol_versions/create_subvol_version_v2', 'tasks/xfstests-dev}'] Failure: Test failure: test_reading_conf (tasks.cephfs.test_cephfs_shell.TestShellOpts) 1 jobs: ['7564313'] suites: ['1-ceph', '2-logrotate}', 'clusters/1-mds-1-client-coloc', 'conf/{client', 'distro/centos_latest', 'fs/shell/{begin/{0-install', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mount/fuse', 'no_client_pidfile}', 'objectstore/bluestore-bitmap', 'osd}', 'overrides/{ignorelist_health', 'tasks/cephfs-shell}'] Failure: Test failure: test_fscrypt_dummy_encryption_with_quick_group (tasks.cephfs.test_fscrypt.TestFscrypt) 1 jobs: ['7564235'] suites: ['1-ceph', '1-tests/fscrypt-common}}', '2-logrotate}', 'bluestore-bitmap', 'clusters/1-mds-1-client', 'conf/{client', 'distro/{centos_latest}', 'fs/fscrypt/{begin/{0-install', 'ignorelist_health_more', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mount', 'mount/kclient/{mount-syntax/{v2}', 'ms-die-on-skipped}}', 'osd', 'osd}', 'overrides/{distro/testing/k-testing', 'overrides/{ignorelist_health', 'pg-warn}', 'tasks/{0-client']