Found 184 jobs Found 13 distinct failure reasons Failure: "2023-01-26T02:24:49.690599+0000 mgr.x (mgr.4108) 1 : cluster [ERR] Failed to load ceph-mgr modules: prometheus" in cluster log 115 jobs suites intersection: [] suites union: ['1-ceph', '1-client', '1-upgrade}}', '2-logrotate}', '2-upgrade', '2-workunit/cfuse_workunit_snaptests}}', '2-workunit/cfuse_workunit_suites_fsstress}}', '2-workunit/cfuse_workunit_suites_pjd}}', '2-workunit/cfuse_workunit_trivial_sync}}', '2-workunit/ffsb}}', '2-workunit/fs/snaps}}', '2-workunit/fs/trivial_sync}}', '2-workunit/iozone}}', '2-workunit/suites/fsstress}}', '2-workunit/suites/iozone}}', '2-workunit/suites/pjd}}', '3-client-upgrade', '3-compat_client/no}}', '3-compat_client/pacific}}', '4-compat_client', '5-client-sanity}}', 'centos_latest', 'clients/mirror', 'cluster/1-node', 'cluster/{1-node}', 'clusters/1-mds-1-client-coloc', 'clusters/1-mds-2-client-micro', 'clusters/1-node-1-mds-1-osd', 'clusters/1a3s-mds-1c-client', 'clusters/1a3s-mds-2c-client', 'clusters/1a3s-mds-4c-client', 'clusters/1a5s-mds-1c-client', 'clusters/fixed-2-ucephfs', 'conf/{client', 'debug', 'distro/{centos_8}', 'distro/{rhel_8}', 'distro/{ubuntu_latest}', 'fs/32bits/{begin/{0-install', 'fs/full/{begin/{0-install', 'fs/functional/{begin/{0-install', 'fs/libcephfs/{begin/{0-install', 'fs/mixed-clients/{begin/{0-install', 'fs/multifs/{begin/{0-install', 'fs/permission/{begin/{0-install', 'fs/snaps/{begin/{0-install', 'fs/thrash/multifs/{begin/{0-install', 'fs/thrash/workloads/{begin/{0-install', 'fs/top/{begin/{0-install', 'fs/traceless/{begin/{0-install', 'fs/upgrade/featureful_client/old_client/{bluestore-bitmap', 'fs/upgrade/featureful_client/upgraded_client/{bluestore-bitmap', 'fs/upgrade/nofs/{bluestore-bitmap', 'fs/valgrind/{begin/{0-install', 'fs/verify/{begin/{0-install', 'fs/volumes/{begin/{0-install', 'ignorelist_health', 'ignorelist_wrongly_marked_down', 'ignorelist_wrongly_marked_down}', 'kclient-overrides/{distro/stock/{k-stock', 'mds', 'mirror/{cephfs-mirror/one-per-cluster', 'mon', 'mon-debug', 'mon-debug}', 'mount', 'mount/fuse', 'mount/kclient/{k-testing', 'mount/kclient/{mount-syntax/{v1}', 'mount/kclient/{mount-syntax/{v2}', 'ms-die-on-skipped}', 'ms-die-on-skipped}}', 'msgr-failures/none', 'msgr-failures/osd-mds-delay', 'multifs', 'multimds/no', 'multimds/yes', 'no-mds-cluster', 'no_client_pidfile}', 'objectstore-ec/bluestore-bitmap', 'objectstore-ec/bluestore-comp', 'objectstore-ec/bluestore-comp-ec-root', 'objectstore-ec/bluestore-ec-root', 'objectstore/bluestore-bitmap', 'objectstore/bluestore-ec-root', 'osd-asserts}', 'osd}', 'overrides', 'overrides/ignorelist_health', 'overrides/whitelist_health', 'overrides/{distro/stock/{k-stock', 'overrides/{distro/testing/k-testing', 'overrides/{faked-ino', 'overrides/{frag', 'overrides/{ignorelist_health', 'pg-warn}', 'prefetch_dirfrags/no', 'prefetch_dirfrags/yes', 'races', 'ranks/1', 'ranks/3', 'ranks/5', 'rhel_8}', 'session_timeout', 'session_timeout}', 'supported-random-distros$/{centos_8}', 'tasks/acls}', 'tasks/admin}', 'tasks/alternate-pool}', 'tasks/auto-repair}', 'tasks/backtrace}', 'tasks/cap-flush}', 'tasks/cfuse_workunit_misc}', 'tasks/cfuse_workunit_suites_dbench', 'tasks/cfuse_workunit_suites_fsstress', 'tasks/cfuse_workunit_suites_fsstress}', 'tasks/cfuse_workunit_suites_pjd}', 'tasks/client-readahead}', 'tasks/client-recovery}', 'tasks/client}', 'tasks/damage}', 'tasks/data-scan}', 'tasks/dbench', 'tasks/exports}', 'tasks/failover}', 'tasks/forward-scrub}', 'tasks/fragment}', 'tasks/fsstress', 'tasks/fstop}', 'tasks/ino_release_cb}', 'tasks/journal-repair}', 'tasks/kernel_cfuse_workunits_dbench_iozone}', 'tasks/mds-flush}', 'tasks/mds-full}', 'tasks/mds_creation_retry}', 'tasks/metrics}', 'tasks/mgr-osd-full}', 'tasks/mirror}}', 'tasks/multifs-auth}', 'tasks/multimds_misc}', 'tasks/openfiletable}', 'tasks/pool-perm}', 'tasks/quota}', 'tasks/recovery-fs}', 'tasks/scrub}', 'tasks/sessionmap}', 'tasks/snap-schedule}', 'tasks/snap_schedule_snapdir}', 'tasks/strays}', 'tasks/test_journal_migration}', 'tasks/truncate_delay}', 'tasks/volumes/{overrides', 'tasks/workunit/dir-max-entries}', 'tasks/workunit/quota}', 'tasks/workunit/snaps}', 'tasks/{0-octopus', 'tasks/{1-thrash/mds', 'tasks/{1-thrash/mon', 'tasks/{1-thrash/osd', 'test/basic}}', 'test/misc}}', 'test/snapshot}}', 'thrashosds-health}', 'traceless/50pc}', 'validater/lockdep}', 'validater/valgrind}'] Failure: Command failed (workunit test suites/pjd.sh) on smithi057 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=d42d65e724427960623d805c4a5a240507876d19 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 6h /home/ubuntu/cephtest/clone.client.0/qa/workunits/suites/pjd.sh' 1 jobs: ['7138260'] suites: ['1-ceph', '2-logrotate}', '2-workunit/cfuse_workunit_suites_pjd}}', 'clusters/1a3s-mds-2c-client', 'conf/{client', 'distro/{centos_8}', 'fs/thrash/multifs/{begin/{0-install', 'ignorelist_health', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mount', 'mount/kclient/{mount-syntax/{v2}', 'ms-die-on-skipped}}', 'msgr-failures/osd-mds-delay', 'multifs', 'objectstore/bluestore-bitmap', 'osd}', 'overrides/{distro/testing/k-testing', 'overrides/{frag', 'session_timeout', 'tasks/{1-thrash/mon', 'thrashosds-health}'] Failure: reached maximum tries (150) after waiting for 900 seconds 1 jobs: ['7138263'] suites: ['1-ceph', '2-logrotate}', 'clusters/fixed-2-ucephfs', 'conf/{client', 'distro/{centos_8}', 'fs/permission/{begin/{0-install', 'ignorelist_wrongly_marked_down}', 'mds', 'mon', 'mount/fuse', 'objectstore-ec/bluestore-comp', 'osd}', 'overrides/{ignorelist_health', 'tasks/cfuse_workunit_suites_pjd}'] Failure: error during scrub thrashing: Command failed on smithi161 with status 1: 'sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.1:0 scrub status' 1 jobs: ['7138294'] suites: ['1-cephadm', '2-logrotate}', 'begin/{0-install', 'clusters/1a11s-mds-1c-client-3node', 'conf/{client', 'fs/workload/{0-rhel_8', 'ignorelist_health', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mount', 'mount/kclient/{mount-syntax/{v1}', 'ms-die-on-skipped}}', 'ms_mode/{legacy}', 'objectstore-ec/bluestore-comp-ec-root', 'omap_limit/10000', 'osd-asserts', 'osd}', 'overrides/{distro/testing/k-testing', 'overrides/{frag', 'ranks/5', 'scrub/yes', 'session_timeout}', 'standby-replay', 'tasks/{0-check-counter', 'workunit/suites/dbench}', 'wsync/{yes}}'] Failure: Command failed (workunit test fs/snaps/snaptest-multiple-capsnaps.sh) on smithi112 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=d42d65e724427960623d805c4a5a240507876d19 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/fs/snaps/snaptest-multiple-capsnaps.sh' 1 jobs: ['7138405'] suites: ['1-ceph', '2-logrotate}', 'clusters/1a3s-mds-1c-client', 'conf/{client', 'distro/{ubuntu_latest}', 'fs/snaps/{begin/{0-install', 'ignorelist_wrongly_marked_down}', 'mds', 'mon', 'mount/fuse', 'objectstore-ec/bluestore-comp-ec-root', 'osd}', 'overrides/{ignorelist_health', 'tasks/workunit/snaps}'] Failure: "2023-01-26T04:12:31.442761+0000 mds.e (mds.0) 1 : cluster [WRN] client.4517 isn't responding to mclientcaps(revoke), ino 0x1000000318b pending pAsLsXsFsx issued pAsLsXsFsxr, sent 303.920557 seconds ago" in cluster log 1 jobs: ['7138423'] suites: ['1-ceph', '2-logrotate}', 'clusters/1a5s-mds-1c-client', 'conf/{client', 'distro/{ubuntu/{latest', 'fs/verify/{begin/{0-install', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mon-debug', 'mount', 'mount/kclient/{k-testing', 'ms-die-on-skipped}', 'objectstore-ec/bluestore-bitmap', 'osd}', 'overrides/{ignorelist_health', 'overrides}}', 'ranks/1', 'session_timeout}', 'tasks/fsstress', 'validater/valgrind}'] Failure: "2023-01-26T01:20:34.355603+0000 osd.2 (osd.2) 12 : cluster [ERR] 5.1fs0 deep-scrub : stat mismatch, got 0/2 objects, 0/0 clones, 0/2 dirty, 0/0 omap, 0/0 pinned, 0/0 hit_set_archive, 0/0 whiteouts, 0/8388608 bytes, 0/0 manifest objects, 0/0 hit_set_archive bytes." in cluster log 1 jobs: ['7138329'] suites: ['1-ceph', '2-logrotate}', '2-workunit/suites/iozone}}', 'clusters/1a5s-mds-1c-client', 'conf/{client', 'distro/{ubuntu_latest}', 'fs/thrash/workloads/{begin/{0-install', 'ignorelist_health', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mount', 'mount/kclient/{mount-syntax/{v1}', 'ms-die-on-skipped}}', 'msgr-failures/osd-mds-delay', 'objectstore-ec/bluestore-ec-root', 'osd}', 'overrides/{distro/testing/k-testing', 'overrides/{frag', 'prefetch_dirfrags/no', 'races', 'ranks/3', 'session_timeout', 'tasks/{1-thrash/osd', 'thrashosds-health}'] Failure: Test failure: test_subvolume_snapshot_info_if_orphan_clone (tasks.cephfs.test_volumes.TestSubvolumeSnapshotClones) 2 jobs: ['7138279', '7138406'] suites intersection: ['1-ceph', '2-logrotate}', 'clusters/1a3s-mds-4c-client', 'conf/{client', 'fs/volumes/{begin/{0-install', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'no_client_pidfile}', 'objectstore/bluestore-bitmap', 'osd}', 'overrides/{ignorelist_health', 'tasks/volumes/{overrides', 'test/clone}}'] suites union: ['1-ceph', '2-logrotate}', 'clusters/1a3s-mds-4c-client', 'conf/{client', 'distro/{rhel_8}', 'distro/{ubuntu_latest}', 'fs/volumes/{begin/{0-install', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mount', 'mount/fuse', 'mount/kclient/{mount-syntax/{v1}', 'ms-die-on-skipped}}', 'no_client_pidfile}', 'objectstore/bluestore-bitmap', 'osd}', 'overrides/{distro/testing/k-testing', 'overrides/{ignorelist_health', 'tasks/volumes/{overrides', 'test/clone}}'] Failure: Failed to fetch package version from https://shaman.ceph.com/api/search/?status=ready&project=ceph&flavor=default&distros=ubuntu%2F22.04%2Fx86_64&sha1=d42d65e724427960623d805c4a5a240507876d19 2 jobs: ['7138413', '7138346'] suites intersection: ['1-ceph', '2-logrotate}', 'clusters/1a3s-mds-2c-client', 'conf/{client', 'fs/mixed-clients/{begin/{0-install', 'ignorelist_wrongly_marked_down', 'kclient-overrides/{distro/testing/k-testing', 'mds', 'mon', 'ms-die-on-skipped}', 'osd-asserts}', 'osd}', 'overrides/{ignorelist_health', 'tasks/kernel_cfuse_workunits_untarbuild_blogbench}'] suites union: ['1-ceph', '2-logrotate}', 'clusters/1a3s-mds-2c-client', 'conf/{client', 'fs/mixed-clients/{begin/{0-install', 'ignorelist_wrongly_marked_down', 'kclient-overrides/{distro/testing/k-testing', 'mds', 'mon', 'ms-die-on-skipped}', 'objectstore-ec/bluestore-comp-ec-root', 'objectstore-ec/bluestore-ec-root', 'osd-asserts}', 'osd}', 'overrides/{ignorelist_health', 'tasks/kernel_cfuse_workunits_untarbuild_blogbench}'] Crash: "2023-01-26T02:54:59.592994+0000 mgr.x (mgr.4101) 1 : cluster [ERR] Failed to load ceph-mgr modules: prometheus" in cluster log ceph version 17.2.5-703-gd42d65e7 (d42d65e724427960623d805c4a5a240507876d19) quincy (stable) 1: /lib64/libpthread.so.0(+0x12b20) [0x7fe46d5e6b20] 2: gsignal() 3: abort() 4: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x18f) [0x7fe46e5e8493] 5: /usr/lib64/ceph/libceph-common.so.2(+0x2695ff) [0x7fe46e5e85ff] 6: (MDSTableClient::handle_request(boost::intrusive_ptr const&)+0x1278) [0x564b06360958] 7: (MDSRank::handle_message(boost::intrusive_ptr const&)+0xd75) [0x564b06068315] 8: (MDSRank::_dispatch(boost::intrusive_ptr const&, bool)+0x5db) [0x564b0606a7ab] 9: (MDSRankDispatcher::ms_dispatch(boost::intrusive_ptr const&)+0x5c) [0x564b0606adec] 10: (MDSDaemon::ms_dispatch2(boost::intrusive_ptr const&)+0x1bf) [0x564b0605445f] 11: (Messenger::ms_deliver_dispatch(boost::intrusive_ptr const&)+0x478) [0x7fe46e862e68] 12: (DispatchQueue::entry()+0x50f) [0x7fe46e8602af] 13: (DispatchQueue::DispatchThread::entry()+0x11) [0x7fe46e927ad1] 14: /lib64/libpthread.so.0(+0x814a) [0x7fe46d5dc14a] 15: clone() 2 jobs: ['7138407', '7138253'] suites intersection: ['1-ceph', '2-logrotate}', 'clusters/1a3s-mds-4c-client', 'conf/{client', 'fs/functional/{begin/{0-install', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mount', 'ms-die-on-skipped}}', 'no_client_pidfile}', 'osd}', 'overrides/{ignorelist_health', 'tasks/snapshots}'] suites union: ['1-ceph', '2-logrotate}', 'clusters/1a3s-mds-4c-client', 'conf/{client', 'distro/{centos_8}', 'distro/{rhel_8}', 'fs/functional/{begin/{0-install', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mount', 'mount/kclient/{mount-syntax/{v1}', 'mount/kclient/{mount-syntax/{v2}', 'ms-die-on-skipped}}', 'no_client_pidfile}', 'objectstore/bluestore-bitmap', 'objectstore/bluestore-ec-root', 'osd}', 'overrides/{distro/stock/{k-stock', 'overrides/{distro/testing/k-testing', 'overrides/{ignorelist_health', 'rhel_8}', 'tasks/snapshots}'] Failure: SELinux denials found on ubuntu@smithi012.front.sepia.ceph.com: ['type=AVC msg=audit(1674688493.679:195): avc: denied { node_bind } for pid=1390 comm="ping" saddr=172.21.15.12 scontext=system_u:system_r:ping_t:s0 tcontext=system_u:object_r:node_t:s0 tclass=icmp_socket permissive=1'] 1 jobs: ['7138259'] suites: ['1-cephadm', '2-logrotate}', 'begin/{0-install', 'clusters/1a11s-mds-1c-client-3node', 'conf/{client', 'fs/workload/{0-rhel_8', 'ignorelist_health', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mount', 'mount/kclient/{mount-syntax/{v2}', 'ms-die-on-skipped}}', 'ms_mode/{crc}', 'objectstore-ec/bluestore-comp', 'omap_limit/10', 'osd-asserts', 'osd}', 'overrides/{distro/testing/k-testing', 'overrides/{frag', 'ranks/5', 'scrub/no', 'session_timeout}', 'standby-replay', 'tasks/{0-check-counter', 'workunit/fs/misc}', 'wsync/{no}}'] Failure: timeout expired in wait_until_healthy 2 jobs: ['7138399', '7138302'] suites intersection: [] suites union: ['1-mount', '1-rename', '2-workload/dbench', 'distro/single-container-host', 'distro/single-container-host}', 'fs/cephadm/multivolume/{0-start', 'fs/cephadm/renamevolume/{0-start', 'overrides/ignorelist_health}'] Dead: b'2023-01-26T13:16:21.825 DEBUG:teuthology.exit:Finished running handlers' 1 jobs: ['7138348'] suites: ['1-ranks/2', '1-volume/{0-create', '2-allow_standby_replay/yes', '2-client', '3-inline/yes', '3-upgrade-with-workload', '4-verify}', '4-verify}}', 'centos_8.stream_container_tools', 'conf/{client', 'fs/upgrade/mds_upgrade_sequence/{bluestore-bitmap', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'osd}', 'overrides/{ignorelist_health', 'pg-warn', 'roles', 'syntax}', 'tasks/{0-from/v16.2.4']