Found 173 jobs Found 7 distinct failure reasons Failure: "2024-01-25T23:34:12.398374+0000 mon.a (mon.0) 586 : cluster [WRN] Health check failed: Reduced data availability: 2 pgs peering (PG_AVAILABILITY)" in cluster log 4 jobs: ['7531734', '7531710', '7531700', '7531720'] suites intersection: ['conf/{client', 'mds', 'mon', 'osd}', 'overrides/{ignorelist_health'] suites union: ['clusters/1a3s-mds-2c-client', 'clusters/1a3s-mds-4c-client', 'clusters/fixed-2-ucephfs', 'conf/{client', 'distro/{centos_8}', 'distro/{ubuntu_latest}', 'fs/functional/{begin', 'fs/multifs/{begin', 'fs/permission/{begin', 'ignorelist_wrongly_marked_down', 'ignorelist_wrongly_marked_down}', 'mds', 'mon', 'mon-debug}', 'mount/fuse', 'mount/kclient/{mount', 'ms-die-on-skipped}}', 'no_client_pidfile}', 'objectstore-ec/bluestore-comp', 'objectstore-ec/bluestore-ec-root', 'objectstore/bluestore-bitmap', 'osd}', 'overrides/{distro/stock/{k-stock', 'overrides/{ignorelist_health', 'rhel_8}', 'tasks/cfuse_workunit_suites_pjd}', 'tasks/metrics}', 'tasks/multifs-auth}', 'tasks/snap-schedule}'] Failure: 2 jobs: ['7531989', '7531915'] suites intersection: ['conf/{client', 'ignorelist_health', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mount/kclient/{mount', 'ms-die-on-skipped}}', 'osd}'] suites union: ['2-workunit/cfuse_workunit_trivial_sync}}', 'clusters/1a11s-mds-1c-client-3node', 'clusters/1a3s-mds-2c-client', 'conf/{client', 'distro/{centos_8}', 'distro/{ubuntu_latest}', 'frag', 'fs/thrash/multifs/{begin', 'fs/workload/{begin', 'ignorelist_health', 'ignorelist_wrongly_marked_down', 'k-testing}', 'mds', 'mon', 'mount/kclient/{mount', 'ms-die-on-skipped}}', 'ms_mode/{crc}', 'msgr-failures/none', 'multifs', 'objectstore-ec/bluestore-comp-ec-root', 'objectstore/bluestore-bitmap', 'omap_limit/10', 'osd-asserts', 'osd}', 'overrides/{client-shutdown', 'overrides/{distro/stock/{k-stock', 'overrides/{distro/testing/{flavor/ubuntu_latest', 'overrides/{frag', 'ranks/5', 'rhel_8}', 'scrub/yes', 'session_timeout', 'session_timeout}', 'standby-replay', 'subvolume/{no-subvolume}', 'tasks/{0-check-counter', 'tasks/{1-thrash/mds', 'thrashosds-health}', 'workunit/suites/blogbench}', 'wsync/{yes}}'] Crash: Command failed (workunit test fs/full/subvolume_snapshot_rm.sh) on smithi123 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=585bdf827237450fb80df4a856eba434a64948fa TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/fs/full/subvolume_snapshot_rm.sh' ceph version 16.2.14-508-ga72a5dac (a72a5dace2353a54147d37bd2699a2a482507966) pacific (stable) 1: (ceph::__ceph_abort(char const*, int, char const*, std::__cxx11::basic_string, std::allocator > const&)+0xe5) [0x557ebe895558] 2: (BlueStore::_txc_add_transaction(BlueStore::TransContext*, ceph::os::Transaction*)+0x1497) [0x557ebeee6bd7] 3: (BlueStore::queue_transactions(boost::intrusive_ptr&, std::vector >&, boost::intrusive_ptr, ThreadPool::TPHandle*)+0x316) [0x557ebef03e86] 4: (non-virtual thunk to PrimaryLogPG::queue_transactions(std::vector >&, boost::intrusive_ptr)+0x58) [0x557ebeb44b28] 5: (ReplicatedBackend::submit_transaction(hobject_t const&, object_stat_sum_t const&, eversion_t const&, std::unique_ptr >&&, eversion_t const&, eversion_t const&, std::vector >&&, std::optional&, Context*, unsigned long, osd_reqid_t, boost::intrusive_ptr)+0xcad) [0x557ebed355dd] 6: (PrimaryLogPG::issue_repop(PrimaryLogPG::RepGather*, PrimaryLogPG::OpContext*)+0xcf0) [0x557ebeab0320] 7: (PrimaryLogPG::execute_ctx(PrimaryLogPG::OpContext*)+0x115d) [0x557ebeb0c79d] 8: (PrimaryLogPG::do_op(boost::intrusive_ptr&)+0x35ca) [0x557ebeb15afa] 9: (PrimaryLogPG::do_request(boost::intrusive_ptr&, ThreadPool::TPHandle&)+0xcbb) [0x557ebeb1c52b] 10: (OSD::dequeue_op(boost::intrusive_ptr, boost::intrusive_ptr, ThreadPool::TPHandle&)+0x309) [0x557ebe99fcc9] 11: (ceph::osd::scheduler::PGOpItem::run(OSD*, OSDShard*, boost::intrusive_ptr&, ThreadPool::TPHandle&)+0x6b) [0x557ebec0249b] 12: (OSD::ShardedOpWQ::_process(unsigned int, ceph::heartbeat_handle_d*)+0xc28) [0x557ebe9bd798] 13: (ShardedThreadPool::shardedthreadpool_worker(unsigned int)+0x5c4) [0x557ebf042ff4] 14: (ShardedThreadPool::WorkThreadSharded::entry()+0x14) [0x557ebf045ed4] 15: /lib64/libpthread.so.0(+0x81cf) [0x7f721c8611cf] 1 jobs: ['7531751'] suites: ['clusters/1-node-1-mds-1-osd', 'conf/{client', 'distro/{rhel_8}', 'fs/full/{begin', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mount/fuse', 'no_client_pidfile}', 'objectstore/bluestore-bitmap', 'osd}', 'overrides', 'overrides/{ignorelist_health', 'tasks/mgr-osd-full}'] Failure: Command failed on smithi192 with status 22: 'sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.4 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c7a35ec0-bc19-11ee-95b2-87774f69a715 -- ceph orch daemon add osd smithi192:vg_nvme/lv_4' 1 jobs: ['7531918'] suites: ['1-ranks/1', '1-volume/{0-create', '2-allow_standby_replay/yes', '2-client', '3-inline/no', '3-upgrade-with-workload', '4-verify}', '4-verify}}', 'centos_8.stream_container_tools', 'conf/{client', 'fs/upgrade/mds_upgrade_sequence/{bluestore-bitmap', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'osd}', 'overrides/{ignorelist_health', 'pg-warn}', 'roles', 'tasks/{0-from/v16.2.4'] Failure: Test failure: test_acls (tasks.cephfs.test_acls.TestACLs) 1 jobs: ['7531774'] suites: ['clusters/1a3s-mds-4c-client', 'conf/{client', 'distro/{centos_8}', 'fs/functional/{begin', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mount/kclient/{mount', 'ms-die-on-skipped}}', 'no_client_pidfile}', 'objectstore/bluestore-bitmap', 'osd}', 'overrides/{distro/stock/{k-stock', 'overrides/{ignorelist_health', 'rhel_8}', 'tasks/acls}'] Failure: Command failed on smithi129 with status 8: 'TESTDIR=/home/ubuntu/cephtest bash -s' 3 jobs: ['7531804', '7531699', '7531704'] suites intersection: ['conf/{client', 'mds', 'mon', 'osd}', 'overrides/{ignorelist_health'] suites union: ['1-ranks/1', '1-volume/{0-create', '2-allow_standby_replay/yes', '2-client', '3-inline/yes', '3-upgrade-with-workload', '4-verify}', '4-verify}}', 'centos_8.stream_container_tools', 'clusters/1-mds-2-client', 'clusters/1-mds-3-client', 'conf/{client', 'distros/ubuntu_latest', 'fs/multiclient/{begin', 'fs/upgrade/mds_upgrade_sequence/{bluestore-bitmap', 'ignorelist_wrongly_marked_down', 'ignorelist_wrongly_marked_down}', 'mds', 'mon', 'mount/fuse', 'objectstore-ec/bluestore-bitmap', 'objectstore-ec/bluestore-ec-root', 'osd}', 'overrides/{ignorelist_health', 'pg-warn}', 'roles', 'tasks/ior-shared-file}', 'tasks/{0-from/v16.2.4'] Failure: timeout expired in wait_until_healthy 2 jobs: ['7531741', '7531772'] suites intersection: ['1-ranks/1', '1-volume/{0-create', '2-allow_standby_replay/no', '2-client', '3-upgrade-with-workload', '4-verify}', '4-verify}}', 'centos_8.stream_container_tools', 'conf/{client', 'fs/upgrade/mds_upgrade_sequence/{bluestore-bitmap', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'osd}', 'overrides/{ignorelist_health', 'pg-warn}', 'roles', 'tasks/{0-from/v16.2.4'] suites union: ['1-ranks/1', '1-volume/{0-create', '2-allow_standby_replay/no', '2-client', '3-inline/no', '3-inline/yes', '3-upgrade-with-workload', '4-verify}', '4-verify}}', 'centos_8.stream_container_tools', 'conf/{client', 'fs/upgrade/mds_upgrade_sequence/{bluestore-bitmap', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'osd}', 'overrides/{ignorelist_health', 'pg-warn}', 'roles', 'tasks/{0-from/v16.2.4']