Found 37 jobs Found 8 distinct failure reasons Failure: "1712992412.8671365 mon.a (mon.0) 502 : cluster [WRN] Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY)" in cluster log 12 jobs: ['7654937', '7654964', '7654949', '7654931', '7654951', '7654954', '7654930', '7654945', '7654958', '7654934', '7654953', '7654950'] suites intersection: [] suites union: ['-', '0-roles', '0-start', '1-ceph-install/quincy', '1-ranks/1', '1-start', '1-tasks', '1-tasks}', '1-volume/{0-create', '2', '2-allow_standby_replay/no', '2-client/kclient', '2-first-half-tasks/radosbench', '2-first-half-tasks/rbd-import-export', '2-first-half-tasks/rbd_api', '2-first-half-tasks/readwrite', '2-first-half-tasks/snaps-few-objects', '3-inline/no', '3-stress-tasks/{radosbench', '3-upgrade-mgr-staggered', '4-config-upgrade/{fail_fs}', '4-second-half-tasks/radosbench', '4-second-half-tasks/rbd-import-export', '4-verify}', '5-upgrade-with-workload', '6-verify}}', 'centos_9.stream', 'conf/{client', 'fail_fs/yes', 'ignorelist_upgrade', 'ignorelist_wrongly_marked_down', 'mds', 'mgr', 'mon', 'mon_election/classic', 'mon_election/classic}', 'mon_election/connectivity', 'mon_election/connectivity}', 'objectstore/filestore-xfs', 'osd}', 'overrides/{ignorelist_health', 'pg-warn', 'pg_health', 'rados_api', 'rados_loadgenbig', 'rbd-cls', 'rbd-import-export', 'rbd_api', 'rbd_import_export', 'readwrite', 'roles', 'snaps-few-objects}', 'start}', 'syntax}', 'tasks/{0-from/quincy', 'test_rbd_api', 'test_rbd_python}}', 'ubuntu_latest}', 'upgrade', 'upgrade-sequence', 'upgrade/cephfs/mds_upgrade_sequence/{bluestore-bitmap', 'upgrade/quincy-x/filestore-remove-check/{0-cluster/{openstack', 'upgrade/quincy-x/parallel/{0-random-distro$/{centos_9.stream}', 'upgrade/quincy-x/parallel/{0-random-distro$/{ubuntu_22.04}', 'upgrade/quincy-x/stress-split/{0-distro/centos_9.stream', 'upgrade/quincy-x/stress-split/{0-distro/centos_9.stream_runc', 'upgrade/reef-x/parallel/{0-random-distro$/{centos_9.stream_runc}', 'upgrade/reef-x/parallel/{0-random-distro$/{centos_9.stream}', 'upgrade/reef-x/stress-split/{0-distro/centos_9.stream_runc', 'upgrade/reef-x/stress-split/{0-distro/ubuntu_22.04', 'upgrade/telemetry-upgrade/quincy-x/{0-random-distro$/{centos_9.stream}', 'workload/{ec-rados-default'] Failure: reached maximum tries (51) after waiting for 300 seconds 3 jobs: ['7654946', '7654963', '7654933'] suites intersection: ['1-ranks/2', '1-volume/{0-create', '2-client/fuse', '3-upgrade-mgr-staggered', '4-config-upgrade/{fail_fs}', '4-verify}', '5-upgrade-with-workload', '6-verify}}', 'centos_9.stream', 'conf/{client', 'ignorelist_upgrade', 'ignorelist_wrongly_marked_down', 'mds', 'mgr', 'mon', 'osd}', 'overrides/{ignorelist_health', 'pg-warn', 'pg_health', 'roles', 'syntax}', 'upgrade/cephfs/mds_upgrade_sequence/{bluestore-bitmap'] suites union: ['1-ranks/2', '1-volume/{0-create', '2-allow_standby_replay/no', '2-allow_standby_replay/yes', '2-client/fuse', '3-inline/no', '3-inline/yes', '3-upgrade-mgr-staggered', '4-config-upgrade/{fail_fs}', '4-verify}', '5-upgrade-with-workload', '6-verify}}', 'centos_9.stream', 'conf/{client', 'fail_fs/no', 'fail_fs/yes', 'ignorelist_upgrade', 'ignorelist_wrongly_marked_down', 'mds', 'mgr', 'mon', 'osd}', 'overrides/{ignorelist_health', 'pg-warn', 'pg_health', 'roles', 'syntax}', 'tasks/{0-from/quincy', 'tasks/{0-from/reef/{v18.2.1}', 'upgrade/cephfs/mds_upgrade_sequence/{bluestore-bitmap'] Failure: "2024-04-13T09:47:53.272579+0000 osd.4 (osd.4) 47 : cluster [ERR] 58.13 soid 58:c93822fb:::smithi005677167-404:head : object info inconsistent , snapset inconsistent , attr value mismatch '__header'" in cluster log 9 jobs: ['7654960', '7654943', '7654947', '7654959', '7654935', '7654961', '7654939', '7654965', '7654938'] suites intersection: ['0-roles', '1-start', '3-stress-tasks/{radosbench', 'rbd-cls', 'rbd-import-export', 'rbd_api', 'readwrite', 'snaps-few-objects}'] suites union: ['0-roles', '1-start', '2-first-half-tasks/radosbench', '2-first-half-tasks/rbd-cls', '2-first-half-tasks/rbd-import-export', '2-first-half-tasks/rbd_api', '2-first-half-tasks/readwrite', '3-stress-tasks/{radosbench', '4-second-half-tasks/radosbench', '4-second-half-tasks/rbd-import-export', 'mon_election/classic}', 'mon_election/connectivity}', 'rbd-cls', 'rbd-import-export', 'rbd_api', 'readwrite', 'snaps-few-objects}', 'upgrade/quincy-x/stress-split/{0-distro/centos_9.stream', 'upgrade/quincy-x/stress-split/{0-distro/centos_9.stream_runc', 'upgrade/quincy-x/stress-split/{0-distro/ubuntu_22.04', 'upgrade/reef-x/stress-split/{0-distro/centos_9.stream', 'upgrade/reef-x/stress-split/{0-distro/centos_9.stream_runc'] Failure: Failed to fetch package version from https://shaman.ceph.com/api/search/?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=pacific 1 jobs: ['7654932'] suites: ['0-start', '1-tasks}', 'upgrade/telemetry-upgrade/pacific-x/{0-random-distro$/{centos_9.stream}'] Failure: "2024-04-13T08:04:05.829364+0000 mon.a (mon.0) 179 : cluster 3 [WRN] MON_DOWN: 1/3 mons down, quorum a,b" in cluster log 1 jobs: ['7654957'] suites: ['0-roles', '1-start', '2-first-half-tasks/radosbench', '3-stress-tasks/{radosbench', '4-second-half-tasks/rbd-import-export', 'mon_election/connectivity}', 'rbd-cls', 'rbd-import-export', 'rbd_api', 'readwrite', 'snaps-few-objects}', 'upgrade/reef-x/stress-split/{0-distro/ubuntu_22.04'] Timeout 3h running clone.client.1/qa/workunits/suites/fsstress.sh 2 jobs: ['7654940', '7654952'] suites intersection: ['1-ranks/1', '1-volume/{0-create', '2-client/kclient', '3-inline/yes', '3-upgrade-mgr-staggered', '4-config-upgrade/{fail_fs}', '4-verify}', '5-upgrade-with-workload', '6-verify}}', 'centos_9.stream', 'conf/{client', 'fail_fs/no', 'ignorelist_upgrade', 'ignorelist_wrongly_marked_down', 'mds', 'mgr', 'mon', 'osd}', 'overrides/{ignorelist_health', 'pg-warn', 'pg_health', 'roles', 'syntax}', 'upgrade/cephfs/mds_upgrade_sequence/{bluestore-bitmap'] suites union: ['1-ranks/1', '1-volume/{0-create', '2-allow_standby_replay/no', '2-allow_standby_replay/yes', '2-client/kclient', '3-inline/yes', '3-upgrade-mgr-staggered', '4-config-upgrade/{fail_fs}', '4-verify}', '5-upgrade-with-workload', '6-verify}}', 'centos_9.stream', 'conf/{client', 'fail_fs/no', 'ignorelist_upgrade', 'ignorelist_wrongly_marked_down', 'mds', 'mgr', 'mon', 'osd}', 'overrides/{ignorelist_health', 'pg-warn', 'pg_health', 'roles', 'syntax}', 'tasks/{0-from/reef/{v18.2.0}', 'tasks/{0-from/reef/{v18.2.1}', 'upgrade/cephfs/mds_upgrade_sequence/{bluestore-bitmap'] Failure: timeout expired in wait_until_healthy 3 jobs: ['7654941', '7654942', '7654955'] suites intersection: ['0-roles', '1-start', '3-stress-tasks/{radosbench', 'rbd-cls', 'rbd-import-export', 'rbd_api', 'readwrite', 'snaps-few-objects}'] suites union: ['0-roles', '1-start', '2-first-half-tasks/rbd-import-export', '2-first-half-tasks/snaps-few-objects', '3-stress-tasks/{radosbench', '4-second-half-tasks/radosbench', '4-second-half-tasks/rbd-import-export', 'mon_election/classic}', 'mon_election/connectivity}', 'rbd-cls', 'rbd-import-export', 'rbd_api', 'readwrite', 'snaps-few-objects}', 'upgrade/quincy-x/stress-split/{0-distro/ubuntu_22.04', 'upgrade/reef-x/stress-split/{0-distro/ubuntu_22.04'] Failure: Command failed on smithi017 with status 22: 'sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell 4.30 deep-scrub' 1 jobs: ['7654956'] suites: ['1-client', '2-upgrade', '3-compat_client/quincy}}', 'centos_8.stream', 'clusters/1-mds-2-client-micro', 'conf/{client', 'ignorelist_wrongly_marked_down', 'mds', 'mgr', 'mon', 'multimds/no', 'multimds/yes', 'osd}', 'overrides/{ignorelist_health', 'pg-warn', 'pg_health}', 'tasks/{0-octopus', 'upgrade/cephfs/featureful_client/old_client/{bluestore-bitmap']