Found 4 jobs Found 4 distinct failure reasons Dead: b'2023-03-16T20:10:10.871 INFO:journalctl@ceph.mon.a.smithi045.stdout:Mar 16 19:50:12 smithi045 ceph-mon[122409]: pgmap v19940: 97 pgs: 97 active+clean; 7.9 MiB data, 239 MiB used, 1.0 TiB / 1.0 TiB avail; 2.5 KiB/s rd, 5 op/s' 1 jobs: ['7210186'] suites: ['1-cephadm', '1-check-counter', '2-logrotate}', '2-scrub/no', '3-snaps/no', '4-flush/yes', '5-workunit/suites/pjd}}', 'begin/{0-install', 'clusters/1a11s-mds-1c-client-3node', 'conf/{client', 'frag', 'fs:workload/{0-rhel_8', 'ignorelist_health', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mount', 'mount/kclient/{base/{mount-syntax/{v2}', 'ms-die-on-skipped}}', 'ms_mode/secure', 'n/3', 'objectstore-ec/bluestore-bitmap', 'omap_limit/10', 'osd-asserts', 'osd}', 'overrides/{cephsqlite-timeout', 'overrides/{distro/testing/k-testing', 'ranks/multi/{export-check', 'replication/always}', 'session_timeout}', 'standby-replay', 'tasks/{0-subvolume/{with-namespace-isolated-and-quota}', 'wsync/yes}'] Dead: b'2023-03-16T20:10:12.977 INFO:journalctl@ceph.osd.7.smithi158.stdout:Mar 16 18:29:03 smithi158 ceph-a2f7421c-c3d1-11ed-9afb-001a4aab830c-osd-7[111504]: 2023-03-16T18:29:03.138+0000 7fdf7b37f700 -1 osd.7 102 heartbeat_check: no reply from 172.21.15.175:6804 osd.2 ever on either front or back, first ping sent 2023-03-16T18:28:02.164420+0000 (oldest deadline 2023-03-16T18:28:22.164420+0000)' 1 jobs: ['7210183'] suites: ['1-cephadm', '1-check-counter', '2-logrotate}', '2-scrub/yes', '3-snaps/no', '4-flush/no', '5-workunit/suites/pjd}}', 'begin/{0-install', 'clusters/1a11s-mds-1c-client-3node', 'conf/{client', 'frag', 'fs:workload/{0-rhel_8', 'ignorelist_health', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mount/fuse', 'n/3', 'objectstore-ec/bluestore-ec-root', 'omap_limit/10000', 'osd-asserts', 'osd}', 'overrides/{cephsqlite-timeout', 'ranks/multi/{export-check', 'replication/always}', 'session_timeout}', 'standby-replay', 'tasks/{0-subvolume/{with-quota}'] Dead: b'2023-03-16T20:10:12.975 INFO:journalctl@ceph.osd.10.smithi135.stdout:Mar 16 17:30:30 smithi135 ceph-264afaba-c3d6-11ed-9afb-001a4aab830c-osd-10[147442]: 2023-03-16T17:30:30.884+0000 7f5452d08700 -1 osd.10 830 heartbeat_check: no reply from 172.21.15.100:6814 osd.3 ever on either front or back, first ping sent 2023-03-16T17:27:40.343912+0000 (oldest deadline 2023-03-16T17:28:00.343912+0000)' 1 jobs: ['7210185'] suites: ['1-cephadm', '1-check-counter', '2-logrotate}', '2-scrub/yes', '3-snaps/yes', '4-flush/no', '5-workunit/suites/pjd}}', 'begin/{0-install', 'clusters/1a11s-mds-1c-client-3node', 'conf/{client', 'frag', 'fs:workload/{0-rhel_8', 'ignorelist_health', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mount', 'mount/kclient/{base/{mount-syntax/{v2}', 'ms-die-on-skipped}}', 'ms_mode/crc', 'n/3', 'objectstore-ec/bluestore-comp-ec-root', 'omap_limit/10000', 'osd-asserts', 'osd}', 'overrides/{cephsqlite-timeout', 'overrides/{distro/testing/k-testing', 'ranks/multi/{export-check', 'replication/always}', 'session_timeout}', 'standby-replay', 'tasks/{0-subvolume/{with-quota}', 'wsync/yes}'] Dead: b'2023-03-16T20:10:12.979 INFO:teuthology.orchestra.run.smithi008.stdout:ceph-mgr 122673 176709 io_contex ceph 25r FIFO 0,12 0t0 659339 pipe' 1 jobs: ['7210184'] suites: ['1-cephadm', '1-check-counter', '2-logrotate}', '2-scrub/no', '3-snaps/yes', '4-flush/yes', '5-workunit/suites/pjd}}', 'begin/{0-install', 'clusters/1a11s-mds-1c-client-3node', 'conf/{client', 'frag', 'fs:workload/{0-rhel_8', 'ignorelist_health', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mount', 'mount/kclient/{base/{mount-syntax/{v2}', 'ms-die-on-skipped}}', 'ms_mode/legacy', 'n/3', 'objectstore-ec/bluestore-comp', 'omap_limit/10', 'osd-asserts', 'osd}', 'overrides/{cephsqlite-timeout', 'overrides/{distro/testing/k-testing', 'ranks/multi/{export-check', 'replication/always}', 'session_timeout}', 'standby-replay', 'tasks/{0-subvolume/{with-no-extra-options}', 'wsync/yes}']