Found 9 jobs Found 4 distinct failure reasons Dead: b'2022-12-24T02:37:50.841 INFO:tasks.ceph.osd.5.smithi070.stderr:2022-12-24T02:37:50.841+0000 7f5d182ae700 -1 osd.5 33 get_health_metrics reporting 256 slow ops, oldest is osd_op(client.4475.0:3212 3.0 3:252a9954:::rbd_data.11859d422e56.0000000000000639:head [write 1613824~4096 in=4096b] snapc 5=[5] ondisk+write+known_if_redirected+supports_pool_eio e33)' 4 jobs: ['7126337', '7126335', '7126334', '7126333'] suites intersection: ['clusters/{fixed-3', 'min-compat-client/octopus', 'msgr-failures/few', 'openstack}', 'pool/small-cache-pool', 'rbd/librbd/{cache/writeback', 'workloads/rbd_fio}'] suites union: ['clusters/{fixed-3', 'config/copy-on-read', 'config/none', 'config/permit-partial-discard', 'min-compat-client/octopus', 'msgr-failures/few', 'objectstore/bluestore-comp-zlib', 'objectstore/bluestore-hybrid', 'objectstore/bluestore-low-osd-mem-target', 'objectstore/bluestore-stupid', 'openstack}', 'pool/small-cache-pool', 'rbd/librbd/{cache/writeback', 'supported-random-distro$/{centos_8}', 'supported-random-distro$/{rhel_8}', 'supported-random-distro$/{ubuntu_latest}', 'workloads/rbd_fio}'] Dead: b'2022-12-24T02:37:51.522 INFO:tasks.ceph.osd.1.smithi023.stderr:2022-12-24T02:37:51.515+0000 7f989050e700 -1 osd.1 32 get_health_metrics reporting 192 slow ops, oldest is osd_op(osd.4.14:72451 1.3 1:d8c43d40:::rbd_data.11a627ed1ea2.00000000000007d3:head [copy-from ver 15295 in=76b] snapc 4=[] ondisk+write+ignore_overlay+enforce_snapc+known_if_redirected+supports_pool_eio e32)' 2 jobs: ['7126340', '7126336'] suites intersection: ['clusters/{fixed-3', 'config/permit-partial-discard', 'min-compat-client/octopus', 'msgr-failures/few', 'openstack}', 'pool/small-cache-pool', 'rbd/librbd/{cache/writeback', 'workloads/rbd_fio}'] suites union: ['clusters/{fixed-3', 'config/permit-partial-discard', 'min-compat-client/octopus', 'msgr-failures/few', 'objectstore/bluestore-comp-lz4', 'objectstore/bluestore-comp-zstd', 'openstack}', 'pool/small-cache-pool', 'rbd/librbd/{cache/writeback', 'supported-random-distro$/{centos_8}', 'supported-random-distro$/{ubuntu_latest}', 'workloads/rbd_fio}'] Dead: b'2022-12-24T01:54:04.931 INFO:tasks.workunit.cluster1.client.mirror.smithi053.stderr:+ rbd --cluster cluster1 -p mirror --namespace ls' ceph version 18.0.0-1585-g8e8a1558 (8e8a1558516ac987e5d040e0feb0770aac7caabd) reef (dev) 1: /lib/x86_64-linux-gnu/libpthread.so.0(+0x14420) [0x7f556e4a3420] 2: gsignal() 3: abort() 4: (ceph::__ceph_abort(char const*, int, char const*, std::__cxx11::basic_string, std::allocator > const&)+0x190) [0x55e687b5e616] 5: (KernelDevice::_aio_thread()+0xe41) [0x55e6887973d1] 6: (KernelDevice::AioCompletionThread::entry()+0x11) [0x55e68879c921] 7: /lib/x86_64-linux-gnu/libpthread.so.0(+0x8609) [0x7f556e497609] 8: clone() 1 jobs: ['7126338'] suites: ['clients/mirror', 'cluster/{2-node', 'msgr-failures/few', 'objectstore/bluestore-stupid', 'openstack}', 'policy/simple', 'rbd-mirror/four-per-cluster', 'rbd/mirror-thrash/{base/install', 'supported-random-distro$/{ubuntu_latest}', 'workloads/rbd-mirror-snapshot-stress-workunit-exclusive-lock}'] Dead: b'2022-12-24T02:37:53.089 INFO:tasks.ceph.osd.5.smithi050.stderr:2022-12-24T02:37:52.986+0000 7f454aadc700 -1 osd.5 30 get_health_metrics reporting 100 slow ops, oldest is osd_op(osd.0.13:37375 1.1 1:878b394b:::rbd_data.116bb6b6c77.000000000000030f:head [copy-from ver 13142 in=75b] snapc 5=[] ondisk+write+ignore_overlay+enforce_snapc+known_if_redirected+supports_pool_eio e30)' 1 jobs: ['7126339'] suites: ['clusters/{fixed-3', 'config/copy-on-read', 'min-compat-client/octopus', 'msgr-failures/few', 'objectstore/bluestore-comp-snappy', 'openstack}', 'pool/small-cache-pool', 'rbd/librbd/{cache/writeback', 'supported-random-distro$/{rhel_8}', 'workloads/rbd_fio}']