Found 9 jobs Found 5 distinct failure reasons Dead: b'2022-12-22T19:43:09.780 INFO:tasks.ceph.osd.1.smithi003.stderr:2022-12-22T19:43:09.779+0000 7f49d4b2f700 -1 osd.1 32 get_health_metrics reporting 256 slow ops, oldest is osd_op(client.4472.0:2105 3.0 3:2450f2d2:::rbd_data.11dab49f1948.0000000000000064:head [call rbd.sparse_copyup in=24b,set-alloc-hint object_size 4194304 write_size 4194304] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e32)' 4 jobs: ['7124672', '7124676', '7124674', '7124670'] suites intersection: ['clusters/{fixed-3', 'min-compat-client/octopus', 'msgr-failures/few', 'openstack}', 'pool/small-cache-pool', 'rbd/librbd/{cache/writeback', 'workloads/rbd_fio}'] suites union: ['clusters/{fixed-3', 'config/copy-on-read', 'config/none', 'config/permit-partial-discard', 'min-compat-client/octopus', 'msgr-failures/few', 'objectstore/bluestore-comp-snappy', 'objectstore/bluestore-comp-zlib', 'objectstore/bluestore-hybrid', 'objectstore/bluestore-stupid', 'openstack}', 'pool/small-cache-pool', 'rbd/librbd/{cache/writeback', 'supported-random-distro$/{centos_8}', 'supported-random-distro$/{rhel_8}', 'supported-random-distro$/{ubuntu_latest}', 'workloads/rbd_fio}'] Dead: b'2022-12-22T19:02:21.438 INFO:tasks.workunit.cluster1.client.mirror.smithi164.stderr:+ rbd --cluster cluster1 -p mirror --namespace ls' ceph version 18.0.0-1585-g8e8a1558 (8e8a1558516ac987e5d040e0feb0770aac7caabd) reef (dev) 1: /lib/x86_64-linux-gnu/libpthread.so.0(+0x14420) [0x7f4f57b4f420] 2: gsignal() 3: abort() 4: (ceph::__ceph_abort(char const*, int, char const*, std::__cxx11::basic_string, std::allocator > const&)+0x190) [0x55c5cf853616] 5: (KernelDevice::_aio_thread()+0xe41) [0x55c5d048c3d1] 6: (KernelDevice::AioCompletionThread::entry()+0x11) [0x55c5d0491921] 7: /lib/x86_64-linux-gnu/libpthread.so.0(+0x8609) [0x7f4f57b43609] 8: clone() 1 jobs: ['7124675'] suites: ['clients/mirror', 'cluster/{2-node', 'msgr-failures/few', 'objectstore/bluestore-stupid', 'openstack}', 'policy/simple', 'rbd-mirror/four-per-cluster', 'rbd/mirror-thrash/{base/install', 'supported-random-distro$/{ubuntu_latest}', 'workloads/rbd-mirror-snapshot-stress-workunit-exclusive-lock}'] Dead: b'2022-12-22T19:43:12.536 INFO:tasks.ceph.osd.4.smithi049.stderr:2022-12-22T19:43:12.532+0000 7f1bb17f9700 -1 osd.4 34 get_health_metrics reporting 256 slow ops, oldest is osd_op(client.4564.0:2552 3.2 3:69ac2672:::rbd_data.11b3434dce07.00000000000006d9:head [stat,write 2662400~4096 in=4096b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e34)' 1 jobs: ['7124677'] suites: ['clusters/{fixed-3', 'config/permit-partial-discard', 'min-compat-client/octopus', 'msgr-failures/few', 'objectstore/bluestore-comp-lz4', 'openstack}', 'pool/small-cache-pool', 'rbd/librbd/{cache/writeback', 'supported-random-distro$/{ubuntu_latest}', 'workloads/rbd_fio}'] Dead: b'2022-12-22T19:43:14.461 INFO:tasks.ceph.osd.0.smithi103.stderr:2022-12-22T19:43:14.444+0000 7f4a59f1e700 -1 osd.0 31 get_health_metrics reporting 256 slow ops, oldest is osd_op(client.4522.0:43878 3.1 3:90d4ecbd:::rbd_data.115c519859c3.000000000000055f:head [write 1413120~4096 in=4096b] snapc 4=[4] ondisk+write+known_if_redirected+supports_pool_eio e31)' 1 jobs: ['7124673'] suites: ['clusters/{fixed-3', 'config/permit-partial-discard', 'min-compat-client/octopus', 'msgr-failures/few', 'objectstore/bluestore-comp-zstd', 'openstack}', 'pool/small-cache-pool', 'rbd/librbd/{cache/writeback', 'supported-random-distro$/{centos_8}', 'workloads/rbd_fio}'] Dead: b'2022-12-22T19:43:16.294 INFO:tasks.ceph.osd.4.smithi115.stderr:2022-12-22T19:43:16.293+0000 7f1977b37700 -1 osd.4 30 get_health_metrics reporting 256 slow ops, oldest is osd_op(client.4578.0:4056 3.2 3:63047c53:::rbd_data.11b3bd76f2bd.0000000000000782:head [stat,set-alloc-hint object_size 4194304 write_size 4194304,write 1703936~4096 in=4096b] snapc 0=[] RETRY=3 ondisk+retry+write+known_if_redirected+supports_pool_eio e30)' 1 jobs: ['7124671'] suites: ['clusters/{fixed-3', 'config/none', 'min-compat-client/octopus', 'msgr-failures/few', 'objectstore/bluestore-low-osd-mem-target', 'openstack}', 'pool/small-cache-pool', 'rbd/librbd/{cache/writeback', 'supported-random-distro$/{rhel_8}', 'workloads/rbd_fio}']