Found 25 jobs
Found 11 distinct failure reasons
Dead: b'2023-09-04T04:12:02.730 DEBUG:teuthology.orchestra.run.smithi140:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 30 ceph --cluster ceph --admin-daemon /var/run/ceph/ceph-osd.6.asok dump_historic_ops'
7 jobs: ['7387863', '7387853', '7387858', '7387867', '7387862', '7387846', '7387865']
suites intersection: ['ceph', 'clusters/{fixed-2', 'openstack}', 'rados', 'thrashosds-health']
suites union: ['1-pg-log-overrides/normal_pg_log', '1-pg-log-overrides/short_pg_log', '2-recovery-overrides/{default}', '2-recovery-overrides/{more-active-recovery}', '2-recovery-overrides/{more-async-partial-recovery}', '2-recovery-overrides/{more-async-recovery}', '3-scrub-overrides/{default}', '3-scrub-overrides/{max-simultaneous-scrubs-2}', '3-scrub-overrides/{max-simultaneous-scrubs-3}', 'backoff/normal', 'backoff/peering', 'ceph', 'clusters/{fixed-2', 'crc-failures/bad_map_crc_failure', 'crc-failures/default', 'd-balancer/crush-compat', 'd-balancer/on', 'mon_election/classic', 'mon_election/connectivity', 'msgr-failures/fastclose', 'msgr-failures/osd-delay', 'msgr-failures/osd-dispatch-delay', 'msgr/async-v1only', 'msgr/async-v2only', 'objectstore/bluestore-bitmap', 'objectstore/bluestore-comp-zstd', 'objectstore/bluestore-hybrid', 'objectstore/bluestore-low-osd-mem-target', 'openstack}', 'rados', 'rados:thrash/{0-size-min-size-overrides/2-size-2-min-size', 'rados:thrash/{0-size-min-size-overrides/3-size-2-min-size', 'supported-random-distro$/{ubuntu_20.04}', 'supported-random-distro$/{ubuntu_latest}', 'thrashers/careful', 'thrashers/mapgap', 'thrashers/morepggrow', 'thrashers/none', 'thrashers/pggrow', 'thrashosds-health', 'workloads/pool-snaps-few-objects-redelete}', 'workloads/snaps-few-objects-redelete}']

Dead: b'2023-09-04T04:12:02.734 DEBUG:teuthology.orchestra.run.smithi082:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 30 ceph --cluster ceph --admin-daemon /var/run/ceph/ceph-osd.1.asok dump_blocked_ops'
4 jobs: ['7387859', '7387854', '7387861', '7387848']
suites intersection: ['ceph', 'clusters/{fixed-2', 'openstack}', 'rados', 'supported-random-distro$/{ubuntu_20.04}', 'thrashosds-health']
suites union: ['1-pg-log-overrides/normal_pg_log', '1-pg-log-overrides/short_pg_log', '2-recovery-overrides/{default}', '2-recovery-overrides/{more-async-partial-recovery}', '2-recovery-overrides/{more-async-recovery}', '3-scrub-overrides/{max-simultaneous-scrubs-2}', '3-scrub-overrides/{max-simultaneous-scrubs-3}', 'backoff/normal', 'backoff/peering', 'ceph', 'clusters/{fixed-2', 'crc-failures/bad_map_crc_failure', 'crc-failures/default', 'd-balancer/crush-compat', 'd-balancer/on', 'mon_election/classic', 'mon_election/connectivity', 'msgr-failures/fastclose', 'msgr-failures/few', 'msgr-failures/osd-dispatch-delay', 'msgr/async-v1only', 'msgr/async-v2only', 'objectstore/bluestore-comp-lz4', 'objectstore/bluestore-comp-zlib', 'objectstore/bluestore-low-osd-mem-target', 'openstack}', 'rados', 'rados:thrash/{0-size-min-size-overrides/2-size-2-min-size', 'rados:thrash/{0-size-min-size-overrides/3-size-2-min-size', 'supported-random-distro$/{ubuntu_20.04}', 'thrashers/default', 'thrashers/none', 'thrashers/pggrow', 'thrashosds-health', 'workloads/pool-snaps-few-objects-redelete}', 'workloads/snaps-few-objects-redelete}']

Failure: Host key for server 'smithi163.front.sepia.ceph.com' does not match: got 'AAAAC3NzaC1lZDI1NTE5AAAAIM4+XQ99VyKyeDug0K+eRk8NwMmBI6EkjBdRaTws3DS0', expected 'AAAAB3NzaC1yc2EAAAADAQABAAABgQCZwie6nKnAdJdo0P27s3XfrupVQjnEK2hmdZPQLVm6tQ0ZjqfSF8tZk1Yw7xEUkRetIivd3R1T5Jy173E10bmpmCTa0HpvG9vPegGQuseVJo16br3ERSdg3qBfLU5IXu6JnP/UIery/Q+cJNn3/93KdndZSrBmp99DyDjPd9FbKcZKcow9P/PC6atUG/116UuLyZhX7ziApjZrdgE7/BXCXzKCB3EFBioq5KxKeM+4xeOJT4WyJ7nk9lQJNPsTrdN10vtbXZsIDUCBOHQBCrUOKvQQRoTv1WmvsDtyluxPggb2l3FA1/YRCp7giEArZTx5s7BbCXxqEfgmYDza9qr1ZILXECnJ7RPCOzAUCpb0/Qax5ABC65Qc5aTSR3feOJxfxa5OIxdxwF0gH8kVvtYDrw/A16XIpqOm3MyCUorAs7yaWbMRNqPhG3AypmRKOVibWZkmwLgV875uRp6OJnnlTSdC571DNjMoLHEdnYyDKzb9XVtFzt4YzoRdnF8ZyzM='
1 jobs: ['7387855']
suites: ['1-pg-log-overrides/short_pg_log', '2-recovery-overrides/{more-partial-recovery}', '3-scrub-overrides/{max-simultaneous-scrubs-2}', 'backoff/peering', 'ceph', 'clusters/{fixed-2', 'crc-failures/default', 'd-balancer/on', 'mon_election/connectivity', 'msgr-failures/osd-dispatch-delay', 'msgr/async-v2only', 'objectstore/bluestore-comp-lz4', 'openstack}', 'rados', 'rados:thrash/{0-size-min-size-overrides/3-size-2-min-size', 'supported-random-distro$/{ubuntu_latest}', 'thrashers/none', 'thrashosds-health', 'workloads/pool-snaps-few-objects-redelete}']

Failure: Host key for server 'smithi093.front.sepia.ceph.com' does not match: got 'AAAAC3NzaC1lZDI1NTE5AAAAILcmqu+o9D8RMfmR0yuF8s650RVR0tJsy7AwJ0+0Vemt', expected 'AAAAB3NzaC1yc2EAAAADAQABAAABgQC+j0sg/d7k45gHLyXPdrqglBc2vULQXHXtxLkidlvcp93AKQM+bRoSIfJjMD+2SJ9om0k7ITh6zISRXewJwZjzmior2RtJaw82YSlEvAEegvJccgvyXGyDflSnfcACCXfWjGN73SYuuR6/YmEEFQ1w8Ss4Mt0qJ7uXZ11xYcErHmD30UApP+xKEFIK3WiuumMPp/rKZcpCjbCNxrpHtQ/SXg89OyABE1Bi/zU8EBw3yuBft/dWQuEJeeVNh6CPFC+RZv0b40hWGNKhrndZhp2l8428IQlW7Xs2+f6LDml9jaW2OthUZUrTzpKJLfLrH7ZAYpGZXufSInLPH1ZjC1w5jk3QwR8YZhclA8hzMllzqJoctLIPjrIqrvJwoHPxnM71u1ubZIbmbUgYSGKXGL8ptOFWZBXQY7KBdUGrn1QEqjRxOEWc8BQNJ/ovEac8UnYXKmfmJk2HGKknSBsw+UTzXJxBdaRL70uSH0UZKTUZ/yk0zgOckqARQWuU8Z3qmrE='
1 jobs: ['7387850']
suites: ['1-pg-log-overrides/short_pg_log', '2-recovery-overrides/{more-partial-recovery}', '3-scrub-overrides/{max-simultaneous-scrubs-2}', 'backoff/normal', 'ceph', 'clusters/{fixed-2', 'crc-failures/default', 'd-balancer/on', 'mon_election/connectivity', 'msgr-failures/few', 'msgr/async-v1only', 'objectstore/bluestore-stupid', 'openstack}', 'rados', 'rados:thrash/{0-size-min-size-overrides/3-size-2-min-size', 'supported-random-distro$/{ubuntu_latest}', 'thrashers/default', 'thrashosds-health', 'workloads/snaps-few-objects-redelete}']

Failure: Host key for server 'smithi062.front.sepia.ceph.com' does not match: got 'AAAAC3NzaC1lZDI1NTE5AAAAIEWCoKzr5EoRJJCr6/OHbS9b+n41q2ODQed3p9WM2Vjr', expected 'AAAAB3NzaC1yc2EAAAADAQABAAABgQCz7v/HKSSv/AByZacmYR19tmYbfenl0PJDVtq3KdXC1EcvHWhYv6hoyjhyJBTEDN5zwZhlO5WW5rUQesHJVJfbXPSpPf2PwAPM7OzL1KFNumS6W6BF8AhkB5a9yoX0V2Bzey940LOgNn3f9uo96tusyIroTVjGCs1YKEvLJYmD0n9qWoMESXTD/XODaugBs0dY7PxFKR01ZcKH21kRbXYu3tSArJaFdD7cuBSscxzScVmwy4WRZHp6FBZ+xfxGAT/5hEP0GYoCcjISIh1DIfHjw6DQ8PHH9aLEusSQ2tZKCeH/en4KVHVN0+1tN2xmcGCRMdjZdTscvecAq/7sGjBsnkeHjRvbWk9AlEWizLqoSSbUzd3zQpWhjIrarChZ4WkD4uUUnwu5h/2gqr1+UbY5RXIWQtXS1R+qYZ6XjXHjViW7pCsG+HP2UzbpP7cALCkBRStWDNtrobVTqnyB5KqBh01UFrYgoEKlqdGmaYgvSnbp36IrDxmmRio4iYLSY8E='
1 jobs: ['7387845']
suites: ['1-pg-log-overrides/normal_pg_log', '2-recovery-overrides/{more-async-partial-recovery}', '3-scrub-overrides/{max-simultaneous-scrubs-3}', 'backoff/normal', 'ceph', 'clusters/{fixed-2', 'crc-failures/bad_map_crc_failure', 'd-balancer/crush-compat', 'mon_election/classic', 'msgr-failures/fastclose', 'msgr/async-v1only', 'objectstore/bluestore-comp-snappy', 'openstack}', 'rados', 'rados:thrash/{0-size-min-size-overrides/2-size-2-min-size', 'supported-random-distro$/{ubuntu_latest}', 'thrashers/pggrow', 'thrashosds-health', 'workloads/snaps-few-objects-redelete}']

Failure: Host key for server 'smithi159.front.sepia.ceph.com' does not match: got 'AAAAC3NzaC1lZDI1NTE5AAAAIMcjjBGfNT9Flq7fKYbjQp65JhwTHSfLGGhicUrkvBg3', expected 'AAAAB3NzaC1yc2EAAAADAQABAAABgQCjzDqpn6Tw0QazSPXjzn/TK6QFoZQOvJjvfcNLmJPE3dymlrVEOvokHIAmGVo0k+PIogdKKJ0nWbcHm59kq7ezD6SEyRsmYewzfiYE9iW+PWbHiexCuqbgwwGQ1RPPNjjpwkmUnpS1MVMf4MISVjC8auJcHjJ3Ixg9V67uhhXxmF6URb/Ck1yAx0E3g4Rj4JZ96xRxoMQDy+FH3lFstQuOoAP3oLReUpkoAgeCeznh1STAoFOSfK7khQMIRpe6ybUfB5MDr7RDgrfWmkomqqwdTspEgJKCKxcMfZDm+NN5HRKYlKuEFH7jMP60KUPf+9FPSWAs6h+Yj/age01e0nk65VHhJRaqXY2KiQ0uT/y7Vk/iax9ZO4qBVEaN0tBPX6hQZEPuOcOMR7Y3aWRtndKjBreJn9ncY9nciW7Cm85JJiBBfZLaZMYvcAgWQixBg1JcK3f/IHUwaCrOZg/qaT4hzjylhEhSLSy7MGHFyOqPwTRHdN//qmb7CiltGIMDXhs='
1 jobs: ['7387869']
suites: ['1-pg-log-overrides/normal_pg_log', '2-recovery-overrides/{more-active-recovery}', '3-scrub-overrides/{max-simultaneous-scrubs-2}', 'backoff/peering', 'ceph', 'clusters/{fixed-2', 'crc-failures/bad_map_crc_failure', 'd-balancer/crush-compat', 'mon_election/classic', 'msgr-failures/osd-delay', 'msgr/async-v2only', 'objectstore/bluestore-comp-zstd', 'openstack}', 'rados', 'rados:thrash/{0-size-min-size-overrides/2-size-2-min-size', 'supported-random-distro$/{ubuntu_latest}', 'thrashers/pggrow', 'thrashosds-health', 'workloads/pool-snaps-few-objects-redelete}']

Dead: b'OSError: Socket is closed'
6 jobs: ['7387852', '7387868', '7387866', '7387851', '7387864', '7387857']
suites intersection: ['ceph', 'clusters/{fixed-2', 'openstack}', 'rados', 'thrashosds-health']
suites union: ['1-pg-log-overrides/normal_pg_log', '1-pg-log-overrides/short_pg_log', '2-recovery-overrides/{default}', '2-recovery-overrides/{more-active-recovery}', '2-recovery-overrides/{more-async-partial-recovery}', '2-recovery-overrides/{more-async-recovery}', '3-scrub-overrides/{default}', '3-scrub-overrides/{max-simultaneous-scrubs-2}', '3-scrub-overrides/{max-simultaneous-scrubs-3}', 'backoff/normal', 'backoff/peering', 'ceph', 'clusters/{fixed-2', 'crc-failures/bad_map_crc_failure', 'crc-failures/default', 'd-balancer/crush-compat', 'd-balancer/on', 'mon_election/classic', 'mon_election/connectivity', 'msgr-failures/fastclose', 'msgr-failures/few', 'msgr-failures/osd-delay', 'msgr-failures/osd-dispatch-delay', 'msgr/async-v1only', 'msgr/async-v2only', 'objectstore/bluestore-bitmap', 'objectstore/bluestore-comp-snappy', 'objectstore/bluestore-hybrid', 'objectstore/bluestore-low-osd-mem-target', 'objectstore/bluestore-stupid', 'openstack}', 'rados', 'rados:thrash/{0-size-min-size-overrides/2-size-2-min-size', 'rados:thrash/{0-size-min-size-overrides/3-size-2-min-size', 'supported-random-distro$/{ubuntu_20.04}', 'supported-random-distro$/{ubuntu_latest}', 'thrashers/careful', 'thrashers/default', 'thrashers/none', 'thrashers/pggrow', 'thrashosds-health', 'workloads/pool-snaps-few-objects-redelete}', 'workloads/snaps-few-objects-redelete}']

Failure: Host key for server 'smithi017.front.sepia.ceph.com' does not match: got 'AAAAC3NzaC1lZDI1NTE5AAAAIBmxnZ0SeL8uCtUnBVVynEL+C+RrycyWdHnjOcp4JP/Z', expected 'AAAAB3NzaC1yc2EAAAADAQABAAABgQDI28eADtCgjJKOL9wnDTMwPBmCBT7xZFoRdYEdwlVqiCPrrtgICUXHCR+nY7OWebYBMtNDDFG9h27ddB6PRR1mUVEEmsEk14O6CAFcxAmRb3JF+5BxoGMPUJTtugqsUAVvmkEq4Jw5lKyvxcZm83VkNqLIjSTMO0vJC5jIEnZ+ArcaZhDwCW2pk7msV1YoydBa489WckEXIwJEf3nmVezA7aVmaRWpI8dbChRFuHeYOsAJVJ8RonOy1x7E/i683pJZCH9RQIo7/xD1FNiKI34ykfD/0ENUkzKnmBHNGzhH42pHKtPlTfOBTj6rBYgMZ1oLET22IAM1fozZ8QgZbAum1npCZ4EgZY3u+qKnJrNhV3KMu9u2c6KFdEKYXbA6B8iYRFET+mHnDcyNteOF7i/B9XbMD85htHcsMB2YsEo3jrQxpztVu6BxSn4boC16BIoWYXuyE5ivFkXWQp8C77OqyxDMIT9vPp4jmEbBWiECaYgR2PyF5f41GxqCep4GHGU='
1 jobs: ['7387847']
suites: ['1-pg-log-overrides/normal_pg_log', '2-recovery-overrides/{more-async-recovery}', '3-scrub-overrides/{max-simultaneous-scrubs-3}', 'backoff/peering', 'ceph', 'clusters/{fixed-2', 'crc-failures/bad_map_crc_failure', 'd-balancer/crush-compat', 'mon_election/classic', 'msgr-failures/osd-delay', 'msgr/async-v2only', 'objectstore/bluestore-bitmap', 'openstack}', 'rados', 'rados:thrash/{0-size-min-size-overrides/2-size-2-min-size', 'supported-random-distro$/{ubuntu_latest}', 'thrashers/mapgap', 'thrashosds-health', 'workloads/pool-snaps-few-objects-redelete}']

Failure: Host key for server 'smithi037.front.sepia.ceph.com' does not match: got 'AAAAC3NzaC1lZDI1NTE5AAAAICuHhOQXGAcwpvNBkqhqd+inEEmL45LN33jgz0MDKiAY', expected 'AAAAB3NzaC1yc2EAAAADAQABAAABgQDvw8jRZkZE1uHkjct49yYCWLEWazLF26/vegqJimHpTTgkoyTzW6TxA0uoey16owXxAR0iM5Kv/14RmHHSf6gbd3lhV3dFZaZStC5WhNZmsvoBaJkPbIUhcHqaY8hl77k1iunj+raqFXdFkEhvOG1vunfc4bGXFxGVuEVy+X0jvjg+AoDNbq11ixksDRKDJ84a/ZLcyHlcDBpcfwQ3MmI239/aZPOpGvtkbNIDy0mnhGjmHIJhPRkx5watDzp6wgPC3r/Iq+Lz6dVFqJw6MZL1ASJ+89IHemJBKekgE+Tk+31jyLAWLSvezoELBJHttyEb4e+4LW5Vfse9cMVMYKFQL9PUKrNICNrVS7600bV9297O5Tb7zvTq/AKcuHtvogxwy3nfjhFFPDLtsBAGKQUg+ZLU7cw7AJmhPEJToztTjB7wFmQNF6nqxdXZsLthVaxJNJ68bNEE+EO5ITxpu9pTyd/cfjNr2ku/NTIzzyY6hMaiJookjwj3o5OPoF2xDvU='
1 jobs: ['7387856']
suites: ['1-pg-log-overrides/normal_pg_log', '2-recovery-overrides/{default}', '3-scrub-overrides/{default}', 'backoff/peering', 'ceph', 'clusters/{fixed-2', 'crc-failures/bad_map_crc_failure', 'd-balancer/crush-compat', 'mon_election/classic', 'msgr-failures/osd-delay', 'msgr/async-v2only', 'objectstore/bluestore-comp-zstd', 'openstack}', 'rados', 'rados:thrash/{0-size-min-size-overrides/2-size-2-min-size', 'supported-random-distro$/{ubuntu_latest}', 'thrashers/default', 'thrashosds-health', 'workloads/pool-snaps-few-objects-redelete}']

Failure: Host key for server 'smithi033.front.sepia.ceph.com' does not match: got 'AAAAC3NzaC1lZDI1NTE5AAAAIK+bfCe7TU42TyJsAihIQ6osnxQX4PVASmWj9ci2PRdC', expected 'AAAAB3NzaC1yc2EAAAADAQABAAABgQCqzZ63lmVPoY25axntAOHs9VbqzGQVYQi8Cc6K97YYlTCFHLdJdRi7QNxGf4P/nNYNtFHVFbxRLFMiaxtdjEQu4q9Mbixb4yrei8EhrHjfEc4v1/WyDP8DTbk1Brw7GaE8zQ/e1fifn3RI0DtLFj1N8f6u6raw575ViwZXNpQLsWkgBbYHJSO3GCbJIOpRXy0fJB/gzP18h/7fvg8lY/6aPjnNQKRzQskZJ6j7boA/UAlxpR/rKyBIEtlZdhGhN/oN41OHEZ2/GYlhusTjefd1152gsCbjrqQKPZThxo/CUf07pn1l+UZzVMaj51mfSMNjWEu0lELvOf3wuBpS3nmafh5CZystzwQmBOL12iVkna1EeR+qlPxnsXqdTSBSp/5eFFsPalcalUdKF8RsMRhpnecq149n1K9NhVqzK3EG2ks69fxYkpP3W/NrytR31KFs1ebmJUe6edRwBDvQDSwlKpCsYQL5ST7O1AvpbMoIpcP5c0LeNXjJe+R2+Hq2dcM='
1 jobs: ['7387860']
suites: ['1-pg-log-overrides/normal_pg_log', '2-recovery-overrides/{more-active-recovery}', '3-scrub-overrides/{default}', 'backoff/normal', 'ceph', 'clusters/{fixed-2', 'crc-failures/bad_map_crc_failure', 'd-balancer/crush-compat', 'mon_election/classic', 'msgr-failures/fastclose', 'msgr/async-v1only', 'objectstore/bluestore-low-osd-mem-target', 'openstack}', 'rados', 'rados:thrash/{0-size-min-size-overrides/2-size-2-min-size', 'supported-random-distro$/{ubuntu_latest}', 'thrashers/mapgap', 'thrashosds-health', 'workloads/snaps-few-objects-redelete}']

Failure: Host key for server 'smithi077.front.sepia.ceph.com' does not match: got 'AAAAC3NzaC1lZDI1NTE5AAAAIAOGEAgU7V7Tnaq9uddsA6gzJOqzz2BX0a5RIJ0zB0Zy', expected 'AAAAB3NzaC1yc2EAAAADAQABAAABgQDJe+H6Td7xX5fPMG7cnvLozXEkIJSqQC88Es8vErndcHet8WkVzJdskkLt6JJ+d0JmpU3/nXILv83lJ3qkwIuFxIq0coQqdCvXxB9giMXzXPNZq/oQwkQgDsglF5aQAk4ILEfRfJIK0D1T/QVK922jcqWrLt+iWwehVWqvtg/lUQGYwmfgFpIxc50ax0RuBBKA74UtjrJiohKua3DHhQYAakJkwo0vvNF0zEGOGR4T0102ktd6HnIcopAoztng8Xs+kT7KQ0RWAOldzjNmvU29ANaUwvSDo7zFUKMtgIpCN++nGdnJHDGeTT/cDKJQ4nzNAqcN5D62oZ+XUKhkF4pB+8+Ab6ZiTFUlrQ0TNazHJLrb08qdlTSnNC5N08l3gK6skXJf4OtoSm04zt2rcKP6zE8R5sZ6vB5MUgQ2fHt5eLn7I4AymS673SaPaesvU/3M3AtAA7XFIC7Os9/oupPUEIab9YMpksWMc1vjY9u8oLQanrqdY2z0mVzqaaXp6CE='
1 jobs: ['7387849']
suites: ['1-pg-log-overrides/normal_pg_log', '2-recovery-overrides/{more-active-recovery}', '3-scrub-overrides/{default}', 'backoff/peering', 'ceph', 'clusters/{fixed-2', 'crc-failures/bad_map_crc_failure', 'd-balancer/crush-compat', 'mon_election/classic', 'msgr-failures/fastclose', 'msgr/async-v2only', 'objectstore/bluestore-low-osd-mem-target', 'openstack}', 'rados', 'rados:thrash/{0-size-min-size-overrides/2-size-2-min-size', 'supported-random-distro$/{ubuntu_latest}', 'thrashers/mapgap', 'thrashosds-health', 'workloads/pool-snaps-few-objects-redelete}']