Found 83 jobs
Missing teuthology log /home/teuthworker/archive/lflores-2024-03-26_17:10:07-fs-wip-batrick-testing-20240320.145742-distro-default-smithi/7623784/teuthology.log
Found 18 distinct failure reasons
Failure: reached maximum tries (51) after waiting for 300 seconds
5 jobs: ['7623756', '7623812', '7623826', '7623796', '7623772']
suites intersection: ['1-volume/{0-create', '2-client/fuse', '3-upgrade-mgr-staggered', '4-config-upgrade/{fail_fs}', '4-verify}', '5-upgrade-with-workload', '6-verify}}', 'centos_9.stream', 'conf/{client', 'fs/upgrade/mds_upgrade_sequence/{bluestore-bitmap', 'ignorelist_upgrade', 'ignorelist_wrongly_marked_down', 'mds', 'mgr', 'mon', 'osd}', 'overrides/{ignorelist_health', 'pg-warn', 'roles', 'syntax}']
suites union: ['1-ranks/1', '1-ranks/2', '1-volume/{0-create', '2-allow_standby_replay/no', '2-allow_standby_replay/yes', '2-client/fuse', '3-inline/no', '3-inline/yes', '3-upgrade-mgr-staggered', '4-config-upgrade/{fail_fs}', '4-verify}', '5-upgrade-with-workload', '6-verify}}', 'centos_9.stream', 'conf/{client', 'fail_fs/no', 'fail_fs/yes', 'fs/upgrade/mds_upgrade_sequence/{bluestore-bitmap', 'ignorelist_upgrade', 'ignorelist_wrongly_marked_down', 'mds', 'mgr', 'mon', 'osd}', 'overrides/{ignorelist_health', 'pg-warn', 'roles', 'syntax}', 'tasks/{0-from/quincy', 'tasks/{0-from/reef/{reef}', 'tasks/{0-from/reef/{v18.2.1}']

Failure: "2024-03-26T17:52:14.058729+0000 mds.b (mds.0) 139 : cluster [WRN] Scrub error on inode 0x10000003c80 (/volumes/qa/sv_0/c5a24205-8212-466d-b5ed-c6d2b325c0fb/client.0/tmp/clients/client8/~dmtmp/COREL) see mds.b log and `damage ls` output for details" in cluster log
2 jobs: ['7623746', '7623818']
suites intersection: ['1-cephadm', '1-check-counter', '2-logrotate', '2-scrub/yes', '3-modules}', 'begin/{0-install', 'clusters/1a11s-mds-1c-client-3node', 'conf/{client', 'export-check', 'frag', 'fs/workload/{0-centos_9.stream', 'ignorelist_health', 'ignorelist_wrongly_marked_down', 'mds', 'mgr', 'mon', 'n/5', 'osd-asserts', 'osd}', 'overrides/{cephsqlite-timeout', 'pg_health', 'replication/always}', 'session_timeout}', 'standby-replay']
suites union: ['1-cephadm', '1-check-counter', '2-logrotate', '2-scrub/yes', '3-modules}', '3-snaps/no', '3-snaps/yes', '4-flush/no', '4-flush/yes', '5-workunit/suites/blogbench}}', '5-workunit/suites/dbench}}', 'begin/{0-install', 'clusters/1a11s-mds-1c-client-3node', 'conf/{client', 'export-check', 'frag', 'fs/workload/{0-centos_9.stream', 'ignorelist_health', 'ignorelist_wrongly_marked_down', 'k-stock}', 'mds', 'mgr', 'mon', 'mount', 'mount/fuse', 'mount/kclient/{base/{mount-syntax/{v2}', 'ms-die-on-skipped}}', 'ms_mode/secure', 'n/5', 'objectstore-ec/bluestore-bitmap', 'objectstore-ec/bluestore-ec-root', 'omap_limit/10', 'omap_limit/10000', 'osd-asserts', 'osd}', 'overrides/{cephsqlite-timeout', 'overrides/{distro/stock/{centos_9.stream', 'pg_health', 'ranks/multi/{balancer/automatic', 'ranks/multi/{balancer/random', 'replication/always}', 'session_timeout}', 'standby-replay', 'tasks/{0-subvolume/{with-namespace-isolated-and-quota}', 'tasks/{0-subvolume/{with-namespace-isolated}', 'wsync/no}']

Dead: None
2 jobs: ['7623784', '7623804']

Failure: Command failed (workunit test suites/fsx.sh) on smithi120 with status 2: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=72e2437271dcafb9826f2f49bb1001426fa6de81 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/suites/fsx.sh'
4 jobs: ['7623751', '7623769', '7623824', '7623806']
suites intersection: ['1-cephadm', '1-check-counter', '2-logrotate', '3-modules}', '5-workunit/suites/fsx}}', 'begin/{0-install', 'clusters/1a11s-mds-1c-client-3node', 'conf/{client', 'export-check', 'frag', 'fs/workload/{0-centos_9.stream', 'ignorelist_health', 'ignorelist_wrongly_marked_down', 'mds', 'mgr', 'mon', 'osd-asserts', 'osd}', 'overrides/{cephsqlite-timeout', 'pg_health', 'replication/default}', 'session_timeout}', 'standby-replay']
suites union: ['1-cephadm', '1-check-counter', '2-logrotate', '2-scrub/no', '2-scrub/yes', '3-modules}', '3-snaps/no', '3-snaps/yes', '4-flush/no', '4-flush/yes', '5-workunit/suites/fsx}}', 'begin/{0-install', 'clusters/1a11s-mds-1c-client-3node', 'conf/{client', 'export-check', 'frag', 'fs/workload/{0-centos_9.stream', 'ignorelist_health', 'ignorelist_wrongly_marked_down', 'k-stock}', 'mds', 'mgr', 'mon', 'mount', 'mount/fuse', 'mount/kclient/{base/{mount-syntax/{v2}', 'ms-die-on-skipped}}', 'ms_mode/crc', 'ms_mode/secure', 'n/3', 'n/5', 'objectstore-ec/bluestore-bitmap', 'objectstore-ec/bluestore-comp-ec-root', 'objectstore-ec/bluestore-ec-root', 'omap_limit/10', 'omap_limit/10000', 'osd-asserts', 'osd}', 'overrides/{cephsqlite-timeout', 'overrides/{distro/stock/{centos_9.stream', 'pg_health', 'ranks/multi/{balancer/automatic', 'ranks/multi/{balancer/random', 'replication/default}', 'session_timeout}', 'standby-replay', 'tasks/{0-subvolume/{with-namespace-isolated}', 'tasks/{0-subvolume/{with-no-extra-options}', 'tasks/{0-subvolume/{with-quota}', 'wsync/no}']

Failure: ['Creating mailbox file: File exists', 'useradd: Not copying any file from skel directory into it.', "useradd: user 'avilez' already exists", "useradd: user 'devansh' already exists", "useradd: user 'poonc3' already exists", 'useradd: warning: the home directory /home/billscales already exists.', 'useradd: warning: the home directory /home/jamiepryde already exists.', 'useradd: warning: the home directory /home/nicowang already exists.']
1 jobs: ['7623786']
suites: ['1-ranks/1', '1-volume/{0-create', '2-allow_standby_replay/no', '2-client/kclient', '3-inline/no', '3-upgrade-mgr-staggered', '4-config-upgrade/{fail_fs}', '4-verify}', '5-upgrade-with-workload', '6-verify}}', 'centos_9.stream', 'conf/{client', 'fail_fs/no', 'fs/upgrade/mds_upgrade_sequence/{bluestore-bitmap', 'ignorelist_upgrade', 'ignorelist_wrongly_marked_down', 'mds', 'mgr', 'mon', 'osd}', 'overrides/{ignorelist_health', 'pg-warn', 'roles', 'syntax}', 'tasks/{0-from/reef/{reef}']

Failure: Command failed on smithi176 with status 1: 'sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ad1641ee-eb99-11ee-95d0-87774f69a715 -e sha1=360516069d9393362c4cc6eb9371680fe16d66ab -- bash -c \'ceph versions | jq -e \'"\'"\'.mgr | keys\'"\'"\' | grep $sha1\''
3 jobs: ['7623766', '7623750', '7623780']
suites intersection: ['1-volume/{0-create', '2-client/kclient', '3-inline/yes', '3-upgrade-mgr-staggered', '4-config-upgrade/{fail_fs}', '4-verify}', '5-upgrade-with-workload', '6-verify}}', 'centos_9.stream', 'conf/{client', 'fail_fs/yes', 'fs/upgrade/mds_upgrade_sequence/{bluestore-bitmap', 'ignorelist_upgrade', 'ignorelist_wrongly_marked_down', 'mds', 'mgr', 'mon', 'osd}', 'overrides/{ignorelist_health', 'pg-warn', 'roles', 'syntax}', 'tasks/{0-from/quincy']
suites union: ['1-ranks/1', '1-ranks/2', '1-volume/{0-create', '2-allow_standby_replay/no', '2-allow_standby_replay/yes', '2-client/kclient', '3-inline/yes', '3-upgrade-mgr-staggered', '4-config-upgrade/{fail_fs}', '4-verify}', '5-upgrade-with-workload', '6-verify}}', 'centos_9.stream', 'conf/{client', 'fail_fs/yes', 'fs/upgrade/mds_upgrade_sequence/{bluestore-bitmap', 'ignorelist_upgrade', 'ignorelist_wrongly_marked_down', 'mds', 'mgr', 'mon', 'osd}', 'overrides/{ignorelist_health', 'pg-warn', 'roles', 'syntax}', 'tasks/{0-from/quincy']

Failure: error during scrub thrashing: rank damage found: {'backtrace'}
5 jobs: ['7623768', '7623765', '7623761', '7623763', '7623815']
suites intersection: ['1-cephadm', '1-check-counter', '2-logrotate', '2-scrub/yes', '3-modules}', 'begin/{0-install', 'clusters/1a11s-mds-1c-client-3node', 'conf/{client', 'export-check', 'frag', 'fs/workload/{0-centos_9.stream', 'ignorelist_health', 'ignorelist_wrongly_marked_down', 'mds', 'mgr', 'mon', 'osd-asserts', 'osd}', 'overrides/{cephsqlite-timeout', 'pg_health', 'replication/always}', 'session_timeout}', 'standby-replay']
suites union: ['1-cephadm', '1-check-counter', '2-logrotate', '2-scrub/yes', '3-modules}', '3-snaps/no', '3-snaps/yes', '4-flush/no', '4-flush/yes', '5-workunit/kernel_untar_build}}', '5-workunit/suites/blogbench}}', '5-workunit/suites/ffsb}}', '5-workunit/suites/fsstress}}', 'begin/{0-install', 'clusters/1a11s-mds-1c-client-3node', 'conf/{client', 'export-check', 'frag', 'fs/workload/{0-centos_9.stream', 'ignorelist_health', 'ignorelist_wrongly_marked_down', 'k-stock}', 'mds', 'mgr', 'mon', 'mount', 'mount/fuse', 'mount/kclient/{base/{mount-syntax/{v1}', 'mount/kclient/{base/{mount-syntax/{v2}', 'ms-die-on-skipped}}', 'ms_mode/crc', 'ms_mode/legacy', 'n/3', 'n/5', 'objectstore-ec/bluestore-bitmap', 'objectstore-ec/bluestore-comp', 'objectstore-ec/bluestore-comp-ec-root', 'omap_limit/10', 'omap_limit/10000', 'osd-asserts', 'osd}', 'overrides/{cephsqlite-timeout', 'overrides/{distro/stock/{centos_9.stream', 'overrides/{distro/testing/k-testing', 'pg_health', 'ranks/multi/{balancer/automatic', 'ranks/multi/{balancer/random', 'replication/always}', 'session_timeout}', 'standby-replay', 'tasks/{0-subvolume/{with-namespace-isolated-and-quota}', 'tasks/{0-subvolume/{with-no-extra-options}', 'tasks/{0-subvolume/{with-quota}', 'wsync/no}', 'wsync/yes}']

Failure: ['Creating mailbox file: File exists', 'useradd: Not copying any file from skel directory into it.', "useradd: user 'avilez' already exists", "useradd: user 'billscales' already exists", "useradd: user 'devansh' already exists", "useradd: user 'jjperez' already exists", "useradd: user 'ljsanders' already exists", "useradd: user 'nicowang' already exists", 'useradd: warning: the home directory /home/adamlyon already exists.', 'useradd: warning: the home directory /home/connorfa already exists.', 'useradd: warning: the home directory /home/jamiepryde already exists.', 'useradd: warning: the home directory /home/luorixin already exists.', 'useradd: warning: the home directory /home/poonc3 already exists.']
1 jobs: ['7623787']
suites: ['1-cephadm', '1-check-counter', '2-logrotate', '2-scrub/yes', '3-modules}', '3-snaps/no', '4-flush/no', '5-workunit/suites/fsstress}}', 'begin/{0-install', 'clusters/1a11s-mds-1c-client-3node', 'conf/{client', 'export-check', 'frag', 'fs/workload/{0-centos_9.stream', 'ignorelist_health', 'ignorelist_wrongly_marked_down', 'k-stock}', 'mds', 'mgr', 'mon', 'mount', 'mount/kclient/{base/{mount-syntax/{v1}', 'ms-die-on-skipped}}', 'ms_mode/crc', 'n/3', 'objectstore-ec/bluestore-comp-ec-root', 'omap_limit/10000', 'osd-asserts', 'osd}', 'overrides/{cephsqlite-timeout', 'overrides/{distro/stock/{centos_9.stream', 'pg_health', 'ranks/multi/{balancer/random', 'replication/default}', 'session_timeout}', 'standby-replay', 'tasks/{0-subvolume/{with-namespace-isolated}', 'wsync/no}']

Failure: "2024-03-26T18:01:47.476275+0000 mon.a (mon.0) 1147 : cluster [WRN] Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY)" in cluster log
3 jobs: ['7623762', '7623758', '7623819']
suites intersection: []
suites union: ['1-cephadm', '1-check-counter', '1-mount', '1-ranks/2', '1-volume/{0-create', '2-allow_standby_replay/no', '2-client/kclient', '2-logrotate', '2-scrub/no', '2-workload/dbench', '3-inline/no', '3-modules}', '3-snaps/yes', '3-upgrade-mgr-staggered', '4-config-upgrade/{fail_fs}', '4-flush/no', '4-verify}', '5-upgrade-with-workload', '5-workunit/postgres}}', '6-verify}}', 'begin/{0-install', 'centos_9.stream', 'clusters/1a11s-mds-1c-client-3node', 'conf/{client', 'distro/single-container-host}', 'export-check', 'fail_fs/no', 'frag', 'fs/cephadm/multivolume/{0-start', 'fs/upgrade/mds_upgrade_sequence/{bluestore-bitmap', 'fs/workload/{0-centos_9.stream', 'ignorelist_health', 'ignorelist_upgrade', 'ignorelist_wrongly_marked_down', 'k-stock}', 'mds', 'mgr', 'mon', 'mount', 'mount/kclient/{base/{mount-syntax/{v2}', 'ms-die-on-skipped}}', 'ms_mode/crc', 'n/5', 'objectstore-ec/bluestore-comp-ec-root', 'omap_limit/10000', 'osd-asserts', 'osd}', 'overrides/{cephsqlite-timeout', 'overrides/{distro/stock/{centos_9.stream', 'overrides/{ignorelist_health', 'pg-warn', 'pg_health', 'ranks/multi/{balancer/random', 'replication/default}', 'roles', 'session_timeout}', 'standby-replay', 'syntax}', 'tasks/{0-from/reef/{v18.2.0}', 'tasks/{0-subvolume/{with-namespace-isolated}', 'wsync/no}']

Failure: Command failed on smithi096 with status 22: 'sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell 4.8 deep-scrub'
2 jobs: ['7623776', '7623816']
suites intersection: ['1-client', '2-upgrade', '3-compat_client/quincy}}', 'centos_8.stream', 'clusters/1-mds-2-client-micro', 'conf/{client', 'fs/upgrade/featureful_client/old_client/{bluestore-bitmap', 'ignorelist_wrongly_marked_down', 'mds', 'mgr', 'mon', 'multimds/yes', 'osd}', 'overrides/{ignorelist_health', 'pg-warn}', 'tasks/{0-octopus']
suites union: ['1-client', '2-upgrade', '3-compat_client/quincy}}', 'centos_8.stream', 'clusters/1-mds-2-client-micro', 'conf/{client', 'fs/upgrade/featureful_client/old_client/{bluestore-bitmap', 'ignorelist_wrongly_marked_down', 'mds', 'mgr', 'mon', 'multimds/yes', 'osd}', 'overrides/{ignorelist_health', 'pg-warn}', 'tasks/{0-octopus']

Failure: ['Failed to manage policy for boolean nagios_run_sudo: [Errno 11] Resource temporarily unavailable']
1 jobs: ['7623788']
suites: ['1-cephadm', '1-check-counter', '2-logrotate', '2-scrub/no', '3-modules}', '3-snaps/yes', '4-flush/yes', '5-workunit/suites/fsx}}', 'begin/{0-install', 'clusters/1a11s-mds-1c-client-3node', 'conf/{client', 'export-check', 'frag', 'fs/workload/{0-centos_9.stream', 'ignorelist_health', 'ignorelist_wrongly_marked_down', 'mds', 'mgr', 'mon', 'mount', 'mount/kclient/{base/{mount-syntax/{v2}', 'ms-die-on-skipped}}', 'ms_mode/legacy', 'n/3', 'objectstore-ec/bluestore-comp', 'omap_limit/10', 'osd-asserts', 'osd}', 'overrides/{cephsqlite-timeout', 'overrides/{distro/testing/k-testing', 'pg_health', 'ranks/multi/{balancer/random', 'replication/always}', 'session_timeout}', 'standby-replay', 'tasks/{0-subvolume/{with-namespace-isolated-and-quota}', 'wsync/yes}']

Failure: Command failed on smithi107 with status 1: 'sudo yum upgrade -y linux-firmware'
1 jobs: ['7623811']
suites: ['1-cephadm', '1-check-counter', '2-logrotate', '2-scrub/no', '3-modules}', '3-snaps/yes', '4-flush/yes', '5-workunit/suites/pjd}}', 'begin/{0-install', 'clusters/1a11s-mds-1c-client-3node', 'conf/{client', 'export-check', 'frag', 'fs/workload/{0-centos_9.stream', 'ignorelist_health', 'ignorelist_wrongly_marked_down', 'mds', 'mgr', 'mon', 'mount', 'mount/kclient/{base/{mount-syntax/{v1}', 'ms-die-on-skipped}}', 'ms_mode/legacy', 'n/5', 'objectstore-ec/bluestore-comp', 'omap_limit/10', 'osd-asserts', 'osd}', 'overrides/{cephsqlite-timeout', 'overrides/{distro/testing/k-testing', 'pg_health', 'ranks/multi/{balancer/random', 'replication/always}', 'session_timeout}', 'standby-replay', 'tasks/{0-subvolume/{with-namespace-isolated}', 'wsync/yes}']

Dead: b'2024-03-27T05:55:54.140 DEBUG:teuthology.exit:Finished running handlers'
2 jobs: ['7623778', '7623814']
suites intersection: ['1-cephadm', '1-check-counter', '2-logrotate', '2-scrub/no', '3-modules}', '4-flush/yes', '5-workunit/fs/misc}}', 'begin/{0-install', 'clusters/1a11s-mds-1c-client-3node', 'conf/{client', 'frag', 'fs/workload/{0-centos_9.stream', 'ignorelist_health', 'ignorelist_wrongly_marked_down', 'mds', 'mgr', 'mon', 'omap_limit/10', 'osd-asserts', 'osd}', 'overrides/{cephsqlite-timeout', 'pg_health', 'session_timeout}', 'standby-replay']
suites union: ['1-cephadm', '1-check-counter', '2-logrotate', '2-scrub/no', '3-modules}', '3-snaps/no', '3-snaps/yes', '4-flush/yes', '5-workunit/fs/misc}}', 'begin/{0-install', 'clusters/1a11s-mds-1c-client-3node', 'conf/{client', 'export-check', 'frag', 'fs/workload/{0-centos_9.stream', 'ignorelist_health', 'ignorelist_wrongly_marked_down', 'mds', 'mgr', 'mon', 'mount', 'mount/fuse', 'mount/kclient/{base/{mount-syntax/{v2}', 'ms-die-on-skipped}}', 'ms_mode/legacy', 'n/5', 'objectstore-ec/bluestore-bitmap', 'objectstore-ec/bluestore-comp', 'omap_limit/10', 'osd-asserts', 'osd}', 'overrides/{cephsqlite-timeout', 'overrides/{distro/testing/k-testing', 'pg_health', 'ranks/1', 'ranks/multi/{balancer/automatic', 'replication/default}', 'session_timeout}', 'standby-replay', 'tasks/{0-subvolume/{with-namespace-isolated-and-quota}', 'tasks/{0-subvolume/{with-quota}', 'wsync/yes}']

Failure: "2024-03-26T18:32:26.478539+0000 mds.b (mds.0) 89 : cluster [WRN] Scrub error on inode 0x1000000ca2f (/volumes/qa/sv_1/747364c3-7326-4053-8fe0-06f70c896825/client.0/tmp/t/linux-6.5.11/drivers/net/wireless/rsi) see mds.b log and `damage ls` output for details" in cluster log
1 jobs: ['7623779']
suites: ['1-cephadm', '1-check-counter', '2-logrotate', '2-scrub/yes', '3-modules}', '3-snaps/no', '4-flush/no', '5-workunit/kernel_untar_build}}', 'begin/{0-install', 'clusters/1a11s-mds-1c-client-3node', 'conf/{client', 'export-check', 'frag', 'fs/workload/{0-centos_9.stream', 'ignorelist_health', 'ignorelist_wrongly_marked_down', 'k-stock}', 'mds', 'mgr', 'mon', 'mount', 'mount/kclient/{base/{mount-syntax/{v2}', 'ms-die-on-skipped}}', 'ms_mode/secure', 'n/5', 'objectstore-ec/bluestore-ec-root', 'omap_limit/10000', 'osd-asserts', 'osd}', 'overrides/{cephsqlite-timeout', 'overrides/{distro/stock/{centos_9.stream', 'pg_health', 'ranks/multi/{balancer/random', 'replication/always}', 'session_timeout}', 'standby-replay', 'tasks/{0-subvolume/{with-namespace-isolated-and-quota}', 'wsync/no}']

Failure: The following counters failed to be set on mds daemons: {'mds.exported', 'mds.imported'}
1 jobs: ['7623817']
suites: ['1-cephadm', '1-check-counter', '2-logrotate', '2-scrub/no', '3-modules}', '3-snaps/no', '4-flush/yes', '5-workunit/postgres}}', 'begin/{0-install', 'clusters/1a11s-mds-1c-client-3node', 'conf/{client', 'export-check', 'frag', 'fs/workload/{0-centos_9.stream', 'ignorelist_health', 'ignorelist_wrongly_marked_down', 'mds', 'mgr', 'mon', 'mount', 'mount/kclient/{base/{mount-syntax/{v1}', 'ms-die-on-skipped}}', 'ms_mode/legacy', 'n/5', 'objectstore-ec/bluestore-comp', 'omap_limit/10', 'osd-asserts', 'osd}', 'overrides/{cephsqlite-timeout', 'overrides/{distro/testing/k-testing', 'pg_health', 'ranks/multi/{balancer/random', 'replication/default}', 'session_timeout}', 'standby-replay', 'tasks/{0-subvolume/{with-no-extra-options}', 'wsync/yes}']

Failure: ['Data could not be sent to remote host "smithi193.front.sepia.ceph.com". Make sure this host can be reached over ssh: ssh: connect to host smithi193.front.sepia.ceph.com port 22: No route to host']
2 jobs: ['7623789', '7623807']
suites intersection: ['1-cephadm', '1-check-counter', '2-logrotate', '3-modules}', '5-workunit/suites/fsync-tester}}', 'begin/{0-install', 'clusters/1a11s-mds-1c-client-3node', 'conf/{client', 'export-check', 'frag', 'fs/workload/{0-centos_9.stream', 'ignorelist_health', 'ignorelist_wrongly_marked_down', 'mds', 'mgr', 'mon', 'mount', 'ms-die-on-skipped}}', 'n/5', 'osd-asserts', 'osd}', 'overrides/{cephsqlite-timeout', 'pg_health', 'ranks/multi/{balancer/automatic', 'session_timeout}', 'standby-replay']
suites union: ['1-cephadm', '1-check-counter', '2-logrotate', '2-scrub/no', '2-scrub/yes', '3-modules}', '3-snaps/no', '3-snaps/yes', '4-flush/no', '4-flush/yes', '5-workunit/suites/fsync-tester}}', 'begin/{0-install', 'clusters/1a11s-mds-1c-client-3node', 'conf/{client', 'export-check', 'frag', 'fs/workload/{0-centos_9.stream', 'ignorelist_health', 'ignorelist_wrongly_marked_down', 'k-stock}', 'mds', 'mgr', 'mon', 'mount', 'mount/kclient/{base/{mount-syntax/{v1}', 'mount/kclient/{base/{mount-syntax/{v2}', 'ms-die-on-skipped}}', 'ms_mode/legacy', 'ms_mode/secure', 'n/5', 'objectstore-ec/bluestore-comp', 'objectstore-ec/bluestore-ec-root', 'omap_limit/10', 'omap_limit/10000', 'osd-asserts', 'osd}', 'overrides/{cephsqlite-timeout', 'overrides/{distro/stock/{centos_9.stream', 'overrides/{distro/testing/k-testing', 'pg_health', 'ranks/multi/{balancer/automatic', 'replication/always}', 'replication/default}', 'session_timeout}', 'standby-replay', 'tasks/{0-subvolume/{with-namespace-isolated-and-quota}', 'tasks/{0-subvolume/{with-no-extra-options}', 'wsync/no}', 'wsync/yes}']

Failure: Command failed (workunit test suites/dbench.sh) on smithi097 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=72e2437271dcafb9826f2f49bb1001426fa6de81 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/suites/dbench.sh'
2 jobs: ['7623764', '7623783']
suites intersection: ['1-cephadm', '1-check-counter', '2-logrotate', '2-scrub/no', '3-modules}', '3-snaps/yes', '5-workunit/suites/dbench}}', 'begin/{0-install', 'clusters/1a11s-mds-1c-client-3node', 'conf/{client', 'export-check', 'frag', 'fs/workload/{0-centos_9.stream', 'ignorelist_health', 'ignorelist_wrongly_marked_down', 'mds', 'mgr', 'mon', 'mount', 'mount/kclient/{base/{mount-syntax/{v1}', 'ms-die-on-skipped}}', 'n/3', 'osd-asserts', 'osd}', 'overrides/{cephsqlite-timeout', 'pg_health', 'session_timeout}', 'standby-replay']
suites union: ['1-cephadm', '1-check-counter', '2-logrotate', '2-scrub/no', '3-modules}', '3-snaps/yes', '4-flush/no', '4-flush/yes', '5-workunit/suites/dbench}}', 'begin/{0-install', 'clusters/1a11s-mds-1c-client-3node', 'conf/{client', 'export-check', 'frag', 'fs/workload/{0-centos_9.stream', 'ignorelist_health', 'ignorelist_wrongly_marked_down', 'k-stock}', 'mds', 'mgr', 'mon', 'mount', 'mount/kclient/{base/{mount-syntax/{v1}', 'ms-die-on-skipped}}', 'ms_mode/legacy', 'ms_mode/secure', 'n/3', 'objectstore-ec/bluestore-comp', 'objectstore-ec/bluestore-ec-root', 'omap_limit/10', 'omap_limit/10000', 'osd-asserts', 'osd}', 'overrides/{cephsqlite-timeout', 'overrides/{distro/stock/{centos_9.stream', 'overrides/{distro/testing/k-testing', 'pg_health', 'ranks/multi/{balancer/automatic', 'ranks/multi/{balancer/random', 'replication/always}', 'replication/default}', 'session_timeout}', 'standby-replay', 'tasks/{0-subvolume/{with-no-extra-options}', 'tasks/{0-subvolume/{with-quota}', 'wsync/no}', 'wsync/yes}']

Failure: SSH connection to smithi107 was lost: 'rm -f /tmp/kernel.x86_64.rpm && echo kernel-6.8.0_g1df0c7e8a235-1.x86_64.rpm | wget -nv -O /tmp/kernel.x86_64.rpm --base=https://3.chacra.ceph.com/r/kernel/testing/1df0c7e8a2354c42d738e5454ba83ea7332deac2/centos/9/flavors/default/x86_64/ --input-file=-'
1 jobs: ['7623802']
suites: ['1-cephadm', '1-check-counter', '2-logrotate', '2-scrub/no', '3-modules}', '3-snaps/yes', '4-flush/yes', '5-workunit/suites/ffsb}}', 'begin/{0-install', 'clusters/1a11s-mds-1c-client-3node', 'conf/{client', 'export-check', 'frag', 'fs/workload/{0-centos_9.stream', 'ignorelist_health', 'ignorelist_wrongly_marked_down', 'mds', 'mgr', 'mon', 'mount', 'mount/kclient/{base/{mount-syntax/{v1}', 'ms-die-on-skipped}}', 'ms_mode/legacy', 'n/3', 'objectstore-ec/bluestore-comp', 'omap_limit/10', 'osd-asserts', 'osd}', 'overrides/{cephsqlite-timeout', 'overrides/{distro/testing/k-testing', 'pg_health', 'ranks/multi/{balancer/automatic', 'replication/default}', 'session_timeout}', 'standby-replay', 'tasks/{0-subvolume/{with-no-extra-options}', 'wsync/yes}']