Found 210 jobs Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Missing teuthology log /home/teuthworker/archive/vshankar-2024-03-07_10:41:12-fs-wip-vshankar-testing-20240307.022905-testing-default-smithi/7584523/teuthology.log Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Found 30 distinct failure reasons Dead: b"teuthology.exceptions.MaxWhileTries: 'get_status smithi008.front.sepia.ceph.com' reached maximum tries (11) after waiting for 32.5 seconds" 8 jobs: ['7584489', '7584491', '7584481', '7584488', '7584487', '7584490', '7584485', '7584480'] suites intersection: ['conf/{client', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'osd}'] suites union: ['1-ceph', '1-cephadm', '1-check-counter', '1-client', '2-logrotate}', '2-scrub/yes', '2-upgrade', '2-workunit/iozone}}', '2-workunit/suites/fsstress}}', '3-compat_client/quincy}}', '3-snaps/yes', '4-flush/no', '5-workunit/kernel_untar_build}}', 'begin/{0-install', 'centos_8.stream', 'clusters/1-mds-2-client-micro', 'clusters/1a11s-mds-1c-client-3node', 'clusters/1a3s-mds-2c-client', 'clusters/1a3s-mds-4c-client', 'clusters/1a5s-mds-1c-client', 'conf/{client', 'distro/{centos_latest}', 'distro/{ubuntu_latest}', 'export-check', 'frag', 'fs/functional/{begin/{0-install', 'fs/thrash/multifs/{begin/{0-install', 'fs/thrash/workloads/{begin/{0-install', 'fs/upgrade/featureful_client/old_client/{bluestore-bitmap', 'fs/verify/{begin/{0-install', 'fs/volumes/{begin/{0-install', 'fs/workload/{0-centos_9.stream', 'ignorelist_health', 'ignorelist_wrongly_marked_down', 'k-stock}', 'mds', 'mon', 'mon-debug', 'mount', 'mount/fuse', 'mount/kclient/{base/{mount-syntax/{v2}', 'mount/kclient/{k-testing', 'mount/kclient/{mount-syntax/{v2}', 'ms-die-on-skipped}', 'ms-die-on-skipped}}', 'ms_mode/legacy', 'msgr-failures/none', 'msgr-failures/osd-mds-delay', 'multifs', 'multimds/yes', 'n/5', 'no_client_pidfile}', 'objectstore-ec/bluestore-bitmap', 'objectstore-ec/bluestore-ec-root', 'objectstore/bluestore-bitmap', 'objectstore/bluestore-ec-root', 'omap_limit/10000', 'osd-asserts', 'osd}', 'overrides/{cephsqlite-timeout', 'overrides/{client-shutdown', 'overrides/{distro/stock/{centos_9.stream', 'overrides/{distro/testing/k-testing', 'overrides/{ignorelist_health', 'pg-warn}', 'prefetch_dirfrags/no', 'prefetch_entire_dirfrags/yes', 'races', 'ranks/3', 'ranks/5', 'ranks/multi/{balancer/automatic', 'replication/default}', 'session_timeout', 'session_timeout}', 'standby-replay', 'subvol_versions/create_subvol_version_v1', 'tasks/fsstress', 'tasks/openfiletable}', 'tasks/quota}', 'tasks/volumes/{overrides', 'tasks/{0-octopus', 'tasks/{0-subvolume/{with-namespace-isolated-and-quota}', 'tasks/{1-thrash/mon', 'tasks/{1-thrash/osd', 'test/snapshot}}', 'thrashosds-health}', 'validater/lockdep}', 'wsync/no}'] Failure: valgrind error: Leak_StillReachable operator new[](unsigned long) rocksdb::ObjectLibrary::AddFactoryEntry(char const*, std::unique_ptr >&&) UnknownInlinedFun 1 jobs: ['7584662'] suites: ['1-ceph', '2-logrotate}', 'clusters/1a5s-mds-1c-client', 'conf/{client', 'distro/{ubuntu/{overrides', 'fs/verify/{begin/{0-install', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mon-debug', 'mount', 'mount/kclient/{k-testing', 'ms-die-on-skipped}', 'objectstore-ec/bluestore-bitmap', 'osd}', 'overrides/{ignorelist_health', 'ranks/1', 'session_timeout}', 'tasks/fsstress', 'ubuntu_latest}}', 'validater/valgrind}'] Failure: Command failed on smithi094 with status 3: 'sudo logrotate /etc/logrotate.d/ceph-test.conf' 3 jobs: ['7584651', '7584499', '7584663'] suites intersection: ['1-ceph', '2-logrotate}', 'conf/{client', 'mds', 'mon', 'mount/fuse', 'osd}'] suites union: ['1-ceph', '2-logrotate}', '2-workunit/cfuse_workunit_snaptests}}', 'clusters/1-mds-2-client', 'clusters/1a3s-mds-2c-client', 'clusters/1a3s-mds-4c-client', 'conf/{client', 'distro/{centos_latest}', 'distro/{ubuntu_latest}', 'distros/ubuntu_latest', 'frag', 'fs/multiclient/{begin/{0-install', 'fs/thrash/multifs/{begin/{0-install', 'fs/volumes/{begin/{0-install', 'ignorelist_health', 'ignorelist_wrongly_marked_down', 'ignorelist_wrongly_marked_down}', 'mds', 'mon', 'mount/fuse', 'msgr-failures/none', 'multifs', 'no_client_pidfile}', 'objectstore-ec/bluestore-bitmap', 'objectstore/bluestore-bitmap', 'osd}', 'overrides/{client-shutdown', 'overrides/{ignorelist_health', 'session_timeout', 'tasks/cephfs_misc_tests}', 'tasks/volumes/{overrides', 'tasks/{1-thrash/mon', 'test/basic}}', 'thrashosds-health}'] Failure: Test failure: test_cd_with_args (tasks.cephfs.test_cephfs_shell.TestCD) 1 jobs: ['7584550'] suites: ['1-ceph', '2-logrotate}', 'clusters/1-mds-1-client-coloc', 'conf/{client', 'distro/ubuntu_latest', 'fs/shell/{begin/{0-install', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mount/fuse', 'no_client_pidfile}', 'objectstore/bluestore-bitmap', 'osd}', 'overrides/{ignorelist_health', 'tasks/cephfs-shell}'] Failure: error during scrub thrashing: rank damage found: {'backtrace'} 2 jobs: ['7584668', '7584675'] suites intersection: ['1-cephadm', '1-check-counter', '2-logrotate}', '2-scrub/yes', '3-snaps/yes', '4-flush/yes', 'begin/{0-install', 'clusters/1a11s-mds-1c-client-3node', 'conf/{client', 'export-check', 'frag', 'fs/workload/{0-centos_9.stream', 'ignorelist_health', 'ignorelist_wrongly_marked_down', 'k-stock}', 'mds', 'mon', 'mount', 'ms-die-on-skipped}}', 'omap_limit/10', 'osd-asserts', 'osd}', 'overrides/{cephsqlite-timeout', 'overrides/{distro/stock/{centos_9.stream', 'ranks/multi/{balancer/automatic', 'replication/always}', 'session_timeout}', 'standby-replay', 'wsync/yes}'] suites union: ['1-cephadm', '1-check-counter', '2-logrotate}', '2-scrub/yes', '3-snaps/yes', '4-flush/yes', '5-workunit/fs/misc}}', '5-workunit/postgres}}', 'begin/{0-install', 'clusters/1a11s-mds-1c-client-3node', 'conf/{client', 'export-check', 'frag', 'fs/workload/{0-centos_9.stream', 'ignorelist_health', 'ignorelist_wrongly_marked_down', 'k-stock}', 'mds', 'mon', 'mount', 'mount/kclient/{base/{mount-syntax/{v1}', 'mount/kclient/{base/{mount-syntax/{v2}', 'ms-die-on-skipped}}', 'ms_mode/crc', 'ms_mode/secure', 'n/3', 'n/5', 'objectstore-ec/bluestore-comp-ec-root', 'objectstore-ec/bluestore-ec-root', 'omap_limit/10', 'osd-asserts', 'osd}', 'overrides/{cephsqlite-timeout', 'overrides/{distro/stock/{centos_9.stream', 'ranks/multi/{balancer/automatic', 'replication/always}', 'session_timeout}', 'standby-replay', 'tasks/{0-subvolume/{with-no-extra-options}', 'tasks/{0-subvolume/{with-quota}', 'wsync/yes}'] Failure: Command failed on smithi086 with status 255: 'mpiexec -f /home/ubuntu/cephtest/mpi-hosts /home/ubuntu/cephtest/binary/usr/local/bin/ior -e -w -r -W -b 10m -a POSIX -o /home/ubuntu/cephtest/gmnt/ior.testfile' 1 jobs: ['7584580'] suites: ['1-ceph', '2-logrotate}', 'clusters/1-mds-3-client', 'conf/{client', 'distros/ubuntu_latest', 'fs/multiclient/{begin/{0-install', 'ignorelist_wrongly_marked_down}', 'mds', 'mon', 'mount/fuse', 'objectstore-ec/bluestore-comp', 'osd}', 'overrides/{ignorelist_health', 'tasks/ior-shared-file}'] Dead: b'2024-03-08T01:44:52.202 DEBUG:teuthology.exit:Finished running handlers' 4 jobs: ['7584482', '7584650', '7584477', '7584543'] suites intersection: ['2-logrotate}', 'conf/{client', 'mds', 'mon', 'osd}'] suites union: ['1-ceph', '1-cephadm', '1-check-counter', '2-logrotate}', '2-scrub/no', '2-workunit/suites/ffsb}}', '3-snaps/no', '4-flush/yes', '5-workunit/suites/iozone}}', 'begin/{0-install', 'clusters/1-mds-2-client', 'clusters/1a11s-mds-1c-client-3node', 'clusters/1a3s-mds-2c-client', 'clusters/1a5s-mds-1c-client', 'conf/{client', 'distro/{ubuntu/{overrides', 'distro/{ubuntu_latest}', 'distros/ubuntu_latest', 'export-check', 'frag', 'fs/mixed-clients/{begin/{0-install', 'fs/multiclient/{begin/{0-install', 'fs/thrash/workloads/{begin/{0-install', 'fs/workload/{0-centos_9.stream', 'ignorelist_health', 'ignorelist_wrongly_marked_down', 'ignorelist_wrongly_marked_down}', 'k-stock}', 'kclient-overrides/{distro/testing/k-testing', 'mds', 'mon', 'mount', 'mount/fuse', 'mount/kclient/{base/{mount-syntax/{v2}', 'ms-die-on-skipped}', 'ms-die-on-skipped}}', 'ms_mode/crc', 'msgr-failures/osd-mds-delay', 'n/3', 'objectstore-ec/bluestore-comp', 'objectstore-ec/bluestore-ec-root', 'omap_limit/10', 'osd-asserts', 'osd-asserts}', 'osd}', 'overrides/{cephsqlite-timeout', 'overrides/{client-shutdown', 'overrides/{distro/stock/{centos_9.stream', 'overrides/{ignorelist_health', 'prefetch_dirfrags/yes', 'prefetch_entire_dirfrags/no', 'races', 'ranks/1', 'ranks/multi/{balancer/automatic', 'replication/default}', 'session_timeout', 'session_timeout}', 'standby-replay', 'tasks/ior-shared-file}', 'tasks/kernel_cfuse_workunits_untarbuild_blogbench}', 'tasks/{0-subvolume/{with-quota}', 'tasks/{1-thrash/mon', 'thrashosds-health}', 'ubuntu_latest}}', 'wsync/yes}'] Failure: smithi195.front.sepia.ceph.com: _ansible_no_log: null changed: false invocation: module_args: allow_downgrade: false allowerasing: false autoremove: false bugfix: false cacheonly: false conf_file: null disable_excludes: null disable_gpg_check: false disable_plugin: [] disablerepo: [] download_dir: null download_only: false enable_plugin: [] enablerepo: [] exclude: [] install_repoquery: true install_weak_deps: true installroot: / list: null lock_timeout: 30 name: - krb5-workstation nobest: false releasever: null security: false skip_broken: false sslverify: true state: present update_cache: false update_only: false validate_certs: true msg: 'Failed to download metadata for repo ''CentOS-AppStream'': Cannot download repomd.xml: Cannot download repodata/repomd.xml: All mirrors were tried' rc: 1 results: [] 2 jobs: ['7584479', '7584486'] suites intersection: ['2-logrotate}', 'conf/{client', 'k-stock}', 'mds', 'mon', 'mount', 'ms-die-on-skipped}}', 'osd}', 'overrides/{distro/stock/{centos_9.stream'] suites union: ['1-ceph', '1-cephadm', '1-check-counter', '2-logrotate}', '2-scrub/no', '3-snaps/no', '4-flush/yes', '5-workunit/postgres}}', 'begin/{0-install', 'clusters/1a11s-mds-1c-client-3node', 'clusters/1a3s-mds-1c-client', 'conf/{client', 'distro/{centos_latest}', 'export-check', 'frag', 'fs/snaps/{begin/{0-install', 'fs/workload/{0-centos_9.stream', 'ignorelist_health', 'ignorelist_wrongly_marked_down', 'ignorelist_wrongly_marked_down}', 'k-stock}', 'mds', 'mon', 'mount', 'mount/kclient/{base/{mount-syntax/{v2}', 'mount/kclient/{mount-syntax/{v2}', 'ms-die-on-skipped}}', 'ms_mode/secure', 'n/5', 'objectstore-ec/bluestore-bitmap', 'objectstore-ec/bluestore-ec-root', 'omap_limit/10', 'osd-asserts', 'osd}', 'overrides/{cephsqlite-timeout', 'overrides/{distro/stock/{centos_9.stream', 'overrides/{ignorelist_health', 'ranks/multi/{balancer/random', 'replication/always}', 'session_timeout}', 'standby-replay', 'tasks/workunit/snaps}', 'tasks/{0-subvolume/{with-namespace-isolated}', 'wsync/yes}'] Failure: Test failure: test_mds_failover (tasks.cephfs.test_exports.TestBalMask) 1 jobs: ['7584618'] suites: ['1-ceph', '2-logrotate}', 'clusters/1a3s-mds-4c-client', 'conf/{client', 'distro/{centos_latest}', 'fs/functional/{begin/{0-install', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mount', 'mount/kclient/{mount-syntax/{v2}', 'ms-die-on-skipped}}', 'no_client_pidfile}', 'objectstore/bluestore-bitmap', 'osd}', 'overrides/{distro/testing/k-testing', 'overrides/{ignorelist_health', 'subvol_versions/create_subvol_version_v2', 'tasks/exports}'] Failure: Test failure: test_reading_conf (tasks.cephfs.test_cephfs_shell.TestShellOpts) 1 jobs: ['7584655'] suites: ['1-ceph', '2-logrotate}', 'clusters/1-mds-1-client-coloc', 'conf/{client', 'distro/centos_latest', 'fs/shell/{begin/{0-install', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mount/fuse', 'no_client_pidfile}', 'objectstore/bluestore-bitmap', 'osd}', 'overrides/{ignorelist_health', 'tasks/cephfs-shell}'] Failure: Command failed (workunit test fs/snaps/untar_snap_rm.sh) on smithi129 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=b0ea271bb96f7f4cec2f2a40ff88a5def1e0530f TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 6h /home/ubuntu/cephtest/clone.client.0/qa/workunits/fs/snaps/untar_snap_rm.sh' 7 jobs: ['7584667', '7584657', '7584529', '7584512', '7584577', '7584616', '7584592'] suites intersection: ['1-ceph', '2-logrotate}', 'conf/{client', 'mds', 'mon', 'mount', 'ms-die-on-skipped}}', 'osd}', 'overrides/{distro/testing/k-testing'] suites union: ['1-ceph', '2-logrotate}', '2-workunit/cfuse_workunit_snaptests}}', '2-workunit/fs/snaps}}', 'clusters/1a3s-mds-1c-client', 'clusters/1a3s-mds-2c-client', 'clusters/1a5s-mds-1c-client', 'conf/{client', 'distro/{centos_latest}', 'distro/{ubuntu_latest}', 'frag', 'fs/snaps/{begin/{0-install', 'fs/thrash/multifs/{begin/{0-install', 'fs/thrash/workloads/{begin/{0-install', 'ignorelist_health', 'ignorelist_wrongly_marked_down', 'ignorelist_wrongly_marked_down}', 'mds', 'mon', 'mount', 'mount/kclient/{mount-syntax/{v1}', 'mount/kclient/{mount-syntax/{v2}', 'ms-die-on-skipped}}', 'msgr-failures/none', 'multifs', 'objectstore-ec/bluestore-bitmap', 'objectstore-ec/bluestore-comp', 'objectstore-ec/bluestore-comp-ec-root', 'objectstore-ec/bluestore-ec-root', 'objectstore/bluestore-bitmap', 'osd}', 'overrides/{client-shutdown', 'overrides/{distro/testing/k-testing', 'overrides/{ignorelist_health', 'prefetch_dirfrags/no', 'prefetch_dirfrags/yes', 'prefetch_entire_dirfrags/no', 'prefetch_entire_dirfrags/yes', 'races', 'ranks/5', 'session_timeout', 'tasks/workunit/snaps}', 'tasks/{1-thrash/mds', 'tasks/{1-thrash/mon', 'tasks/{1-thrash/osd', 'thrashosds-health}'] Failure: Test failure: test_generic (tasks.cephfs.tests_from_xfstests_dev.TestXFSTestsDev) 1 jobs: ['7584542'] suites: ['1-ceph', '2-logrotate}', 'clusters/1a3s-mds-4c-client', 'conf/{client', 'distro/{ubuntu_latest}', 'fs/functional/{begin/{0-install', 'ignorelist_wrongly_marked_down', 'k-stock}', 'mds', 'mon', 'mount', 'mount/kclient/{mount-syntax/{v2}', 'ms-die-on-skipped}}', 'no_client_pidfile}', 'objectstore/bluestore-bitmap', 'osd}', 'overrides/{distro/stock/{centos_9.stream', 'overrides/{ignorelist_health', 'subvol_versions/create_subvol_version_v2', 'tasks/xfstests-dev}'] Failure: Test failure: test_cephfs_mirror_cancel_mirroring_and_readd (tasks.cephfs.test_mirroring.TestMirroring) 1 jobs: ['7584661'] suites: ['1-ceph', '2-logrotate}', 'centos_latest', 'clients/mirror', 'cluster/1-node', 'debug', 'fs/valgrind/{begin/{0-install', 'mirror/{cephfs-mirror/one-per-cluster', 'mount/fuse', 'overrides/ignorelist_health', 'tasks/mirror}}'] Failure: valgrind error: Leak_StillReachable calloc calloc _dl_check_map_versions 2 jobs: ['7584526', '7584607'] suites intersection: ['1-ceph', '2-logrotate}', 'clusters/1a5s-mds-1c-client', 'conf/{client', 'fs/verify/{begin/{0-install', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mon-debug', 'mount/fuse', 'osd}', 'overrides/{ignorelist_health', 'session_timeout}', 'tasks/dbench', 'validater/valgrind}'] suites union: ['1-ceph', '2-logrotate}', 'clusters/1a5s-mds-1c-client', 'conf/{client', 'distro/{centos_latest}', 'distro/{ubuntu/{overrides', 'fs/verify/{begin/{0-install', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mon-debug', 'mount/fuse', 'objectstore-ec/bluestore-comp-ec-root', 'objectstore-ec/bluestore-ec-root', 'osd}', 'overrides/{ignorelist_health', 'ranks/1', 'ranks/5', 'session_timeout}', 'tasks/dbench', 'ubuntu_latest}}', 'validater/valgrind}'] Dead: None 2 jobs: ['7584523', '7584630'] Failure: SSH connection to smithi040 was lost: 'cd /lib/firmware/updates && sudo git fetch origin && sudo git reset --hard origin/main' 1 jobs: ['7584524'] suites: ['1-ceph', '2-logrotate}', '2-workunit/cfuse_workunit_suites_pjd}}', 'clusters/1a3s-mds-2c-client', 'conf/{client', 'distro/{ubuntu_latest}', 'frag', 'fs/thrash/multifs/{begin/{0-install', 'ignorelist_health', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mount', 'mount/kclient/{mount-syntax/{v1}', 'ms-die-on-skipped}}', 'msgr-failures/none', 'multifs', 'objectstore/bluestore-bitmap', 'osd}', 'overrides/{client-shutdown', 'overrides/{distro/testing/k-testing', 'session_timeout', 'tasks/{1-thrash/mon', 'thrashosds-health}'] Failure: reached maximum tries (51) after waiting for 300 seconds 4 jobs: ['7584665', '7584600', '7584563', '7584634'] suites intersection: ['1-volume/{0-create', '2-client/fuse', '3-upgrade-mgr-staggered', '4-config-upgrade/{fail_fs}', '4-verify}', '5-upgrade-with-workload', '6-verify}}', 'centos_9.stream', 'conf/{client', 'fs/upgrade/mds_upgrade_sequence/{bluestore-bitmap', 'ignorelist_upgrade', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'osd}', 'overrides/{ignorelist_health', 'pg-warn', 'roles', 'syntax}'] suites union: ['1-ranks/1', '1-ranks/2', '1-volume/{0-create', '2-allow_standby_replay/no', '2-allow_standby_replay/yes', '2-client/fuse', '3-inline/no', '3-inline/yes', '3-upgrade-mgr-staggered', '4-config-upgrade/{fail_fs}', '4-verify}', '5-upgrade-with-workload', '6-verify}}', 'centos_9.stream', 'conf/{client', 'fail_fs/no', 'fail_fs/yes', 'fs/upgrade/mds_upgrade_sequence/{bluestore-bitmap', 'ignorelist_upgrade', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'osd}', 'overrides/{ignorelist_health', 'pg-warn', 'roles', 'syntax}', 'tasks/{0-from/reef/{reef}', 'tasks/{0-from/reef/{v18.2.0}', 'tasks/{0-from/reef/{v18.2.1}'] Failure: Command failed (workunit test suites/pjd.sh) on smithi040 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=b0ea271bb96f7f4cec2f2a40ff88a5def1e0530f TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 6h /home/ubuntu/cephtest/clone.client.0/qa/workunits/suites/pjd.sh' 1 jobs: ['7584555'] suites: ['1-cephadm', '1-check-counter', '2-logrotate}', '2-scrub/yes', '3-snaps/yes', '4-flush/no', '5-workunit/suites/pjd}}', 'begin/{0-install', 'clusters/1a11s-mds-1c-client-3node', 'conf/{client', 'frag', 'fs/workload/{0-centos_9.stream', 'ignorelist_health', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mount', 'mount/kclient/{base/{mount-syntax/{v1}', 'ms-die-on-skipped}}', 'ms_mode/legacy', 'objectstore-ec/bluestore-ec-root', 'omap_limit/10000', 'osd-asserts', 'osd}', 'overrides/{cephsqlite-timeout', 'overrides/{distro/testing/k-testing', 'ranks/1', 'session_timeout}', 'standby-replay', 'tasks/{0-subvolume/{with-namespace-isolated-and-quota}', 'wsync/no}'] Failure: Command failed (workunit test suites/fsx.sh) on smithi044 with status 2: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=b0ea271bb96f7f4cec2f2a40ff88a5def1e0530f TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/suites/fsx.sh' 2 jobs: ['7584522', '7584612'] suites intersection: ['1-cephadm', '1-check-counter', '2-logrotate}', '5-workunit/suites/fsx}}', 'begin/{0-install', 'clusters/1a11s-mds-1c-client-3node', 'conf/{client', 'export-check', 'frag', 'fs/workload/{0-centos_9.stream', 'ignorelist_health', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'n/3', 'osd-asserts', 'osd}', 'overrides/{cephsqlite-timeout', 'session_timeout}', 'standby-replay'] suites union: ['1-cephadm', '1-check-counter', '2-logrotate}', '2-scrub/no', '2-scrub/yes', '3-snaps/no', '3-snaps/yes', '4-flush/no', '4-flush/yes', '5-workunit/suites/fsx}}', 'begin/{0-install', 'clusters/1a11s-mds-1c-client-3node', 'conf/{client', 'export-check', 'frag', 'fs/workload/{0-centos_9.stream', 'ignorelist_health', 'ignorelist_wrongly_marked_down', 'k-stock}', 'mds', 'mon', 'mount', 'mount/fuse', 'mount/kclient/{base/{mount-syntax/{v1}', 'ms-die-on-skipped}}', 'ms_mode/crc', 'n/3', 'objectstore-ec/bluestore-comp', 'objectstore-ec/bluestore-comp-ec-root', 'omap_limit/10', 'omap_limit/10000', 'osd-asserts', 'osd}', 'overrides/{cephsqlite-timeout', 'overrides/{distro/stock/{centos_9.stream', 'ranks/multi/{balancer/automatic', 'ranks/multi/{balancer/random', 'replication/always}', 'replication/default}', 'session_timeout}', 'standby-replay', 'tasks/{0-subvolume/{with-namespace-isolated}', 'tasks/{0-subvolume/{with-no-extra-options}', 'wsync/yes}'] Failure: Test failure: test_acls (tasks.cephfs.test_acls.TestACLs) 1 jobs: ['7584558'] suites: ['1-ceph', '2-logrotate}', 'clusters/1a3s-mds-4c-client', 'conf/{client', 'distro/{ubuntu_latest}', 'fs/functional/{begin/{0-install', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mount', 'mount/kclient/{mount-syntax/{v1}', 'ms-die-on-skipped}}', 'no_client_pidfile}', 'objectstore/bluestore-bitmap', 'osd}', 'overrides/{distro/testing/k-testing', 'overrides/{ignorelist_health', 'subvol_versions/create_subvol_version_v2', 'tasks/acls}'] Failure: Command failed (workunit test fs/fscrypt.sh) on smithi122 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=b0ea271bb96f7f4cec2f2a40ff88a5def1e0530f TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 6h /home/ubuntu/cephtest/clone.client.0/qa/workunits/fs/fscrypt.sh none pjd' 1 jobs: ['7584622'] suites: ['1-ceph', '1-tests/fscrypt-pjd}}', '2-logrotate}', 'bluestore-bitmap', 'clusters/1-mds-1-client', 'conf/{client', 'distro/{centos_latest}', 'fs/fscrypt/{begin/{0-install', 'ignorelist_health_more', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mount', 'mount/kclient/{mount-syntax/{v2}', 'ms-die-on-skipped}}', 'osd', 'osd}', 'overrides/{distro/testing/k-testing', 'overrides/{ignorelist_health', 'pg-warn}', 'tasks/{0-client'] Failure: Command failed (workunit test kernel_untar_build.sh) on smithi060 with status 2: 'mkdir -p -- /home/ubuntu/cephtest/mnt.1/client.1/tmp && cd -- /home/ubuntu/cephtest/mnt.1/client.1/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=b0ea271bb96f7f4cec2f2a40ff88a5def1e0530f TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="1" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.1 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.1 CEPH_MNT=/home/ubuntu/cephtest/mnt.1 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.1/qa/workunits/kernel_untar_build.sh' 1 jobs: ['7584574'] suites: ['1-ceph', '2-logrotate}', 'clusters/1a3s-mds-2c-client', 'conf/{client', 'distro/{ubuntu/{overrides', 'fs/mixed-clients/{begin/{0-install', 'ignorelist_wrongly_marked_down', 'kclient-overrides/{distro/testing/k-testing', 'mds', 'mon', 'ms-die-on-skipped}', 'objectstore-ec/bluestore-bitmap', 'osd-asserts}', 'osd}', 'overrides/{ignorelist_health', 'tasks/kernel_cfuse_workunits_untarbuild_blogbench}', 'ubuntu_latest}}'] Failure: Failed to reconnect to smithi158 1 jobs: ['7584515'] suites: ['1-ceph', '2-logrotate}', 'clusters/1a3s-mds-4c-client', 'conf/{client', 'distro/{centos_latest}', 'fs/functional/{begin/{0-install', 'ignorelist_wrongly_marked_down', 'k-stock}', 'mds', 'mon', 'mount', 'mount/kclient/{mount-syntax/{v2}', 'ms-die-on-skipped}}', 'no_client_pidfile}', 'objectstore/bluestore-bitmap', 'osd}', 'overrides/{distro/stock/{centos_9.stream', 'overrides/{ignorelist_health', 'subvol_versions/create_subvol_version_v2', 'tasks/snap_schedule_snapdir}'] Failure: "2024-03-07T20:20:00.000297+0000 mon.smithi064 (mon.0) 331 : cluster 3 [WRN] PG_DEGRADED: Degraded data redundancy: 33/228 objects degraded (14.474%), 13 pgs degraded" in cluster log 2 jobs: ['7584541', '7584620'] suites intersection: ['1-volume/{0-create', '2-allow_standby_replay/yes', '2-client/kclient', '3-inline/no', '3-upgrade-mgr-staggered', '4-config-upgrade/{fail_fs}', '4-verify}', '5-upgrade-with-workload', '6-verify}}', 'centos_9.stream', 'conf/{client', 'fail_fs/yes', 'fs/upgrade/mds_upgrade_sequence/{bluestore-bitmap', 'ignorelist_upgrade', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'osd}', 'overrides/{ignorelist_health', 'pg-warn', 'roles', 'syntax}', 'tasks/{0-from/quincy'] suites union: ['1-ranks/1', '1-ranks/2', '1-volume/{0-create', '2-allow_standby_replay/yes', '2-client/kclient', '3-inline/no', '3-upgrade-mgr-staggered', '4-config-upgrade/{fail_fs}', '4-verify}', '5-upgrade-with-workload', '6-verify}}', 'centos_9.stream', 'conf/{client', 'fail_fs/yes', 'fs/upgrade/mds_upgrade_sequence/{bluestore-bitmap', 'ignorelist_upgrade', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'osd}', 'overrides/{ignorelist_health', 'pg-warn', 'roles', 'syntax}', 'tasks/{0-from/quincy'] Failure: Test failure: test_clean_stale_subvolume_snapshot_metadata (tasks.cephfs.test_volumes.TestSubvolumeSnapshots) 1 jobs: ['7584623'] suites: ['1-ceph', '2-logrotate}', 'clusters/1a3s-mds-4c-client', 'conf/{client', 'distro/{centos_latest}', 'fs/volumes/{begin/{0-install', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mount', 'mount/kclient/{mount-syntax/{v2}', 'ms-die-on-skipped}}', 'no_client_pidfile}', 'objectstore/bluestore-bitmap', 'osd}', 'overrides/{distro/testing/k-testing', 'overrides/{ignorelist_health', 'tasks/volumes/{overrides', 'test/snapshot}}'] Failure: SSH connection to smithi143 was lost: 'rm -f /tmp/kernel.x86_64.rpm && echo kernel-6.8.0_rc7_gdd5c1cd17167-1.x86_64.rpm | wget -nv -O /tmp/kernel.x86_64.rpm --base=https://2.chacra.ceph.com/r/kernel/testing/dd5c1cd171679b4e0ceeab3e8d62f25c2b4e6412/centos/9/flavors/default/x86_64/ --input-file=-' 1 jobs: ['7584498'] suites: ['1-ceph', '2-logrotate}', 'clusters/fixed-2-ucephfs', 'conf/{client', 'distro/{centos_latest}', 'fs/traceless/{begin/{0-install', 'ignorelist_health', 'ignorelist_wrongly_marked_down}', 'mds', 'mon', 'mount/fuse', 'objectstore-ec/bluestore-bitmap', 'osd}', 'overrides/{frag', 'tasks/cfuse_workunit_suites_ffsb', 'traceless/50pc}'] Failure: Test failure: test_pool_perm (tasks.cephfs.test_pool_perm.TestPoolPerm) 1 jobs: ['7584676'] suites: ['1-ceph', '2-logrotate}', 'clusters/1a3s-mds-4c-client', 'conf/{client', 'distro/{ubuntu_latest}', 'fs/functional/{begin/{0-install', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mount/fuse', 'no_client_pidfile}', 'objectstore/bluestore-bitmap', 'osd}', 'overrides/{ignorelist_health', 'subvol_versions/create_subvol_version_v2', 'tasks/pool-perm}'] Timeout 3h running clone.client.1/qa/workunits/suites/fsstress.sh 1 jobs: ['7584685'] suites: ['1-ranks/2', '1-volume/{0-create', '2-allow_standby_replay/yes', '2-client/kclient', '3-inline/yes', '3-upgrade-mgr-staggered', '4-config-upgrade/{fail_fs}', '4-verify}', '5-upgrade-with-workload', '6-verify}}', 'centos_9.stream', 'conf/{client', 'fail_fs/no', 'fs/upgrade/mds_upgrade_sequence/{bluestore-bitmap', 'ignorelist_upgrade', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'osd}', 'overrides/{ignorelist_health', 'pg-warn', 'roles', 'syntax}', 'tasks/{0-from/quincy'] Failure: [Errno 104] Connection reset by peer 1 jobs: ['7584476'] suites: ['1-ceph', '2-logrotate}', '2-workunit/ffsb}}', 'clusters/1a3s-mds-2c-client', 'conf/{client', 'distro/{ubuntu_latest}', 'frag', 'fs/thrash/multifs/{begin/{0-install', 'ignorelist_health', 'ignorelist_wrongly_marked_down', 'k-stock}', 'mds', 'mon', 'mount', 'mount/kclient/{mount-syntax/{v1}', 'ms-die-on-skipped}}', 'msgr-failures/none', 'multifs', 'objectstore/bluestore-bitmap', 'osd}', 'overrides/{client-shutdown', 'overrides/{distro/stock/{centos_9.stream', 'session_timeout', 'tasks/{1-thrash/mds', 'thrashosds-health}'] Failure: Command failed (workunit test suites/ffsb.sh) on smithi008 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=b0ea271bb96f7f4cec2f2a40ff88a5def1e0530f TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/suites/ffsb.sh' 1 jobs: ['7584504'] suites: ['1-cephadm', '1-check-counter', '2-logrotate}', '2-scrub/yes', '3-snaps/yes', '4-flush/no', '5-workunit/suites/ffsb}}', 'begin/{0-install', 'clusters/1a11s-mds-1c-client-3node', 'conf/{client', 'export-check', 'frag', 'fs/workload/{0-centos_9.stream', 'ignorelist_health', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mount', 'mount/kclient/{base/{mount-syntax/{v2}', 'ms-die-on-skipped}}', 'ms_mode/legacy', 'n/3', 'objectstore-ec/bluestore-ec-root', 'omap_limit/10000', 'osd-asserts', 'osd}', 'overrides/{cephsqlite-timeout', 'overrides/{distro/testing/k-testing', 'ranks/multi/{balancer/automatic', 'replication/always}', 'session_timeout}', 'standby-replay', 'tasks/{0-subvolume/{with-quota}', 'wsync/no}']