Found 200 jobs Misunderstood line: Found 8 distinct failure reasons Failure: "1686267822.7953215 osd.1 (osd.1) 2 : cluster [WRN] WaitReplicas::react(const DigestUpdate&): Unexpected DigestUpdate event" in cluster log 1 jobs: ['7299092'] suites: ['mon_election/classic', 'msgr-failures/many', 'msgr/async-v1only', 'objectstore/bluestore-hybrid', 'rados', 'rados/singleton/{all/osd-recovery', 'supported-random-distro$/{ubuntu_latest}}'] Failure: Command failed on smithi079 with status 100: "sudo apt update && sudo apt install -y apt-transport-https ca-certificates curl && sudo curl -fsSLo /usr/share/keyrings/kubernetes-archive-keyring.gpg https://packages.cloud.google.com/apt/doc/apt-key.gpg && echo 'deb [signed-by=/usr/share/keyrings/kubernetes-archive-keyring.gpg] https://apt.kubernetes.io/ kubernetes-xenial main' | sudo tee /etc/apt/sources.list.d/kubernetes.list && sudo apt update && sudo apt install -y kubelet kubeadm kubectl bridge-utils" 2 jobs: ['7299035', '7299118'] suites intersection: ['0-kubeadm', '0-nvme-loop', '1-rook', 'k8s/1.21', 'rados/rook/smoke/{0-distro/ubuntu_20.04'] suites union: ['0-kubeadm', '0-nvme-loop', '1-rook', '2-workload/none', '2-workload/radosbench', 'cluster/1-node', 'cluster/3-node', 'k8s/1.21', 'net/flannel', 'net/host', 'rados/rook/smoke/{0-distro/ubuntu_20.04', 'rook/1.7.2}', 'rook/master}'] Failure: Command failed on smithi112 with status 5: 'sudo systemctl stop ceph-eb80fe7a-0638-11ee-9b1e-001a4aab830c@mon.b' 2 jobs: ['7299040', '7298998'] suites intersection: ['mon_election/connectivity'] suites union: ['0-size-min-size-overrides/2-size-2-min-size', '1-install/nautilus-v1only', 'agent/on', 'backoff/normal', 'ceph', 'clusters/{openstack', 'd-balancer/crush-compat', 'mon_election/connectivity', 'msgr-failures/few', 'rados', 'rados/cephadm/workunits/{0-distro/centos_8.stream_container_tools', 'rados/thrash-old-clients/{0-distro$/{centos_8.stream_container_tools}', 'task/test_orch_cli_mon}', 'thrashers/mapgap', 'thrashosds-health', 'three-plus-one}', 'workloads/cache-snaps}'] Failure: Command failed (workunit test cls/test_cls_rbd.sh) on smithi159 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=quincy TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cls/test_cls_rbd.sh' 1 jobs: ['7299148'] suites: ['0-start', '1-tasks', 'mon_election/connectivity', 'rados/upgrade/parallel/{0-random-distro$/{rhel_8.6_container_tools_rhel8}', 'rados_api', 'rados_loadgenbig', 'rbd_import_export', 'test_rbd_api', 'test_rbd_python}}', 'upgrade-sequence', 'workload/{ec-rados-default'] Failure: Command failed (workunit test misc/mclock-config.sh) on smithi189 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=9564655a5e7b3237dff14304a9dff15cf1a8439a TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/standalone/misc/mclock-config.sh' 1 jobs: ['7299127'] suites: ['rados/standalone/{supported-random-distro$/{ubuntu_20.04}', 'workloads/misc}'] Valgrind: osd (InvalidRead) 1 jobs: ['7299017'] suites: ['ceph', 'clusters/{fixed-2', 'd-thrash/default/{default', 'mon_election/connectivity', 'msgr-failures/few', 'msgr/async-v1only', 'objectstore/bluestore-comp-snappy', 'openstack}', 'rados', 'rados/verify/{centos_latest', 'tasks/rados_api_tests', 'thrashosds-health}', 'validater/valgrind}'] Failure: Test failure: test_cephfs_mirror (tasks.cephadm_cases.test_cli.TestCephadmCLI) 1 jobs: ['7299164'] suites: ['agent/on', 'mon_election/connectivity', 'rados/cephadm/workunits/{0-distro/centos_8.stream_container_tools_crun', 'task/test_orch_cli}'] Failure: Command failed (workunit test cephadm/test_dashboard_e2e.sh) on smithi177 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=9564655a5e7b3237dff14304a9dff15cf1a8439a TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cephadm/test_dashboard_e2e.sh' 1 jobs: ['7299062'] suites: ['debug/mgr', 'mon_election/connectivity', 'rados/dashboard/{0-single-container-host', 'random-objectstore$/{bluestore-stupid}', 'tasks/e2e}']