Found 95 jobs
Missing teuthology log /home/teuthworker/archive/adking-2024-02-22_03:34:55-orch:cephadm-wip-adk-testing-2024-02-21-1648-distro-default-smithi/7570681/teuthology.log
Saw end of BT but not start
Saw end of BT but not start
Saw end of BT but not start
Saw end of BT but not start
Saw end of BT but not start
Saw end of BT but not start
Found 12 distinct failure reasons
Failure: Command failed on smithi063 with status 2: "sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'grep thebody /mnt/foo/foobucket/myobject'"
4 jobs: ['7570669', '7570619', '7570602', '7570665']
suites intersection: []
suites union: ['0-nvme-loop', '1-ranks/1', '1-start', '1-volume/{0-create', '2-allow_standby_replay/yes', '2-client/kclient', '2-services/nfs-ingress-rgw-bucket', '2-services/nfs-ingress-rgw-user', '3-final}', '3-inline/no', '3-upgrade-mgr-staggered', '4-config-upgrade/{fail_fs}', '4-verify}', '5-upgrade-with-workload', '6-verify}}', 'centos_9.stream', 'conf/{client', 'fail_fs/no', 'ignorelist_upgrade', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'orch:cephadm/mds_upgrade_sequence/{bluestore-bitmap', 'orch:cephadm/smoke-roleless/{0-distro/centos_9.stream', 'orch:cephadm/smoke-roleless/{0-distro/ubuntu_22.04', 'osd}', 'overrides/{ignorelist_health', 'pg-warn', 'roles', 'syntax}', 'tasks/{0-from/quincy']

Failure: "2024-02-22T17:00:00.000149+0000 mon.a (mon.0) 1129 : cluster 3 [WRN] OSDMAP_FLAGS: noscrub flag(s) set" in cluster log
21 jobs: ['7570643', '7570658', '7570626', '7570680', '7570673', '7570612', '7570640', '7570627', '7570644', '7570682', '7570662', '7570620', '7570603', '7570695', '7570660', '7570634', '7570663', '7570659', '7570672', '7570685', '7570638']
suites intersection: []
suites union: ['0-nvme-loop', '1-ranks/2', '1-start', '1-volume/{0-create', '2-allow_standby_replay/no', '2-allow_standby_replay/yes', '2-client/kclient', '2-repo_digest/defaut', '2-repo_digest/repo_digest', '2-services/nfs-ingress', '2-thrash', '3-final}', '3-inline/no', '3-tasks/rados_api_tests', '3-tasks/radosbench', '3-tasks/small-objects', '3-tasks/snaps-few-objects', '3-upgrade-mgr-staggered', '3-upgrade/simple', '3-upgrade/staggered', '4-config-upgrade/{fail_fs}', '4-verify}', '4-wait', '5-upgrade-ls', '5-upgrade-with-workload', '6-verify}}', 'agent/off', 'agent/on', 'centos_9.stream', 'conf/{client', 'fail_fs/no', 'fixed-2', 'ignorelist_upgrade', 'ignorelist_wrongly_marked_down', 'mds', 'mode/packaged', 'mode/root', 'mon', 'mon_election/classic', 'mon_election/classic}', 'mon_election/connectivity', 'mon_election/connectivity}', 'msgr/async', 'msgr/async-v1only', 'msgr/async-v2only', 'orch:cephadm/mds_upgrade_sequence/{bluestore-bitmap', 'orch:cephadm/smoke-roleless/{0-distro/centos_9.stream_runc', 'orch:cephadm/smoke-small/{0-distro/centos_9.stream_runc', 'orch:cephadm/thrash/{0-distro/centos_9.stream', 'orch:cephadm/thrash/{0-distro/centos_9.stream_runc', 'orch:cephadm/thrash/{0-distro/ubuntu_22.04', 'orch:cephadm/upgrade/{1-start-distro/1-start-centos_9.stream', 'orch:cephadm/upgrade/{1-start-distro/1-start-ubuntu_20.04', 'orch:cephadm/with-work/{0-distro/centos_9.stream', 'orch:cephadm/with-work/{0-distro/centos_9.stream_runc', 'orch:cephadm/with-work/{0-distro/ubuntu_22.04', 'orch:cephadm/workunits/{0-distro/centos_9.stream_runc', 'osd}', 'overrides/{ignorelist_health', 'pg-warn', 'roles', 'root}', 'start', 'start}', 'syntax}', 'task/test_extra_daemon_features}', 'tasks/rados_api_tests}', 'tasks/rados_python}', 'tasks/rotate-keys}', 'tasks/{0-from/quincy']

Failure: "2024-02-22T16:26:32.477667+0000 mon.smithi078 (mon.0) 506 : cluster [WRN] Health check failed: Degraded data redundancy: 2/6 objects degraded (33.333%), 1 pg degraded (PG_DEGRADED)" in cluster log
1 jobs: ['7570606']
suites: ['1-bootstrap/17.2.0', '1-start', '2-nfs', '3-upgrade-with-workload', '4-final}', 'orch:cephadm/mgr-nfs-upgrade/{0-centos_9.stream']

Failure: Command failed on smithi165 with status 1: 'sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:81cd474a3ea0396e13ed818bfa0e15ca8c93c81d pull'
1 jobs: ['7570622']
suites: ['0-nvme-loop', '1-start', '2-services/nfs', '3-final}', 'orch:cephadm/smoke-roleless/{0-distro/centos_9.stream']

Failure: Command failed on smithi077 with status 1: 'sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 6e75cb24-d1a1-11ee-95bf-87774f69a715 -e sha1=81cd474a3ea0396e13ed818bfa0e15ca8c93c81d -- bash -c \'ceph versions | jq -e \'"\'"\'.mgr | keys\'"\'"\' | grep $sha1\''
1 jobs: ['7570630']
suites: ['1-ranks/2', '1-volume/{0-create', '2-allow_standby_replay/no', '2-client/fuse', '3-inline/yes', '3-upgrade-mgr-staggered', '4-config-upgrade/{fail_fs}', '4-verify}', '5-upgrade-with-workload', '6-verify}}', 'centos_9.stream', 'conf/{client', 'fail_fs/yes', 'ignorelist_upgrade', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'orch:cephadm/mds_upgrade_sequence/{bluestore-bitmap', 'osd}', 'overrides/{ignorelist_health', 'pg-warn', 'roles', 'syntax}', 'tasks/{0-from/reef/{reef}']

Failure: Command failed (workunit test cephadm/test_repos.sh) on smithi106 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=81cd474a3ea0396e13ed818bfa0e15ca8c93c81d TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cephadm/test_repos.sh'
1 jobs: ['7570692']
suites: ['agent/on', 'mon_election/connectivity', 'orch:cephadm/workunits/{0-distro/ubuntu_22.04', 'task/test_cephadm_repos}']

Failure: reached maximum tries (51) after waiting for 300 seconds
3 jobs: ['7570605', '7570671', '7570648']
suites intersection: ['1-volume/{0-create', '2-client/fuse', '3-inline/yes', '3-upgrade-mgr-staggered', '4-config-upgrade/{fail_fs}', '4-verify}', '5-upgrade-with-workload', '6-verify}}', 'centos_9.stream', 'conf/{client', 'fail_fs/yes', 'ignorelist_upgrade', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'orch:cephadm/mds_upgrade_sequence/{bluestore-bitmap', 'osd}', 'overrides/{ignorelist_health', 'pg-warn', 'roles', 'syntax}']
suites union: ['1-ranks/1', '1-ranks/2', '1-volume/{0-create', '2-allow_standby_replay/no', '2-allow_standby_replay/yes', '2-client/fuse', '3-inline/yes', '3-upgrade-mgr-staggered', '4-config-upgrade/{fail_fs}', '4-verify}', '5-upgrade-with-workload', '6-verify}}', 'centos_9.stream', 'conf/{client', 'fail_fs/yes', 'ignorelist_upgrade', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'orch:cephadm/mds_upgrade_sequence/{bluestore-bitmap', 'osd}', 'overrides/{ignorelist_health', 'pg-warn', 'roles', 'syntax}', 'tasks/{0-from/reef/{v18.2.0}', 'tasks/{0-from/reef/{v18.2.1}']

Dead: None
1 jobs: ['7570681']

Timeout 3h running clone.client.1/qa/workunits/suites/fsstress.sh
1 jobs: ['7570688']
suites: ['1-ranks/2', '1-volume/{0-create', '2-allow_standby_replay/yes', '2-client/kclient', '3-inline/yes', '3-upgrade-mgr-staggered', '4-config-upgrade/{fail_fs}', '4-verify}', '5-upgrade-with-workload', '6-verify}}', 'centos_9.stream', 'conf/{client', 'fail_fs/yes', 'ignorelist_upgrade', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'orch:cephadm/mds_upgrade_sequence/{bluestore-bitmap', 'osd}', 'overrides/{ignorelist_health', 'pg-warn', 'roles', 'syntax}', 'tasks/{0-from/quincy']

Dead: b'2024-02-23T04:49:26.339 DEBUG:teuthology.exit:Finished running handlers'
2 jobs: ['7570654', '7570677']
suites intersection: []
suites union: ['1-ranks/1', '1-volume/{0-create', '2-allow_standby_replay/no', '2-client/fuse', '3-inline/no', '3-upgrade-mgr-staggered', '4-config-upgrade/{fail_fs}', '4-verify}', '5-upgrade-with-workload', '6-verify}}', 'agent/on', 'centos_9.stream', 'conf/{client', 'fail_fs/no', 'ignorelist_upgrade', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mon_election/connectivity', 'orch:cephadm/mds_upgrade_sequence/{bluestore-bitmap', 'orch:cephadm/workunits/{0-distro/centos_9.stream', 'osd}', 'overrides/{ignorelist_health', 'pg-warn', 'roles', 'syntax}', 'task/test_iscsi_container/{centos_9.stream', 'tasks/{0-from/reef/{reef}', 'test_iscsi_container}}']

Failure: Command failed on smithi167 with status 22: 'sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:81cd474a3ea0396e13ed818bfa0e15ca8c93c81d shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a440d544-d1a3-11ee-95bf-87774f69a715 -- bash -c \'set -ex\nHOSTNAMES=$(ceph orch host ls --format json | jq -r \'"\'"\'.[] | .hostname\'"\'"\')\nfor host in $HOSTNAMES; do\n  # find the hostname for "host.c" which will have no mgr\n  HAS_MGRS=$(ceph orch ps --hostname ${host} --format json | jq \'"\'"\'any(.daemon_type == "mgr")\'"\'"\')\n  if [ "$HAS_MGRS" == "false" ]; then\n    HOST_C="${host}"\n  fi\ndone\n# One last thing to worry about before draining the host\n# is that the teuthology test tends to put the explicit\n# hostnames in the placement for the mon service.\n# We want to make sure we can drain without providing\n# --force and there is a check for the host being removed\n# being listed explicitly in the placements. Therefore,\n# we should remove it from the mon placement.\nceph orch ls mon --export > mon.yaml\nsed /"$HOST_C"/d mon.yaml > mon_adjusted.yaml\nceph orch apply -i mon_adjusted.yaml\n# now drain that host\nceph orch host drain $HOST_C --zap-osd-devices\n# wait for drain to complete\nHOST_C_DAEMONS=$(ceph orch ps --hostname $HOST_C)\nwhile [ "$HOST_C_DAEMONS" != "No daemons reported" ]; do\n  sleep 15\n  HOST_C_DAEMONS=$(ceph orch ps --hostname $HOST_C)\ndone\n# we want to check the ability to remove the host from\n# the CRUSH map, so we should first verify the host is in\n# the CRUSH map.\nceph osd getcrushmap -o compiled-crushmap\ncrushtool -d compiled-crushmap -o crushmap.txt\nCRUSH_MAP=$(cat crushmap.txt)\nif ! grep -q "$HOST_C" <<< "$CRUSH_MAP"; then\n  printf "Expected to see $HOST_C in CRUSH map. Saw:\\n\\n$CRUSH_MAP"\n  exit 1\nfi\n# If the drain was successful, we should be able to remove the\n# host without force with no issues. If there are still daemons\n# we will get a response telling us to drain the host and a\n# non-zero return code\nceph orch host rm $HOST_C --rm-crush-entry\n# verify we\'"\'"\'ve successfully removed the host from the CRUSH map\nsleep 30\nceph osd getcrushmap -o compiled-crushmap\ncrushtool -d compiled-crushmap -o crushmap.txt\nCRUSH_MAP=$(cat crushmap.txt)\nif grep -q "$HOST_C" <<< "$CRUSH_MAP"; then\n  printf "Saw $HOST_C in CRUSH map after it should have been removed.\\n\\n$CRUSH_MAP"\n  exit 1\nfi\n\''
1 jobs: ['7570650']
suites: ['agent/off', 'mon_election/classic', 'orch:cephadm/workunits/{0-distro/ubuntu_22.04', 'task/test_host_drain}']

Failure: Test failure: test_create_and_delete_export (tasks.cephfs.test_nfs.TestNFS)
1 jobs: ['7570607']
suites: ['conf/{client', 'mds', 'mon', 'orch:cephadm/nfs/{cluster/{1-node}', 'osd}', 'overrides/ignorelist_health', 'supported-random-distros$/{centos_latest}', 'tasks/nfs}']