Found 100 jobs
Found 9 distinct failure reasons
Failure: Command failed on trial051 with status 1: 'sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c \'grep \'"\'"\'Number of nodes:3 (including 1 deleted nodes)\'"\'"\' /tmp/ctdb_status\''
1 jobs: ['62922']
suites: ['orch:cephadm/smb/{0-distro/ubuntu_22.04', 'tasks/deploy_smb_ctdb_node_gone_state}']

Failure: Command failed on trial064 with status 1: 'sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:squid shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 6bce4e93-0f60-11f1-906f-d404e6e7d460 -e sha1=042df16cd9ee8a8df553b188fef8a8c4a9e7ab01 -- bash -c \'ceph versions | jq -e \'"\'"\'.overall | length == 1\'"\'"\'\''
7 jobs: ['62910', '62924', '62961', '62870', '62889', '62934', '62890']
suites intersection: []
suites union: ['1-bootstrap/squid', '1-ranks/1', '1-ranks/2', '1-start', '1-volume/{0-create', '2-allow_standby_replay/no', '2-allow_standby_replay/yes', '2-client/fuse', '2-client/kclient', '2-nfs', '2-repo_digest/defaut', '2-repo_digest/repo_digest', '3-inline/yes', '3-upgrade-mgr-staggered', '3-upgrade-with-workload', '3-upgrade/simple', '3-upgrade/staggered', '4-config-upgrade/{fail_fs}', '4-final}', '4-verify}', '4-wait', '5-upgrade-ls', '5-upgrade-with-workload', '6-verify}}', 'agent/off', 'agent/on', 'centos_9.stream', 'conf/{client', 'fail_fs/yes', 'ignorelist_upgrade', 'ignorelist_wrongly_marked_down', 'kernel', 'mds', 'mgr', 'mon', 'mon_election/classic}', 'mon_election/connectivity}', 'orch:cephadm/mds_upgrade_sequence/{bluestore-bitmap', 'orch:cephadm/mgr-nfs-upgrade/{0-centos_9.stream', 'orch:cephadm/upgrade/{1-start-distro/1-start-centos_9.stream-squid', 'orch:cephadm/upgrade/{1-start-distro/1-start-ubuntu_22.04-squid', 'orch:cephadm/upgrade/{1-start-distro/1-start-ubuntu_22.04-tentacle', 'osd}', 'overrides/{ignorelist_health', 'pg-warn', 'pg_health', 'roles', 'syntax}', 'tasks/{0-from/tentacle']

Failure: Test failure: test_error (tasks.mgr.test_orchestrator_cli.TestOrchestratorCli.test_error)
2 jobs: ['62878', '62936']
suites intersection: ['2-node-mgr', 'orchestrator_cli}']
suites union: ['2-node-mgr', 'agent/off', 'agent/on', 'orch:cephadm/orchestrator_cli/{0-random-distro$/{centos_9.stream_runc}', 'orch:cephadm/orchestrator_cli/{0-random-distro$/{centos_9.stream}', 'orchestrator_cli}']

Failure: reached maximum tries (300) after waiting for 300 seconds
3 jobs: ['62923', '62931', '62911']
suites intersection: []
suites union: ['agent/off', 'agent/on', 'mon_election/classic', 'mon_election/connectivity', 'orch:cephadm/workunits/{0-distro/centos_9.stream_runc', 'orch:cephadm/workunits/{0-distro/ubuntu_22.04', 'task/test_mgmt_gateway}', 'task/test_nvmeof_basic_mtls_disabled}', 'task/test_nvmeof_mtls_enabled_cephadm_signed_certs}']

Failure: Command failed on trial093 with status 1: 'sudo yum -y install ceph-radosgw'
1 jobs: ['62906']
suites: ['1-ranks/1', '1-volume/{0-create', '2-allow_standby_replay/no', '2-client/kclient', '3-inline/no', '3-upgrade-mgr-staggered', '4-config-upgrade/{fail_fs}', '4-verify}', '5-upgrade-with-workload', '6-verify}}', 'centos_9.stream', 'conf/{client', 'fail_fs/no', 'ignorelist_upgrade', 'ignorelist_wrongly_marked_down', 'kernel', 'mds', 'mgr', 'mon', 'orch:cephadm/mds_upgrade_sequence/{bluestore-bitmap', 'osd}', 'overrides/{ignorelist_health', 'pg-warn', 'pg_health', 'roles', 'syntax}', 'tasks/{0-from/squid']

Failure: Command failed on trial080 with status 1: 'test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm'
2 jobs: ['62875', '62907']
suites intersection: ['1-start', '2-nfs', '3-upgrade-with-workload', '4-final}', 'orch:cephadm/mgr-nfs-upgrade/{0-centos_9.stream']
suites union: ['1-bootstrap/19.2.0', '1-bootstrap/20.2.0', '1-start', '2-nfs', '3-upgrade-with-workload', '4-final}', 'orch:cephadm/mgr-nfs-upgrade/{0-centos_9.stream']

Failure: Command failed on trial030 with status 4: 'sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:042df16cd9ee8a8df553b188fef8a8c4a9e7ab01 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bd8aa3d3-0f60-11f1-938b-d404e6e7d460 -- bash -c \'set -e\nset -x\nceph orch apply node-exporter\nceph orch apply grafana\nceph orch apply alertmanager\nceph orch apply prometheus\nsleep 240\nceph orch ls\nceph orch ps\nceph orch host ls\nMON_DAEMON=$(ceph orch ps --daemon-type mon -f json | jq -r \'"\'"\'last | .daemon_name\'"\'"\')\nGRAFANA_HOST=$(ceph orch ps --daemon-type grafana -f json | jq -e \'"\'"\'.[]\'"\'"\' | jq -r \'"\'"\'.hostname\'"\'"\')\nPROM_HOST=$(ceph orch ps --daemon-type prometheus -f json | jq -e \'"\'"\'.[]\'"\'"\' | jq -r \'"\'"\'.hostname\'"\'"\')\nALERTM_HOST=$(ceph orch ps --daemon-type alertmanager -f json | jq -e \'"\'"\'.[]\'"\'"\' | jq -r \'"\'"\'.hostname\'"\'"\')\nGRAFANA_IP=$(ceph orch host ls -f json | jq -r --arg GRAFANA_HOST "$GRAFANA_HOST" \'"\'"\'.[] | select(.hostname==$GRAFANA_HOST) | .addr\'"\'"\')\nPROM_IP=$(ceph orch host ls -f json | jq -r --arg PROM_HOST "$PROM_HOST" \'"\'"\'.[] | select(.hostname==$PROM_HOST) | .addr\'"\'"\')\nALERTM_IP=$(ceph orch host ls -f json | jq -r --arg ALERTM_HOST "$ALERTM_HOST" \'"\'"\'.[] | select(.hostname==$ALERTM_HOST) | .addr\'"\'"\')\n# check each host node-exporter metrics endpoint is responsive\nALL_HOST_IPS=$(ceph orch host ls -f json | jq -r \'"\'"\'.[] | .addr\'"\'"\')\nfor ip in $ALL_HOST_IPS; do\n  curl -s http://${ip}:9100/metric\ndone\n# check grafana endpoints are responsive and database health is okay\ncurl -k -s https://${GRAFANA_IP}:3000/api/health\ncurl -k -s https://${GRAFANA_IP}:3000/api/health | jq -e \'"\'"\'.database == "ok"\'"\'"\'\n# stop mon daemon in order to trigger an alert\nceph orch daemon stop $MON_DAEMON\nsleep 120\n# check prometheus endpoints are responsive and mon down alert is firing\ncurl -s http://${PROM_IP}:9095/api/v1/status/config\ncurl -s http://${PROM_IP}:9095/api/v1/status/config | jq -e \'"\'"\'.status == "success"\'"\'"\'\ncurl -s http://${PROM_IP}:9095/api/v1/alerts\ncurl -s http://${PROM_IP}:9095/api/v1/alerts | jq -e \'"\'"\'.data | .alerts | .[] | select(.labels | .alertname == "CephMonDown") | .state == "firing"\'"\'"\'\n# check alertmanager endpoints are responsive and mon down alert is active\ncurl -s http://${ALERTM_IP}:9093/api/v2/status\ncurl -s http://${ALERTM_IP}:9093/api/v2/alerts\ncurl -s http://${ALERTM_IP}:9093/api/v2/alerts | jq -e \'"\'"\'.[] | select(.labels | .alertname == "CephMonDown") | .status | .state == "active"\'"\'"\'\n# check prometheus metrics endpoint is not empty and make sure we can get metrics\nMETRICS_URL=$(ceph mgr services | jq -r .prometheus)\n[ -n "$METRICS_URL" ] || exit 1\ncurl -s "${METRICS_URL}metrics" | grep -q \'"\'"\'^ceph_health_status\'"\'"\'\n\''
1 jobs: ['62916']
suites: ['agent/on', 'mon_election/connectivity', 'orch:cephadm/workunits/{0-distro/centos_9.stream', 'task/test_monitoring_stack_basic}']

Failure: "2026-02-21T20:03:19.756478+0000 mon.a (mon.0) 324 : cluster [WRN] Health check failed: 1 osds down (OSD_DOWN)" in cluster log
1 jobs: ['62880']
suites: ['cluster/{fixed-3}', 'conf/{disable-pool-app}', 'orch:cephadm/rbd_iscsi/{base/install', 'supported-container-hosts$/{ubuntu_22.04}', 'workloads/cephadm_iscsi}']

Dead: b'2026-02-22T02:00:47.907 DEBUG:teuthology.exit:Finished running handlers'
1 jobs: ['62900']
suites: ['agent/on', 'mon_election/connectivity', 'orch:cephadm/workunits/{0-distro/centos_9.stream_runc', 'task/test_iscsi_container/{centos_9.stream', 'test_iscsi_container}}']