Found 100 jobs
Missing teuthology log /home/teuthworker/mnt/teuthology/shwetabhosale-2026-02-14_03:52:45-orch:cephadm-wip-shweta-testing-2026-02-14-0018-distro-default-trial/50680/teuthology.log
Found 13 distinct failure reasons
Failure: Command failed on trial042 with status 1: 'sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:f8acada04a9facc6b0d4adc891b97eb878425a98 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b46a8126-0b93-11f1-a92d-d404e6e7d460 -- bash -c \'set -ex\n# since we don\'"\'"\'t know the real hostnames before the test, the next\n# bit is in order to replace the fake hostnames "host.a/b/c" with\n# the actual names cephadm knows the host by within the mon spec\nceph orch host ls --format json | jq -r \'"\'"\'.[] | .hostname\'"\'"\' > realnames\necho $\'"\'"\'host.a\\nhost.b\\nhost.c\'"\'"\' > fakenames\necho $\'"\'"\'a\\nb\\nc\'"\'"\' > mon_ids\necho $\'"\'"\'{datacenter=a}\\n{datacenter=b,rack=2}\\n{datacenter=a,rack=3}\'"\'"\' > crush_locs\nceph orch ls --service-name mon --export > mon.yaml\nMONSPEC=`cat mon.yaml`\necho "$MONSPEC"\nwhile read realname <&3 && read fakename <&4; do\n  MONSPEC="${MONSPEC//$fakename/$realname}"\ndone 3<realnames 4<fakenames\necho "$MONSPEC" > mon.yaml\ncat mon.yaml\n# now the spec should have the real hostnames, so let\'"\'"\'s re-apply\nceph orch apply -i mon.yaml\nsleep 90\nceph orch ps --refresh\nceph orch ls --service-name mon --export > mon.yaml; ceph orch apply -i mon.yaml\nsleep 90\nceph mon dump\nceph mon dump --format json\n# verify all the crush locations got set from "ceph mon dump" output\nwhile read monid <&3 && read crushloc <&4; do\n  ceph mon dump --format json | jq --arg monid "$monid" --arg crushloc "$crushloc" -e \'"\'"\'.mons | .[] | select(.name == $monid) | .crush_location == $crushloc\'"\'"\'\ndone 3<mon_ids 4<crush_locs\n\''
1 jobs: ['50707']
suites: ['agent/on', 'mon_election/connectivity', 'orch:cephadm/workunits/{0-distro/centos_9.stream_runc', 'task/test_set_mon_crush_locations}']

Failure: failed to connect to AD DC SMB share
1 jobs: ['50673']
suites: ['orch:cephadm/smb/{0-distro/centos_9.stream_runc', 'tasks/deploy_smb_proxy_enabled}']

Failure: Command failed on trial078 with status 1: 'test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm'
3 jobs: ['50637', '50641', '50635']
suites intersection: ['1-start']
suites union: ['0-nvme-loop', '1-bootstrap/18.2.0', '1-start', '2-nfs', '2-ops/rm-zap-flag}', '2-services/nfs-ingress-rgw-user', '3-final}', '3-upgrade-with-workload', '4-final}', 'orch:cephadm/mgr-nfs-upgrade/{0-centos_9.stream', 'orch:cephadm/osds/{0-distro/ubuntu_22.04', 'orch:cephadm/smoke-roleless/{0-distro/ubuntu_22.04']

Failure: Command failed on trial005 with status 32: "sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'mount -t nfs 192.168.17.5:/fake /mnt/foo'"
2 jobs: ['50723', '50683']
suites intersection: []
suites union: ['0-nvme-loop', '1-start', '2-services/nfs-keepalive-only', '3-final}', 'orch:cephadm/smb/{0-distro/ubuntu_22.04', 'orch:cephadm/smoke-roleless/{0-distro/centos_9.stream', 'tasks/deploy_smb_ctdb_node_gone_state}']

Failure: Command failed on trial105 with status 1: 'sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:squid shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 79a39165-0b92-11f1-9e57-d404e6e7d460 -e sha1=f8acada04a9facc6b0d4adc891b97eb878425a98 -- bash -c \'ceph versions | jq -e \'"\'"\'.overall | length == 1\'"\'"\'\''
3 jobs: ['50652', '50705', '50720']
suites intersection: []
suites union: ['1-ranks/1', '1-ranks/2', '1-volume/{0-create', '2-allow_standby_replay/no', '2-client/fuse', '2-client/kclient', '2-repo_digest/defaut', '3-inline/yes', '3-upgrade-mgr-staggered', '3-upgrade/simple', '4-config-upgrade/{fail_fs}', '4-verify}', '4-wait', '5-upgrade-ls', '5-upgrade-with-workload', '6-verify}}', 'agent/on', 'centos_9.stream', 'conf/{client', 'fail_fs/yes', 'ignorelist_upgrade', 'ignorelist_wrongly_marked_down', 'kernel', 'mds', 'mgr', 'mon', 'mon_election/classic}', 'orch:cephadm/mds_upgrade_sequence/{bluestore-bitmap', 'orch:cephadm/upgrade/{1-start-distro/1-start-ubuntu_22.04-reef', 'osd}', 'overrides/{ignorelist_health', 'pg-warn', 'pg_health', 'roles', 'syntax}', 'tasks/{0-from/squid']

Failure: "2026-02-17T00:07:00.182530+0000 mon.trial119 (mon.0) 180 : cluster [WRN] Health check failed: Failed to apply 1 service(s): mon (CEPHADM_APPLY_SPEC_FAIL)" in cluster log
21 jobs: ['50731', '50706', '50694', '50660', '50665', '50724', '50693', '50661', '50645', '50710', '50700', '50716', '50662', '50650', '50717', '50726', '50730', '50690', '50654', '50691', '50719']
suites intersection: []
suites union: ['0-nvme-loop', '1-start', '2-ops/repave-all}', '2-ops/rm-zap-wait}', '2-services/jaeger', '2-services/nfs-ingress-rgw-bucket', '2-services/nfs-ingress2', '2-services/nfs-keepalive-only', '2-services/nfs2', '2-services/nvmeof', '3-final}', 'agent/off', 'fixed-2', 'mode/packaged', 'mode/root', 'mon_election/classic', 'mon_election/connectivity', 'msgr/async-v1only', 'orch:cephadm/no-agent-workunits/{0-distro/centos_9.stream', 'orch:cephadm/no-agent-workunits/{0-distro/centos_9.stream_runc', 'orch:cephadm/no-agent-workunits/{0-distro/ubuntu_22.04', 'orch:cephadm/osds/{0-distro/centos_9.stream', 'orch:cephadm/osds/{0-distro/ubuntu_22.04', 'orch:cephadm/smb/{0-distro/ubuntu_22.04', 'orch:cephadm/smoke-roleless/{0-distro/centos_9.stream_runc', 'orch:cephadm/smoke-roleless/{0-distro/ubuntu_22.04', 'orch:cephadm/smoke-small/{0-distro/centos_9.stream_runc', 'orch:cephadm/with-work/{0-distro/ubuntu_22.04', 'orch:cephadm/workunits/{0-distro/ubuntu_22.04', 'start', 'start}', 'task/test_ca_signed_key}', 'task/test_cephadm_timeout}', 'task/test_host_drain}', 'task/test_orch_cli_mon}', 'tasks/deploy_smb_mgr_clustering_ips}', 'tasks/deploy_smb_mgr_ctdb_res_ips}', 'tasks/rados_api_tests}', 'tasks/rotate-keys}']

Failure: 502 Server Error: Bad Gateway for url: https://1.chacra.ceph.com/repos/ceph/wip-shweta-testing-2026-02-14-0018/f8acada04a9facc6b0d4adc891b97eb878425a98/ubuntu/jammy/flavors/default/repo
2 jobs: ['50638', '50640']
suites intersection: []
suites union: ['2-node-mgr', 'agent/off', 'conf/{client', 'ignorelist_health', 'mds', 'mgr', 'mon', 'nfs-version/{unset}', 'orch:cephadm/nfs/{cluster/{1-node}', 'orch:cephadm/orchestrator_cli/{0-random-distro$/{ubuntu_22.04}', 'orchestrator_cli}', 'osd}', 'overrides/{ignore_mgr_down', 'pg_health}', 'supported-random-distros$/{ubuntu_latest}', 'tasks/nfs}']

Dead: b'2026-02-17T05:47:49.981 DEBUG:teuthology.exit:Finished running handlers'
1 jobs: ['50672']
suites: ['agent/on', 'mon_election/connectivity', 'orch:cephadm/workunits/{0-distro/centos_9.stream', 'task/test_iscsi_container/{centos_9.stream', 'test_iscsi_container}}']

Failure: Test failure: test_error (tasks.mgr.test_orchestrator_cli.TestOrchestratorCli.test_error)
1 jobs: ['50695']
suites: ['2-node-mgr', 'agent/on', 'orch:cephadm/orchestrator_cli/{0-random-distro$/{centos_9.stream}', 'orchestrator_cli}']

Failure: Command failed (workunit test cephadm/test_cephadm.sh) on trial078 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=f8acada04a9facc6b0d4adc891b97eb878425a98 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cephadm/test_cephadm.sh'
1 jobs: ['50725']
suites: ['agent/on', 'mon_election/connectivity', 'orch:cephadm/workunits/{0-distro/centos_9.stream', 'task/test_cephadm}']

Failure: Command failed on trial080 with status 4: 'sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:f8acada04a9facc6b0d4adc891b97eb878425a98 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f20f7214-0b92-11f1-aa4f-d404e6e7d460 -- bash -c \'set -e\nset -x\nceph orch apply node-exporter\nceph orch apply grafana\nceph orch apply alertmanager\nceph orch apply prometheus\nsleep 240\nceph orch ls\nceph orch ps\nceph orch host ls\nMON_DAEMON=$(ceph orch ps --daemon-type mon -f json | jq -r \'"\'"\'last | .daemon_name\'"\'"\')\nGRAFANA_HOST=$(ceph orch ps --daemon-type grafana -f json | jq -e \'"\'"\'.[]\'"\'"\' | jq -r \'"\'"\'.hostname\'"\'"\')\nPROM_HOST=$(ceph orch ps --daemon-type prometheus -f json | jq -e \'"\'"\'.[]\'"\'"\' | jq -r \'"\'"\'.hostname\'"\'"\')\nALERTM_HOST=$(ceph orch ps --daemon-type alertmanager -f json | jq -e \'"\'"\'.[]\'"\'"\' | jq -r \'"\'"\'.hostname\'"\'"\')\nGRAFANA_IP=$(ceph orch host ls -f json | jq -r --arg GRAFANA_HOST "$GRAFANA_HOST" \'"\'"\'.[] | select(.hostname==$GRAFANA_HOST) | .addr\'"\'"\')\nPROM_IP=$(ceph orch host ls -f json | jq -r --arg PROM_HOST "$PROM_HOST" \'"\'"\'.[] | select(.hostname==$PROM_HOST) | .addr\'"\'"\')\nALERTM_IP=$(ceph orch host ls -f json | jq -r --arg ALERTM_HOST "$ALERTM_HOST" \'"\'"\'.[] | select(.hostname==$ALERTM_HOST) | .addr\'"\'"\')\n# check each host node-exporter metrics endpoint is responsive\nALL_HOST_IPS=$(ceph orch host ls -f json | jq -r \'"\'"\'.[] | .addr\'"\'"\')\nfor ip in $ALL_HOST_IPS; do\n  curl -s http://${ip}:9100/metric\ndone\n# check grafana endpoints are responsive and database health is okay\ncurl -k -s https://${GRAFANA_IP}:3000/api/health\ncurl -k -s https://${GRAFANA_IP}:3000/api/health | jq -e \'"\'"\'.database == "ok"\'"\'"\'\n# stop mon daemon in order to trigger an alert\nceph orch daemon stop $MON_DAEMON\nsleep 120\n# check prometheus endpoints are responsive and mon down alert is firing\ncurl -s http://${PROM_IP}:9095/api/v1/status/config\ncurl -s http://${PROM_IP}:9095/api/v1/status/config | jq -e \'"\'"\'.status == "success"\'"\'"\'\ncurl -s http://${PROM_IP}:9095/api/v1/alerts\ncurl -s http://${PROM_IP}:9095/api/v1/alerts | jq -e \'"\'"\'.data | .alerts | .[] | select(.labels | .alertname == "CephMonDown") | .state == "firing"\'"\'"\'\n# check alertmanager endpoints are responsive and mon down alert is active\ncurl -s http://${ALERTM_IP}:9093/api/v2/status\ncurl -s http://${ALERTM_IP}:9093/api/v2/alerts\ncurl -s http://${ALERTM_IP}:9093/api/v2/alerts | jq -e \'"\'"\'.[] | select(.labels | .alertname == "CephMonDown") | .status | .state == "active"\'"\'"\'\n# check prometheus metrics endpoint is not empty and make sure we can get metrics\nMETRICS_URL=$(ceph mgr services | jq -r .prometheus)\n[ -n "$METRICS_URL" ] || exit 1\ncurl -s "${METRICS_URL}metrics" | grep -q \'"\'"\'^ceph_health_status\'"\'"\'\n\''
1 jobs: ['50687']
suites: ['agent/on', 'mon_election/connectivity', 'orch:cephadm/workunits/{0-distro/ubuntu_22.04', 'task/test_monitoring_stack_basic}']

Failure: reached maximum tries (120) after waiting for 120 seconds
2 jobs: ['50678', '50684']
suites intersection: []
suites union: ['0-nvme-loop', '1-ranks/2', '1-start', '1-volume/{0-create', '2-allow_standby_replay/yes', '2-client/fuse', '2-ops/deploy-raw}', '3-inline/yes', '3-upgrade-mgr-staggered', '4-config-upgrade/{fail_fs}', '4-verify}', '5-upgrade-with-workload', '6-verify}}', 'centos_9.stream', 'conf/{client', 'fail_fs/yes', 'ignorelist_upgrade', 'ignorelist_wrongly_marked_down', 'kernel', 'mds', 'mgr', 'mon', 'orch:cephadm/mds_upgrade_sequence/{bluestore-bitmap', 'orch:cephadm/osds/{0-distro/centos_9.stream_runc', 'osd}', 'overrides/{ignorelist_health', 'pg-warn', 'pg_health', 'roles', 'syntax}', 'tasks/{0-from/squid']

Dead: None
1 jobs: ['50680']