Found 272 jobs
Found 24 distinct failure reasons
Failure: 
1 jobs: ['59126']
suites: ['base', 'mem$/{normal-2}', 'mon_election/classic', 'msgr-failures/many', 'msgr/async-v2only', 'objectstore/{bluestore/{alloc$/{bitmap}', 'onode-segment$/{512K-onoff}', 'rados', 'rados/singleton/{all/backfill-toofull', 'supported-random-distro$/{centos_latest}}', 'v1}}}}', 'write$/{v1/{compr$/{yes$/{snappy}}']

Failure: Command failed (workunit test cephadm/test_cephadm.sh) on trial171 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=802bf111cd55e57f5842153d7ba2b8b964aa818a TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cephadm/test_cephadm.sh'
2 jobs: ['58980', '59218']
suites intersection: ['agent/on', 'mon_election/connectivity', 'task/test_cephadm}']
suites union: ['agent/on', 'mon_election/connectivity', 'rados/cephadm/workunits/{0-distro/centos_9.stream', 'rados/cephadm/workunits/{0-distro/ubuntu_22.04', 'task/test_cephadm}']

Failure: Command failed on trial171 with status 1: 'yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_2'
2 jobs: ['59197', '59060']
suites intersection: ['debug/mgr', 'rados/dashboard/{0-single-container-host', 'tasks/e2e}']
suites union: ['debug/mgr', 'mon_election/classic', 'mon_election/connectivity', 'rados/dashboard/{0-single-container-host', 'random-objectstore$/{bluestore-bitmap}', 'random-objectstore$/{bluestore-hybrid}', 'tasks/e2e}']

Failure: Command failed on trial125 with status 127: 'sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:802bf111cd55e57f5842153d7ba2b8b964aa818a shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 1b0b9f8b-0db5-11f1-9da3-d404e6e7d460 -- ceph config generate-minimal-conf'
1 jobs: ['59174']
suites: ['agent/on', 'mon_election/connectivity', 'rados/cephadm/workunits/{0-distro/centos_9.stream_runc', 'task/test_set_mon_crush_locations}']

Failure: Command failed (workunit test osd-backfill/osd-backfill-prio.sh) on trial053 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=802bf111cd55e57f5842153d7ba2b8b964aa818a TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/standalone/osd-backfill/osd-backfill-prio.sh'
1 jobs: ['59186']
suites: ['rados/standalone/{supported-random-distro$/{centos_latest}', 'workloads/osd-backfill}']

Failure: Command failed (workunit test cephtool/test_crimson_dump_metrics.sh) on trial066 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=802bf111cd55e57f5842153d7ba2b8b964aa818a TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cephtool/test_crimson_dump_metrics.sh'
5 jobs: ['58962', '59145', '59027', '59085', '59209']
suites intersection: ['base', 'rados', 'rados/singleton-bluestore/{all/cephtool']
suites union: ['base', 'mem$/{low}', 'mem$/{normal-2}', 'mon_election/classic', 'mon_election/connectivity', 'msgr-failures/few', 'msgr-failures/many', 'msgr-failures/none', 'msgr/async', 'msgr/async-v1only', 'msgr/async-v2only', 'objectstore/bluestore/{alloc$/{bitmap}', 'objectstore/bluestore/{alloc$/{hybrid}', 'objectstore/bluestore/{alloc$/{stupid}', 'onode-segment$/{256K}', 'onode-segment$/{512K}', 'onode-segment$/{none}', 'rados', 'rados/singleton-bluestore/{all/cephtool', 'random}}}', 'supported-random-distro$/{centos_latest}}', 'supported-random-distro$/{ubuntu_latest}}', 'v1}}}', 'v2}}}', 'write$/{random/{compr$/{yes$/{lz4}}', 'write$/{v1/{compr$/{no$/{no}}', 'write$/{v2/{compr$/{no$/{no}}']

Failure: The WatchDog barked due to Thrasher OSDThrasher threw exception wait_for_recovery: failed before timeout expired
1 jobs: ['59168']
suites: ['base', 'ceph', 'clusters/{fixed-4}', 'ec_optimizations/ec_optimizations_on', 'mem$/{normal-2}', 'mon_election/connectivity', 'msgr-failures/osd-dispatch-delay', 'objectstore/{bluestore/{alloc$/{avl}', 'onode-segment$/{256K}', 'rados', 'rados/thrash-erasure-code-isa/{arch/x86_64', 'random}}}}', 'recovery-overrides/{more-active-recovery}', 'supported-random-distro$/{centos_latest}', 'thrashers/pggrow_host', 'thrashosds-health', 'workloads/ec-rados-plugin=isa-k=10-m=4}', 'write$/{random/{compr$/{yes$/{lz4}}']

Timeout 3h running clone.client.0/qa/standalone/misc/test-ceph-helpers.sh
1 jobs: ['59103']
suites: ['rados/standalone/{supported-random-distro$/{centos_latest}', 'workloads/misc}']

Failure: "2026-02-19T16:53:01.764793+0000 mon.a (mon.0) 497 : cluster [WRN] Health check failed: No InfluxDB server configured (MGR_INFLUX_NO_SERVER)" in cluster log
1 jobs: ['59094']
suites: ['2-ceph', '3-mgrmodules', '4-units/module_selftest}}', 'base', 'debug/mgr', 'distro/{centos_latest}', 'mem$/{normal-1}', 'mgr_ttl_cache/enable', 'mon_election/connectivity', 'onode-segment$/{512K-onoff}', 'rados/mgr/{clusters/{2-node-mgr}', 'random-objectstore$/{bluestore/{alloc$/{stupid}', 'tasks/{1-install', 'v1}}}}', 'write$/{v1/{compr$/{yes$/{zstd}}']

Failure: HTTPSConnectionPool(host='shaman.ceph.com', port=443): Max retries exceeded with url: /api/search?status=ready&project=ceph&flavor=default&distros=ubuntu%2F22.04%2Fx86_64&ref=wip-yuri-testing-2026-02-18-1820 (Caused by NewConnectionError('<urllib3.connection.HTTPSConnection object at 0x71bd97acaf00>: Failed to establish a new connection: [Errno -3] Temporary failure in name resolution'))
2 jobs: ['59148', '59130']
suites: ['mon_election/connectivity', 'rados', 'rados/singleton-nomsgr/{all/admin_socket_output', 'supported-random-distro$/{ubuntu_latest}}']

Dead: b'2026-02-19T22:26:27.682 DEBUG:teuthology.exit:Finished running handlers'
1 jobs: ['59084']
suites: ['agent/on', 'mon_election/connectivity', 'rados/cephadm/workunits/{0-distro/centos_9.stream', 'task/test_iscsi_container/{centos_9.stream', 'test_iscsi_container}}']

Failure: Command failed (workunit test mon/misc.sh) on trial085 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=802bf111cd55e57f5842153d7ba2b8b964aa818a TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/standalone/mon/misc.sh'
1 jobs: ['59159']
suites: ['rados/standalone/{supported-random-distro$/{ubuntu_latest}', 'workloads/mon}']

Failure: Command failed on trial016 with status 1: 'sudo yum -y install ceph-radosgw ceph-test ceph ceph-base cephadm ceph-immutable-object-cache ceph-mgr ceph-mgr-dashboard ceph-mgr-diskprediction-local ceph-mgr-rook ceph-mgr-cephadm ceph-osd ceph-osd-classic ceph-fuse ceph-volume librados-devel libcephfs2 libcephfs-devel librados2 librbd1 python3-rados python3-rgw python3-cephfs python3-rbd rbd-fuse rbd-mirror rbd-nbd sqlite-devel'
1 jobs: ['59117']
suites: ['2-ceph', '3-mgrmodules', '4-units/per_module_finisher_stats}}', 'base', 'debug/mgr', 'distro/{centos_latest}', 'mem$/{normal-2}', 'mgr_ttl_cache/disable', 'mon_election/classic', 'onode-segment$/{512K}', 'rados/mgr/{clusters/{2-node-mgr}', 'random-objectstore$/{bluestore/{alloc$/{hybrid}', 'tasks/{1-install', 'v2}}}}', 'write$/{v2/{compr$/{yes$/{lz4}}']

Failure: Command failed (workunit test post-file.sh) on trial042 with status 255: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=802bf111cd55e57f5842153d7ba2b8b964aa818a TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/post-file.sh'
2 jobs: ['59155', '59017']
suites intersection: ['rados', 'rados/singleton-nomsgr/{all/ceph-post-file', 'supported-random-distro$/{centos_latest}}']
suites union: ['mon_election/classic', 'mon_election/connectivity', 'rados', 'rados/singleton-nomsgr/{all/ceph-post-file', 'supported-random-distro$/{centos_latest}}']

Crash: Command failed (workunit test cephtool/test_kvstore_tool.sh) on trial168 with status 139: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=802bf111cd55e57f5842153d7ba2b8b964aa818a TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cephtool/test_kvstore_tool.sh'
ceph version 20.3.0-5336-g802bf111 (802bf111cd55e57f5842153d7ba2b8b964aa818a) tentacle (dev - None)
 1: /lib/x86_64-linux-gnu/libc.so.6(+0x42520) [0x7fd9c372b520]
 2: (RocksDBStore::get_wholespace_iterator(unsigned int)+0xc7) [0x561821ed06b7]
 3: (RocksDBStore::get_iterator(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, unsigned int, KeyValueDB::IteratorBounds)+0x53f) [0x561821ec9c2f]
 4: (BlueStore::_close_db()+0x324) [0x561821d2f1f4]
 5: (BlueStore::umount()+0x195) [0x561821d37f65]
 6: ceph-kvstore-tool(+0x20e513) [0x561821cd6513]
 7: main()
 8: /lib/x86_64-linux-gnu/libc.so.6(+0x29d90) [0x7fd9c3712d90]
 9: __libc_start_main()
 10: _start()
2 jobs: ['59011', '59149']
suites intersection: ['rados', 'rados/singleton-nomsgr/{all/ceph-kvstore-tool']
suites union: ['mon_election/classic', 'mon_election/connectivity', 'rados', 'rados/singleton-nomsgr/{all/ceph-kvstore-tool', 'supported-random-distro$/{centos_latest}}', 'supported-random-distro$/{ubuntu_latest}}']

Failure: Command failed (workunit test dencoder/test-dencoder.sh) on trial012 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=802bf111cd55e57f5842153d7ba2b8b964aa818a TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/dencoder/test-dencoder.sh'
1 jobs: ['58991']
suites: ['1-tasks', 'rados/encoder/{0-start', 'supported-random-distro$/{ubuntu_latest}}']

Failure: Command failed (workunit test osd/osd-bluefs-volume-ops.sh) on trial119 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=802bf111cd55e57f5842153d7ba2b8b964aa818a TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/standalone/osd/osd-bluefs-volume-ops.sh'
1 jobs: ['59215']
suites: ['rados/standalone/{supported-random-distro$/{ubuntu_latest}', 'workloads/osd}']

Failure: Found coredumps on ubuntu@trial160.front.sepia.ceph.com
2 jobs: ['59123', '59220']
suites intersection: ['base', 'objectstore/{bluestore/{alloc$/{bitmap}', 'rados', 'random}}}}', 'supported-random-distro$/{ubuntu_latest}', 'write$/{random/{compr$/{no$/{no}}']
suites union: ['base', 'cluster/{12-osds}', 'clusters/{fixed-2}', 'ec_optimizations/ec_optimizations_off', 'mem$/{low}', 'mem$/{normal-1}', 'mon_election/classic', 'mon_election/connectivity', 'msgr-failures/few', 'msgr-failures/osd-delay', 'msgr/async-v2only', 'objectstore/{bluestore/{alloc$/{bitmap}', 'onode-segment$/{512K}', 'onode-segment$/{none}', 'rados', 'rados/basic/{ceph', 'rados/thrash-erasure-code-big/{ceph', 'random}}}}', 'recovery-overrides/{default}', 'supported-random-distro$/{ubuntu_latest}', 'tasks/libcephsqlite}', 'thrashers/default', 'thrashosds-health', 'workloads/ec-rados-plugin=jerasure-k=4-m=2-technique=cauchy}', 'write$/{random/{compr$/{no$/{no}}']

Failure: failed to complete snap trimming before timeout
1 jobs: ['59063']
suites: ['1-pg-log-overrides/short_pg_log', '2-recovery-overrides/{more-async-partial-recovery}', '3-scrub-overrides/{max-simultaneous-scrubs-3}', 'backoff/normal', 'base', 'ceph', 'clusters/{fixed-4}', 'crc-failures/default', 'd-balancer/crush-compat', 'mem$/{normal-1}', 'mon_election/connectivity', 'msgr-failures/osd-dispatch-delay', 'msgr/async-v1only', 'objectstore/{bluestore/{alloc$/{stupid}', 'onode-segment$/{512K}', 'rados', 'rados/thrash/{0-size-min-size-overrides/3-size-2-min-size', 'random}}}}', 'supported-random-distro$/{ubuntu_latest}', 'thrashers/morepggrow', 'thrashosds-health', 'workloads/pool-snaps-few-objects}', 'write$/{random/{compr$/{yes$/{zlib}}']

Failure: Test failure: test_list_enabled_module (tasks.mgr.dashboard.test_mgr_module.MgrModuleTest.test_list_enabled_module)
2 jobs: ['59128', '58990']
suites intersection: ['debug/mgr', 'rados/dashboard/{0-single-container-host', 'tasks/dashboard}']
suites union: ['debug/mgr', 'mon_election/classic', 'mon_election/connectivity', 'rados/dashboard/{0-single-container-host', 'random-objectstore$/{bluestore-comp-lz4}', 'random-objectstore$/{bluestore-low-osd-mem-target}', 'tasks/dashboard}']

Failure: Command failed on trial163 with status 128: 'rm -rf /home/ubuntu/cephtest/clone.client.0 && git clone https://github.com/ceph/ceph.git /home/ubuntu/cephtest/clone.client.0 && cd /home/ubuntu/cephtest/clone.client.0 && git checkout 802bf111cd55e57f5842153d7ba2b8b964aa818a'
3 jobs: ['59112', '59162', '59049']
suites intersection: []
suites union: ['0-size-min-size-overrides/2-size-2-min-size', '1-install/reef', 'backoff/peering_and_degraded', 'base', 'ceph', 'clusters/{three-plus-one}', 'd-balancer/crush-compat', 'mem$/{normal-1}', 'mon_election/classic', 'mon_election/connectivity', 'msgr-failures/few', 'msgr-failures/none', 'msgr/async', 'objectstore/{bluestore/{alloc$/{btree}', 'onode-segment$/{none}', 'rados', 'rados/objectstore/{backends/fusestore', 'rados/singleton/{all/3-az-stretch-cluster-netsplit', 'rados/thrash-old-clients/{0-distro$/{centos_9.stream}', 'random}}}}', 'supported-random-distro$/{ubuntu_latest}}', 'thrashers/careful', 'thrashosds-health', 'workloads/rbd_cls}', 'write$/{random/{compr$/{no$/{no}}']

Failure: Test failure: test_standby (tasks.mgr.test_prometheus.TestPrometheus.test_standby)
1 jobs: ['59157']
suites: ['2-ceph', '3-mgrmodules', '4-units/prometheus}}', 'base', 'debug/mgr', 'distro/{centos_latest}', 'mem$/{low}', 'mgr_ttl_cache/disable', 'mon_election/classic', 'onode-segment$/{512K}', 'rados/mgr/{clusters/{2-node-mgr}', 'random-objectstore$/{bluestore/{alloc$/{avl}', 'random}}}}', 'tasks/{1-install', 'write$/{random/{compr$/{no$/{no}}']

Failure: "grep: /var/log/ceph/81b0a644-0dae-11f1-99b4-d404e6e7d460/ceph.log: No such file or directory" in cluster log
1 jobs: ['59047']
suites: ['0-size-min-size-overrides/3-size-2-min-size', '1-install/reef', 'backoff/peering_and_degraded', 'ceph', 'clusters/{three-plus-one}', 'd-balancer/on', 'mon_election/classic', 'msgr-failures/few', 'rados', 'rados/thrash-old-clients/{0-distro$/{centos_9.stream}', 'thrashers/morepggrow', 'thrashosds-health', 'workloads/snaps-few-objects}']

Failure: Command failed on trial161 with status 4: 'sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:802bf111cd55e57f5842153d7ba2b8b964aa818a shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid fa9960c8-0db1-11f1-a65a-d404e6e7d460 -- bash -c \'set -e\nset -x\nceph orch apply node-exporter\nceph orch apply grafana\nceph orch apply alertmanager\nceph orch apply prometheus\nsleep 240\nceph orch ls\nceph orch ps\nceph orch host ls\nMON_DAEMON=$(ceph orch ps --daemon-type mon -f json | jq -r \'"\'"\'last | .daemon_name\'"\'"\')\nGRAFANA_HOST=$(ceph orch ps --daemon-type grafana -f json | jq -e \'"\'"\'.[]\'"\'"\' | jq -r \'"\'"\'.hostname\'"\'"\')\nPROM_HOST=$(ceph orch ps --daemon-type prometheus -f json | jq -e \'"\'"\'.[]\'"\'"\' | jq -r \'"\'"\'.hostname\'"\'"\')\nALERTM_HOST=$(ceph orch ps --daemon-type alertmanager -f json | jq -e \'"\'"\'.[]\'"\'"\' | jq -r \'"\'"\'.hostname\'"\'"\')\nGRAFANA_IP=$(ceph orch host ls -f json | jq -r --arg GRAFANA_HOST "$GRAFANA_HOST" \'"\'"\'.[] | select(.hostname==$GRAFANA_HOST) | .addr\'"\'"\')\nPROM_IP=$(ceph orch host ls -f json | jq -r --arg PROM_HOST "$PROM_HOST" \'"\'"\'.[] | select(.hostname==$PROM_HOST) | .addr\'"\'"\')\nALERTM_IP=$(ceph orch host ls -f json | jq -r --arg ALERTM_HOST "$ALERTM_HOST" \'"\'"\'.[] | select(.hostname==$ALERTM_HOST) | .addr\'"\'"\')\n# check each host node-exporter metrics endpoint is responsive\nALL_HOST_IPS=$(ceph orch host ls -f json | jq -r \'"\'"\'.[] | .addr\'"\'"\')\nfor ip in $ALL_HOST_IPS; do\n  curl -s http://${ip}:9100/metric\ndone\n# check grafana endpoints are responsive and database health is okay\ncurl -k -s https://${GRAFANA_IP}:3000/api/health\ncurl -k -s https://${GRAFANA_IP}:3000/api/health | jq -e \'"\'"\'.database == "ok"\'"\'"\'\n# stop mon daemon in order to trigger an alert\nceph orch daemon stop $MON_DAEMON\nsleep 120\n# check prometheus endpoints are responsive and mon down alert is firing\ncurl -s http://${PROM_IP}:9095/api/v1/status/config\ncurl -s http://${PROM_IP}:9095/api/v1/status/config | jq -e \'"\'"\'.status == "success"\'"\'"\'\ncurl -s http://${PROM_IP}:9095/api/v1/alerts\ncurl -s http://${PROM_IP}:9095/api/v1/alerts | jq -e \'"\'"\'.data | .alerts | .[] | select(.labels | .alertname == "CephMonDown") | .state == "firing"\'"\'"\'\n# check alertmanager endpoints are responsive and mon down alert is active\ncurl -s http://${ALERTM_IP}:9093/api/v2/status\ncurl -s http://${ALERTM_IP}:9093/api/v2/alerts\ncurl -s http://${ALERTM_IP}:9093/api/v2/alerts | jq -e \'"\'"\'.[] | select(.labels | .alertname == "CephMonDown") | .status | .state == "active"\'"\'"\'\n# check prometheus metrics endpoint is not empty and make sure we can get metrics\nMETRICS_URL=$(ceph mgr services | jq -r .prometheus)\n[ -n "$METRICS_URL" ] || exit 1\ncurl -s "${METRICS_URL}metrics" | grep -q \'"\'"\'^ceph_health_status\'"\'"\'\n\''
1 jobs: ['59114']
suites: ['agent/on', 'mon_election/connectivity', 'rados/cephadm/workunits/{0-distro/ubuntu_22.04', 'task/test_monitoring_stack_basic}']