archive_path: /home/teuthworker/mnt/teuthology/yuriw-2026-02-17_20:43:43-rados-wip-yuri6-testing-2026-02-17-1732-squid-distro-default-trial/53883 branch: wip-yuri6-testing-2026-02-17-1732-squid description: rados/cephadm/workunits/{0-distro/ubuntu_22.04 agent/on mon_election/connectivity task/test_monitoring_stack_basic} email: null first_in_suite: false flavor: default job_id: '53883' kernel: &id001 branch: distro kdb: true sha1: distro ktype: distro last_in_suite: false machine_type: trial name: yuriw-2026-02-17_20:43:43-rados-wip-yuri6-testing-2026-02-17-1732-squid-distro-default-trial no_nested_subset: false os_type: ubuntu os_version: '22.04' overrides: admin_socket: branch: wip-yuri6-testing-2026-02-17-1732-squid ceph: conf: global: mon election default strategy: 3 mgr: debug mgr: 20 debug ms: 1 mgr/cephadm/use_agent: true mon: debug mon: 20 debug ms: 1 debug paxos: 20 osd: debug ms: 1 debug osd: 20 flavor: default log-ignorelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) - MON_DOWN - mons down - mon down - out of quorum - CEPHADM_STRAY_DAEMON - CEPHADM_FAILED_DAEMON log-only-match: - CEPHADM_ sha1: d855f53b89fdcec760fd9232a5fb55ed4fb111a1 ceph-deploy: conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: {} install: ceph: flavor: default sha1: d855f53b89fdcec760fd9232a5fb55ed4fb111a1 workunit: branch: wip-yuri6-testing-2026-02-17-1732-squid sha1: d855f53b89fdcec760fd9232a5fb55ed4fb111a1 owner: scheduled_yuriw@soko04.front.sepia.ceph.com priority: 69 repo: https://github.com/ceph/ceph-ci.git roles: - - host.a - mon.a - mgr.a - osd.0 - - host.b - mon.b - mgr.b - osd.1 - - host.c - mon.c - osd.2 seed: 2987 sha1: d855f53b89fdcec760fd9232a5fb55ed4fb111a1 sleep_before_teardown: 0 subset: 111/120000 suite: rados suite_branch: wip-yuri6-testing-2026-02-17-1732-squid suite_path: /home/teuthworker/src/github.com_ceph_ceph-c_d855f53b89fdcec760fd9232a5fb55ed4fb111a1/qa suite_relpath: qa suite_repo: https://github.com/ceph/ceph-ci.git suite_sha1: d855f53b89fdcec760fd9232a5fb55ed4fb111a1 targets: trial127.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBLgT8IbYuSrdZXrMiCZLlhazDJFGR62s/IgHcLFcB2Cm2YKskMEmPz3FjUkZXm7ByvTHc5yfUBcPR7b9UsSSAhA= trial142.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBGqgyib+50/7IBA83H1VOqetcMNHYwyLk0Gsnubf4USmEi6uwcc+kGCg7QU+kvAxrKugd+FkEbVcLyORJj/puX4= trial145.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBGa6Sz/t2Vp8ccKkn8korSfvcnbxzMXV2LVFZJuUmGb3LXPBtogCWUPkeEZs6pZG3jHmJNuu2XKFW55P/lcdhMI= tasks: - internal.check_packages: null - internal.buildpackages_prep: null - internal.save_config: null - internal.check_lock: null - internal.add_remotes: null - console_log: null - internal.connect: null - internal.push_inventory: null - internal.serialize_remote_roles: null - internal.check_conflict: null - internal.check_ceph_data: null - internal.vm_setup: null - kernel: *id001 - internal.base: null - internal.archive_upload: null - internal.archive: null - internal.coredump: null - internal.sudo: null - internal.syslog: null - internal.timer: null - pcp: null - selinux: null - ansible.cephlab: null - clock: null - install: null - cephadm: null - cephadm.shell: host.a: - "set -e\nset -x\nceph orch apply node-exporter\nceph orch apply grafana\nceph\ \ orch apply alertmanager\nceph orch apply prometheus\nsleep 240\nceph orch\ \ ls\nceph orch ps\nceph orch host ls\nMON_DAEMON=$(ceph orch ps --daemon-type\ \ mon -f json | jq -r 'last | .daemon_name')\nGRAFANA_HOST=$(ceph orch ps --daemon-type\ \ grafana -f json | jq -e '.[]' | jq -r '.hostname')\nPROM_HOST=$(ceph orch\ \ ps --daemon-type prometheus -f json | jq -e '.[]' | jq -r '.hostname')\nALERTM_HOST=$(ceph\ \ orch ps --daemon-type alertmanager -f json | jq -e '.[]' | jq -r '.hostname')\n\ GRAFANA_IP=$(ceph orch host ls -f json | jq -r --arg GRAFANA_HOST \"$GRAFANA_HOST\"\ \ '.[] | select(.hostname==$GRAFANA_HOST) | .addr')\nPROM_IP=$(ceph orch host\ \ ls -f json | jq -r --arg PROM_HOST \"$PROM_HOST\" '.[] | select(.hostname==$PROM_HOST)\ \ | .addr')\nALERTM_IP=$(ceph orch host ls -f json | jq -r --arg ALERTM_HOST\ \ \"$ALERTM_HOST\" '.[] | select(.hostname==$ALERTM_HOST) | .addr')\n# check\ \ each host node-exporter metrics endpoint is responsive\nALL_HOST_IPS=$(ceph\ \ orch host ls -f json | jq -r '.[] | .addr')\nfor ip in $ALL_HOST_IPS; do\n\ \ curl -s http://${ip}:9100/metric\ndone\n# check grafana endpoints are responsive\ \ and database health is okay\ncurl -k -s https://${GRAFANA_IP}:3000/api/health\n\ curl -k -s https://${GRAFANA_IP}:3000/api/health | jq -e '.database == \"ok\"\ '\n# stop mon daemon in order to trigger an alert\nceph orch daemon stop $MON_DAEMON\n\ sleep 120\n# check prometheus endpoints are responsive and mon down alert is\ \ firing\ncurl -s http://${PROM_IP}:9095/api/v1/status/config\ncurl -s http://${PROM_IP}:9095/api/v1/status/config\ \ | jq -e '.status == \"success\"'\ncurl -s http://${PROM_IP}:9095/api/v1/alerts\n\ curl -s http://${PROM_IP}:9095/api/v1/alerts | jq -e '.data | .alerts | .[]\ \ | select(.labels | .alertname == \"CephMonDown\") | .state == \"firing\"'\n\ # check alertmanager endpoints are responsive and mon down alert is active\n\ curl -s http://${ALERTM_IP}:9093/api/v2/status\ncurl -s http://${ALERTM_IP}:9093/api/v2/alerts\n\ curl -s http://${ALERTM_IP}:9093/api/v2/alerts | jq -e '.[] | select(.labels\ \ | .alertname == \"CephMonDown\") | .status | .state == \"active\"'\n" teuthology: fragments_dropped: [] meta: {} postmerge: [] teuthology_branch: reimage-ssh-down teuthology_sha1: 5f66ecfb34c0370410e78b3ee641753d19da653b timestamp: 2026-02-17_20:43:43 tube: trial user: yuriw verbose: true worker_log: /home/teuthworker/mnt/teuthology/worker_logs/dispatcher.trial.3682