#!/usr/bin/env bash
# Purpose: prove the monitoring stack is doing its job - Prometheus healthy, the rules
#          loaded, at least one engine target up, the gateway target up, a GPU exporter
#          answering, and Grafana serving the provisioned dashboard - with one pass or
#          fail line for each
# Platform: all (spark, strix, mac, nvidia)
# Minimum memory: 8 GB, which is what the service being watched needs
# Assumes: curl on PATH, the stack from compose-monitoring.yaml running, and a .env beside
#          this script. Nothing is printed that would reveal the Grafana password: the
#          Grafana check uses its unauthenticated health endpoint only.
set -euo pipefail

HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
ENV_FILE="${ENV_FILE:-${HERE}/.env}"

if [ -f "$ENV_FILE" ]; then
    set -a
    # shellcheck source=/dev/null
    . "$ENV_FILE"
    set +a
fi

: "${MONITORING_HOST:=127.0.0.1}"
: "${PROMETHEUS_PORT:=9090}"
: "${GRAFANA_PORT:=3000}"

PROM="http://${MONITORING_HOST}:${PROMETHEUS_PORT}"
GRAFANA="http://${MONITORING_HOST}:${GRAFANA_PORT}"

command -v curl >/dev/null 2>&1 || { echo "curl is not on PATH." >&2; exit 1; }

PASSES=0
FAILURES=0
SKIPS=0

skip() {
    # skip <name> <reason> - a check that could not be made, which is not a failure
    SKIPS=$((SKIPS + 1))
    printf '  SKIP  %s  (%s)\n' "$1" "$2"
}

report() {
    # report <name> <ok:0|1> [detail]
    if [ "$2" -eq 0 ]; then
        PASSES=$((PASSES + 1))
        printf '  PASS  %s\n' "$1"
    else
        FAILURES=$((FAILURES + 1))
        printf '  FAIL  %s%s\n' "$1" "${3:+  (${3})}"
    fi
}

# Ask Prometheus a question and return the first sample value, or an empty string.
# The instant-query response puts each sample as "value":[<timestamp>,"<value>"], and the
# value is the quoted second element. No jq, because this has to run on a bare machine.
promql() {
    curl -sf --get "${PROM}/api/v1/query" --data-urlencode "query=$1" 2>/dev/null \
        | grep -oE '"value":\[[0-9.eE+-]+,"[^"]+"' \
        | head -n 1 \
        | sed -e 's/.*,"//' -e 's/"$//' || true
}

# Count the series a query returns, which is the only thing most checks here need.
count_series() {
    curl -sf --get "${PROM}/api/v1/query" --data-urlencode "query=$1" 2>/dev/null \
        | grep -o '"metric"' | wc -l | tr -d ' '
}

echo "==> checking ${PROM}"

if curl -sf "${PROM}/-/healthy" >/dev/null; then
    report "Prometheus is healthy" 0
else
    report "Prometheus is healthy" 1 "is the stack up?"
    echo "  Nothing else can pass while Prometheus is down. Stopping here." >&2
    exit 1
fi

RULES="$(curl -sf "${PROM}/api/v1/rules" || true)"
for rule in GatewayUnreachable RequestsQueueing AcceleratorMemoryHigh; do
    if printf '%s' "$RULES" | grep -q -- "$rule"; then
        report "alert rule ${rule} is loaded" 0
    else
        report "alert rule ${rule} is loaded" 1 "check rule_files and the container mount"
    fi
done

# The gateway from Part 9. If this is down, the dashboard has no request data at all.
if [ "$(promql 'up{job="gateway"}')" = "1" ]; then
    report "gateway target is up" 0
else
    report "gateway target is up" 1 "is the Part 9 gateway running on this machine?"
fi

# At least one engine, whichever it is. A machine only needs one for the dashboard to work.
ENGINES_UP="$(count_series 'up{job=~"llama-server|vllm|sglang"} == 1')"
if [ "${ENGINES_UP:-0}" -ge 1 ]; then
    report "at least one engine target is up" 0
else
    report "at least one engine target is up" 1 "llama-server needs --metrics to publish anything"
fi

# One GPU exporter, whichever your track uses.
GPU_SERIES="$(count_series 'DCGM_FI_DEV_POWER_USAGE or local_llm_accelerator_power_watts')"
if [ "${GPU_SERIES:-0}" -ge 1 ]; then
    report "a GPU or accelerator exporter is publishing power" 0
else
    report "a GPU or accelerator exporter is publishing power" 1 "run your track's exporter"
fi

MEM_SERIES="$(count_series 'DCGM_FI_DEV_FB_USED or local_llm_accelerator_memory_used_bytes')"
if [ "${MEM_SERIES:-0}" -ge 1 ]; then
    report "accelerator memory is being recorded" 0
else
    report "accelerator memory is being recorded" 1 "the memory panel will stay empty"
fi

# The host itself, which is what tells you the machine ran out of memory rather than the
# engine deciding not to allocate.
if [ "$(promql 'up{job="node"}')" = "1" ]; then
    report "node exporter target is up" 0
else
    report "node exporter target is up" 1 "host memory and disk will be missing"
fi

echo "==> checking ${GRAFANA}"

if curl -sf "${GRAFANA}/api/health" >/dev/null; then
    report "Grafana is healthy" 0
else
    report "Grafana is healthy" 1 "check the container log for a provisioning error"
fi

# This endpoint needs a login, so it is only checked when the admin password is available
# from .env. The password is passed to curl and never printed.
if [ -n "${GRAFANA_ADMIN_PASSWORD:-}" ]; then
    if curl -sf -u "admin:${GRAFANA_ADMIN_PASSWORD}" \
        "${GRAFANA}/api/dashboards/uid/local-llm-service" >/dev/null 2>&1; then
        report "the provisioned dashboard is present" 0
    else
        report "the provisioned dashboard is present" 1 \
            "check the Grafana container log for a provisioning error"
    fi
else
    skip "the provisioned dashboard is present" \
        "GRAFANA_ADMIN_PASSWORD is not set; open the dashboard in a browser instead"
fi

echo ""
echo "    ${PASSES} passed, ${FAILURES} failed, ${SKIPS} skipped"
if [ "$FAILURES" -gt 0 ]; then
    echo "    Open ${PROM}/targets to see which scrape jobs are failing and why." >&2
    exit 1
fi