#!/usr/bin/env bash # Purpose: prove the gateway is doing its job - liveness, readiness, the three model # aliases, an OpenAI-shaped call, an embedding, an Anthropic-shaped call, and a # rejected anonymous request - and print a pass or fail line for each # Platform: all (spark, strix, mac, nvidia) # Minimum memory: 8 GB, which is what the models behind the gateway need # Assumes: curl on PATH, the gateway running, and a .env beside this script holding # LITELLM_MASTER_KEY. The key is read from the environment and never printed. # The first model call loads a model, so allow a minute for it on a cold start. set -euo pipefail HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" ENV_FILE="${ENV_FILE:-${HERE}/.env}" if [ -f "$ENV_FILE" ]; then set -a # shellcheck source=/dev/null . "$ENV_FILE" set +a fi : "${GATEWAY_HOST:=127.0.0.1}" : "${GATEWAY_PORT:=4000}" BASE="http://${GATEWAY_HOST}:${GATEWAY_PORT}" KEY="${LITELLM_MASTER_KEY:-}" if [ -z "$KEY" ]; then echo "LITELLM_MASTER_KEY is not set. Put it in ${ENV_FILE} or export it." >&2 exit 1 fi if ! command -v curl >/dev/null 2>&1; then echo "curl is not on PATH." >&2 exit 1 fi PASSES=0 FAILURES=0 report() { # report [detail] if [ "$2" -eq 0 ]; then PASSES=$((PASSES + 1)) printf ' PASS %s\n' "$1" else FAILURES=$((FAILURES + 1)) printf ' FAIL %s%s\n' "$1" "${3:+ (${3})}" fi } check_contains() { # check_contains if printf '%s' "$2" | grep -q -- "$3"; then report "$1" 0 else report "$1" 1 "did not contain ${3}" fi } echo "==> checking ${BASE}" # 1. Liveness: the process is up. No dependencies are checked. if curl -sf "${BASE}/health/liveliness" >/dev/null; then report "liveness endpoint answers" 0 else report "liveness endpoint answers" 1 "is the gateway running?" echo " Nothing else can pass while the gateway is down. Stopping here." >&2 exit 1 fi # 2. Readiness: the worker is ready to accept traffic. if curl -sf "${BASE}/health/readiness" >/dev/null; then report "readiness endpoint answers" 0 else report "readiness endpoint answers" 1 "check the database connection on the container path" fi # 3. The three aliases every later part of the course expects. MODELS_BODY="$(curl -sf -H "Authorization: Bearer ${KEY}" "${BASE}/v1/models" || true)" for alias_name in "local/chat" "local/coder" "local/embed"; do check_contains "alias ${alias_name} is published" "$MODELS_BODY" "$alias_name" done # 4. An OpenAI-shaped chat completion. This is the call that loads a model, so it is the # slow one on a cold start. CHAT_BODY="$(curl -sf --max-time 300 "${BASE}/v1/chat/completions" \ -H "Authorization: Bearer ${KEY}" \ -H "Content-Type: application/json" \ -d '{"model":"local/chat", "messages":[{"role":"user","content":"Reply with one word: ready"}], "max_tokens":8}' || true)" check_contains "OpenAI-shaped chat completion returns content" "$CHAT_BODY" '"content"' # 5. An embedding, which proves the second engine is reachable and the router picked it. EMBED_BODY="$(curl -sf --max-time 300 "${BASE}/v1/embeddings" \ -H "Authorization: Bearer ${KEY}" \ -H "Content-Type: application/json" \ -d '{"model":"local/embed","input":"a sentence to embed"}' || true)" check_contains "embeddings endpoint returns a vector" "$EMBED_BODY" '"embedding"' # 6. The Anthropic-shaped surface, over the same alias and the same engine. MSG_BODY="$(curl -sf --max-time 300 "${BASE}/v1/messages" \ -H "x-api-key: ${KEY}" \ -H "anthropic-version: 2023-06-01" \ -H "Content-Type: application/json" \ -d '{"model":"local/chat", "messages":[{"role":"user","content":"Reply with one word: ready"}], "max_tokens":8}' || true)" check_contains "Anthropic-shaped messages endpoint answers" "$MSG_BODY" '"content"' # 7. The gateway must refuse an unauthenticated request. A gateway that answers without a # key is not a gateway. ANON_STATUS="$(curl -s -o /dev/null -w '%{http_code}' "${BASE}/v1/chat/completions" \ -H "Content-Type: application/json" \ -d '{"model":"local/chat","messages":[{"role":"user","content":"hello"}],"max_tokens":4}' || true)" if [ "$ANON_STATUS" = "401" ] || [ "$ANON_STATUS" = "403" ]; then report "anonymous request is refused" 0 else report "anonymous request is refused" 1 "got HTTP ${ANON_STATUS}" fi echo "" echo " ${PASSES} passed, ${FAILURES} failed" if [ "$FAILURES" -gt 0 ]; then echo " See gateway-logs/ on the native path, or 'docker compose logs' on the" >&2 echo " container path, for what the router and the engines actually said." >&2 exit 1 fi