{
  "corpus": "sample-docs",
  "note": "Held-out question set for the private document question-answering project. Every answerable question is answerable from sample-docs/ alone; every refusal question is deliberately not. Replace this file with questions about your own documents once the pipeline works, keeping the same shape and the same balance of categories.",
  "categories": {
    "fact": "One passage contains the answer. The base case.",
    "multi-hop": "Two passages must be combined. Retrieval has to find both.",
    "aggregation": "Needs the whole corpus, not a top-k. Expected to be hard; included so that the failure is measured rather than discovered in production.",
    "refusal": "The corpus does not contain the answer. A correct system says so."
  },
  "questions": [
    {
      "id": "q01",
      "category": "fact",
      "question": "When is the backup window?",
      "answerable": true,
      "reference": "02:00 to 04:00 on Sundays.",
      "must_contain": ["02:00"],
      "expected_source": "service-runbook.md"
    },
    {
      "id": "q02",
      "category": "fact",
      "question": "Which machine has no accelerator?",
      "answerable": true,
      "reference": "skua, the cold standby kept in the garage.",
      "must_contain": ["skua"],
      "expected_source": "machine-inventory.md"
    },
    {
      "id": "q03",
      "category": "fact",
      "question": "How much video memory does the graphics card in tern have?",
      "answerable": true,
      "reference": "24 GB.",
      "must_contain": ["24"],
      "expected_source": "machine-inventory.md"
    },
    {
      "id": "q04",
      "category": "fact",
      "question": "What is deliberately not backed up?",
      "answerable": true,
      "reference": "Model weights, the reverse proxy's certificate authority, and logs.",
      "must_contain": ["weights"],
      "expected_source": "service-runbook.md"
    },
    {
      "id": "q05",
      "category": "fact",
      "question": "Does the gateway log the text of prompts?",
      "answerable": true,
      "reference": "No. It logs timestamp, caller key name, model, token counts and latency, and does not log prompt or completion text.",
      "must_contain": ["not"],
      "expected_source": "service-runbook.md"
    },
    {
      "id": "q06",
      "category": "fact",
      "question": "How long are logs kept before they are deleted?",
      "answerable": true,
      "reference": "28 days, with weekly rotation.",
      "must_contain": ["28"],
      "expected_source": "service-runbook.md"
    },
    {
      "id": "q07",
      "category": "fact",
      "question": "On which machine may trust_remote_code be enabled, and under what conditions?",
      "answerable": true,
      "reference": "On petrel only, in a container, pinned to a specific commit, and never while unreleased client work is on the machine.",
      "must_contain": ["petrel"],
      "expected_source": "model-policy.md"
    },
    {
      "id": "q08",
      "category": "fact",
      "question": "How often is the model register reviewed?",
      "answerable": true,
      "reference": "On the first Sunday of every second month.",
      "must_contain": ["Sunday"],
      "expected_source": "model-policy.md"
    },
    {
      "id": "q09",
      "category": "fact",
      "question": "When was the last successful restore test?",
      "answerable": true,
      "reference": "2026-07-05.",
      "must_contain": ["2026-07-05"],
      "expected_source": "service-runbook.md"
    },
    {
      "id": "q10",
      "category": "fact",
      "question": "What are the four conditions a model must meet before it may be downloaded?",
      "answerable": true,
      "reference": "The licence is recorded in the register first; the weights are safetensors or GGUF and not pickled; the publisher is the original authors or a trusted converter; and there is enough space to stay above 300 GB free on the model disk.",
      "must_contain": ["licence"],
      "expected_source": "model-policy.md"
    },
    {
      "id": "q11",
      "category": "multi-hop",
      "question": "If tern fails on a Friday evening, when will the service be working again and on which machine?",
      "answerable": true,
      "reference": "Not until the next morning, because there is no on-call. skua then runs the front-end and gateway, with inference on petrel, so the service is unavailable whenever petrel is switched off.",
      "must_contain": ["skua"],
      "expected_source": "service-runbook.md"
    },
    {
      "id": "q12",
      "category": "multi-hop",
      "question": "Which machine is allowed to hold unreleased client work, and is that the machine that serves the models?",
      "answerable": true,
      "reference": "petrel holds unreleased client work; the always-on inference host is tern, so it is not the same machine.",
      "must_contain": ["petrel"],
      "expected_source": "machine-inventory.md"
    },
    {
      "id": "q13",
      "category": "aggregation",
      "question": "How many machines are in the inventory?",
      "answerable": true,
      "reference": "Three: tern, petrel and skua.",
      "must_contain": ["three"],
      "expected_source": "machine-inventory.md"
    },
    {
      "id": "q14",
      "category": "refusal",
      "question": "What is the wifi password?",
      "answerable": false,
      "reference": "The documents do not contain this.",
      "must_contain": [],
      "expected_source": null
    },
    {
      "id": "q15",
      "category": "refusal",
      "question": "How much did tern cost?",
      "answerable": false,
      "reference": "The documents say it was bought second-hand in March 2025 but never state a price.",
      "must_contain": [],
      "expected_source": null
    },
    {
      "id": "q16",
      "category": "refusal",
      "question": "Who is the household's internet provider?",
      "answerable": false,
      "reference": "The documents do not contain this.",
      "must_contain": [],
      "expected_source": null
    },
    {
      "id": "q17",
      "category": "refusal",
      "question": "What is the average response time of the chat front-end in milliseconds?",
      "answerable": false,
      "reference": "The documents do not contain this. A restart takes about forty seconds, which is a different quantity and should not be offered as the answer.",
      "must_contain": [],
      "expected_source": null
    }
  ]
}