{
  "name": "Part 26 agent task suite",
  "version": "1.0.0",
  "comment": "Fifteen tasks with checkable outcomes, over the three course-authored documents from Part 10's sample-docs (machine-inventory.md, model-policy.md, service-runbook.md). Copy those three files into the agent's workspace, or point the agent at the Part 10 index built from them, or both. Every expected answer is stated in one of those documents, so nothing here can be answered from a model's own memory, and every check is a string, a regular expression, a tool name or a step count rather than a judgement. The suite is deliberately small: it detects a large difference between two scaffolds and cannot detect a small one.",
  "workspace": "the three documents from src/labs/part-10-models-at-work/sample-docs/",
  "tasks": [
    {
      "id": "r-backup-window",
      "category": "retrieval",
      "task": "During which hours, and on which day of the week, does the backup window run? Answer with the times and the day.",
      "expect_in_answer": ["02:00", "04:00", "sunday"],
      "max_steps": 6,
      "notes": "One passage in the runbook. The shortest possible successful trajectory: one search, one finish."
    },
    {
      "id": "r-gateway-port",
      "category": "retrieval",
      "task": "Which port does the model gateway listen on? Answer with the number.",
      "expect_in_answer": ["4000"],
      "max_steps": 6,
      "notes": "The runbook's service table. Four ports appear in it, so a wrong answer is a retrieval or reading failure rather than a refusal."
    },
    {
      "id": "r-log-retention",
      "category": "retrieval",
      "task": "How many days are gateway logs kept before they are deleted?",
      "expect_in_answer": ["28"],
      "max_steps": 6,
      "notes": "Tests whether a number survives the trip from a passage into an answer."
    },
    {
      "id": "r-not-backed-up",
      "category": "retrieval",
      "task": "Name one thing the runbook says is deliberately not backed up, and say why.",
      "expect_regex": "(model weights|certificate authority|logs)",
      "max_steps": 6,
      "notes": "Three acceptable answers, so the check is a regular expression rather than a fixed string."
    },
    {
      "id": "r-client-work",
      "category": "retrieval",
      "task": "Which machine is the only one allowed to hold unreleased client work?",
      "expect_in_answer": ["petrel"],
      "expect_not_in_answer": ["tern", "skua"],
      "max_steps": 6,
      "notes": "The forbidden strings catch an answer that hedges by naming every machine."
    },
    {
      "id": "r-restore-test",
      "category": "retrieval",
      "task": "On what date was the last successful restore test?",
      "expect_in_answer": ["2026-07-05"],
      "max_steps": 6,
      "notes": "A date that appears once. A model answering from its own memory cannot produce it."
    },
    {
      "id": "t-list-workspace",
      "category": "tools",
      "task": "List the file names in your workspace. Give every name in your final answer.",
      "expect_in_answer": ["machine-inventory.md", "model-policy.md", "service-runbook.md"],
      "expect_tool": "run_command",
      "max_steps": 5,
      "notes": "Checks that the agent reaches for a command rather than guessing at a directory listing."
    },
    {
      "id": "t-read-review-date",
      "category": "tools",
      "task": "Read the file model-policy.md and give the date on which it was last reviewed.",
      "expect_in_answer": ["2026-09-01"],
      "expect_tool": "read_file",
      "max_steps": 5,
      "notes": "Named file, named field. Failure here is usually a path the file tool rejected."
    },
    {
      "id": "t-grep-port",
      "category": "tools",
      "task": "Search the documents for the port number 8090 and say which service listens on it.",
      "expect_in_answer": ["document index"],
      "max_steps": 6,
      "notes": "Either search_documents or a grep through run_command is a valid route, so no tool is required."
    },
    {
      "id": "t-count-machines",
      "category": "tools",
      "task": "How many machines are described in the machine inventory? Answer with the number and name them.",
      "expect_in_answer": ["3", "tern", "petrel", "skua"],
      "max_steps": 7,
      "notes": "Counting requires reading the whole document rather than the first matching passage."
    },
    {
      "id": "x-escalation",
      "category": "reasoning",
      "task": "If tern becomes unrecoverable, which machine runs the front end and gateway, and which machine serves inference?",
      "expect_in_answer": ["skua", "petrel"],
      "max_steps": 8,
      "notes": "Two hops: the runbook names the escalation, the inventory says why skua cannot serve a model."
    },
    {
      "id": "x-window-length",
      "category": "reasoning",
      "task": "How many hours long is the backup window? Answer with the number of hours.",
      "expect_regex": "\\b(two|2)\\b",
      "max_steps": 6,
      "notes": "Retrieval plus one subtraction. The regular expression accepts the word or the digit."
    },
    {
      "id": "x-pickled-checkpoint",
      "category": "reasoning",
      "task": "A model you want is published only as a pickled checkpoint. Under the model policy, may it be downloaded? Answer yes or no, and give the rule.",
      "expect_in_answer": ["no"],
      "expect_regex": "(pickled|safetensors|GGUF)",
      "max_steps": 7,
      "notes": "Applies a written rule to a case the document does not state directly."
    },
    {
      "id": "x-untrusted-machine",
      "category": "reasoning",
      "task": "Which machine is treated as physically untrusted, and what is the reason given?",
      "expect_in_answer": ["skua", "garage"],
      "max_steps": 7,
      "notes": "The reason and the conclusion are in different sections of the inventory."
    },
    {
      "id": "x-refusal",
      "category": "reasoning",
      "task": "What is the wireless network password for the house?",
      "expect_regex": "(cannot|can't|not (?:in|stated|recorded|available|given)|no (?:answer|information|record)|does not (?:say|contain|state))",
      "expect_not_in_answer": ["password is"],
      "max_steps": 8,
      "notes": "The documents do not contain this. A confident invented answer fails; saying so passes. This is the single most informative task in the suite."
    }
  ]
}