{
  "name": "Part 27 trajectory collection set",
  "version": "1.0.0",
  "comment": "Sixteen tasks over the same three course-authored documents Part 26's suite uses (machine-inventory.md, model-policy.md and service-runbook.md from src/labs/part-10-models-at-work/sample-docs/), asking different questions on purpose. This is the set you COLLECT trajectories from; Part 26's agent-tasks.json is the set you MEASURE with, and the two must not overlap or the measurement is of memorisation. trajectories-to-sft.py checks that for you and will tell you if you drift. Replace these with tasks over your own repository as soon as the pipeline runs end to end: a fine-tune on questions about three sample documents improves a model at answering questions about three sample documents, which is not what you want from it.",
  "workspace": "the three documents from src/labs/part-10-models-at-work/sample-docs/",
  "tasks": [
    {
      "id": "c-front-end-port",
      "category": "retrieval",
      "task": "On which port is the chat front-end published to the house network? Give the number.",
      "expect_in_answer": ["443"],
      "max_steps": 6,
      "notes": "The runbook's service table lists four ports, so a wrong number is a reading failure rather than a refusal."
    },
    {
      "id": "c-restart-seconds",
      "category": "retrieval",
      "task": "About how long does restarting one service take, and what accounts for most of that time?",
      "expect_regex": "(forty|40)",
      "expect_in_answer": ["model"],
      "max_steps": 6,
      "notes": "Two facts from one sentence. A trajectory that answers half of it is a useful negative example."
    },
    {
      "id": "c-tern-model-disk",
      "category": "retrieval",
      "task": "How much NVMe storage does the always-on inference host set aside for models?",
      "expect_in_answer": ["2 tb"],
      "max_steps": 6,
      "notes": "The agent has to work out which machine is the always-on host before it can answer."
    },
    {
      "id": "c-petrel-memory",
      "category": "retrieval",
      "task": "How much system memory does the workstation have?",
      "expect_in_answer": ["128"],
      "max_steps": 6,
      "notes": "Three memory figures appear in the inventory, so the wrong machine gives the wrong number."
    },
    {
      "id": "c-skua-accelerator",
      "category": "retrieval",
      "task": "What accelerator does the cold-standby machine have?",
      "expect_regex": "(none|no accelerator)",
      "max_steps": 6,
      "notes": "An answer of 'none' is correct and short, which is exactly the shape a small model tends to pad."
    },
    {
      "id": "c-free-space-floor",
      "category": "retrieval",
      "task": "The download rules set a floor on free space. What is the floor, and on which disk?",
      "expect_in_answer": ["300"],
      "max_steps": 6,
      "notes": "One clause of a four-part rule."
    },
    {
      "id": "c-register-cadence",
      "category": "reasoning",
      "task": "How often is the model register reviewed, and what happens at a review to a model nobody has used?",
      "expect_regex": "(second month|every second)",
      "expect_in_answer": ["delete"],
      "max_steps": 8,
      "notes": "Two sentences that sit together. The exception clause is the trap."
    },
    {
      "id": "c-remote-code",
      "category": "reasoning",
      "task": "A model you want requires custom code to load. Where may it be run, and under what conditions?",
      "expect_in_answer": ["petrel", "container"],
      "expect_not_in_answer": ["tern"],
      "max_steps": 8,
      "notes": "The forbidden string catches an answer that lists both machines to be safe."
    },
    {
      "id": "c-standing-exception",
      "category": "reasoning",
      "task": "The model policy records exactly one standing exception. What is it, and what has to happen afterwards?",
      "expect_regex": "(speech|transcription)",
      "expect_in_answer": ["delete"],
      "max_steps": 8,
      "notes": "The word 'exception' appears twice in the document with different meanings."
    },
    {
      "id": "c-retire-after",
      "category": "retrieval",
      "task": "After how long without being powered on is a machine retired?",
      "expect_regex": "(six months|6 months)",
      "max_steps": 6,
      "notes": "A single fact at the end of a document, which is where retrieval tends to miss."
    },
    {
      "id": "c-oncall",
      "category": "reasoning",
      "task": "Who is on call overnight, and what is the stated reason for that arrangement?",
      "expect_regex": "(no one|nobody|no on-call|there is no)",
      "max_steps": 8,
      "notes": "The correct answer is a negative. Models that pattern-match on 'on-call' invent a rota."
    },
    {
      "id": "c-log-fields",
      "category": "retrieval",
      "task": "Name two of the fields the gateway writes for each request, and say what it does not write.",
      "expect_regex": "(timestamp|token|latency|model)",
      "expect_in_answer": ["not"],
      "max_steps": 6,
      "notes": "The second half matters more than the first: the document is explicit about what is not logged."
    },
    {
      "id": "c-external-drive",
      "category": "retrieval",
      "task": "How often is a copy of the backup taken off the machine it was written on?",
      "expect_regex": "(first sunday|monthly|each month|every month)",
      "max_steps": 6,
      "notes": "Two 'first Sunday' rules exist in different documents; only one is about backups."
    },
    {
      "id": "c-front-end-stop",
      "category": "reasoning",
      "task": "Why is the front-end stopped during the backup window, and for roughly how long?",
      "expect_regex": "(sqlite|database)",
      "expect_in_answer": ["ninety"],
      "max_steps": 8,
      "notes": "Cause and duration from one paragraph."
    },
    {
      "id": "c-grep-runlab",
      "category": "tools",
      "task": "Search the documents for the command that starts every service, and give the command exactly as written.",
      "expect_in_answer": ["runlab up"],
      "max_steps": 6,
      "notes": "A search task whose answer is a literal string, so a paraphrase fails the check."
    },
    {
      "id": "c-count-md",
      "category": "tools",
      "task": "List the markdown files in your workspace and say how many of them there are.",
      "expect_regex": "(three|3)",
      "expect_in_answer": ["service-runbook.md"],
      "max_steps": 6,
      "notes": "The shortest successful trajectory is one listing and one finish, which makes it a good check that the collection loop works at all."
    }
  ]
}