[
  {
    "prompt": "Sherlock Holmes lives at ",
    "expected": "221B Baker Street",
    "why": "A fact that appears many times in the corpus. If the from-scratch model has it, the corpus taught it; if not, the run was too short."
  },
  {
    "prompt": "The detective's companion and chronicler is Doctor ",
    "expected": "Watson",
    "why": "A single common token. The easiest item in the set, and the one to check first when everything scores badly."
  },
  {
    "prompt": "Holmes turned to me and said, \"My dear ",
    "expected": "Watson",
    "why": "The same fact reached through a stylistic pattern rather than a definition, which is a different thing to have learned."
  },
  {
    "prompt": "It is a capital mistake to theorise before one has ",
    "expected": "data",
    "why": "A memorable line from the corpus. Scoring well here is partly memorisation, which is worth saying out loud in the write-up."
  },
  {
    "prompt": "The capital of France is ",
    "expected": "Paris",
    "why": "A general fact that is probably not in a narrow corpus. The from-scratch model should do badly and the pretrained model should not, which is the whole point of the comparison."
  },
  {
    "prompt": "Water boils at one hundred degrees ",
    "expected": "Celsius",
    "why": "A second out-of-domain fact, so that one lucky guess does not decide the out-of-domain column."
  },
  {
    "prompt": "The opposite of hot is ",
    "expected": "cold",
    "why": "General language rather than general knowledge. Even a small model trained on English prose has a chance here, which separates two kinds of failure."
  },
  {
    "prompt": "If yesterday was Friday, then tomorrow will be ",
    "expected": "Sunday",
    "why": "A small piece of reasoning. Expect both models to fail at this scale; record it anyway, because it is the item that improves last."
  }
]