{ "name": "personal-eval", "version": 1, "note": "The template for your own evaluation set. Fifteen course-authored tasks are supplied as worked examples of each category; replace them with twenty to fifty tasks drawn from work you actually do. Every task needs a prompt, a reference answer and a rubric. Deterministic checks (must_contain, must_not_contain, max_words) are scored without a model and are the cheapest signal in the file, so use them wherever the requirement can be expressed mechanically.", "settings": { "temperature": 0.0, "top_p": 1.0, "seed": 7, "max_tokens": 512, "comment": "Fixed for every run. Change them only deliberately, and record the change: two runs at different settings are not comparable." }, "categories": { "format": "Does it produce the shape you asked for, every time?", "extraction": "Can it pull stated facts out of text without inventing any?", "reasoning": "Can it carry a short chain of arithmetic or logic to a definite answer?", "code": "Does it write code that would run, in the language and style asked for?", "refusal": "Does it decline when the answer is unknowable or the premise is false?", "summarise": "Can it compress a passage without adding to it?", "language": "Does it hold a non-English requirement and leave names alone?", "instruction": "Does it hold several constraints at once?" }, "tasks": [ { "id": "t01", "category": "format", "prompt": "List the five stages of one training-loop iteration. Reply with exactly five lines, each starting with a hyphen and a space, each under eight words. No preamble, no closing sentence.", "reference": "- Take the next batch\n- Forward pass to predictions\n- Compute the loss\n- Backward pass for gradients\n- Step the parameters", "rubric": "Five hyphen-prefixed lines, nothing else, each under eight words, naming batch, forward, loss, backward and step in that order.", "must_contain": ["loss"], "must_not_contain": ["Here", "Sure"], "max_words": 45 }, { "id": "t02", "category": "extraction", "prompt": "From this line, return JSON with keys host, port, status and ms, and nothing else:\n2026-08-14T09:12:03Z tern:8080 answered 200 in 431 ms", "reference": "{\"host\": \"tern\", \"port\": 8080, \"status\": 200, \"ms\": 431}", "rubric": "A single JSON object with exactly those four keys and those four values. Any prose outside the object is a failure.", "must_contain": ["431", "8080"], "must_not_contain": ["```"], "max_words": 40 }, { "id": "t03", "category": "instruction", "prompt": "In one sentence, explain what a key-value cache stores during decoding. Do not use the words 'model', 'AI' or 'token'.", "reference": "It stores the attention keys and values already computed for the earlier positions of the sequence, so they are not recomputed at every step.", "rubric": "One sentence; correct account of caching earlier keys and values to avoid recomputation; none of the three banned words appears in any form.", "must_contain": ["key"], "must_not_contain": ["model", "AI", "token"], "max_words": 45 }, { "id": "t04", "category": "reasoning", "prompt": "A model must read 4.2 GB of weights for every output word. The machine moves 210 GB per second from memory. Ignoring everything else, how many output words per second is the ceiling? Give the number and one line of working.", "reference": "50. 210 divided by 4.2 is 50.", "rubric": "The answer 50, with the division 210 / 4.2 shown. Any other number is wrong regardless of how the working reads.", "must_contain": ["50"], "must_not_contain": [], "max_words": 60 }, { "id": "t05", "category": "code", "prompt": "Write a Python function `nth_word(text, n)` returning the nth whitespace-separated word of text, counting from 1, and returning None when there are fewer than n words. No explanation, code only.", "reference": "def nth_word(text, n):\n words = text.split()\n return words[n - 1] if 1 <= n <= len(words) else None", "rubric": "A function of the right name and signature; one-based indexing; returns None rather than raising when out of range; no prose around the code.", "must_contain": ["def nth_word", "None"], "must_not_contain": [], "max_words": 90 }, { "id": "t06", "category": "refusal", "prompt": "What was the exact wall-clock time of the third request my server handled yesterday?", "reference": "That cannot be answered: I have no access to your server or its logs.", "rubric": "States plainly that the information is not available to it. Any invented time, or any confident answer, is a failure. Offering how the reader could find out is fine and not required.", "must_contain": [], "must_not_contain": [], "max_words": 60 }, { "id": "t07", "category": "summarise", "prompt": "Summarise the passage in at most 25 words. Add nothing that is not stated.\n\nThe garage machine, skua, has never served a model. It has 32 GB of memory, no accelerator, and is kept powered down. If the main host fails, skua runs only the front-end and the gateway, and the work of generating text moves to the workstation, which is switched off most nights. The household was told before the service opened that there is no overnight cover.", "reference": "skua is a powered-down spare with no accelerator; on failover it runs only the front-end and gateway, with generation moving to the workstation, and there is no overnight cover.", "rubric": "At most 25 words. Must convey that skua has no accelerator and is a standby, and must not add facts absent from the passage.", "must_contain": ["skua"], "must_not_contain": [], "max_words": 28 }, { "id": "t08", "category": "extraction", "prompt": "Classify this message as exactly one of: bug, question, feature-request, documentation, other. Reply with the single label and nothing else.\n\n\"After the upgrade, long conversations return an empty reply once they pass about eight thousand tokens. Nothing in the log looks wrong.\"", "reference": "bug", "rubric": "The single word 'bug'. Any additional prose, punctuation or explanation is a failure of the format even if the label is right.", "must_contain": ["bug"], "must_not_contain": ["question", "feature"], "max_words": 3 }, { "id": "t09", "category": "language", "prompt": "Translate into French. Leave the product name 'llama-server' exactly as written: \"Start llama-server on port 8080 and check the log before opening the firewall.\"", "reference": "Démarrez llama-server sur le port 8080 et vérifiez le journal avant d'ouvrir le pare-feu.", "rubric": "Fluent French, correct meaning, and the string llama-server unchanged and untranslated. Translating or hyphenating the product name differently is a failure.", "must_contain": ["llama-server"], "must_not_contain": [], "max_words": 40 }, { "id": "t10", "category": "extraction", "prompt": "Here are five asset records. Which host has the asset tag ending in 341?\n\npetrel RL-0117\nskua RL-0223\ntern RL-0341\nprinter RL-0009\ndoorbell RL-0055\n\nAnswer with the host name only.", "reference": "tern", "rubric": "The single word 'tern'. Naming any other host is a retrieval failure; adding a sentence is a format failure.", "must_contain": ["tern"], "must_not_contain": ["petrel", "skua"], "max_words": 3 }, { "id": "t11", "category": "instruction", "prompt": "Rewrite this so it could be sent to a colleague, keeping every fact and the same level of urgency: \"Whoever restarted the gateway during a request obviously did not read the runbook and has broken the front-end again.\"", "reference": "The gateway was restarted during a request, which has broken the front-end again. The runbook asks us not to restart it while the front-end is serving; could we check before restarting next time?", "rubric": "Same facts: gateway restarted mid-request, front-end broken again, runbook covers it. Blame removed but urgency kept. Inventing a cause or softening the problem itself is a failure.", "must_contain": ["gateway"], "must_not_contain": ["obviously"], "max_words": 70 }, { "id": "t12", "category": "format", "prompt": "Reply with a JSON object with exactly the keys \"answer\" (a string) and \"confidence\" (a number between 0 and 1). The question is: how many bytes are in a kibibyte?", "reference": "{\"answer\": \"1024\", \"confidence\": 0.99}", "rubric": "Valid JSON, exactly two keys with those names, answer conveying 1024, confidence a number in range. A code fence around it is a failure.", "must_contain": ["1024", "confidence"], "must_not_contain": ["```"], "max_words": 30 }, { "id": "t13", "category": "instruction", "prompt": "Write three sentences about backups. Constraints: British spelling; no sentence longer than twelve words; do not use the word 'data'; the third sentence must be a question.", "reference": "Backups protect what cannot be downloaded again. Test a restore before you trust the archive. When did you last restore one into a scratch volume?", "rubric": "Exactly three sentences; each twelve words or fewer; the word 'data' absent; the third sentence ends in a question mark; British spelling throughout.", "must_contain": ["?"], "must_not_contain": ["data"], "max_words": 40 }, { "id": "t14", "category": "format", "prompt": "Turn these into a Markdown table with the headers Service, Host and Port, in this order, and output nothing else.\n\nchat front-end on tern, port 443\ngateway on tern, port 4000\nengine on tern, port 8080", "reference": "| Service | Host | Port |\n| --- | --- | --- |\n| chat front-end | tern | 443 |\n| gateway | tern | 4000 |\n| engine | tern | 8080 |", "rubric": "A Markdown table with those three headers in that order and three data rows with the correct values. Any surrounding prose is a failure.", "must_contain": ["| Service | Host | Port |", "4000"], "must_not_contain": [], "max_words": 60 }, { "id": "t15", "category": "refusal", "prompt": "Since quantising a model to 4 bits always makes it faster on every machine, why do people still run 8-bit weights?", "reference": "The premise is not right: 4-bit is not faster on every machine. It reduces the bytes read per token, which helps where memory bandwidth is the limit, but on a machine limited by something else, or with a kernel that has no fast path for that format, it can be no faster or slower. People run 8-bit weights when the quality cost of 4-bit matters more than the speed, or when 4-bit is not faster there.", "rubric": "Must challenge the 'always faster on every machine' premise rather than accepting it. A good answer explains that the gain comes from fewer bytes read per token and is conditional on the bottleneck and the kernel. Accepting the premise and answering the question as asked is a failure however fluent the rest is.", "must_contain": [], "must_not_contain": [], "max_words": 160 } ] }