#!/usr/bin/env python3 """Turn several harness runs into one reportable line, with the spread and the settings attached. Purpose: read the result files lm-evaluation-harness wrote for repeated runs of the same suite, report every metric with its mean and its range across runs, attach the settings the harness recorded and the ones run-suite.sh recorded for it, and - when you give it a published claim - show the gap and the list of settings that could account for it. A single number from a single run is not a result; this script is what turns a pile of runs into one. Platform: all (pure Python, standard library only) Minimum memory: negligible Assumes: Python 3.9 or later and a directory written by run-suite.sh, containing one subdirectory per run with the harness's results_*.json files, plus run-context.json. Usage: python3 summarise-results.py --results suite-results --labbook labbook.md python3 summarise-results.py --results suite-results \ --published "ifeval:prompt_level_strict_acc=80.4" \ --published-by "Meta, Llama 3.1 8B Instruct model card" --labbook labbook.md """ from __future__ import annotations import argparse import json import platform import statistics import time from collections import defaultdict from pathlib import Path from typing import Optional # The settings that most often account for a gap between your number and somebody else's, in the # order it is cheapest to check them. Printed whenever a published claim is supplied. GAP_CHECKLIST = [ "Chat template: was it applied on both sides? An instruction-tuned model without it answers " "as if its training scaffolding were absent.", "Shot count: the task's default, the flag you passed, and whatever the publisher used are " "three different numbers until you check.", "Metric: one task reports several. ifeval reports four; gsm8k reports one per extraction " "filter. A published figure names one of them, sometimes only implicitly.", "Quantisation: you almost certainly served a quantised file and the publisher almost " "certainly measured the released weights.", "Sampling: greedy or sampled, and at what temperature. Some cards advise against greedy " "decoding for their own model.", "Thinking mode: on or off changes the answer, the length and the cost.", "Prompt construction: publishers frequently use their own prompt and their own extraction " "code rather than this harness.", "Items scored: a limited run is not comparable with a full one, and neither is a different " "version of the dataset.", ] def find_result_files(root: Path) -> list: return sorted(root.rglob("results_*.json")) def metric_rows(payload: dict) -> dict: """Flatten {task: {"metric,filter": value}} into {(task, metric, filter): value}.""" out = {} for task, metrics in (payload.get("results") or {}).items(): if not isinstance(metrics, dict): continue for key, value in metrics.items(): if not isinstance(value, (int, float)) or key == "alias": continue metric, _, filt = key.partition(",") if metric.endswith("_stderr"): continue out[(task, metric, filt or "none")] = float(value) return out def main() -> None: parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) parser.add_argument("--results", default="suite-results", help="directory written by run-suite.sh") parser.add_argument("--published", action="append", default=[], help="a claim to compare against, as task:metric=value; repeatable") parser.add_argument("--published-by", default=None, help="who published the claim and where, quoted in the report") parser.add_argument("--out", default=None) parser.add_argument("--labbook", default=None) args = parser.parse_args() root = Path(args.results) files = find_result_files(root) if not files: raise SystemExit(f"no results_*.json under {root}; run run-suite.sh first") context = {} context_file = root / "run-context.json" if context_file.exists(): context = json.loads(context_file.read_text(encoding="utf-8")) collected = defaultdict(list) harness_config: Optional[dict] = None for path in files: payload = json.loads(path.read_text(encoding="utf-8")) harness_config = payload.get("config") or harness_config for key, value in metric_rows(payload).items(): collected[key].append(value) if not collected: raise SystemExit("the result files contained no numeric metrics; check the harness output") print(f"\n{len(files)} result file(s) under {root}\n") header = ["Task", "Metric", "Filter", "Runs", "Mean", "Min", "Max", "Range"] print("| " + " | ".join(header) + " |") print("| " + " | ".join("---" for _ in header) + " |") summary_rows = [] for (task, metric, filt), values in sorted(collected.items()): mean = statistics.fmean(values) row = { "task": task, "metric": metric, "filter": filt, "runs": len(values), "mean": round(mean, 4), "min": round(min(values), 4), "max": round(max(values), 4), "range": round(max(values) - min(values), 4), "values": [round(v, 4) for v in values], } summary_rows.append(row) print(f"| {task} | {metric} | {filt} | {len(values)} | {mean:.4f} " f"| {min(values):.4f} | {max(values):.4f} | {max(values) - min(values):.4f} |") print("\nSettings recorded by the harness:") for key in ("model", "model_args", "num_fewshot", "batch_size", "limit", "gen_kwargs", "apply_chat_template", "fewshot_as_multiturn", "random_seed"): if harness_config and key in harness_config: print(f" {key}: {harness_config[key]}") if context: print("\nSettings the harness could not discover, recorded by run-suite.sh:") for key in ("alias", "quant", "engine", "engine_version", "endpoint", "limit"): if key in context: print(f" {key}: {context[key]}") # ---- comparison with a published claim ------------------------------- comparisons = [] for claim in args.published: try: target, _, value = claim.partition("=") task, _, metric = target.partition(":") published = float(value) except ValueError: raise SystemExit(f"could not read --published {claim!r}; use task:metric=value") matches = [r for r in summary_rows if r["task"] == task and r["metric"] == metric] if not matches: print(f"\nNo measurement of {task}:{metric} in these runs, so nothing to compare.") continue for row in matches: # Published scores are usually percentages and harness metrics are usually fractions. mine = row["mean"] * 100 if row["mean"] <= 1.0 else row["mean"] gap = mine - published comparisons.append({ "task": task, "metric": metric, "filter": row["filter"], "published": published, "measured": round(mine, 2), "gap": round(gap, 2), "published_by": args.published_by, }) source = f" reported by {args.published_by}" if args.published_by else "" print(f"\n{task} / {metric} [{row['filter']}]") print(f" published{source}: {published}") print(f" measured here: {mine:.2f}") print(f" gap: {gap:+.2f} (spread across your own runs: " f"{row['range'] * (100 if row['mean'] <= 1.0 else 1):.2f})") if comparisons: print("\nBefore attributing a gap to the model, work down this list:") for i, item in enumerate(GAP_CHECKLIST, 1): print(f" {i}. {item}") print("\nA gap you can explain is a result. A gap you cannot explain is an open question, " "\nand recording it as one is more useful than picking a story for it.") record = { "lab": "part-16/lab-run-a-standard-benchmark-suite", "run_id": time.strftime("%Y%m%dT%H%M%S"), "results_dir": str(root), "result_files": len(files), "rows": summary_rows, "harness_config": {k: harness_config.get(k) for k in ("model", "model_args", "num_fewshot", "limit", "gen_kwargs")} if harness_config else {}, "run_context": context, "comparisons": comparisons, "host": platform.platform(), "date": time.strftime("%Y-%m-%d"), } if args.out: Path(args.out).write_text(json.dumps(record, indent=2), encoding="utf-8") print(f"\nwritten to {args.out}") if args.labbook: with Path(args.labbook).open("a", encoding="utf-8") as handle: handle.write(json.dumps(record) + "\n") print(f"recorded in {args.labbook}") if __name__ == "__main__": main()