"""Put the sequence-level student and the logit student side by side, with what each cost. Purpose: the logit lab's conclusion. Runs Part 10's harness over the same tasks for both students, asks the judge to compare them with the order swapped so position bias shows as a flip rate rather than hiding inside the answer, and reads the lab notebook for what each route cost in tokens, seconds and watt-hours. The output is a decision, not a score: which of the two routes to use next time, and on what evidence. Platform: all (pure Python over HTTP; both students are reached through an OpenAI-compatible API, so they may be served by any engine on any track) Minimum memory: 12 GB on the machine serving the students; this script needs very little Assumes: Python 3.10 or newer; Part 10's run-eval.py and judge.py in --harness-dir; both students reachable at --base-url under the names given; distillog.py next to this file; the lab notebook holding the generate, filter, train and train-logit records written by the other scripts in this part. Usage: python3 compare-students.py --harness-dir ~/eval --tasks my-tasks.json \\ --base-url http://127.0.0.1:4000/v1 \\ --sequence-student local/student-seq --logit-student local/student-logit \\ --judge-model local/chat --quant Q4_K_M --engine llama.cpp \\ --engine-version v0.4.0 --out-dir compare-out --labbook labbook.md Add --skip-eval to reuse results files already in --out-dir, which is what you want when only the judging step needs rerunning. """ from __future__ import annotations import argparse import json import subprocess import sys from pathlib import Path from typing import Any import distillog # Which run-log stage paid for which route. The sequence-level student paid for # generation and filtering as well as its own training; the logit student paid # only for its training run, because it never generated a dataset. ROUTE_STAGES = { "sequence": ("generate", "filter", "train"), "logit": ("train-logit",), } def run(script: Path, arguments: list[str]) -> None: command = [sys.executable, str(script), *arguments] print("+ " + " ".join(command)) subprocess.run(command, check=True) def checks(results_path: Path) -> tuple[int, int]: data = json.loads(results_path.read_text(encoding="utf-8")) return sum(1 for r in data["results"] if r["checks"]["passed"]), len(data["results"]) def route_cost(labbook: str, route: str) -> dict[str, Any]: """Add up the cost blocks of the stages that belong to one route. A stage that recorded null for a field contributes nothing to that field and is counted under `incomplete`, so a total is never quietly made up of half the run. """ totals = {"prompt_tokens": 0, "completion_tokens": 0, "seconds": 0.0, "watt_hours": 0.0} present = {k: 0 for k in totals} stages_found = [] for record in distillog.read_stages(labbook): if record.get("stage") not in ROUTE_STAGES[route]: continue stages_found.append(record["stage"]) cost = record.get("cost") or {} for key in totals: value = cost.get(key) if isinstance(value, (int, float)): totals[key] += value present[key] += 1 return { "stages": stages_found, "totals": {k: (round(v, 2) if isinstance(v, float) else v) for k, v in totals.items()}, "incomplete": [k for k in totals if present[k] == 0], } def main() -> None: parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) parser.add_argument("--harness-dir", required=True) parser.add_argument("--tasks", required=True) parser.add_argument("--base-url", default="http://127.0.0.1:8080/v1") parser.add_argument("--api-key", default=None) parser.add_argument("--sequence-student", required=True, help="the student trained on teacher text") parser.add_argument("--logit-student", required=True, help="the student trained against teacher logits") parser.add_argument("--judge-model", default=None) parser.add_argument("--quant", default="unknown") parser.add_argument("--engine", default="llama.cpp") parser.add_argument("--engine-version", default="unknown") parser.add_argument("--out-dir", default="compare-out") parser.add_argument("--skip-eval", action="store_true", help="reuse results files already in --out-dir") parser.add_argument("--labbook", default="labbook.md") args = parser.parse_args() harness = Path(args.harness_dir) run_eval, judge = harness / "run-eval.py", harness / "judge.py" if not run_eval.is_file(): raise SystemExit(f"{run_eval} not found; point --harness-dir at your Part 10 evaluation directory") out = Path(args.out_dir) out.mkdir(parents=True, exist_ok=True) files = {"sequence": out / "results-sequence.json", "logit": out / "results-logit.json"} models = {"sequence": args.sequence_student, "logit": args.logit_student} for route, path in files.items(): if args.skip_eval and path.is_file(): print(f"reusing {path}") continue run(run_eval, [ "--base-url", args.base_url, *(["--api-key", args.api_key] if args.api_key else []), "--model", models[route], "--quant", args.quant, "--engine", args.engine, "--engine-version", args.engine_version, "--tasks", args.tasks, "--out", str(path), "--notes", f"Part 15 student comparison, {route} route", ]) if args.judge_model: if not judge.is_file(): raise SystemExit(f"{judge} not found, but --judge-model was given") run(judge, [ "compare", "--base-url", args.base_url, *(["--api-key", args.api_key] if args.api_key else []), "--judge-model", args.judge_model, "--results-a", str(files["sequence"]), "--results-b", str(files["logit"]), "--labbook", args.labbook, ]) print("\nRead the flip rate the comparison printed before reading the winner. A high " "flip rate means the judge is answering the order of the answers rather than " "their quality, and the head-to-head number means little.") summary: dict[str, Any] = {} print("\n" + "=" * 72) print(f"{'route':<12}{'served as':<26}{'checks':>10}") for route, path in files.items(): passed, total = checks(path) cost = route_cost(args.labbook, route) summary[route] = {"model": models[route], "checks_passed": passed, "checks_total": total, "cost": cost} print(f"{route:<12}{models[route]:<26}{passed:>4}/{total:<5}") print("\nWhat each route cost, from the run log:") for route in files: cost = summary[route]["cost"] totals = cost["totals"] stages = ", ".join(cost["stages"]) or "no stages found in the notebook" print(f" {route:<10} stages: {stages}") print(f" completion tokens {totals['completion_tokens']}, " f"{totals['seconds'] / 60:.1f} minutes, " f"{totals['watt_hours']} Wh") if cost["incomplete"]: print(f" not recorded anywhere: {', '.join(cost['incomplete'])}") seq, log = summary["sequence"], summary["logit"] print() if seq["checks_passed"] == log["checks_passed"]: print("The two students passed the same number of deterministic checks. Choose on " "cost, on the judge comparison, and on which route you can run again next " "month, not on a difference that is not there.") else: better = "sequence" if seq["checks_passed"] > log["checks_passed"] else "logit" print(f"The {better} route passed more deterministic checks on this task set. That is " "one task set on one machine: report it with the settings above and treat it " "as evidence rather than as a general result.") report_path = out / "comparison.json" report_path.write_text(json.dumps(summary, indent=2), encoding="utf-8") print(f"\nwritten to {report_path}") rec = distillog.record( labbook=args.labbook, lab="part-15/compare-students", stage="compare", teacher=None, student={"sequence": models["sequence"], "logit": models["logit"]}, dataset={"tasks": args.tasks, "tasks_sha256": distillog.file_sha256(args.tasks)}, hyperparameters={"engine": args.engine, "engine_version": args.engine_version, "quant": args.quant, "judge_model": args.judge_model}, seed=None, scores=summary, config_path=__file__, notes=None, ) print(f"recorded {rec['stage']} as {rec['run_id']} in {args.labbook}") if __name__ == "__main__": main()