#!/usr/bin/env python3 """Assemble the capstone report's evidence tables from the lab notebook. Purpose: read labbook.md, find every machine-readable record the course's labs appended to it, group them by the lab that wrote them, and print the tables the capstone report needs - as Markdown for the report itself, or as the course's shape so that a number and the context that makes it meaningful cannot be separated. The script never invents a field: it prints the fields it found, names the records whose shape it does not recognise, and lists the context keys you still have to fill in by hand. Platform: all (pure Python; no third-party package, no network, no model) Minimum memory: 8 GB (it reads a text file; the floor is the machine you have) Assumes: Python 3.9 or newer, and the lab notebook started in Part 1, in which the labs of Parts 6, 9, 11, 16, 18, 22, 23 and 26 have appended one JSON object per line. Lines that are not JSON objects are your own prose and are skipped. Usage: python3 collect-capstone-evidence.py --labbook labbook.md python3 collect-capstone-evidence.py --labbook labbook.md --format mdx python3 collect-capstone-evidence.py --labbook labbook.md --group part-09 python3 collect-capstone-evidence.py --labbook labbook.md --list python3 collect-capstone-evidence.py --labbook labbook.md --format json > evidence.json Endpoints are dropped from the output by default, because a report is a document you give to somebody else and a base URL is an address on your network. Pass --keep-endpoints if you are writing the report only for yourself. """ from __future__ import annotations import argparse import json import sys from pathlib import Path from typing import Any, Iterable # --------------------------------------------------------------------------- # What the labs of this course write. # # Each entry is a group key prefix and the columns worth putting in a report, # in the order a reader wants them. A field named here that is absent from the # records is simply not printed: the preference is an ordering, never a promise # that the field exists. A record whose group matches nothing below still gets # a table; its columns are then every scalar field the records actually carry, # in the order they first appeared. # --------------------------------------------------------------------------- KNOWN_SHAPES: dict[str, dict[str, Any]] = { "part-06/": { "what": "llama.cpp benchmark rows (llama-bench)", "columns": [ "model", "quant", "test", "backend", "n_gpu_layers", "file_gb", "tokens_per_s", "tokens_per_s_stddev", "reps", ], "context": { "hardware": ["host"], "engine": ["engine"], "version": ["build"], "model": ["model"], "quant": ["quant"], "date": ["measured_on"], }, }, "part-09/": { "what": "load-test rows, one per concurrency level", "columns": [ "label", "concurrency", "requests", "completed", "failed", "output_tokens_per_s", "requests_per_s", "ttft_s.p50", "ttft_s.p90", "tpot_s.p50", "tpot_s.p90", ], "context": { "engine": ["engine"], "version": ["engine_version"], "model": ["model"], "date": ["recorded_at"], }, }, "part-11/": { "what": "training run records", "columns": [ "run_id", "model", "dataset", "seed", "config_commit", "date", ], "context": { "model": ["model"], "version": ["config_commit"], "date": ["date"], }, }, "part-18/": { "what": "cluster topology and link records", "columns": [ "machine", "roles", "cluster_interface", "intended_mtu", "peers", "date", ], "context": {"date": ["date"], "hardware": ["machine"]}, }, "part-26/": { "what": "agent evaluation runs", "columns": [ "run_id", "scaffold", "task_set", "model", "router_model", "repeats", "summary.runs", "summary.passed", "summary.success_rate", "summary.mean_steps", "summary.mean_tokens", "summary.mean_seconds", ], "context": { "engine": ["engine"], "version": ["engine_version"], "model": ["model"], "quant": ["quant"], "date": ["date"], "hardware": ["host"], }, }, } # The keys requires. The component refuses to render without them, # which is ADR-004 enforced at build time rather than by good intentions. BENCHMARK_CONTEXT = ("hardware", "engine", "version", "model", "quant", "contextLength", "date") # Field names that carry an address on your network rather than a measurement. ENDPOINT_FIELDS = {"base_url", "url", "endpoint", "host_url", "agent_entry_point"} # Generic fallbacks tried for every group, after any shape-specific mapping. GENERIC_CONTEXT: dict[str, list[str]] = { "hardware": ["host", "machine", "hardware.device_name", "hardware.machine"], "os": ["os", "hardware.os"], "engine": ["engine", "scaffold"], "version": ["engine_version", "build", "version", "config_commit"], "model": ["model", "model_path"], "quant": ["quant", "quantisation", "quantization"], "contextLength": ["context_length", "ctx_size", "n_ctx", "contextLength"], "date": ["date", "measured_on", "recorded_at", "recorded"], } FILL_IN = "FILL-IN" # --------------------------------------------------------------------------- # Reading # --------------------------------------------------------------------------- def read_records(path: Path) -> tuple[list[dict], int, int]: """Every JSON object on its own line, plus the counts of what was skipped. The notebook is a Markdown file that labs append JSON lines to, so most of it is prose. A line that starts with "{" and does not parse is reported rather than silently dropped: a truncated record is worth knowing about. """ records: list[dict] = [] prose = 0 broken = 0 with path.open("r", encoding="utf-8") as handle: for line in handle: stripped = line.strip() if not stripped.startswith("{"): if stripped: prose += 1 continue try: value = json.loads(stripped) except json.JSONDecodeError: broken += 1 continue if isinstance(value, dict): records.append(value) else: broken += 1 return records, prose, broken def group_key(record: dict) -> str: """Group by the lab that wrote the record, then by the kind of record it is. Part 18 writes topology records and link records under one lab name, so the `record` field splits them; every other lab in the course writes one shape per `lab` value. A record with neither field is grouped under a name that says so, because inventing a group for it would hide it. """ lab = record.get("lab") kind = record.get("record") if lab and kind: return f"{lab} [{kind}]" if lab: return str(lab) if kind: return f"(no lab field) [{kind}]" return "(unidentified records)" def shape_for(key: str) -> dict[str, Any] | None: for prefix, shape in KNOWN_SHAPES.items(): if key.startswith(prefix): return shape return None # --------------------------------------------------------------------------- # Flattening # --------------------------------------------------------------------------- def flatten(record: dict, prefix: str = "") -> dict[str, Any]: """One level of dotted keys per nested object; lists are summarised, not expanded. load-test.py nests its percentiles under ttft_s and tpot_s, and agent-eval.py nests its aggregates under summary, so a report table needs ttft_s.p90 to be a column. A list becomes its length in a "n item(s)" string unless it is short and entirely scalar, in which case it is joined: roles and peers read better that way and neither is a measurement. """ out: dict[str, Any] = {} for name, value in record.items(): full = f"{prefix}{name}" if isinstance(value, dict): out.update(flatten(value, prefix=f"{full}.")) elif isinstance(value, list): scalars = [v for v in value if isinstance(v, (str, int, float, bool))] if value and len(scalars) == len(value) and len(value) <= 6: out[full] = ", ".join(str(v) for v in value) else: out[full] = f"{len(value)} item(s)" else: out[full] = value return out def columns_for(rows: list[dict[str, Any]], shape: dict[str, Any] | None, keep_endpoints: bool, all_fields: bool) -> list[str]: """The columns a report wants: the shape's preference where one is known. A load-test record carries two dozen fields and a report table with two dozen columns is unreadable, so a recognised shape prints the fields chosen for a report and --all-fields prints the rest. An unrecognised shape always prints everything, because the alternative would be to decide on your behalf which of your own fields matter. """ present: list[str] = [] for row in rows: for name in row: if name not in present: present.append(name) if not keep_endpoints: present = [c for c in present if c.split(".")[0] not in ENDPOINT_FIELDS] preferred = [c for c in (shape or {}).get("columns", []) if c in present] if preferred and not all_fields: return preferred rest = [c for c in present if c not in preferred and c not in ("lab", "record")] return preferred + rest def cell(value: Any) -> str: if value is None: return "not recorded" if isinstance(value, bool): return "yes" if value else "no" if isinstance(value, float): return f"{value:.4g}" text = str(value) return text.replace("|", "\\|") # --------------------------------------------------------------------------- # Context for # --------------------------------------------------------------------------- def first_present(rows: list[dict[str, Any]], names: Iterable[str]) -> Any: for name in names: for row in rows: if row.get(name) not in (None, ""): return row[name] return None def context_for(rows: list[dict[str, Any]], shape: dict[str, Any] | None) -> tuple[dict[str, Any], list[str]]: """The context this group can fill, and the keys it cannot. Nothing is guessed. A key that no record carries comes back as FILL-IN and is also returned in the second value, so the caller can say out loud which parts of the context the notebook never recorded. That list is the useful output of this function: it is the set of things a future reader of your report would have had to take on trust. """ mapping = dict(GENERIC_CONTEXT) for key, names in (shape or {}).get("context", {}).items(): mapping[key] = list(names) + [n for n in GENERIC_CONTEXT.get(key, []) if n not in names] context: dict[str, Any] = {} missing: list[str] = [] for key in BENCHMARK_CONTEXT + ("os",): value = first_present(rows, mapping.get(key, [key])) if value in (None, ""): if key in BENCHMARK_CONTEXT: context[key] = FILL_IN missing.append(key) continue context[key] = value return context, missing # --------------------------------------------------------------------------- # Output # --------------------------------------------------------------------------- def print_markdown(key: str, rows: list[dict[str, Any]], shape: dict[str, Any] | None, keep_endpoints: bool, all_fields: bool) -> list[str]: cols = columns_for(rows, shape, keep_endpoints, all_fields) what = (shape or {}).get("what", "shape not recognised; every field found is shown") print(f"\n### {key}\n") print(f"{len(rows)} record(s). {what}.\n") if not cols: print("No fields to show: the records in this group are empty.\n") return [] print("| " + " | ".join(cols) + " |") print("| " + " | ".join("---" for _ in cols) + " |") for row in rows: print("| " + " | ".join(cell(row.get(c)) for c in cols) + " |") context, missing = context_for(rows, shape) known = [f"{k}: {context[k]}" for k in context if context[k] != FILL_IN] print("\n*Context found in the records:* " + ("; ".join(known) if known else "none")) if missing: print(f"\n*Context the notebook never recorded, to be supplied by hand:* {', '.join(missing)}") return missing def print_mdx(key: str, rows: list[dict[str, Any]], shape: dict[str, Any] | None, keep_endpoints: bool, all_fields: bool) -> list[str]: """A block for the report, with the context the records carry. The component refuses to render an incomplete context, so a FILL-IN left in the output breaks the build rather than shipping a number with nothing attached to it. That is the intended behaviour and it is why the marker is loud. """ cols = columns_for(rows, shape, keep_endpoints, all_fields) if not cols: print(f"\n") return [] context, missing = context_for(rows, shape) context.setdefault("contextLength", FILL_IN) print(f"\n') if missing: print(f"") return missing def main(argv: list[str]) -> int: parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) parser.add_argument("--labbook", default="labbook.md", help="the notebook the labs have been appending to") parser.add_argument("--format", choices=["markdown", "mdx", "json"], default="markdown", help="markdown for the report, mdx for blocks, json for a pipe") parser.add_argument("--group", action="append", default=[], help="only groups whose key contains this text; repeatable") parser.add_argument("--list", action="store_true", help="list the groups and their record counts, then stop") parser.add_argument("--keep-endpoints", action="store_true", help="keep base URLs and entry points in the tables") parser.add_argument("--all-fields", action="store_true", help="every field found, not just the ones a report wants") args = parser.parse_args(argv) path = Path(args.labbook) if not path.is_file(): print(f"{path}: not found. This is Part 1's lab notebook, the one every lab " f"appends to; the capstone is written from it.", file=sys.stderr) return 2 records, prose, broken = read_records(path) if not records: print(f"{path}: no JSON records found. The labs append one JSON object per line; " f"if you kept your results as prose, the report tables have to be typed by hand.", file=sys.stderr) return 1 groups: dict[str, list[dict[str, Any]]] = {} for record in records: groups.setdefault(group_key(record), []).append(flatten(record)) if args.group: groups = {k: v for k, v in groups.items() if any(g in k for g in args.group)} if not groups: print(f"No group matched {args.group}. Run with --list to see what is there.", file=sys.stderr) return 1 if args.format == "json": json.dump({k: v for k, v in sorted(groups.items())}, sys.stdout, indent=2, sort_keys=True) sys.stdout.write("\n") return 0 print(f"==> {path}") print(f" {len(records)} record(s) in {len(groups)} group(s); " f"{prose} line(s) of prose skipped; {broken} line(s) did not parse.") unknown = [k for k in groups if shape_for(k) is None] if unknown: print(f" {len(unknown)} group(s) have a shape this script does not know; " f"their tables show every field found.") if args.list: for key in sorted(groups): shape = shape_for(key) label = (shape or {}).get("what", "shape not recognised") print(f" {len(groups[key]):>4} {key} ({label})") return 0 missing_anywhere: dict[str, list[str]] = {} emit = print_mdx if args.format == "mdx" else print_markdown for key in sorted(groups): missing = emit(key, groups[key], shape_for(key), args.keep_endpoints, args.all_fields) if missing: missing_anywhere[key] = missing print("\n---\n") if missing_anywhere: print("Context missing from the notebook, by group.") print("Each one is a number in your report that a reader could not judge.") print("Fill it in from the notebook's machine section, or write \"not recorded\"") print("in the table and say so in the report's final section.\n") for key, missing in missing_anywhere.items(): print(f" {key}: {', '.join(missing)}") return 1 print("Every group carried a complete context. Nothing to fill in by hand.") return 0 if __name__ == "__main__": sys.exit(main(sys.argv[1:]))