{
  "$comment": "The schema summarise-architecture.py reads. Copy this file to measurements.json, replace every value with your own, and delete this comment. Every number here is a placeholder written as null or as an obviously fictional value: nothing in this file is a measurement, and the script refuses to summarise a file whose measurements are still null.",
  "schema": "part-22/tiered-architecture/1",
  "author_note": "One or two sentences on what this design is for: who uses it, for what, and what the latency requirement is. A design with no requirement cannot be judged.",
  "requirement": {
    "ttft_budget_s": 2.0,
    "tpot_budget_s": 0.06,
    "concurrent_users": 4,
    "typical_prompt_tokens": 4096,
    "typical_answer_tokens": 300
  },
  "model": {
    "id": "qwen3-8b",
    "quant": "bf16",
    "context_length": 8192,
    "kv_bytes_per_token": 147456
  },
  "machines": [
    {
      "name": "node-a",
      "track": "nvidia",
      "accelerator": "a discrete GPU; name it as your own system reports it",
      "device_memory_gb": 24,
      "host_memory_gb": 64,
      "roles": ["prefill", "router"],
      "notes": "Chosen for prefill because it has the most arithmetic throughput and the least memory."
    },
    {
      "name": "node-b",
      "track": "strix",
      "accelerator": "an integrated GPU on unified memory",
      "device_memory_gb": 96,
      "host_memory_gb": 128,
      "roles": ["decode", "cache", "storage"],
      "notes": "Chosen for decode because every conversation in flight keeps its cache here."
    }
  ],
  "links": [
    {
      "from": "node-a",
      "to": "node-b",
      "class": "2.5 gigabit Ethernet",
      "carries": "per-request",
      "measured_gbps": null,
      "measured_rtt_ms": null,
      "source": "Part 18 lab, labbook line label"
    },
    {
      "from": "node-a",
      "to": "clients",
      "class": "house network",
      "carries": "client",
      "measured_gbps": null,
      "measured_rtt_ms": null,
      "source": "Part 18 lab, labbook line label"
    }
  ],
  "tiers": [
    {
      "tier": "device",
      "where": "node-b device memory",
      "size_gb": 20,
      "measured_hit_rate": null,
      "source": "labbook line label"
    },
    {
      "tier": "host",
      "where": "node-b host memory, through the engine's offloading backend",
      "size_gb": 8,
      "measured_hit_rate": null,
      "source": "labbook line label"
    },
    {
      "tier": "shared-store",
      "where": "the Part 18 shared mount, as a directory of key-value blocks",
      "size_gb": 50,
      "measured_hit_rate": null,
      "source": "labbook line label"
    }
  ],
  "routing": {
    "layer": "litellm",
    "policy": "Describe the rule in one sentence. Round-robin is a rule; prefer-the-backend-that-already-holds-most-of-this-prompt is a better one.",
    "source": "Part 9 project, or vLLM's example proxy"
  },
  "measurements": [
    {
      "label": "baseline",
      "configuration": "one machine, no split, no offload",
      "concurrency": 4,
      "ttft_p50_s": null,
      "tpot_p50_s": null,
      "output_tokens_per_s": null,
      "source": "labbook.md, label baseline"
    },
    {
      "label": "disagg",
      "configuration": "prefill on node-a, decode on node-b, shared-directory connector",
      "concurrency": 4,
      "ttft_p50_s": null,
      "tpot_p50_s": null,
      "output_tokens_per_s": null,
      "source": "labbook.md, label disagg"
    },
    {
      "label": "offload-on",
      "configuration": "one machine, host-memory cache tier of 8 GiB",
      "concurrency": 4,
      "ttft_p50_s": null,
      "tpot_p50_s": null,
      "output_tokens_per_s": null,
      "source": "labbook.md, label offload-on"
    }
  ],
  "decisions": [
    {
      "decision": "State one decision, such as: decode runs on node-b.",
      "because": "State why, in one sentence that a reader could disagree with.",
      "evidence": ["baseline", "disagg"]
    },
    {
      "decision": "State the next decision, such as: the phases are not split.",
      "because": "State why.",
      "evidence": ["disagg", "offload-on"]
    }
  ],
  "one_more_machine": "What you would change if you had one more machine of the kind you would actually buy, and which measurement makes you say so. A design that cannot answer this has not been thought about."
}