#!/usr/bin/env python3 """Build index.html, the public page of the MCP savings study. Every number on the page is read from the files in study/data/ (pulled from Honeycomb and analysed by study/harness/mcp_savings_analyze.py and mcp_savings_explore.py). Nothing numeric is typed by hand here. Standard library only: python3 study/build_site.py # from studies/mcp-savings/ """ from __future__ import annotations import html import json from collections import defaultdict from datetime import datetime from pathlib import Path HERE = Path(__file__).resolve().parent DATA = HERE / "data" OUT = HERE.parent / "index.html" WITH, WITHOUT = "with-mcp", "no-mcp" SETUP = {WITH: "MCP", WITHOUT: "Direct API"} REPO = "https://github.com/pedro-angel/mcp-for-kibana" def load_json(name: str): return json.loads((DATA / name).read_text()) def load_lines(name: str) -> list[dict]: return [json.loads(x) for x in (DATA / name).read_text().splitlines() if x.strip()] A = load_json("analysis.json") X = load_json("exploratory.json") ROWS = load_lines("rows.jsonl") CFG = json.loads((HERE / "harness" / "mcp_savings.json").read_text()) MODELS = CFG["study"]["models"] e = html.escape def model_name(m: str) -> str: """claude-haiku-5-5 -> Haiku 5.5""" parts = m.split("-") return parts[1].capitalize() + " " + ".".join(parts[2:]) def r2(v: float) -> str: return f"{v:.2f}" def ci(lo: float, hi: float, fmt) -> str: return f"{fmt(lo)}–{fmt(hi)}" def pct_saved(ratio: float) -> int: return round((1 - ratio) * 100) def ts(s: str) -> datetime: return datetime.fromisoformat(s[:26].rstrip("Z") + "+00:00") # ---------------------------------------------------------------- derived facts counted_rows = [r for r in ROWS if r.get("study.valid") and not r.get("study.aborted")] assert len(counted_rows) == A["counted"], (len(counted_rows), A["counted"]) workers = sorted({r["study.worker"] for r in counted_rows}) blocks = sorted({r["study.block"] for r in counted_rows}) checks_total = sorted({r["study.checks_total"] for r in counted_rows}) first_t, last_t = ts(A["first_row"]), ts(A["last_row"]) ratios = A["ratios"] cost_saved = {m: pct_saved(ratios[m]["cost_usd"]["ratio"]) for m in MODELS} time_saved = {m: pct_saved(ratios[m]["time_s"]["ratio"]) for m in MODELS} all_below_1 = all(ratios[m][k]["ratio_ci95"][1] < 1 for m in MODELS for k in ("cost_usd", "time_s")) passed = sum(v["passed"] for v in A["success"].values()) runs_total = sum(v["runs"] for v in A["success"].values()) cold = X["cold_start"] cache_life_min = cold["cache_life_s"] // 60 cold_runs = [] # (cell, run_id, reason) for cell, c in cold["cells"].items(): if c["cold"]: cold_runs.append((cell, c["first"]["run_id"], "first")) for rid in c["others_at_floor"]: cold_runs.append((cell, rid, "later")) warm_n = A["counted"] - len(cold_runs) warm_pct = round(100 * warm_n / A["counted"]) later_cold = [x for x in cold_runs if x[2] == "later"] # gap before each later cold run: its start minus the latest end of an earlier # run of the same model and setup (end = start + wall_s), from the rows by_cell = defaultdict(list) for r in counted_rows: by_cell[f'{r["study.model_requested"]}/{r["study.arm"]}'].append(r) later_cold_gaps = [] for cell, rid, _ in later_cold: runs = sorted(by_cell[cell], key=lambda r: r["time"]) i = next(k for k, r in enumerate(runs) if r["study.run_id"] == rid) last_end = max(ts(r["time"]).timestamp() + r["study.wall_s"] for r in runs[:i]) later_cold_gaps.append((cell, round(ts(runs[i]["time"]).timestamp() - last_end))) # MCP tool definitions in the cached prompt: the least cache read per request # of a warm run, MCP minus Direct API, per model tool_def_tokens = sorted( { cold["cells"][f"{m}/{WITH}"]["others_median_min_request_cache_read"] - cold["cells"][f"{m}/{WITHOUT}"]["others_median_min_request_cache_read"] for m in MODELS } ) cold_ratios = cold["ratios"] cold_costlier = [m for m in MODELS if cold_ratios[m]["cold_cost_usd"] > 1] cold_cheaper = [m for m in MODELS if cold_ratios[m]["cold_cost_usd"] <= 1] cold_faster_all = all(cold_ratios[m]["cold_time_s"] < 1 for m in MODELS) tb = X["time_breakdown"] # ---------------------------------------------------------------- charts def fmt_ms(v: float) -> str: return f"{v:,.0f}" def names(ms: list[str]) -> str: n = [model_name(m) for m in ms] return n[0] if len(n) == 1 else ", ".join(n[:-1]) + " and " + n[-1] DOMAIN = 1.5 # shared ratio scale for every ratio chart: 0 to 1.5 def ratio_panel(rows: list[dict], aria: str) -> str: """Dot-and-whisker panel on the shared 0-1.5 ratio scale, reference at 1. rows: [{"label": str, "marks": [{"v", "lo"?, "hi"?, "kind": "interval"|"point", "tip"}]}] """ W, left, right = 360, 86, 344 top, row_h, mark_h = 30, 0, 26 heights = [18 + mark_h * len(r["marks"]) for r in rows] H = top + sum(heights) + 30 def x(v: float) -> float: return left + (min(v, DOMAIN) / DOMAIN) * (right - left) out = [ f'', ] plot_bottom = H - 26 for t in (0, 0.5, 1, 1.5): cls = "ref" if t == 1 else "grid" out.append(f'') out.append(f'{t:g}') out.append(f'← MCP lower') out.append(f'MCP higher →') y = top for r, h in zip(rows, heights): cy0 = y + 18 + mark_h / 2 out.append( f'{e(r["label"])}' ) for k, mk in enumerate(r["marks"]): cy = cy0 + k * mark_h cx = x(mk["v"]) out.append(f"{e(mk['tip'])}") if mk["kind"] == "interval": cls = "good" if mk["hi"] < 1 else "acc" out.append( f'' ) for end in (mk["lo"], mk["hi"]): out.append( f'' ) out.append(f'') label = f'{r2(mk["v"])} ({ci(mk["lo"], mk["hi"], r2)})' else: d = 6 out.append( f'' ) label = r2(mk["v"]) # value label above the mark, kept inside the plot lx = min(max(cx, left + 40), right - 40) out.append(f'{e(label)}') out.append("" % (cy - mark_h / 2, W, mark_h)) out.append("") y += h out.append("") return "\n".join(out) def savings_panel(key: str, what: str) -> str: rows = [] for m in MODELS: r = ratios[m][key] lo, hi = r["ratio_ci95"] tip = ( f"{model_name(m)}, {what}: MCP ÷ Direct API = {r2(r['ratio'])}, " f"95% interval {ci(lo, hi, r2)}" ) rows.append({"label": model_name(m), "marks": [{"kind": "interval", "v": r["ratio"], "lo": lo, "hi": hi, "tip": tip}]}) return ratio_panel(rows, f"{what} ratio MCP ÷ Direct API per model") def cache_panel(key_warm: str, key_cold: str, what: str) -> str: rows = [] for m in MODELS: w = ratios[m][key_warm] lo, hi = w["ratio_ci95"] c = cold_ratios[m][key_cold] rows.append( { "label": model_name(m), "marks": [ { "kind": "interval", "v": w["ratio"], "lo": lo, "hi": hi, "tip": f"{model_name(m)}, {what}, warm: median ÷ median = {r2(w['ratio'])} ({ci(lo, hi, r2)})", }, { "kind": "point", "v": c, "tip": f"{model_name(m)}, {what}, cold: first MCP run ÷ first Direct API run = {r2(c)} (one run each)", }, ], } ) return ratio_panel(rows, f"{what} ratio, warm and cold, per model") def time_chart(W: int) -> str: """Stacked bars: median model requests + tool time outside them + idle, per model and setup.""" lw, bar_h, gap, head = 82, 16, 7, 20 right = W - 58 cells = [(m, a) for m in MODELS for a in (WITH, WITHOUT)] vmax = max(tb["by_model_arm"][f"{m}/{a}"]["wall_s"]["median"] for m, a in cells) dmax = 30 * (int(vmax // 30) + 1) H = 8 + len(MODELS) * (head + 2 * bar_h + gap + 14) + 22 def x(v: float) -> float: return lw + v / dmax * (right - lw) out = [f''] y0 = 8 axis_bottom = H - 22 for t in range(0, dmax + 1, 30): out.append(f'') out.append(f'{t} s') y = y0 segs = (("model_clock_s", "seg-model", "model requests"), ("tool_outside_model_s", "seg-tool", "tools outside requests"), ("idle_s", "seg-idle", "idle")) for m in MODELS: out.append(f'{e(model_name(m))}') y += head for a in (WITH, WITHOUT): c = tb["by_model_arm"][f"{m}/{a}"] out.append(f'{SETUP[a]}') xx = 0.0 for key, cls, label in segs: v = c[key]["median"] w = max(x(xx + v) - x(xx) - 1.5, 0.5) lo, hi = c[key]["median_ci95"] out.append( f'' f"{e(model_name(m))}, {SETUP[a]}: {label} {v:.1f} s (95% interval {lo:.1f}–{hi:.1f})" ) xx += v wall = c["wall_s"]["median"] out.append(f'{wall:.1f} s') y += bar_h + gap y += 14 - gap + gap out.append("") return "\n".join(out) # ---------------------------------------------------------------- tables def cell_table() -> str: pm = A["per_model_arm"] head = ( "SetupCost per run" "Time per runTool calls" ) body = [] for m in MODELS: body.append(f"{e(model_name(m))}") for a in (WITH, WITHOUT): c = pm[f"{m}/{a}"] cost, tm, tc = c["cost_usd"], c["time_s"], c["tool_calls"] body.append( f"{SETUP[a]}" f"${cost['median']:.4f}{ci(*cost['median_ci95'], lambda v: f'{v:.4f}')}" f"{tm['median']:.1f} s{ci(*tm['median_ci95'], lambda v: f'{v:.1f}')}" f"{tc['median']:.0f}{ci(*tc['median_ci95'], lambda v: f'{v:g}')}" ) return f"
{head}{''.join(body)}
" def call_table() -> str: sc = tb["single_call_ms"] rows = [] for m in MODELS: a = sc[f"{m}/{WITH}/mcp"] b = sc[f"{m}/{WITHOUT}/bash"] rows.append( f"{e(model_name(m))}" f"{fmt_ms(a['median'])} ms{ci(*a['median_ci95'], fmt_ms)} · n = {a['n']:,}" f"{fmt_ms(b['median'])} ms{ci(*b['median_ci95'], fmt_ms)} · n = {b['n']:,}" ) return ( "
" "" + "".join(rows) + "
ModelMCP: one tool callDirect API: one shell command
" ) def cold_table() -> str: rows = [] for m in MODELS: rows.append(f"{e(model_name(m))}") for a in (WITH, WITHOUT): f = cold["cells"][f"{m}/{a}"]["first"] med = cold["cells"][f"{m}/{a}"]["median"] rows.append( f"{SETUP[a]}" f"${f['cost_usd']:.4f}${med['cost_usd']:.4f}" f"{f['cache_write_tokens']:,}{med['cache_write_tokens']:,.0f}" ) return ( "
" "" + "".join(rows) + "
SetupCost, first runCost, medianCache write, first run (tokens)Cache write, median (tokens)
" ) # ---------------------------------------------------------------- page lo_cost = min(MODELS, key=lambda m: cost_saved[m]) hi_cost = max(MODELS, key=lambda m: cost_saved[m]) lo_time = min(MODELS, key=lambda m: time_saved[m]) hi_time = max(MODELS, key=lambda m: time_saved[m]) spend = A["spend_usd"]["all_runs"] cli = A["claude_cli"]["values"][0].split(" ")[0] kib = A["kibana_versions"]["values"][0] srv = A["server_versions"][WITH]["values"][0] harness = A["harness_sha"] boot = A["bootstrap"] n_cell = A["per_model_arm"][f"{MODELS[0]}/{WITH}"]["cost_usd"]["n"] steps = CFG["task"]["steps"] spans = tb["spans"] mt = {m: (tb["by_model_arm"][f"{m}/{WITH}"]["model_clock_s"]["median"], tb["by_model_arm"][f"{m}/{WITHOUT}"]["model_clock_s"]["median"]) for m in MODELS} tool_out = [tb["by_model_arm"][f"{m}/{a}"]["tool_outside_model_s"]["median"] for m in MODELS for a in (WITH, WITHOUT)] overlap_with = [tb["by_model_arm"][f"{m}/{WITH}"]["overlap_s_median"] for m in MODELS] call_x = [tb["single_call_ms"][f"{m}/{WITH}/mcp"]["median"] / tb["single_call_ms"][f"{m}/{WITHOUT}/bash"]["median"] for m in MODELS] cli_ratio = X["time_ratio_cli"] gap_txt = "; ".join( f"{model_name(c.split('/')[0])} on {SETUP[c.split('/')[1]]}, started {g} s after the previous run of its model and setup had ended" for c, g in later_cold_gaps ) date_txt = f"{first_t:%-d} to {last_t:%-d %B %Y}, {first_t:%H:%M} to {last_t:%H:%M} UTC" aborted_n = len(A["aborted"]) STEP_LABELS = { "space": "create a Kibana space and work inside it", "data_views": "create data views on the web-logs and flights indices, each with its time field", "weblogs": "build a two-panel web-logs dashboard and a server-errors alert rule", "flights": "build a two-panel flights dashboard and a cancelled-flights alert rule", "change": "change the web-logs dashboard in place: retitle a panel, add a third", "lookup": "read back the dashboard's panel count and the rule's settings", } step_items = "".join(f"
  • {e(STEP_LABELS.get(s['id'], s['id']))}
    • " for s in steps) CSS = """ :root{color-scheme:light dark;--bg:#fbfbf9;--card:#ffffff;--fg:#1c1d1f;--muted:#55595f;--faint:#868b92;--rule:#e2e2de;--grid:#ebebe7; --accent:#2d5bc8;--good:#16794a;--seg-tool:#9097a0;--seg-idle:#cfd3d8;--code:#f1f1ee} @media (prefers-color-scheme:dark){:root{--bg:#131416;--card:#1a1b1e;--fg:#e7e7e4;--muted:#a9adb3;--faint:#7c818a;--rule:#2d2f33;--grid:#24262a; --accent:#86a8ff;--good:#4cc48a;--seg-tool:#7c838c;--seg-idle:#3e4248;--code:#1f2124}} *{box-sizing:border-box} html{-webkit-text-size-adjust:100%} body{margin:0;background:var(--bg);color:var(--fg);font:16px/1.55 system-ui,-apple-system,"Segoe UI",Roboto,"Helvetica Neue",Arial,sans-serif} main{max-width:760px;margin:0 auto;padding:40px 16px 64px} h1{font-size:2rem;line-height:1.2;margin:0 0 6px;letter-spacing:-.01em} .sub{color:var(--muted);margin:0 0 28px} h2{font-size:1.3rem;margin:48px 0 10px;padding-top:20px;border-top:1px solid var(--rule)} h3{font-size:1.02rem;margin:28px 0 6px} p,li{max-width:68ch} .lead{font-size:1.14rem;line-height:1.5} .tag{display:inline-block;font-size:.72rem;letter-spacing:.04em;text-transform:uppercase;color:var(--muted);border:1px solid var(--rule);border-radius:4px;padding:1px 6px;margin-left:8px;vertical-align:3px} .defs{background:var(--card);border:1px solid var(--rule);border-radius:8px;padding:12px 16px;margin:20px 0} .defs p{margin:6px 0} .grid2{display:grid;grid-template-columns:1fr 1fr;gap:8px 28px;margin:12px 0 4px} .panel h4{margin:0 0 2px;font-size:.92rem;color:var(--muted);font-weight:600} figure{margin:20px 0 8px} figcaption{font-size:.84rem;color:var(--muted);margin-top:8px;max-width:72ch} figcaption code{font-size:.8rem} .chart{width:100%;height:auto;display:block;overflow:visible} .chart text{font-family:inherit} .chart .tick{fill:var(--faint);font-size:11px} .chart .note{fill:var(--faint);font-size:11px} .chart .lab{fill:var(--fg);font-size:13px;font-weight:600} .chart .lab2{fill:var(--muted);font-size:12px} .chart .val{fill:var(--fg);font-size:12px;font-variant-numeric:tabular-nums} .chart .val,.chart .note,.chart .lab{paint-order:stroke;stroke:var(--bg);stroke-width:4px;stroke-linejoin:round} .chart .grid{stroke:var(--grid);stroke-width:1} .chart .ref{stroke:var(--fg);stroke-width:1.25;stroke-dasharray:4 3;opacity:.7} .chart .whisk{stroke-width:2;stroke-linecap:round} .chart .whisk.good{stroke:var(--good)}.chart .dot.good{fill:var(--good);stroke:var(--card);stroke-width:2} .chart .whisk.acc{stroke:var(--accent)}.chart .dot.acc{fill:var(--accent);stroke:var(--card);stroke-width:2} .chart .hollow{fill:var(--card);stroke:var(--accent);stroke-width:2} .chart .hit{fill:transparent} .chart .seg-model{fill:var(--accent)}.chart .seg-tool{fill:var(--seg-tool)}.chart .seg-idle{fill:var(--seg-idle)} .legend{display:flex;flex-wrap:wrap;gap:6px 18px;font-size:.84rem;color:var(--muted);margin:10px 0 0;padding:0;list-style:none} .legend li{display:flex;align-items:center;gap:6px} .sw{display:inline-block;width:12px;height:12px;border-radius:3px} .sw.model{background:var(--accent)}.sw.tool{background:var(--seg-tool)}.sw.idle{background:var(--seg-idle)} .sw.good{background:var(--good);border-radius:50%;width:10px;height:10px} .sw.cold{border:2px solid var(--accent);transform:rotate(45deg);width:9px;height:9px;border-radius:1px;margin:0 2px} .tw{overflow-x:auto;margin:14px 0} table{border-collapse:collapse;width:100%;font-size:.92rem;font-variant-numeric:tabular-nums} th,td{text-align:left;padding:7px 8px;border-bottom:1px solid var(--rule);vertical-align:top} thead th{font-size:.8rem;color:var(--muted);font-weight:600;border-bottom:1px solid var(--faint)} tbody th{font-weight:600} tr.gh th{padding-top:14px;border-bottom:none;font-size:.95rem} td{white-space:nowrap}.ci{white-space:nowrap} .num{text-align:right} .ci{display:block;color:var(--faint);font-size:.78rem} table.small{font-size:.86rem} .only-narrow{display:none} code,pre{font-family:ui-monospace,SFMono-Regular,Menlo,Consolas,monospace} code{font-size:.88em;background:var(--code);padding:1px 4px;border-radius:4px;overflow-wrap:anywhere} pre{background:var(--code);border:1px solid var(--rule);border-radius:8px;padding:12px 14px;font-size:.8rem;line-height:1.5;overflow-x:auto;white-space:pre} pre code{background:none;padding:0} ul.facts{padding-left:20px} ul.facts li{margin:4px 0} a{color:var(--accent)} .home{margin:0 0 18px;font-size:.88rem}.home a{color:var(--muted);text-decoration:none}.home a:hover{color:var(--accent);text-decoration:underline} footer{margin-top:56px;padding-top:16px;border-top:1px solid var(--rule);font-size:.84rem;color:var(--muted)} @media (max-width:640px){ main{padding-top:28px} h1{font-size:1.6rem} .lead{font-size:1.05rem} .grid2{grid-template-columns:1fr;gap:18px} .only-wide{display:none}.only-narrow{display:block} th,td{padding:6px 5px} table{font-size:.86rem} table.small{font-size:.8rem} } """ def caption(text: str) -> str: return f"
    {text}
    " def data_link(path: str) -> str: return f'study/data/{path}' cost_rng = f"{cost_saved[lo_cost]}% ({model_name(lo_cost)}) to {cost_saved[hi_cost]}% ({model_name(hi_cost)})" time_rng = f"{time_saved[lo_time]}% ({model_name(lo_time)}) to {time_saved[hi_time]}% ({model_name(hi_time)})" page = f""" MCP savings study

    ← mcp-for-kibana

    MCP savings study

    Claude Code on one Kibana job, {A['counted']} runs, three models: through the mcp-for-kibana MCP server, or through Kibana's REST API.

    With MCP, Claude Code did the same Kibana job cheaper and faster than with Direct API, with each of the three models. Median cost per run fell {cost_rng}; median time per run fell {time_rng}.{' All six 95% intervals lie wholly below 1.' if all_below_1 else ''}

    These are warm-cache numbers: {warm_n} of {A['counted']} runs ({warm_pct}%) started with the prompt already cached. On a cold cache, with one run each, MCP cost more for {names(cold_costlier)}{', and still saved for ' + names(cold_cheaper) if cold_cheaper else ''}.

    The two setups. MCP: Claude Code reaches Kibana through the tools of the mcp-for-kibana server ({srv}). Direct API: Claude Code calls Kibana's REST API itself from the shell, with the pinned Kibana API reference. Everything else is the same.

    Results describe Claude Code with each model, not the model alone. A ratio is the median on MCP ÷ the median on Direct API: below 1, MCP is cheaper or faster.

    Savings per model

    {model_name(hi_cost)} saved the most on both measures. With {n_cell} runs per model and setup, the intervals are narrow.

    Cost per run, MCP ÷ Direct API

    {savings_panel('cost_usd', 'cost')}

    Time per run, MCP ÷ Direct API

    {savings_panel('time_s', 'time')}
    {caption(f"Medians with 95% percentile-bootstrap intervals ({boot['resamples']:,} resamples). Shared scale; dashed line at 1 = same as Direct API. Data: {data_link('analysis.json')} (ratios).")}

    Per model and setup

    On Direct API, {model_name(MODELS[0])} took the longest of all six combinations; on MCP it was the cheapest by far. All {passed} of {runs_total} counted runs passed every check.

    {cell_table()} {caption(f"Median per run, 95% interval below it. Cost is list-price cost (Anthropic's published prices, not a bill), as each run came; caches were not controlled. Time runs from the CLI's start to its result. A tool call is one MCP tool call on MCP, or one shell command on Direct API (a command can bundle several Kibana requests). Data: {data_link('analysis.json')} (per_model_arm).")}

    Where the time goes exploratory

    Not planned before the runs: it suggests where to look, not what holds. On MCP, the time saved is the model's time. A median run spent {mt[MODELS[0]][0]:.1f} s in model requests instead of {mt[MODELS[0]][1]:.1f} s with {model_name(MODELS[0])}, {mt[MODELS[1]][0]:.1f} s instead of {mt[MODELS[1]][1]:.1f} s with {model_name(MODELS[1])}, and {mt[MODELS[2]][0]:.1f} s instead of {mt[MODELS[2]][1]:.1f} s with {model_name(MODELS[2])}. Tool time outside those requests stayed between {min(tool_out):.1f} and {max(tool_out):.1f} s.

    {time_chart(680)}
    {time_chart(360)}
    {caption(f"Median seconds per run on the clock, without double counting (model requests + tool time outside them + idle = wall time); each part is its own median, so the parts need not add up exactly to the median wall time printed at the end. From {spans['attached']:,} of {spans['pulled']:,} CLI spans (claude_code.llm_request, claude_code.tool.execution) attached to the counted runs. Data: {data_link('exploratory.json')} (time_breakdown), from {data_link('spans/spans.jsonl')}.")}

    One MCP tool call takes {min(call_x):.0f} to {max(call_x):.0f} times as long as one shell command, yet tool time does not grow: the CLI starts a tool while the model's response is still streaming, so on MCP a median {min(overlap_with):.1f} to {max(overlap_with):.1f} s of tool time per run runs under the model's requests.

    {call_table()} {caption(f"Median duration of one call, 95% interval and number of calls below it. The interval resamples calls as if independent, so it is narrower than a run-level one. Data: {data_link('exploratory.json')} (time_breakdown.single_call_ms).")}

    The time saving is not harness overhead: by the CLI's own duration the ratios are {r2(cli_ratio[MODELS[0]]['ratio'])} ({model_name(MODELS[0])}), {r2(cli_ratio[MODELS[1]]['ratio'])} ({model_name(MODELS[1])}) and {r2(cli_ratio[MODELS[2]]['ratio'])} ({model_name(MODELS[2])}), each interval {'wholly below 1' if all(cli_ratio[m]['ratio_ci95'][1] < 1 for m in MODELS) else 'shown in the data file'} (time_ratio_cli in {data_link('exploratory.json')}).

    Cache: cold against warm exploratory

    On MCP, the cached prompt carries the server's tool definitions: a warm run's smallest cache read per request is {tool_def_tokens[0]:,} tokens larger than on Direct API. On a warm cache they cost little; a cold run first has to write them to the cache. {warm_n} of {A['counted']} runs started warm. The {len(cold_runs)} cold ones are the first run of each model and setup, and one more ({gap_txt}).

    Cost, MCP ÷ Direct API

    {cache_panel('cost_usd', 'cold_cost_usd', 'cost')}

    Time, MCP ÷ Direct API

    {cache_panel('time_s', 'cold_time_s', 'time')}
    {caption(f"Same scale as above; dashed line at 1. A first run counts as cold when its least-read model request read no more than the lowest per-request cache read of any run of its model and setup. Data: {data_link('exploratory.json')} (cold_start), from {data_link('cache/min-request-cache-read.json')} and {data_link('cache/pilot-root-spans.json')}; warm ratios from {data_link('analysis.json')}.")}

    Cold against cold, MCP cost {names(cold_costlier)} more{', and still saved for ' + names(cold_cheaper) if cold_cheaper else ''}{'; all three cold runs finished sooner on MCP' if cold_faster_all else ''}. One run per model and setup is an observation, not a result.

    {cold_table()} {caption(f"The first run of each model and setup against the median of its {n_cell} runs. Data: {data_link('exploratory.json')} (cold_start.cells).")}

    How it ran

    Limits

    Reproduce

    The rows, spans, analysis output and the frozen harness are in study/ on this branch. To recompute every number from the rows, with uv:

    git clone -b gh-pages {REPO}.git mcp-savings-study
cd mcp-savings-study/studies/mcp-savings/study/harness
uv run python mcp_savings_analyze.py analyze --phase study \\
  --rows ../data/rows.jsonl --harness-sha {harness} --out /tmp/mcp-savings
uv run python mcp_savings_explore.py --phase study \\
  --rows ../data/rows.jsonl --harness-sha {harness} --out /tmp/mcp-savings \\
  --spans ../data/spans/spans.jsonl \\
  --min-reads ../data/cache/min-request-cache-read.json \\
  --pilot-rows ../data/cache/pilot-root-spans.json
cmp /tmp/mcp-savings/analysis.json ../data/analysis.json
cmp /tmp/mcp-savings/exploratory.json ../data/exploratory.json
cd ../.. && python3 study/build_site.py   # rebuilds this page

    How the rows were pulled from Honeycomb: study/docs/PULL_ROWS.md. What each file is: study/README.md.

    """ OUT.write_text(page) print(f"wrote {OUT} ({len(page.encode()):,} bytes)")