index.html

<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>Agentic Skill-Composition Benchmark</title>
<meta name="description" content="Token cost and success rate of skill-based agent workflows vs monolithic prompts">
<link rel="icon" href="data:image/svg+xml,<svg xmlns=%22http://www.w3.org/2000/svg%22 viewBox=%220 0 100 100%22><text y=%22.9em%22 font-size=%2290%22>⚖️</text></svg>">
<style>
  :root {
    color-scheme: light;
    --page:           #f9f9f7;
    --surface-1:      #fcfcfb;
    --text-primary:   #0b0b0b;
    --text-secondary: #52514e;
    --muted:          #898781;
    --grid:           #e1e0d9;
    --baseline:       #c3c2b7;
    --border:         rgba(11,11,11,0.10);
    --good:           #006300;
    --series-mono:    #2a78d6;   /* categorical slot 1: monolithic */
    --series-comp:    #008300;   /* categorical slot 2: skill-composed */
  }
  @media (prefers-color-scheme: dark) {
    :root:where(:not([data-theme="light"])) {
      color-scheme: dark;
      --page: #0d0d0d; --surface-1: #1a1a19;
      --text-primary: #ffffff; --text-secondary: #c3c2b7;
      --muted: #898781; --grid: #2c2c2a; --baseline: #383835;
      --border: rgba(255,255,255,0.10); --good: #0ca30c;
      --series-mono: #3987e5; --series-comp: #008300;
    }
  }
  :root[data-theme="dark"] {
    color-scheme: dark;
    --page: #0d0d0d; --surface-1: #1a1a19;
    --text-primary: #ffffff; --text-secondary: #c3c2b7;
    --muted: #898781; --grid: #2c2c2a; --baseline: #383835;
    --border: rgba(255,255,255,0.10); --good: #0ca30c;
    --series-mono: #3987e5; --series-comp: #008300;
  }
  * { margin: 0; padding: 0; box-sizing: border-box; }
  body {
    background: var(--page); color: var(--text-primary);
    font: 15px/1.55 system-ui, -apple-system, "Segoe UI", sans-serif;
    padding: 32px 20px 64px;
  }
  .wrap { max-width: 1060px; margin: 0 auto; }
  header { margin-bottom: 28px; display: flex; justify-content: space-between; align-items: flex-start; gap: 16px; }
  h1 { font-size: 26px; font-weight: 700; letter-spacing: -0.01em; }
  .sub { color: var(--text-secondary); max-width: 62ch; margin-top: 6px; }
  .sub a { color: inherit; }
  #themeBtn {
    background: var(--surface-1); color: var(--text-secondary);
    border: 1px solid var(--border); border-radius: 8px;
    padding: 6px 12px; font: inherit; font-size: 13px; cursor: pointer; flex-shrink: 0;
  }
  .tiles { display: grid; grid-template-columns: repeat(auto-fit, minmax(220px, 1fr)); gap: 14px; margin-bottom: 26px; }
  .tile {
    background: var(--surface-1); border: 1px solid var(--border);
    border-radius: 14px; padding: 18px 20px;
  }
  .tile .label { font-size: 13px; color: var(--text-secondary); }
  .tile .value { font-size: 34px; font-weight: 700; letter-spacing: -0.02em; margin: 2px 0; }
  .tile .value.good { color: var(--good); }
  .tile .detail { font-size: 12.5px; color: var(--muted); }
  .card {
    background: var(--surface-1); border: 1px solid var(--border);
    border-radius: 14px; padding: 22px 24px; margin-bottom: 22px;
  }
  .card h2 { font-size: 17px; font-weight: 650; letter-spacing: -0.01em; }
  .card .note { font-size: 13px; color: var(--text-secondary); margin: 3px 0 4px; max-width: 72ch; }
  .legend { display: flex; gap: 18px; margin: 10px 0 4px; font-size: 13px; color: var(--text-secondary); flex-wrap: wrap; }
  .legend span { display: inline-flex; align-items: center; gap: 6px; }
  .swatch { width: 12px; height: 12px; border-radius: 3px; display: inline-block; }
  .badge {
    display: inline-block; font-size: 11px; font-weight: 650; letter-spacing: 0.04em;
    color: var(--text-secondary); border: 1px solid var(--border);
    border-radius: 999px; padding: 2px 9px; vertical-align: 2px; margin-left: 8px;
  }
  svg { display: block; width: 100%; height: auto; }
  svg text { font-family: system-ui, -apple-system, "Segoe UI", sans-serif; }
  .tick { font-variant-numeric: tabular-nums; }
  #tooltip {
    position: fixed; pointer-events: none; z-index: 10; display: none;
    background: var(--surface-1); color: var(--text-primary);
    border: 1px solid var(--border); border-radius: 8px;
    box-shadow: 0 4px 14px rgba(0,0,0,0.18);
    padding: 8px 11px; font-size: 12.5px; max-width: 280px;
  }
  #tooltip .t { font-weight: 650; margin-bottom: 2px; }
  #tooltip .r { color: var(--text-secondary); font-variant-numeric: tabular-nums; }
  details { margin-top: 14px; }
  summary { cursor: pointer; font-size: 13px; color: var(--text-secondary); }
  table { border-collapse: collapse; width: 100%; margin-top: 10px; font-size: 13px; }
  th, td { text-align: right; padding: 6px 10px; border-bottom: 1px solid var(--grid); font-variant-numeric: tabular-nums; }
  th:first-child, td:first-child { text-align: left; }
  th { color: var(--text-secondary); font-weight: 600; }
  code, pre { font: 12.5px/1.6 ui-monospace, SFMono-Regular, Menlo, monospace; }
  pre { background: var(--page); border: 1px solid var(--border); border-radius: 8px; padding: 12px 14px; overflow-x: auto; margin: 8px 0; }
  .method p { margin: 8px 0; color: var(--text-secondary); max-width: 78ch; }
  .method strong { color: var(--text-primary); }
  footer { color: var(--muted); font-size: 13px; margin-top: 30px; }
  footer a { color: var(--text-secondary); }
</style>
</head>
<body>
<div class="wrap">
  <header>
    <div>
      <h1 id="headline">Skill routing cuts agent input tokens</h1>
      <div class="sub">One agent, two ways to carry its instructions: a <strong>monolithic prompt</strong> inlining every skill on every turn, vs <strong>skill composition</strong> — a small index plus on-demand loading of only the skills each task needs. Same tasks, same conversation dynamics; only the instruction context differs. <a href="https://github.com/aabhimittal/Agentic-skill-composition-benchmark">Source &amp; harness on GitHub</a>.</div>
    </div>
    <button id="themeBtn" aria-label="Toggle color theme">◐ Theme</button>
  </header>

  <div class="tiles" id="tiles"></div>

  <div class="card">
    <h2 id="barsTitle"></h2>
    <div class="note">Total tokens the API re-reads across all agent turns of each task (context is re-sent every turn, so instruction weight compounds with turn count). Bars start at zero.</div>
    <div class="legend">
      <span><span class="swatch" style="background:var(--series-mono)"></span>Monolithic prompt</span>
      <span><span class="swatch" style="background:var(--series-comp)"></span>Skill-composed</span>
    </div>
    <div id="taskBars"></div>
    <details><summary>Data table</summary><div id="taskTable"></div></details>
  </div>

  <div class="card">
    <h2>The system prompt is 12× smaller with a skill index</h2>
    <div class="note">What each condition carries on <em>every single turn</em>. The composed condition ships name + description per skill and loads full bodies only when a task needs them.</div>
    <div id="anatomy"></div>
  </div>

  <div class="card">
    <h2 id="successTitle">Success rate holds or improves under skill composition <span class="badge" id="simBadge">SIMULATED</span></h2>
    <div class="note" id="successNote">These success rates come from a documented interference model (more co-resident unrelated instructions → more degradation), <strong>not live model runs</strong>. Run <code>python -m benchmark.runner run</code> with an API key to replace them with measured rates — this chart re-renders from whatever <code>results/results.json</code> contains.</div>
    <div class="legend">
      <span><span class="swatch" style="background:var(--series-mono); border-radius:50%"></span>Monolithic prompt</span>
      <span><span class="swatch" style="background:var(--series-comp); border-radius:50%"></span>Skill-composed</span>
    </div>
    <div id="dumbbell"></div>
  </div>

  <div class="card method">
    <h2>Methodology &amp; reproduce</h2>
    <p><strong>Setup.</strong> A library of 6 skills (PDF extraction, spreadsheets, API integration, SQL analytics, data viz, report writing — ~700 tokens each) and 10 agentic tasks that each genuinely require 1–3 of them, with an estimated agent-turn count per task.</p>
    <p><strong>Accounting.</strong> Per turn the API re-reads system prompt + task + accumulated history. Both conditions share identical conversation dynamics (<span id="assumptions"></span>); the skill-composed condition additionally pays a per-load tool-call overhead. Token counts are computed from the actual prompt files in the repo (<span id="counting"></span>).</p>
    <pre>git clone https://github.com/aabhimittal/Agentic-skill-composition-benchmark
cd Agentic-skill-composition-benchmark
python -m benchmark.runner measure            # deterministic token accounting
python -m benchmark.runner measure --exact    # exact counts via count_tokens API
python -m benchmark.runner run --trials 5     # live success-rate runs (needs ANTHROPIC_API_KEY)</pre>
    <p><strong>Caveats.</strong> Savings grow with library size and shrink when a task needs most of the library. Prompt caching reduces the absolute cost of both conditions but not the cached-read and cache-invalidation asymmetry; success-rate deltas are the contested claim — measure them yourself with <code>run</code> mode.</p>
  </div>

  <footer>Costs priced at <span id="pricing"></span>. Built as a reproducible harness — swap in your own skills and tasks under <code>benchmark/</code>. MIT licensed.</footer>
</div>
<div id="tooltip" role="status"></div>

<script>
const FALLBACK = null; // populated at deploy time if fetch is unavailable
const $ = (id) => document.getElementById(id);
const fmtK = (n) => n >= 1000 ? (n/1000).toFixed(n >= 10000 ? 0 : 1) + "k" : String(n);
const fmt = (n) => n.toLocaleString("en-US");
const css = (v) => `var(${v})`;

// --- theme toggle (stamps data-theme; CSS handles both scopes) ---
$("themeBtn").onclick = () => {
  const r = document.documentElement;
  const dark = getComputedStyle(document.body).backgroundColor === "rgb(13, 13, 13)";
  r.setAttribute("data-theme", dark ? "light" : "dark");
};

// --- tooltip ---
const tip = $("tooltip");
function bindTips(container) {
  container.querySelectorAll("[data-tip]").forEach((el) => {
    el.addEventListener("mousemove", (e) => {
      tip.innerHTML = el.dataset.tip;
      tip.style.display = "block";
      const w = tip.offsetWidth;
      tip.style.left = Math.min(e.clientX + 14, innerWidth - w - 8) + "px";
      tip.style.top = (e.clientY + 14) + "px";
    });
    el.addEventListener("mouseleave", () => (tip.style.display = "none"));
  });
}

function render(R) {
  const tasks = [...R.tasks].sort((a, b) => b.monolithic.input_tokens - a.monolithic.input_tokens);
  const T = R.totals, C = R.context_sizes;

  $("headline").textContent = `Skill routing cuts agent input tokens ${T.saved_pct}%`;
  $("barsTitle").textContent = `Input tokens per task: skill routing saves ${T.saved_pct}% overall`;

  // --- hero tiles ---
  const suiteCost = R.totals.monolithic.cost_usd, compCost = R.totals.composed.cost_usd;
  $("tiles").innerHTML = [
    { l: "Input tokens saved", v: T.saved_pct + "%", g: 1, d: `${fmt(T.saved_input_tokens)} of ${fmt(T.monolithic.input_tokens)} across 10 tasks` },
    { l: "System prompt size", v: "12× smaller", g: 0, d: `${fmt(C.monolithic_system_tokens)} → ${fmt(C.composed_system_tokens)} tokens per turn` },
    { l: "Cost per suite run", v: "$" + compCost.toFixed(2), g: 0, d: `vs $${suiteCost.toFixed(2)} monolithic (−${Math.round(100 * (suiteCost - compCost) / suiteCost)}%)` },
    { l: "Skill library", v: "6 skills", g: 0, d: "10 tasks · 1–3 skills each · 3–8 turns" },
  ].map((t) => `<div class="tile"><div class="label">${t.l}</div><div class="value${t.g ? " good" : ""}">${t.v}</div><div class="detail">${t.d}</div></div>`).join("");

  // --- paired horizontal bars: input tokens per task ---
  const W = 1000, LEFT = 220, RIGHT = 84, BW = 13, GAP = 2, GROUP = 44, TOP = 24;
  const H = TOP + tasks.length * GROUP + 30;
  const maxV = Math.max(...tasks.map((t) => t.monolithic.input_tokens));
  const x = (v) => LEFT + (v / maxV) * (W - LEFT - RIGHT);
  let s = `<svg viewBox="0 0 ${W} ${H}" role="img" aria-label="Paired bar chart of input tokens per task">`;
  // gridlines + ticks
  const step = maxV > 40000 ? 20000 : 10000;
  for (let v = 0; v <= maxV; v += step) {
    s += `<line x1="${x(v)}" y1="${TOP - 8}" x2="${x(v)}" y2="${H - 26}" stroke="${css("--grid")}" stroke-width="1"/>`;
    s += `<text class="tick" x="${x(v)}" y="${H - 10}" text-anchor="middle" font-size="11" fill="${css("--muted")}">${fmtK(v)}</text>`;
  }
  tasks.forEach((t, i) => {
    const y0 = TOP + i * GROUP;
    s += `<text x="${LEFT - 10}" y="${y0 + BW + GAP}" text-anchor="end" font-size="12.5" fill="${css("--text-secondary")}">${t.title}</text>`;
    const rows = [
      ["--series-mono", t.monolithic.input_tokens, "Monolithic", y0],
      ["--series-comp", t.composed.input_tokens, "Skill-composed", y0 + BW + GAP],
    ];
    for (const [col, v, name, y] of rows) {
      const wPx = Math.max(2, x(v) - LEFT);
      s += `<path d="M${LEFT},${y} h${wPx - 4} a4,4 0 0 1 4,4 v${BW - 8} a4,4 0 0 1 -4,4 h${4 - wPx} z" fill="${css(col)}" data-tip="<div class='t'>${t.title}</div><div class='r'>${name}: ${fmt(v)} input tokens<br>${t.turns} turns · needs ${t.skills.join(", ")}</div>"/>`;
    }
    s += `<text class="tick" x="${x(t.monolithic.input_tokens) + 8}" y="${y0 + BW + GAP + 4}" font-size="11.5" font-weight="650" fill="${css("--good")}">−${t.saved_pct}%</text>`;
  });
  s += `<line x1="${LEFT}" y1="${TOP - 8}" x2="${LEFT}" y2="${H - 26}" stroke="${css("--baseline")}" stroke-width="1"/></svg>`;
  $("taskBars").innerHTML = s;
  bindTips($("taskBars"));

  // --- data table ---
  $("taskTable").innerHTML = `<table><thead><tr><th>Task</th><th>Skills</th><th>Turns</th><th>Monolithic in-tok</th><th>Composed in-tok</th><th>Saved</th><th>Mono $</th><th>Comp $</th></tr></thead><tbody>` +
    tasks.map((t) => `<tr><td>${t.title}</td><td style="text-align:left">${t.skills.join(", ")}</td><td>${t.turns}</td><td>${fmt(t.monolithic.input_tokens)}</td><td>${fmt(t.composed.input_tokens)}</td><td>${t.saved_pct}%</td><td>${t.monolithic.cost_usd.toFixed(3)}</td><td>${t.composed.cost_usd.toFixed(3)}</td></tr>`).join("") +
    `</tbody></table>`;

  // --- context anatomy: two stacked horizontal bars ---
  const skills = Object.entries(C.skill_body_tokens);
  const monoTotal = C.monolithic_system_tokens;
  const aW = 1000, aLEFT = 150, aRIGHT = 90, aBW = 26;
  const ax = (v) => aLEFT + (v / monoTotal) * (aW - aLEFT - aRIGHT);
  let a = `<svg viewBox="0 0 ${aW} 130" role="img" aria-label="System prompt composition per condition">`;
  // monolithic row: base + each skill body, 2px gaps
  let acc = 0;
  a += `<text x="${aLEFT - 10}" y="42" text-anchor="end" font-size="12.5" fill="${css("--text-secondary")}">Monolithic</text>`;
  const segs = [["base prompt", C.base_prompt_tokens, "--muted"], ...skills.map(([k, v]) => [k, v, "--series-mono"])];
  segs.forEach(([name, v], si) => {
    const x0 = ax(acc), x1 = ax(acc + v);
    a += `<rect x="${x0}" y="26" width="${Math.max(1, x1 - x0 - 2)}" height="${aBW}" rx="3" fill="${css(si === 0 ? "--muted" : "--series-mono")}" opacity="${si === 0 ? 0.75 : 0.5 + 0.5 * (si % 2)}" data-tip="<div class='t'>${name}</div><div class='r'>${fmt(v)} tokens, on every turn of every task</div>"/>`;
    acc += v;
  });
  a += `<text class="tick" x="${ax(monoTotal) + 8}" y="43" font-size="12" font-weight="650" fill="${css("--text-primary")}">${fmt(monoTotal)}</text>`;
  // composed row
  a += `<text x="${aLEFT - 10}" y="96" text-anchor="end" font-size="12.5" fill="${css("--text-secondary")}">Skill-composed</text>`;
  const compSegs = [["base prompt", C.base_prompt_tokens], ["skill index (6 × name + description)", C.composed_system_tokens - C.base_prompt_tokens]];
  acc = 0;
  compSegs.forEach(([name, v], si) => {
    const x0 = ax(acc), x1 = ax(acc + v);
    a += `<rect x="${x0}" y="80" width="${Math.max(1, x1 - x0 - 2)}" height="${aBW}" rx="3" fill="${css(si === 0 ? "--muted" : "--series-comp")}" opacity="${si === 0 ? 0.75 : 1}" data-tip="<div class='t'>${name}</div><div class='r'>${fmt(v)} tokens; full bodies load only when a task needs them</div>"/>`;
    acc += v;
  });
  a += `<text class="tick" x="${ax(C.composed_system_tokens) + 8}" y="97" font-size="12" font-weight="650" fill="${css("--text-primary")}">${fmt(C.composed_system_tokens)} (−${Math.round(100 * (1 - C.composed_system_tokens / monoTotal))}%)</text>`;
  a += `<line x1="${aLEFT}" y1="18" x2="${aLEFT}" y2="114" stroke="${css("--baseline")}" stroke-width="1"/></svg>`;
  $("anatomy").innerHTML = a;
  bindTips($("anatomy"));

  // --- success dumbbell ---
  const measured = R.tasks.every((t) => t.success && t.success.simulated === false);
  if (measured) {
    $("simBadge").textContent = "MEASURED";
    $("successTitle").firstChild.textContent = "Measured success rate by condition ";
    $("successNote").innerHTML = "Live model runs — see <code>results/results.json</code> meta for trial counts and protocol.";
  }
  const dW = 1000, dLEFT = 220, dRIGHT = 40, dROW = 34, dTOP = 30;
  const dH = dTOP + tasks.length * dROW + 34;
  const lo = 0.5, hi = 1.0;
  const dx = (v) => dLEFT + ((v - lo) / (hi - lo)) * (dW - dLEFT - dRIGHT);
  let d = `<svg viewBox="0 0 ${dW} ${dH}" role="img" aria-label="Success rate per task and condition">`;
  d += `<text x="${dLEFT}" y="14" font-size="11" fill="${css("--muted")}">axis starts at 50% — dot plot, not bars</text>`;
  for (let v = lo; v <= hi + 1e-9; v += 0.1) {
    d += `<line x1="${dx(v)}" y1="${dTOP - 6}" x2="${dx(v)}" y2="${dH - 28}" stroke="${css("--grid")}" stroke-width="1"/>`;
    d += `<text class="tick" x="${dx(v)}" y="${dH - 12}" text-anchor="middle" font-size="11" fill="${css("--muted")}">${Math.round(v * 100)}%</text>`;
  }
  tasks.forEach((t, i) => {
    const y = dTOP + i * dROW + 12;
    const m = t.success.monolithic, c = t.success.composed;
    d += `<text x="${dLEFT - 10}" y="${y + 4}" text-anchor="end" font-size="12.5" fill="${css("--text-secondary")}">${t.title}</text>`;
    d += `<line x1="${dx(m)}" y1="${y}" x2="${dx(c)}" y2="${y}" stroke="${css("--baseline")}" stroke-width="2"/>`;
    d += `<circle cx="${dx(m)}" cy="${y}" r="6" fill="${css("--series-mono")}" stroke="${css("--surface-1")}" stroke-width="2" data-tip="<div class='t'>${t.title}</div><div class='r'>Monolithic: ${Math.round(m * 100)}% success</div>"/>`;
    d += `<circle cx="${dx(c)}" cy="${y}" r="6" fill="${css("--series-comp")}" stroke="${css("--surface-1")}" stroke-width="2" data-tip="<div class='t'>${t.title}</div><div class='r'>Skill-composed: ${Math.round(c * 100)}% success</div>"/>`;
  });
  d += `</svg>`;
  $("dumbbell").innerHTML = d;
  bindTips($("dumbbell"));

  // --- meta strings ---
  const A = R.meta.assumptions;
  $("assumptions").textContent = `${A.assistant_tokens_per_turn} assistant + ${A.tool_result_tokens_per_turn} tool-result tokens/turn; ${A.skill_load_overhead_tokens} tokens per skill load`;
  $("counting").textContent = R.meta.token_counting;
  $("pricing").textContent = `${R.meta.pricing.model}: $${R.meta.pricing.input_per_mtok}/M input, $${R.meta.pricing.output_per_mtok}/M output`;
}

fetch("results/results.json")
  .then((r) => r.json())
  .then(render)
  .catch(() => { if (FALLBACK) render(FALLBACK); else document.body.insertAdjacentHTML("beforeend", "<p style='color:var(--muted)'>Could not load results/results.json — serve this page over HTTP (e.g. python -m http.server).</p>"); });
</script>
</body>
</html>