| <!doctype html> |
| <html lang="en" data-theme="dark"> |
| <head> |
| <meta charset="utf-8"> |
| <meta name="viewport" content="width=device-width, initial-scale=1"> |
| <meta name="description" content="Interactive companion for Token Identity as a Routing Signal for Residual MLP Experts."> |
| <title>Token Identity Routing — Interactive Paper</title> |
| <link rel="stylesheet" href="./visualize.css"> |
| <script defer src="https://unpkg.com/lucide@1.17.0/dist/umd/lucide.js"></script> |
| <style> |
| html { background: var(--background); } |
| html > body { |
| width: 100%; |
| max-width: none; |
| margin: 0; |
| padding: clamp(14px, 2vw, 34px); |
| } |
| </style> |
| </head> |
| <body> |
| <div id="tr-paper-demo"> |
| <style> |
| #tr-paper-demo { |
| --tr-gap: clamp(12px, 2.2vw, 22px); |
| color: var(--foreground); |
| width: 100%; |
| } |
| #tr-paper-demo * { box-sizing: border-box; } |
| #tr-paper-demo .tr-shell { display: grid; gap: var(--tr-gap); } |
| #tr-paper-demo .tr-topbar, |
| #tr-paper-demo .tr-brand, |
| #tr-paper-demo .tr-actions, |
| #tr-paper-demo .tr-tabs, |
| #tr-paper-demo .tr-chip-row, |
| #tr-paper-demo .tr-run-head, |
| #tr-paper-demo .tr-legend, |
| #tr-paper-demo .tr-node-head { |
| display: flex; |
| align-items: center; |
| gap: 10px; |
| flex-wrap: wrap; |
| } |
| #tr-paper-demo .tr-topbar { justify-content: space-between; } |
| #tr-paper-demo .tr-mark { |
| display: grid; |
| place-items: center; |
| width: 34px; |
| height: 34px; |
| border-radius: 10px; |
| color: var(--primary); |
| background: color-mix(in srgb, var(--primary) 12%, transparent); |
| } |
| #tr-paper-demo .tr-brand-copy { display: grid; gap: 1px; } |
| #tr-paper-demo .tr-brand-copy strong { font-weight: 500; letter-spacing: .08em; } |
| #tr-paper-demo .tr-brand-copy span { color: var(--muted-foreground); } |
| #tr-paper-demo .tr-hero { |
| display: grid; |
| grid-template-columns: minmax(0, 1.35fr) minmax(250px, .65fr); |
| gap: var(--tr-gap); |
| align-items: end; |
| padding: clamp(10px, 2vw, 18px) 0; |
| } |
| #tr-paper-demo .tr-hero-copy { display: grid; gap: 12px; } |
| #tr-paper-demo .tr-kicker { |
| color: var(--muted-foreground); |
| letter-spacing: .12em; |
| text-transform: uppercase; |
| } |
| #tr-paper-demo .tr-hero h2 { |
| margin: 0; |
| max-width: 760px; |
| font-size: clamp(1.7rem, 4.6vw, 3.25rem); |
| line-height: 1.02; |
| letter-spacing: -.045em; |
| font-weight: 500; |
| } |
| #tr-paper-demo .tr-hero p { |
| margin: 0; |
| max-width: 680px; |
| color: var(--muted-foreground); |
| } |
| #tr-paper-demo .tr-note { |
| border-left: 2px solid var(--primary); |
| padding-left: 12px; |
| color: var(--muted-foreground); |
| } |
| #tr-paper-demo .tr-tabs { |
| border-bottom: 1px solid var(--border); |
| padding-bottom: 10px; |
| } |
| #tr-paper-demo .tr-tab[aria-selected="true"] { |
| color: var(--primary-foreground); |
| background: var(--primary); |
| } |
| #tr-paper-demo .tr-view { display: none; } |
| #tr-paper-demo .tr-view.is-active { display: grid; gap: var(--tr-gap); } |
| #tr-paper-demo .tr-overview { |
| grid-template-columns: minmax(0, 1.45fr) minmax(230px, .55fr); |
| align-items: stretch; |
| } |
| #tr-paper-demo .tr-architecture { |
| min-height: 370px; |
| position: relative; |
| display: grid; |
| grid-template-columns: repeat(12, 1fr); |
| grid-template-rows: repeat(7, minmax(36px, auto)); |
| gap: 10px; |
| padding: 18px 0; |
| } |
| #tr-paper-demo .tr-lines { |
| position: absolute; |
| inset: 0; |
| width: 100%; |
| height: 100%; |
| pointer-events: none; |
| color: var(--border); |
| } |
| #tr-paper-demo .tr-lines .active { |
| color: var(--viz-series-1); |
| transition: color .2s ease, stroke-width .2s ease; |
| stroke-width: 2.5; |
| } |
| #tr-paper-demo .tr-node { |
| position: relative; |
| z-index: 1; |
| display: grid; |
| align-content: center; |
| gap: 4px; |
| min-width: 0; |
| padding: 12px; |
| border: 1px solid var(--border); |
| border-radius: 12px; |
| background: color-mix(in srgb, var(--card) 88%, transparent); |
| color: var(--card-foreground); |
| } |
| #tr-paper-demo .tr-node strong { font-weight: 500; } |
| #tr-paper-demo .tr-node span { color: var(--muted-foreground); } |
| #tr-paper-demo .tr-node.route-active { |
| border-color: var(--viz-series-1); |
| background: color-mix(in srgb, var(--viz-series-1) 12%, var(--card)); |
| } |
| #tr-paper-demo .tr-context { grid-column: 2 / 7; grid-row: 1 / 2; } |
| #tr-paper-demo .tr-token { grid-column: 8 / 12; grid-row: 1 / 2; } |
| #tr-paper-demo .tr-shared { grid-column: 1 / 6; grid-row: 3 / 5; } |
| #tr-paper-demo .tr-table { grid-column: 9 / 13; grid-row: 3 / 5; } |
| #tr-paper-demo .tr-experts { |
| grid-column: 6 / 9; |
| grid-row: 3 / 5; |
| display: grid; |
| gap: 6px; |
| grid-template-columns: repeat(2, minmax(0, 1fr)); |
| } |
| #tr-paper-demo .tr-expert { |
| display: grid; |
| place-items: center; |
| min-height: 42px; |
| border-radius: 8px; |
| border: 1px solid var(--border); |
| color: var(--muted-foreground); |
| background: color-mix(in srgb, var(--muted) 45%, transparent); |
| transition: background .2s ease, color .2s ease, border-color .2s ease; |
| } |
| #tr-paper-demo .tr-expert.route-active { |
| color: var(--foreground); |
| border-color: var(--viz-series-1); |
| background: color-mix(in srgb, var(--viz-series-1) 18%, transparent); |
| } |
| #tr-paper-demo .tr-sum { |
| grid-column: 5 / 9; |
| grid-row: 6 / 7; |
| place-self: center; |
| width: 46px; |
| height: 46px; |
| border-radius: 50%; |
| text-align: center; |
| padding: 0; |
| font-size: 1.3rem; |
| } |
| #tr-paper-demo .tr-output { grid-column: 9 / 13; grid-row: 6 / 7; } |
| #tr-paper-demo .tr-side { display: grid; gap: 12px; align-content: start; } |
| #tr-paper-demo .tr-metric-grid { |
| display: grid; |
| grid-template-columns: repeat(2, minmax(0, 1fr)); |
| gap: 10px; |
| } |
| #tr-paper-demo .tr-route-detail { min-height: 52px; } |
| #tr-paper-demo .tr-routing-layout, |
| #tr-paper-demo .tr-results-layout, |
| #tr-paper-demo .tr-compare-grid { |
| display: grid; |
| grid-template-columns: repeat(2, minmax(0, 1fr)); |
| gap: var(--tr-gap); |
| } |
| #tr-paper-demo .tr-token-grid { |
| display: grid; |
| grid-template-columns: repeat(4, minmax(0, 1fr)); |
| gap: 8px; |
| } |
| #tr-paper-demo .tr-token-btn { justify-content: center; } |
| #tr-paper-demo .tr-token-btn.is-selected { |
| background: var(--primary); |
| color: var(--primary-foreground); |
| } |
| #tr-paper-demo .tr-route-matrix { |
| display: grid; |
| grid-template-columns: 70px repeat(4, minmax(0, 1fr)); |
| gap: 7px; |
| align-items: center; |
| } |
| #tr-paper-demo .tr-route-cell { |
| display: grid; |
| place-items: center; |
| min-height: 38px; |
| border-radius: 8px; |
| background: color-mix(in srgb, var(--muted) 55%, transparent); |
| color: var(--muted-foreground); |
| } |
| #tr-paper-demo .tr-route-cell.is-on { |
| background: color-mix(in srgb, var(--viz-series-1) 20%, transparent); |
| color: var(--foreground); |
| } |
| #tr-paper-demo .tr-result-chart { |
| min-height: 250px; |
| display: flex; |
| align-items: end; |
| gap: 22px; |
| padding: 30px 18px 24px; |
| border-bottom: 1px solid var(--border); |
| position: relative; |
| } |
| #tr-paper-demo .tr-bar-group { |
| flex: 1; |
| display: grid; |
| grid-template-columns: repeat(2, minmax(24px, 60px)); |
| justify-content: center; |
| gap: 8px; |
| align-items: end; |
| height: 190px; |
| position: relative; |
| } |
| #tr-paper-demo .tr-bar { |
| border-radius: 8px 8px 2px 2px; |
| min-height: 20px; |
| background: color-mix(in srgb, var(--viz-series-2) 58%, var(--muted)); |
| position: relative; |
| transition: height .25s ease; |
| } |
| #tr-paper-demo .tr-bar.routed { |
| background: color-mix(in srgb, var(--viz-series-1) 68%, var(--muted)); |
| } |
| #tr-paper-demo .tr-bar span { |
| position: absolute; |
| left: 50%; |
| bottom: calc(100% + 7px); |
| transform: translateX(-50%); |
| color: var(--foreground); |
| white-space: nowrap; |
| } |
| #tr-paper-demo .tr-bar-label { |
| position: absolute; |
| top: calc(100% + 9px); |
| left: 0; |
| right: 0; |
| text-align: center; |
| color: var(--muted-foreground); |
| } |
| #tr-paper-demo .tr-legend { justify-content: center; } |
| #tr-paper-demo .tr-swatch { |
| width: 10px; |
| height: 10px; |
| border-radius: 3px; |
| background: color-mix(in srgb, var(--viz-series-2) 58%, var(--muted)); |
| } |
| #tr-paper-demo .tr-swatch.routed { |
| background: color-mix(in srgb, var(--viz-series-1) 68%, var(--muted)); |
| } |
| #tr-paper-demo .tr-generation { |
| min-height: 172px; |
| white-space: pre-wrap; |
| overflow-wrap: anywhere; |
| } |
| #tr-paper-demo .tr-caret { |
| display: inline-block; |
| width: 1px; |
| height: 1em; |
| margin-left: 2px; |
| vertical-align: -.12em; |
| background: var(--primary); |
| animation: tr-blink .8s steps(1) infinite; |
| } |
| #tr-paper-demo .tr-prompt-row { |
| display: grid; |
| grid-template-columns: minmax(0, 1fr) auto; |
| gap: 10px; |
| align-items: end; |
| } |
| #tr-paper-demo .tr-status { color: var(--muted-foreground); } |
| #tr-paper-demo .tr-status.is-live { color: var(--foreground); } |
| @keyframes tr-blink { 50% { opacity: 0; } } |
| @media (prefers-reduced-motion: reduce) { |
| #tr-paper-demo .tr-caret { animation: none; } |
| } |
| @media (max-width: 680px) { |
| #tr-paper-demo .tr-hero, |
| #tr-paper-demo .tr-overview, |
| #tr-paper-demo .tr-routing-layout, |
| #tr-paper-demo .tr-results-layout, |
| #tr-paper-demo .tr-compare-grid { |
| grid-template-columns: 1fr; |
| } |
| #tr-paper-demo .tr-actions { display: none; } |
| #tr-paper-demo .tr-architecture { min-height: 440px; } |
| #tr-paper-demo .tr-context { grid-column: 1 / 8; } |
| #tr-paper-demo .tr-token { grid-column: 8 / 13; } |
| #tr-paper-demo .tr-shared { grid-column: 1 / 6; grid-row: 3 / 5; } |
| #tr-paper-demo .tr-experts { grid-column: 6 / 10; grid-row: 3 / 5; } |
| #tr-paper-demo .tr-table { grid-column: 10 / 13; grid-row: 3 / 5; } |
| #tr-paper-demo .tr-sum { grid-column: 4 / 7; } |
| #tr-paper-demo .tr-output { grid-column: 7 / 13; } |
| #tr-paper-demo .tr-token-grid { grid-template-columns: repeat(2, minmax(0, 1fr)); } |
| #tr-paper-demo .tr-prompt-row { grid-template-columns: 1fr; } |
| } |
| @media (max-width: 430px) { |
| #tr-paper-demo .tr-tabs { gap: 6px; } |
| #tr-paper-demo .tr-tab { padding-inline: 9px; } |
| #tr-paper-demo .tr-architecture { |
| display: flex; |
| flex-direction: column; |
| min-height: auto; |
| } |
| #tr-paper-demo .tr-lines, #tr-paper-demo .tr-sum { display: none; } |
| #tr-paper-demo .tr-node, |
| #tr-paper-demo .tr-context, |
| #tr-paper-demo .tr-token, |
| #tr-paper-demo .tr-shared, |
| #tr-paper-demo .tr-table, |
| #tr-paper-demo .tr-experts, |
| #tr-paper-demo .tr-output { |
| width: 100%; |
| } |
| #tr-paper-demo .tr-route-matrix { grid-template-columns: 54px repeat(4, minmax(0, 1fr)); } |
| } |
| </style> |
|
|
| <div class="tr-shell"> |
| <header class="tr-topbar"> |
| <div class="tr-brand"> |
| <span class="tr-mark"><i data-lucide="git-branch" aria-hidden="true"></i></span> |
| <span class="tr-brand-copy"> |
| <strong>COMPLEXITY</strong> |
| <span class="text-small">Research companion</span> |
| </span> |
| </div> |
| <nav class="tr-actions" aria-label="Research links"> |
| <a class="btn btn-ghost" href="https://www.complexity-ai.fr/papers/token-identity-routing-residual-experts.pdf" target="_blank" rel="noopener noreferrer"><i data-lucide="file-text" aria-hidden="true"></i> Paper</a> |
| <a class="btn btn-ghost" href="https://github.com/Complexity-ML/tmlr-paper-pool" target="_blank" rel="noopener noreferrer"><i data-lucide="package-open" aria-hidden="true"></i> Supplement</a> |
| <a class="btn btn-ghost" href="https://github.com/Complexity-ML/tmlr-paper-pool" target="_blank" rel="noopener noreferrer"><i data-lucide="github" aria-hidden="true"></i> Code</a> |
| </nav> |
| </header> |
|
|
| <section class="tr-hero"> |
| <div class="tr-hero-copy"> |
| <span class="tr-kicker text-small">Interactive architecture study</span> |
| <h2>Token identity as a routing signal for residual MLP experts</h2> |
| <p>Explore how a fixed lexical lookup selects a narrow residual parameter subspace while a shared dense SwiGLU path handles contextual computation.</p> |
| <div class="tr-chip-row"> |
| <span class="viz-badge">306.5M parameters</span> |
| <span class="viz-badge">8B training tokens</span> |
| <span class="viz-badge">fixed top-2 routing</span> |
| </div> |
| </div> |
| <div class="tr-note text-small"> |
| Interactive companion — not additional experimental evidence. |
| </div> |
| </section> |
|
|
| <nav class="tr-tabs" aria-label="Interactive paper sections"> |
| <button class="btn tr-tab" type="button" role="tab" aria-selected="true" data-view="overview">Architecture</button> |
| <button class="btn tr-tab" type="button" role="tab" aria-selected="false" data-view="routing">Routing lab</button> |
| <button class="btn tr-tab" type="button" role="tab" aria-selected="false" data-view="results">Results</button> |
| <button class="btn tr-tab" type="button" role="tab" aria-selected="false" data-view="compare">Live compare</button> |
| </nav> |
|
|
| <section class="tr-view tr-overview is-active" data-panel="overview" role="tabpanel"> |
| <div> |
| <div class="tr-chip-row" aria-label="Select a token"> |
| <span class="text-small text-muted">Token</span> |
| <button class="btn tr-token-choice is-selected" type="button" data-token="routing">routing</button> |
| <button class="btn tr-token-choice" type="button" data-token="context">context</button> |
| <button class="btn tr-token-choice" type="button" data-token="expert">expert</button> |
| <button class="btn tr-token-choice" type="button" data-token="shared">shared</button> |
| </div> |
|
|
| <div class="tr-architecture" role="img" aria-label="The contextual hidden state enters both a shared SwiGLU branch and two selected residual experts. Token identity controls only the fixed route table."> |
| <svg class="tr-lines" aria-hidden="true"> |
| <path data-from=".tr-context" data-from-anchor="bottom" data-to=".tr-shared" data-to-anchor="top" fill="none" stroke="currentColor" stroke-width="1.5"/> |
| <path class="active" data-from=".tr-context" data-from-anchor="bottom" data-to=".tr-experts" data-to-anchor="top" fill="none" stroke="currentColor"/> |
| <path class="active" data-from=".tr-token" data-from-anchor="bottom" data-to=".tr-table" data-to-anchor="top" fill="none" stroke="currentColor"/> |
| <path class="active" data-from=".tr-table" data-from-anchor="left" data-to=".tr-experts" data-to-anchor="right" fill="none" stroke="currentColor" stroke-dasharray="7 7"/> |
| <path data-from=".tr-shared" data-from-anchor="bottom" data-to=".tr-sum" data-to-anchor="left" fill="none" stroke="currentColor" stroke-width="1.5"/> |
| <path class="active" data-from=".tr-experts" data-from-anchor="bottom" data-to=".tr-sum" data-to-anchor="top" fill="none" stroke="currentColor"/> |
| <path class="active" data-from=".tr-sum" data-from-anchor="right" data-to=".tr-output" data-to-anchor="left" fill="none" stroke="currentColor"/> |
| </svg> |
|
|
| <div class="tr-node tr-context"> |
| <strong>Contextual hidden state <em>x<sub>t</sub></em></strong> |
| <span class="text-small">The same contextual vector enters both branches</span> |
| </div> |
| <div class="tr-node tr-token route-active"> |
| <strong>Token identifier <em>t</em></strong> |
| <span class="text-small" data-token-id>routing · tokenizer ID 22447</span> |
| </div> |
| <div class="tr-node tr-shared"> |
| <strong>Shared SwiGLU</strong> |
| <span class="text-small">Width 3,840 · all tokens · contextual computation</span> |
| </div> |
| <div class="tr-node tr-table route-active"> |
| <strong>Fixed route table <em>r<sub>l</sub>(t)</em></strong> |
| <span class="text-small">Selects two of four experts</span> |
| </div> |
| <div class="tr-node tr-experts route-active"> |
| <strong style="grid-column:1/-1">Residual experts</strong> |
| <span class="tr-expert" data-expert="0">E0</span> |
| <span class="tr-expert route-active" data-expert="1">E1</span> |
| <span class="tr-expert route-active" data-expert="2">E2</span> |
| <span class="tr-expert" data-expert="3">E3</span> |
| </div> |
| <button class="btn tr-sum" type="button" data-tooltip="Element-wise branch sum" aria-label="Add the branch outputs">+</button> |
| <div class="tr-node tr-output"> |
| <strong>Feed-forward output</strong> |
| <span class="text-small">Shared path + 0.5 E<span data-route-a>1</span> + 0.5 E<span data-route-b>2</span></span> |
| </div> |
| </div> |
| </div> |
|
|
| <aside class="tr-side"> |
| <div class="tr-metric-grid"> |
| <div class="card viz-stat"> |
| <span class="text-muted text-small">Matched pair</span> |
| <span class="viz-stat-value">306.5M</span> |
| <span class="text-small">trainable parameters</span> |
| </div> |
| <div class="card viz-stat"> |
| <span class="text-muted text-small">Primary run</span> |
| <span class="viz-stat-value">8B</span> |
| <span class="text-small">FineWeb-Edu tokens</span> |
| </div> |
| </div> |
| <div class="card tr-route-detail"> |
| <div class="tr-node-head"> |
| <i data-lucide="route" aria-hidden="true"></i> |
| <strong>Selected route</strong> |
| </div> |
| <p data-route-copy>“routing” selects E1 + E2 in layer 1. The primary route is checkpoint-derived; the published inference runtime derives the cyclic secondary route.</p> |
| </div> |
| <div class="card"> |
| <strong>Scope of evidence</strong> |
| <p class="text-muted">One matched 306.5M run pair. The result is reported as an observation, not a claim of statistical significance.</p> |
| </div> |
| </aside> |
| </section> |
|
|
| <section class="tr-view" data-panel="routing" role="tabpanel"> |
| <div class="tr-routing-layout"> |
| <div> |
| <div class="tr-run-head"> |
| <div> |
| <strong>Layer-specific deterministic routes</strong> |
| <div class="text-small text-muted">Checkpoint-derived tokenizer IDs and fixed top-2 assignments.</div> |
| </div> |
| </div> |
| <div class="tr-token-grid" style="margin-top:12px"> |
| <button class="btn tr-token-btn is-selected" type="button" data-lab-token="routing">routing</button> |
| <button class="btn tr-token-btn" type="button" data-lab-token="context">context</button> |
| <button class="btn tr-token-btn" type="button" data-lab-token="expert">expert</button> |
| <button class="btn tr-token-btn" type="button" data-lab-token="shared">shared</button> |
| <button class="btn tr-token-btn" type="button" data-lab-token="token">token</button> |
| <button class="btn tr-token-btn" type="button" data-lab-token="model">model</button> |
| <button class="btn tr-token-btn" type="button" data-lab-token="dense">dense</button> |
| <button class="btn tr-token-btn" type="button" data-lab-token="learned">learned</button> |
| </div> |
| </div> |
| <div class="card"> |
| <div class="tr-route-matrix" aria-label="Expert assignment matrix"> |
| <span></span><span class="text-small text-center">E0</span><span class="text-small text-center">E1</span><span class="text-small text-center">E2</span><span class="text-small text-center">E3</span> |
| <span class="text-small">Layer 1</span><span class="tr-route-cell" data-cell="1-0">—</span><span class="tr-route-cell is-on" data-cell="1-1">0.5</span><span class="tr-route-cell is-on" data-cell="1-2">0.5</span><span class="tr-route-cell" data-cell="1-3">—</span> |
| <span class="text-small">Layer 6</span><span class="tr-route-cell" data-cell="6-0">—</span><span class="tr-route-cell" data-cell="6-1">—</span><span class="tr-route-cell is-on" data-cell="6-2">0.5</span><span class="tr-route-cell is-on" data-cell="6-3">0.5</span> |
| <span class="text-small">Layer 12</span><span class="tr-route-cell is-on" data-cell="12-0">0.5</span><span class="tr-route-cell" data-cell="12-1">—</span><span class="tr-route-cell" data-cell="12-2">—</span><span class="tr-route-cell is-on" data-cell="12-3">0.5</span> |
| <span class="text-small">Layer 18</span><span class="tr-route-cell" data-cell="18-0">—</span><span class="tr-route-cell" data-cell="18-1">—</span><span class="tr-route-cell is-on" data-cell="18-2">0.5</span><span class="tr-route-cell is-on" data-cell="18-3">0.5</span> |
| </div> |
| </div> |
| </div> |
| <div class="tr-note text-small" data-lab-detail>Token “routing” follows a reproducible, layer-specific route. A different occurrence of the same token keeps the same route, while its hidden-state input still changes with context.</div> |
| </section> |
|
|
| <section class="tr-view" data-panel="results" role="tabpanel"> |
| <div class="tr-results-layout"> |
| <div> |
| <div class="tr-run-head"> |
| <div> |
| <strong>Last common evaluation point</strong> |
| <div class="text-small text-muted">Fixed evaluation stream from the FineWeb-Edu training split · lower is better</div> |
| </div> |
| </div> |
| <div class="tr-result-chart" role="img" aria-label="Evaluation NLL: dense 2.9482, token-routed 2.9329."> |
| <div class="tr-bar-group"> |
| <div class="tr-bar" style="height:81%"><span>2.9482</span></div> |
| <div class="tr-bar routed" style="height:68%"><span>2.9329</span></div> |
| <div class="tr-bar-label text-small">Evaluation NLL</div> |
| </div> |
| <div class="tr-bar-group"> |
| <div class="tr-bar" style="height:75%"><span>1.00×</span></div> |
| <div class="tr-bar routed" style="height:96%"><span>0.79×</span></div> |
| <div class="tr-bar-label text-small">Training throughput</div> |
| </div> |
| </div> |
| <div class="tr-legend text-small" style="margin-top:30px"> |
| <span class="tr-swatch"></span><span>Dense</span> |
| <span class="tr-swatch routed"></span><span>Token-routed</span> |
| </div> |
| </div> |
| <div class="card"> |
| <table class="table table-sm"> |
| <thead> |
| <tr><th>Measurement</th><th class="text-end">Dense</th><th class="text-end">Routed</th></tr> |
| </thead> |
| <tbody> |
| <tr><td>Parameters</td><td class="text-end">306.5M</td><td class="text-end">306.5M</td></tr> |
| <tr><td>Evaluation NLL</td><td class="text-end">2.9482</td><td class="text-end">2.9329</td></tr> |
| <tr><td>NLL difference</td><td class="text-end">—</td><td class="text-end">−0.0153</td></tr> |
| <tr><td>Training tokens</td><td class="text-end">8B</td><td class="text-end">8B</td></tr> |
| <tr><td>Seeds</td><td class="text-end">1</td><td class="text-end">1</td></tr> |
| </tbody> |
| </table> |
| <p class="text-small text-muted">Single-seed matched-run observation. Evaluation is not held out; downstream multiple-choice differences are within one reported standard error.</p> |
| </div> |
| </div> |
| </section> |
|
|
| <section class="tr-view" data-panel="compare" role="tabpanel"> |
| <div class="tr-prompt-row"> |
| <label class="form-label"> |
| Prompt |
| <input class="form-control" data-prompt value="Machine learning is a branch of artificial intelligence that" /> |
| </label> |
| <button class="btn btn-primary" type="button" data-run> |
| <i data-lucide="play" aria-hidden="true"></i> Run comparison |
| </button> |
| </div> |
| <div class="tr-compare-grid"> |
| <div class="card"> |
| <div class="tr-run-head"> |
| <strong>Dense-306</strong> |
| <span class="viz-badge" data-dense-state>ready</span> |
| </div> |
| <p class="tr-generation" data-dense-output aria-live="polite">Dense output will stream here token by token.</p> |
| </div> |
| <div class="card"> |
| <div class="tr-run-head"> |
| <strong>TR-MOE-306</strong> |
| <span class="viz-badge" data-routed-state>ready</span> |
| </div> |
| <p class="tr-generation" data-routed-output aria-live="polite">Token-routed output will stream here token by token.</p> |
| </div> |
| </div> |
| <div class="tr-status text-small" data-status>Public CPU inference · SSE streaming · qualitative comparison only</div> |
| </section> |
| </div> |
|
|
| <script> |
| (() => { |
| const root = document.getElementById("tr-paper-demo"); |
| if (!root) return; |
| |
| const routes = { |
| routing: { id: 22447, layers: [[1, 2], [2, 3], [3, 0], [2, 3]] }, |
| context: { id: 4057, layers: [[0, 1], [1, 2], [2, 3], [0, 1]] }, |
| expert: { id: 5217, layers: [[0, 1], [1, 2], [2, 3], [0, 1]] }, |
| shared: { id: 5586, layers: [[3, 0], [0, 1], [1, 2], [3, 0]] }, |
| token: { id: 23394, layers: [[3, 0], [0, 1], [1, 2], [3, 0]] }, |
| model: { id: 2181, layers: [[0, 1], [1, 2], [2, 3], [0, 1]] }, |
| dense: { id: 8581, layers: [[0, 1], [1, 2], [2, 3], [0, 1]] }, |
| learned: { id: 4273, layers: [[0, 1], [1, 2], [2, 3], [0, 1]] } |
| }; |
| |
| const tabs = [...root.querySelectorAll(".tr-tab")]; |
| const panels = [...root.querySelectorAll(".tr-view")]; |
| const architecture = root.querySelector(".tr-architecture"); |
| const lineLayer = root.querySelector(".tr-lines"); |
| |
| const anchorPoint = (element, anchor, bounds) => { |
| const rect = element.getBoundingClientRect(); |
| const points = { |
| top: { x: rect.left + rect.width / 2, y: rect.top }, |
| right: { x: rect.right, y: rect.top + rect.height / 2 }, |
| bottom: { x: rect.left + rect.width / 2, y: rect.bottom }, |
| left: { x: rect.left, y: rect.top + rect.height / 2 } |
| }; |
| const point = points[anchor] || points.bottom; |
| return { x: point.x - bounds.left, y: point.y - bounds.top }; |
| }; |
| |
| const updateConnectorPaths = () => { |
| if (!architecture || !lineLayer || !architecture.offsetParent) return; |
| const bounds = architecture.getBoundingClientRect(); |
| if (!bounds.width || !bounds.height) return; |
| lineLayer.setAttribute("viewBox", `0 0 ${bounds.width} ${bounds.height}`); |
| lineLayer.querySelectorAll("[data-from][data-to]").forEach((path) => { |
| const from = architecture.querySelector(path.dataset.from); |
| const to = architecture.querySelector(path.dataset.to); |
| if (!from || !to) return; |
| const start = anchorPoint(from, path.dataset.fromAnchor, bounds); |
| const end = anchorPoint(to, path.dataset.toAnchor, bounds); |
| const dx = end.x - start.x; |
| const dy = end.y - start.y; |
| let d; |
| if (Math.abs(dx) > Math.abs(dy)) { |
| const direction = Math.sign(dx) || 1; |
| const bend = Math.min(Math.max(16, Math.abs(dx) * 0.42), Math.abs(dx) / 2); |
| d = `M ${start.x} ${start.y} C ${start.x + direction * bend} ${start.y}, ${end.x - direction * bend} ${end.y}, ${end.x} ${end.y}`; |
| } else { |
| const direction = Math.sign(dy) || 1; |
| const bend = Math.min(Math.max(16, Math.abs(dy) * 0.42), Math.abs(dy) / 2); |
| d = `M ${start.x} ${start.y} C ${start.x} ${start.y + direction * bend}, ${end.x} ${end.y - direction * bend}, ${end.x} ${end.y}`; |
| } |
| path.setAttribute("d", d); |
| }); |
| }; |
| |
| tabs.forEach((tab) => { |
| tab.addEventListener("click", () => { |
| tabs.forEach((item) => item.setAttribute("aria-selected", String(item === tab))); |
| panels.forEach((panel) => panel.classList.toggle("is-active", panel.dataset.panel === tab.dataset.view)); |
| if (tab.dataset.view === "overview") requestAnimationFrame(updateConnectorPaths); |
| }); |
| }); |
| |
| if (architecture && "ResizeObserver" in window) { |
| const connectorResizeObserver = new ResizeObserver(() => requestAnimationFrame(updateConnectorPaths)); |
| connectorResizeObserver.observe(architecture); |
| } |
| window.addEventListener("resize", updateConnectorPaths); |
| document.fonts?.ready.then(updateConnectorPaths); |
| requestAnimationFrame(updateConnectorPaths); |
| |
| const updateArchitecture = (token) => { |
| const route = routes[token]; |
| if (!route) return; |
| const pair = route.layers[0]; |
| root.querySelector("[data-token-id]").textContent = `${token} · tokenizer ID ${route.id}`; |
| root.querySelector("[data-route-a]").textContent = String(pair[0]); |
| root.querySelector("[data-route-b]").textContent = String(pair[1]); |
| root.querySelector("[data-route-copy]").textContent = |
| `“${token}” selects E${pair[0]} + E${pair[1]} in layer 1. The primary route is checkpoint-derived; the published inference runtime derives the cyclic secondary route.`; |
| root.querySelectorAll("[data-expert]").forEach((expert) => { |
| expert.classList.toggle("route-active", pair.includes(Number(expert.dataset.expert))); |
| }); |
| root.querySelectorAll(".tr-token-choice").forEach((button) => { |
| button.classList.toggle("is-selected", button.dataset.token === token); |
| }); |
| }; |
| |
| root.querySelectorAll(".tr-token-choice").forEach((button) => { |
| button.addEventListener("click", () => updateArchitecture(button.dataset.token)); |
| }); |
| |
| const updateMatrix = (token) => { |
| const route = routes[token]; |
| if (!route) return; |
| const layers = [1, 6, 12, 18]; |
| layers.forEach((layer, layerIndex) => { |
| const [a, b] = route.layers[layerIndex]; |
| for (let expert = 0; expert < 4; expert += 1) { |
| const cell = root.querySelector(`[data-cell="${layer}-${expert}"]`); |
| const active = expert === a || expert === b; |
| cell.classList.toggle("is-on", active); |
| cell.textContent = active ? "0.5" : "—"; |
| } |
| }); |
| root.querySelectorAll(".tr-token-btn").forEach((button) => { |
| button.classList.toggle("is-selected", button.dataset.labToken === token); |
| }); |
| root.querySelector("[data-lab-detail]").textContent = |
| `Token “${token}” (ID ${route.id}) follows these routes in the published inference runtime. Primary routes come from the checkpoint; cyclic secondary routes are deterministic.`; |
| }; |
| |
| root.querySelectorAll(".tr-token-btn").forEach((button) => { |
| button.addEventListener("click", () => updateMatrix(button.dataset.labToken)); |
| }); |
| |
| const COMPARE_ENDPOINT = "https://pacific-i64-compare-306.hf.space/v1/compare"; |
| let activeController = null; |
| |
| const consumeSSE = async (response, onEvent) => { |
| if (!response.body) throw new Error("Streaming response body is unavailable."); |
| const reader = response.body.getReader(); |
| const decoder = new TextDecoder(); |
| let buffer = ""; |
| while (true) { |
| const { done, value } = await reader.read(); |
| if (done) break; |
| buffer += decoder.decode(value, { stream: true }); |
| const lines = buffer.split("\n"); |
| buffer = lines.pop() || ""; |
| for (const line of lines) { |
| const trimmed = line.trim(); |
| if (!trimmed.startsWith("data:")) continue; |
| const payload = trimmed.slice(5).trim(); |
| if (payload === "[DONE]") return; |
| try { |
| onEvent(JSON.parse(payload)); |
| } catch { |
| |
| } |
| } |
| } |
| }; |
| |
| root.querySelector("[data-run]").addEventListener("click", async () => { |
| const runButton = root.querySelector("[data-run]"); |
| const denseOutput = root.querySelector("[data-dense-output]"); |
| const routedOutput = root.querySelector("[data-routed-output]"); |
| const denseState = root.querySelector("[data-dense-state]"); |
| const routedState = root.querySelector("[data-routed-state]"); |
| const status = root.querySelector("[data-status]"); |
| const prompt = root.querySelector("[data-prompt]").value.trim(); |
| |
| if (activeController) { |
| activeController.abort(); |
| activeController = null; |
| runButton.textContent = "Run comparison"; |
| denseState.textContent = "stopped"; |
| routedState.textContent = "stopped"; |
| status.textContent = "Generation stopped."; |
| status.classList.remove("is-live"); |
| return; |
| } |
| if (!prompt) { |
| status.textContent = "Enter a prompt before running the comparison."; |
| return; |
| } |
| |
| const controller = new AbortController(); |
| activeController = controller; |
| denseOutput.textContent = ""; |
| routedOutput.textContent = ""; |
| denseState.textContent = "connecting"; |
| routedState.textContent = "connecting"; |
| runButton.textContent = "Stop generation"; |
| status.textContent = "Connecting to both public 306.5M CPU endpoints…"; |
| status.classList.add("is-live"); |
| |
| try { |
| const response = await fetch(COMPARE_ENDPOINT, { |
| method: "POST", |
| headers: { "Content-Type": "application/json" }, |
| body: JSON.stringify({ |
| prompt, |
| max_tokens: 128, |
| temperature: 0.7, |
| stream: true |
| }), |
| signal: controller.signal |
| }); |
| if (!response.ok) { |
| const detail = (await response.text()).slice(0, 240); |
| throw new Error(`Compare endpoint returned HTTP ${response.status}${detail ? `: ${detail}` : ""}`); |
| } |
| denseState.textContent = "streaming"; |
| routedState.textContent = "streaming"; |
| status.textContent = "Receiving two real SSE streams token by token…"; |
| await consumeSSE(response, (event) => { |
| if (event.error) { |
| if (event.type === "dense") denseState.textContent = "error"; |
| if (event.type === "tr_moe") routedState.textContent = "error"; |
| return; |
| } |
| if (!event.text) return; |
| if (event.type === "dense") denseOutput.textContent += event.text; |
| if (event.type === "tr_moe") routedOutput.textContent += event.text; |
| }); |
| denseState.textContent = denseOutput.textContent ? "complete" : "no output"; |
| routedState.textContent = routedOutput.textContent ? "complete" : "no output"; |
| status.textContent = "Comparison complete · qualitative output only"; |
| } catch (error) { |
| if (error.name === "AbortError") return; |
| denseState.textContent = "unavailable"; |
| routedState.textContent = "unavailable"; |
| status.textContent = `Inference temporarily unavailable: ${error.message}`; |
| } finally { |
| if (activeController === controller) activeController = null; |
| runButton.textContent = "Run comparison"; |
| status.classList.remove("is-live"); |
| } |
| }); |
| |
| window.addEventListener("DOMContentLoaded", () => { |
| globalThis.lucide?.createIcons({ attrs: { width: 16, height: 16 } }); |
| }, { once: true }); |
| })(); |
| </script> |
| </div> |
| </body> |
| </html> |
|
|
