| <?xml version="1.0" encoding="UTF-8" standalone="no"?> |
| <svg |
| width="1460" |
| height="900" |
| viewBox="0 0 1460 900" |
| role="img" |
| aria-labelledby="title desc" |
| version="1.1" |
| id="svg194" |
| xmlns="http://www.w3.org/2000/svg" |
| xmlns:svg="http://www.w3.org/2000/svg"> |
| <title |
| id="title">Three-level nested byte Mamba-2 architecture</title> |
| <desc |
| id="desc">Bytes are grouped by learned close heads into fine pools of 1 to 96 bytes, which are grouped into level-2 pools of 4 to 16 fine pools, which are grouped into level-3 pools of 2 to 16 level-2 pools. Six, twenty, and thirty Mamba-2 blocks run at the three resolutions. Each byte is decoded from five 2000-dimensional readouts: the local byte context, the latest completed fine-pool readout available at the current byte, the fine readout used at the previous byte, the latest completed level-2 state, and the latest completed level-3 state. These are fused by an 8000-wide decoder into 260 next-byte logits. A causal controller emitted at t-1 can adjust the close decision at t.</desc> |
| <defs |
| id="defs15"> |
| <linearGradient |
| id="bg" |
| x1="0" |
| y1="0" |
| x2="1" |
| y2="1"> |
| <stop |
| stop-color="#08111f" |
| id="stop1" /> |
| <stop |
| offset="1" |
| stop-color="#13233d" |
| id="stop2" /> |
| </linearGradient> |
| <linearGradient |
| id="fine" |
| x1="0" |
| y1="0" |
| x2="1" |
| y2="1"> |
| <stop |
| stop-color="#0f766e" |
| id="stop3" /> |
| <stop |
| offset="1" |
| stop-color="#14b8a6" |
| id="stop4" /> |
| </linearGradient> |
| <linearGradient |
| id="l2" |
| x1="0" |
| y1="0" |
| x2="1" |
| y2="1"> |
| <stop |
| stop-color="#9a5b13" |
| id="stop5" /> |
| <stop |
| offset="1" |
| stop-color="#f59e0b" |
| id="stop6" /> |
| </linearGradient> |
| <linearGradient |
| id="l3" |
| x1="0" |
| y1="0" |
| x2="1" |
| y2="1"> |
| <stop |
| stop-color="#5b21b6" |
| id="stop7" /> |
| <stop |
| offset="1" |
| stop-color="#8b5cf6" |
| id="stop8" /> |
| </linearGradient> |
| <linearGradient |
| id="dec" |
| x1="0" |
| y1="0" |
| x2="1" |
| y2="1"> |
| <stop |
| stop-color="#1d4ed8" |
| id="stop9" /> |
| <stop |
| offset="1" |
| stop-color="#38bdf8" |
| id="stop10" /> |
| </linearGradient> |
| <radialGradient |
| id="headerGlow" |
| cx="50%" |
| cy="0%" |
| r="80%"> |
| <stop |
| stop-color="#38bdf8" |
| stop-opacity=".13" |
| id="stop11" /> |
| <stop |
| offset="1" |
| stop-color="#38bdf8" |
| stop-opacity="0" |
| id="stop12" /> |
| </radialGradient> |
| <pattern |
| id="grid" |
| width="32" |
| height="32" |
| patternUnits="userSpaceOnUse"> |
| <path |
| d="M32 0H0V32" |
| fill="none" |
| stroke="#dbeafe" |
| stroke-opacity=".025" |
| stroke-width="1" |
| id="path12" /> |
| </pattern> |
| <filter |
| id="shadow" |
| x="0" |
| y="0" |
| width="1" |
| height="1"> |
| <feDropShadow |
| dx="0" |
| dy="7" |
| stdDeviation="8" |
| flood-opacity=".28" /> |
| </filter> |
| <filter |
| id="cyanGlow" |
| x="-0.1" |
| y="-0.1" |
| width="1.2" |
| height="1.2"> |
| <feDropShadow |
| dx="0" |
| dy="0" |
| stdDeviation="3" |
| flood-color="#67e8f9" |
| flood-opacity=".85" /> |
| </filter> |
| <marker |
| id="arrow" |
| markerWidth="10" |
| markerHeight="10" |
| refX="9" |
| refY="3" |
| orient="auto" |
| markerUnits="strokeWidth"> |
| <path |
| d="M0,0 L0,6 L9,3 z" |
| fill="#a8bddb" |
| id="path13" /> |
| </marker> |
| <marker |
| id="arrowS" |
| markerWidth="8" |
| markerHeight="8" |
| refX="7" |
| refY="3" |
| orient="auto" |
| markerUnits="strokeWidth"> |
| <path |
| d="M0,0 L0,6 L7,3 z" |
| fill="#64748b" |
| id="path14" /> |
| </marker> |
| <marker |
| id="arrowC" |
| markerWidth="10" |
| markerHeight="10" |
| refX="9" |
| refY="3" |
| orient="auto" |
| markerUnits="strokeWidth"> |
| <path |
| d="M0,0 L0,6 L9,3 z" |
| fill="#67e8f9" |
| id="path15" /> |
| </marker> |
| <style |
| id="style15"> |
| text{font-variant-numeric:tabular-nums;text-rendering:geometricPrecision}.title{font:700 28px system-ui,sans-serif;letter-spacing:-.35px;fill:#f4f8ff}.sub{font:14px system-ui,sans-serif;letter-spacing:.08px;fill:#b4c7df}.head{font:700 17px system-ui,sans-serif;fill:#f4f8ff}.headw{font:700 15px system-ui,sans-serif;fill:white}.body{font:13px system-ui,sans-serif;fill:#eef6ff}.tiny{font:11.5px system-ui,sans-serif;fill:#c5d5e9}.micro{font:10.5px system-ui,sans-serif;fill:#a9bfd9} |
| .box{stroke:#ffffff2c;stroke-width:1.5;filter:url(#shadow)} |
| .flow{fill:none;stroke:#a8bddb;stroke-width:2.5;marker-end:url(#arrow)} |
| .bracket{fill:none;stroke:#64748b;stroke-width:1.5} |
| .bracketOpen{fill:none;stroke:#94a3b8;stroke-width:1.5;stroke-dasharray:4 4} |
| .feedback{fill:none;stroke:#67e8f9;stroke-width:2.5;stroke-dasharray:8 7;marker-end:url(#arrowC)} |
| .poolbr{fill:none;stroke:#2dd4bf;stroke-width:1.5;opacity:.85} |
| .byte{fill:#33475f;stroke:#5b7796;stroke-width:.6} |
| .stateF{fill:url(#fine);stroke:#ffffff40;stroke-width:1} |
| .stateFopen{fill:#0f766e33;stroke:#2dd4bf;stroke-width:1.5;stroke-dasharray:5 4} |
| .state2{fill:url(#l2);stroke:#ffffff40;stroke-width:1} |
| .state2open{fill:#9a5b1333;stroke:#fbbf24;stroke-width:1.5;stroke-dasharray:5 4} |
| .state3{fill:url(#l3);stroke:#ffffff40;stroke-width:1} |
| .state3open{fill:#5b21b633;stroke:#c4b5fd;stroke-width:1.5;stroke-dasharray:5 4} |
| .boe{fill:#1e293b;stroke:#94a3b8;stroke-width:1} |
| .lblF{font:700 11px system-ui,sans-serif;fill:white}.lbl2{font:700 11px system-ui,sans-serif;fill:white}.lbl3{font:700 11px system-ui,sans-serif;fill:white} |
| .openlbl{font:9px system-ui,sans-serif;fill:#cbd5e1} |
| .pool{font:10.5px system-ui,sans-serif;fill:#99f6e4} |
| </style> |
| </defs> |
| <rect |
| width="1460" |
| height="900" |
| rx="24" |
| fill="url(#bg)" |
| id="rect15" /> |
| <rect |
| width="1460" |
| height="900" |
| rx="24" |
| fill="url(#grid)" |
| id="rect16" /> |
| <rect |
| width="1460" |
| height="170" |
| rx="24" |
| fill="url(#headerGlow)" |
| id="rect17" /> |
| |
| <text |
| x="40" |
| y="56" |
| class="title" |
| id="text17">Three-Level Nested Byte Mamba-2 — causal hierarchy</text> |
| <text |
| x="40" |
| y="84" |
| class="sub" |
| id="text18">2.478B parameters · dim 2000 · 56 Mamba-2 blocks (6 + 20 + 30) · d_state 64 · 260-ID byte vocabulary · tokenizer-free · BF16</text> |
| <path |
| d="M40 111 H1440" |
| stroke="#7dd3fc" |
| stroke-opacity=".18" |
| id="path18" /> |
| <rect |
| x="990" |
| y="24" |
| width="450" |
| height="76" |
| rx="12" |
| fill="#101d33" |
| stroke="#ffffff22" |
| id="rect18" /> |
| <text |
| x="1006" |
| y="46" |
| class="headw" |
| font-size="13" |
| id="text19">Mamba-2 block — ×6 fine · ×20 level 2 · ×30 level 3</text> |
| <text |
| x="1006" |
| y="66" |
| class="body" |
| font-size="12" |
| id="text20">x ← x + Mamba2(LN(x)) then x ← x + FFN(x)</text> |
| <text |
| x="1006" |
| y="82" |
| class="micro" |
| id="text21">Mamba2: d_state 64 · headdim 100 · d_conv 4 · expand 2</text> |
| <text |
| x="1006" |
| y="95" |
| class="micro" |
| id="text22">FFN: Linear(2000→4000) → GELU → Linear(4000→2000)</text> |
| |
| <rect |
| x="160" |
| y="140" |
| width="790" |
| height="95" |
| rx="12" |
| fill="#24364f" |
| opacity=".35" |
| stroke="#dbeafe" |
| stroke-opacity=".16" |
| id="rect22" /> |
| <rect |
| x="160" |
| y="250" |
| width="790" |
| height="90" |
| rx="12" |
| fill="#0f766e" |
| opacity=".16" |
| stroke="#5eead4" |
| stroke-opacity=".28" |
| id="rect23" /> |
| <rect |
| x="160" |
| y="355" |
| width="790" |
| height="90" |
| rx="12" |
| fill="#9a5b13" |
| opacity=".16" |
| stroke="#fbbf24" |
| stroke-opacity=".28" |
| id="rect24" /> |
| <rect |
| x="160" |
| y="460" |
| width="790" |
| height="90" |
| rx="12" |
| fill="#5b21b6" |
| opacity=".18" |
| stroke="#c4b5fd" |
| stroke-opacity=".28" |
| id="rect25" /> |
| |
| <text |
| x="40" |
| y="172" |
| class="head" |
| id="text25">Byte stream</text> |
| <text |
| x="40" |
| y="192" |
| class="tiny" |
| id="text26">260-ID vocab</text> |
| <text |
| x="40" |
| y="208" |
| class="tiny" |
| id="text27">1 pos / byte</text> |
| <text |
| x="40" |
| y="278" |
| class="head" |
| fill="#5eead4" |
| id="text28">Fine level 1</text> |
| <text |
| x="40" |
| y="298" |
| class="tiny" |
| id="text29">6 × Mamba-2</text> |
| <text |
| x="40" |
| y="314" |
| class="tiny" |
| id="text30">dim 2000</text> |
| <rect |
| x="40" |
| y="326" |
| width="14" |
| height="9" |
| class="stateF" |
| id="rect30" /> |
| <text |
| x="60" |
| y="334" |
| class="micro" |
| id="text31">completed (readable)</text> |
| <rect |
| x="40" |
| y="340" |
| width="14" |
| height="9" |
| class="stateFopen" |
| id="rect31" /> |
| <text |
| x="60" |
| y="348" |
| class="micro" |
| id="text32">open (not readable)</text> |
| <text |
| x="40" |
| y="383" |
| class="head" |
| fill="#fbbf24" |
| id="text33">Level 2</text> |
| <text |
| x="40" |
| y="403" |
| class="tiny" |
| id="text34">20 × Mamba-2</text> |
| <text |
| x="40" |
| y="419" |
| class="tiny" |
| id="text35">dim 2000</text> |
| <text |
| x="40" |
| y="488" |
| class="head" |
| fill="#c4b5fd" |
| id="text36">Level 3</text> |
| <text |
| x="40" |
| y="508" |
| class="tiny" |
| id="text37">30 × Mamba-2</text> |
| <text |
| x="40" |
| y="524" |
| class="tiny" |
| id="text38">dim 2000</text> |
| |
| <g |
| id="g104"> |
| <rect |
| x="170" |
| y="165" |
| width="10" |
| height="10" |
| class="byte" |
| id="rect38" /> |
| <rect |
| x="181" |
| y="165" |
| width="10" |
| height="10" |
| class="byte" |
| id="rect39" /> |
| <rect |
| x="192" |
| y="165" |
| width="10" |
| height="10" |
| class="byte" |
| id="rect40" /> |
| <rect |
| x="203" |
| y="165" |
| width="10" |
| height="10" |
| class="byte" |
| id="rect41" /> |
| <rect |
| x="214" |
| y="165" |
| width="10" |
| height="10" |
| class="byte" |
| id="rect42" /> |
| <rect |
| x="225" |
| y="165" |
| width="10" |
| height="10" |
| class="byte" |
| id="rect43" /> |
| <rect |
| x="236" |
| y="165" |
| width="10" |
| height="10" |
| class="byte" |
| id="rect44" /> |
| <rect |
| x="247" |
| y="165" |
| width="10" |
| height="10" |
| class="byte" |
| id="rect45" /> |
| <rect |
| x="258" |
| y="165" |
| width="10" |
| height="10" |
| class="byte" |
| id="rect46" /> |
| <rect |
| x="269" |
| y="165" |
| width="10" |
| height="10" |
| class="byte" |
| id="rect47" /> |
| <rect |
| x="280" |
| y="165" |
| width="10" |
| height="10" |
| class="byte" |
| id="rect48" /> |
| <rect |
| x="291" |
| y="165" |
| width="10" |
| height="10" |
| class="byte" |
| id="rect49" /> |
| <rect |
| x="302" |
| y="165" |
| width="10" |
| height="10" |
| class="byte" |
| id="rect50" /> |
| <rect |
| x="313" |
| y="165" |
| width="10" |
| height="10" |
| class="byte" |
| id="rect51" /> |
| <rect |
| x="324" |
| y="165" |
| width="10" |
| height="10" |
| class="byte" |
| id="rect52" /> |
| <rect |
| x="335" |
| y="165" |
| width="10" |
| height="10" |
| class="byte" |
| id="rect53" /> |
| <rect |
| x="346" |
| y="165" |
| width="10" |
| height="10" |
| class="byte" |
| id="rect54" /> |
| <rect |
| x="357" |
| y="165" |
| width="10" |
| height="10" |
| class="byte" |
| id="rect55" /> |
| <rect |
| x="368" |
| y="165" |
| width="10" |
| height="10" |
| class="byte" |
| id="rect56" /> |
| <rect |
| x="379" |
| y="165" |
| width="10" |
| height="10" |
| class="byte" |
| id="rect57" /> |
| <rect |
| x="390" |
| y="165" |
| width="10" |
| height="10" |
| class="byte" |
| id="rect58" /> |
| <rect |
| x="401" |
| y="165" |
| width="10" |
| height="10" |
| class="byte" |
| id="rect59" /> |
| <rect |
| x="412" |
| y="165" |
| width="10" |
| height="10" |
| class="byte" |
| id="rect60" /> |
| <rect |
| x="423" |
| y="165" |
| width="10" |
| height="10" |
| class="byte" |
| id="rect61" /> |
| <rect |
| x="434" |
| y="165" |
| width="10" |
| height="10" |
| class="byte" |
| id="rect62" /> |
| <rect |
| x="445" |
| y="165" |
| width="10" |
| height="10" |
| class="byte" |
| id="rect63" /> |
| <rect |
| x="456" |
| y="165" |
| width="10" |
| height="10" |
| class="byte" |
| id="rect64" /> |
| <rect |
| x="467" |
| y="165" |
| width="10" |
| height="10" |
| class="byte" |
| id="rect65" /> |
| <rect |
| x="478" |
| y="165" |
| width="10" |
| height="10" |
| class="byte" |
| id="rect66" /> |
| <rect |
| x="489" |
| y="165" |
| width="10" |
| height="10" |
| class="byte" |
| id="rect67" /> |
| <rect |
| x="500" |
| y="165" |
| width="10" |
| height="10" |
| class="byte" |
| id="rect68" /> |
| <rect |
| x="511" |
| y="165" |
| width="10" |
| height="10" |
| class="byte" |
| id="rect69" /> |
| <rect |
| x="522" |
| y="165" |
| width="10" |
| height="10" |
| class="byte" |
| id="rect70" /> |
| <rect |
| x="533" |
| y="165" |
| width="10" |
| height="10" |
| class="byte" |
| id="rect71" /> |
| <rect |
| x="544" |
| y="165" |
| width="10" |
| height="10" |
| class="byte" |
| id="rect72" /> |
| <rect |
| x="555" |
| y="165" |
| width="10" |
| height="10" |
| class="byte" |
| id="rect73" /> |
| <rect |
| x="566" |
| y="165" |
| width="10" |
| height="10" |
| class="byte" |
| id="rect74" /> |
| <rect |
| x="577" |
| y="165" |
| width="10" |
| height="10" |
| class="byte" |
| id="rect75" /> |
| <rect |
| x="588" |
| y="165" |
| width="10" |
| height="10" |
| class="byte" |
| id="rect76" /> |
| <rect |
| x="599" |
| y="165" |
| width="10" |
| height="10" |
| class="byte" |
| id="rect77" /> |
| <rect |
| x="610" |
| y="165" |
| width="10" |
| height="10" |
| class="byte" |
| id="rect78" /> |
| <rect |
| x="621" |
| y="165" |
| width="10" |
| height="10" |
| class="byte" |
| id="rect79" /> |
| <rect |
| x="632" |
| y="165" |
| width="10" |
| height="10" |
| class="byte" |
| id="rect80" /> |
| <rect |
| x="643" |
| y="165" |
| width="10" |
| height="10" |
| class="byte" |
| id="rect81" /> |
| <rect |
| x="654" |
| y="165" |
| width="10" |
| height="10" |
| class="byte" |
| id="rect82" /> |
| <rect |
| x="665" |
| y="165" |
| width="10" |
| height="10" |
| class="byte" |
| id="rect83" /> |
| <rect |
| x="676" |
| y="165" |
| width="10" |
| height="10" |
| class="byte" |
| id="rect84" /> |
| <rect |
| x="687" |
| y="165" |
| width="10" |
| height="10" |
| class="byte" |
| id="rect85" /> |
| <rect |
| x="698" |
| y="165" |
| width="10" |
| height="10" |
| class="byte" |
| id="rect86" /> |
| <rect |
| x="709" |
| y="165" |
| width="10" |
| height="10" |
| class="byte" |
| id="rect87" /> |
| <rect |
| x="720" |
| y="165" |
| width="10" |
| height="10" |
| class="byte" |
| id="rect88" /> |
| <rect |
| x="731" |
| y="165" |
| width="10" |
| height="10" |
| class="byte" |
| id="rect89" /> |
| <rect |
| x="742" |
| y="165" |
| width="10" |
| height="10" |
| class="byte" |
| id="rect90" /> |
| <rect |
| x="753" |
| y="165" |
| width="10" |
| height="10" |
| class="byte" |
| id="rect91" /> |
| <rect |
| x="764" |
| y="165" |
| width="10" |
| height="10" |
| class="byte" |
| id="rect92" /> |
| <rect |
| x="775" |
| y="165" |
| width="10" |
| height="10" |
| fill="#f8fafc" |
| stroke="#67e8f9" |
| stroke-width="2" |
| filter="url(#cyanGlow)" |
| id="rect93" /> |
| <rect |
| x="786" |
| y="165" |
| width="10" |
| height="10" |
| class="byte" |
| id="rect94" /> |
| <rect |
| x="797" |
| y="165" |
| width="10" |
| height="10" |
| class="byte" |
| id="rect95" /> |
| <rect |
| x="808" |
| y="165" |
| width="10" |
| height="10" |
| class="byte" |
| id="rect96" /> |
| <rect |
| x="819" |
| y="165" |
| width="10" |
| height="10" |
| class="byte" |
| id="rect97" /> |
| <rect |
| x="830" |
| y="165" |
| width="10" |
| height="10" |
| class="byte" |
| id="rect98" /> |
| <rect |
| x="841" |
| y="165" |
| width="10" |
| height="10" |
| class="byte" |
| id="rect99" /> |
| <rect |
| x="852" |
| y="165" |
| width="10" |
| height="10" |
| class="byte" |
| id="rect100" /> |
| <rect |
| x="863" |
| y="165" |
| width="10" |
| height="10" |
| class="byte" |
| id="rect101" /> |
| <rect |
| x="874" |
| y="165" |
| width="10" |
| height="10" |
| class="byte" |
| id="rect102" /> |
| <rect |
| x="885" |
| y="165" |
| width="10" |
| height="10" |
| class="byte" |
| id="rect103" /> |
| <rect |
| x="896" |
| y="165" |
| width="10" |
| height="10" |
| class="byte" |
| id="rect104" /> |
| </g> |
| |
| <g |
| id="g113"> |
| <path |
| d="M170 192 V198 M224 192 V198 M170 192 H224" |
| class="poolbr" |
| id="path104" /> |
| <text |
| x="197" |
| y="212" |
| text-anchor="middle" |
| class="pool" |
| id="text104">p1</text> |
| <path |
| d="M225 192 V198 M257 192 V198 M225 192 H257" |
| class="poolbr" |
| id="path105" /> |
| <text |
| x="241" |
| y="212" |
| text-anchor="middle" |
| class="pool" |
| id="text105">p2</text> |
| <path |
| d="M258 192 V198 M367 192 V198 M258 192 H367" |
| class="poolbr" |
| id="path106" /> |
| <text |
| x="312" |
| y="212" |
| text-anchor="middle" |
| class="pool" |
| id="text106">p3</text> |
| <path |
| d="M368 192 V198 M441 192 V198 M368 192 H441" |
| class="poolbr" |
| id="path107" /> |
| <text |
| x="404" |
| y="212" |
| text-anchor="middle" |
| class="pool" |
| id="text107">p4</text> |
| <path |
| d="M445 192 V198 M476 192 V198 M445 192 H476" |
| class="poolbr" |
| id="path108" /> |
| <text |
| x="460" |
| y="212" |
| text-anchor="middle" |
| class="pool" |
| id="text108">p5</text> |
| <path |
| d="M479 192 V198 M599 192 V198 M479 192 H599" |
| class="poolbr" |
| id="path109" /> |
| <text |
| x="539" |
| y="212" |
| text-anchor="middle" |
| class="pool" |
| id="text109">p6</text> |
| <path |
| d="M600 192 V198 M663 192 V198 M600 192 H663" |
| class="poolbr" |
| id="path110" /> |
| <text |
| x="631" |
| y="212" |
| text-anchor="middle" |
| class="pool" |
| id="text110">p7</text> |
| <path |
| d="M664 192 V198 M750 192 V198 M664 192 H750" |
| class="poolbr" |
| id="path111" /> |
| <text |
| x="707" |
| y="212" |
| text-anchor="middle" |
| class="pool" |
| id="text111">p8</text> |
| <path |
| d="M751 192 V198 M805 192 V198 M751 192 H805" |
| class="poolbr" |
| id="path112" /> |
| <text |
| x="778" |
| y="212" |
| text-anchor="middle" |
| class="pool" |
| id="text112">p9</text> |
| <path |
| d="M806 192 V198 M905 192 V198 M806 192 H905" |
| class="poolbr" |
| id="path113" /> |
| <text |
| x="855" |
| y="212" |
| text-anchor="middle" |
| class="pool" |
| id="text113">p10</text> |
| </g> |
| |
| <path |
| d="M780 187 L788 195 L780 203 L772 195 Z" |
| fill="#0e7490" |
| stroke="#67e8f9" |
| stroke-width="1.5" |
| filter="url(#cyanGlow)" |
| id="path114" /> |
| <text |
| x="780.58655" |
| y="229.92215" |
| text-anchor="middle" |
| class="micro" |
| fill="#67e8f9" |
| id="text114" |
| style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;font-size:10.5px;line-height:normal;font-family:system-ui, sans-serif">close head: close pool after byte t if score ≥ 0.98</text> |
| |
| <circle |
| cx="166" |
| cy="285" |
| r="9" |
| class="boe" |
| id="circle114" /> |
| <text |
| x="166" |
| y="288" |
| text-anchor="middle" |
| font-size="7" |
| fill="#cbd5e1" |
| id="text115">BOE</text> |
| <rect |
| x="177" |
| y="265" |
| width="40" |
| height="40" |
| rx="8" |
| class="stateF" |
| id="rect115" /> |
| <text |
| x="197" |
| y="289" |
| text-anchor="middle" |
| class="lblF" |
| id="text116">S1</text> |
| <rect |
| x="221" |
| y="265" |
| width="40" |
| height="40" |
| rx="8" |
| class="stateF" |
| id="rect116" /> |
| <text |
| x="241" |
| y="289" |
| text-anchor="middle" |
| class="lblF" |
| id="text117">S2</text> |
| <rect |
| x="292" |
| y="265" |
| width="40" |
| height="40" |
| rx="8" |
| class="stateF" |
| id="rect117" /> |
| <text |
| x="312" |
| y="289" |
| text-anchor="middle" |
| class="lblF" |
| id="text118">S3</text> |
| <rect |
| x="384" |
| y="265" |
| width="40" |
| height="40" |
| rx="8" |
| class="stateF" |
| id="rect118" /> |
| <text |
| x="404" |
| y="289" |
| text-anchor="middle" |
| class="lblF" |
| id="text119">S4</text> |
| <rect |
| x="440" |
| y="265" |
| width="40" |
| height="40" |
| rx="8" |
| class="stateF" |
| id="rect119" /> |
| <text |
| x="460" |
| y="289" |
| text-anchor="middle" |
| class="lblF" |
| id="text120">S5</text> |
| <rect |
| x="519" |
| y="265" |
| width="40" |
| height="40" |
| rx="8" |
| class="stateF" |
| id="rect120" /> |
| <text |
| x="539" |
| y="289" |
| text-anchor="middle" |
| class="lblF" |
| id="text121">S6</text> |
| <rect |
| x="611" |
| y="265" |
| width="40" |
| height="40" |
| rx="8" |
| class="stateF" |
| id="rect121" /> |
| <text |
| x="631" |
| y="289" |
| text-anchor="middle" |
| class="lblF" |
| id="text122">S7</text> |
| <rect |
| x="687" |
| y="265" |
| width="40" |
| height="40" |
| rx="8" |
| class="stateF" |
| id="rect122" /> |
| <text |
| x="707" |
| y="289" |
| text-anchor="middle" |
| class="lblF" |
| id="text123">S8</text> |
| <rect |
| x="758" |
| y="265" |
| width="40" |
| height="40" |
| rx="8" |
| class="stateFopen" |
| id="rect123" /> |
| <text |
| x="778" |
| y="285" |
| text-anchor="middle" |
| class="lblF" |
| id="text124">S9</text> |
| <text |
| x="778" |
| y="298" |
| text-anchor="middle" |
| class="openlbl" |
| id="text125">open</text> |
| <rect |
| x="835" |
| y="265" |
| width="40" |
| height="40" |
| rx="8" |
| class="stateF" |
| id="rect125" /> |
| <text |
| x="855" |
| y="289" |
| text-anchor="middle" |
| class="lblF" |
| id="text126">S10</text> |
| |
| <path |
| d="M197 309 V334 M404 309 V334 M197 334 H404" |
| class="bracket" |
| id="path126" /> |
| <path |
| d="M300 334 V371" |
| class="bracket" |
| marker-end="url(#arrowS)" |
| id="path127" /> |
| <path |
| d="M460 309 V334 M707 309 V334 M460 334 H707" |
| class="bracket" |
| id="path128" /> |
| <path |
| d="M583 334 V371" |
| class="bracket" |
| marker-end="url(#arrowS)" |
| id="path129" /> |
| <path |
| d="M778 309 V334 M855 309 V334 M778 334 H855" |
| class="bracketOpen" |
| id="path130" /> |
| <path |
| d="M816 334 V371" |
| class="bracketOpen" |
| marker-end="url(#arrowS)" |
| id="path131" /> |
| <circle |
| cx="166" |
| cy="395" |
| r="9" |
| class="boe" |
| id="circle131" /> |
| <text |
| x="166" |
| y="398" |
| text-anchor="middle" |
| font-size="7" |
| fill="#cbd5e1" |
| id="text131">BOE</text> |
| <rect |
| x="278" |
| y="375" |
| width="44" |
| height="40" |
| rx="8" |
| class="state2" |
| id="rect131" /> |
| <text |
| x="300" |
| y="399" |
| text-anchor="middle" |
| class="lbl2" |
| id="text132">G1</text> |
| <rect |
| x="561" |
| y="375" |
| width="44" |
| height="40" |
| rx="8" |
| class="state2" |
| id="rect132" /> |
| <text |
| x="583" |
| y="399" |
| text-anchor="middle" |
| class="lbl2" |
| id="text133">G2</text> |
| <rect |
| x="794" |
| y="375" |
| width="44" |
| height="40" |
| rx="8" |
| class="state2open" |
| id="rect133" /> |
| <text |
| x="816" |
| y="393" |
| text-anchor="middle" |
| class="lbl2" |
| id="text134">G3</text> |
| <text |
| x="816" |
| y="406" |
| text-anchor="middle" |
| class="openlbl" |
| id="text135">open 2/4</text> |
| |
| <path |
| d="M300 419 V446 M583 419 V446 M300 446 H583" |
| class="bracket" |
| id="path135" /> |
| <path |
| d="M441 446 V476" |
| class="bracket" |
| marker-end="url(#arrowS)" |
| id="path136" /> |
| <path |
| d="M816 419 V446" |
| class="bracketOpen" |
| id="path137" /> |
| <path |
| d="M816 446 V476" |
| class="bracketOpen" |
| marker-end="url(#arrowS)" |
| id="path138" /> |
| <circle |
| cx="166" |
| cy="500" |
| r="9" |
| class="boe" |
| id="circle138" /> |
| <text |
| x="166" |
| y="503" |
| text-anchor="middle" |
| font-size="7" |
| fill="#cbd5e1" |
| id="text138">BOE</text> |
| <rect |
| x="419" |
| y="480" |
| width="44" |
| height="40" |
| rx="8" |
| class="state3" |
| id="rect138" /> |
| <text |
| x="441" |
| y="504" |
| text-anchor="middle" |
| class="lbl3" |
| id="text139">H1</text> |
| <rect |
| x="794" |
| y="480" |
| width="44" |
| height="40" |
| rx="8" |
| class="state3open" |
| id="rect139" /> |
| <text |
| x="816" |
| y="498" |
| text-anchor="middle" |
| class="lbl3" |
| id="text140">H2</text> |
| <text |
| x="816" |
| y="511" |
| text-anchor="middle" |
| class="openlbl" |
| id="text141">open 1/2</text> |
| |
| <text |
| x="759.46875" |
| y="131.92615" |
| text-anchor="middle" |
| class="micro" |
| fill="#e2e8f0" |
| id="text142" |
| style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;font-size:10.5px;line-height:normal;font-family:system-ui, sans-serif">byte t</text> |
| |
| <path |
| d="M785 171 H990" |
| class="flow" |
| id="path142" /> |
| <path |
| d="M707 265 V240 H958 V222 H990" |
| class="flow" |
| id="path143" /> |
| <path |
| d="M707 265 V252 H948 V266 H990" |
| class="flow" |
| id="path144" /> |
| <path |
| d="M605 395 L990 310" |
| class="flow" |
| id="path145" /> |
| <path |
| d="m 440.98328,520.01916 v 45.01355 H 937.65747 V 353.96917 l 55.49027,-0.49025" |
| class="flow" |
| id="path146" /> |
| |
| <text |
| x="990" |
| y="148" |
| class="head" |
| id="text146">Decode at byte t</text> |
| <rect |
| x="990" |
| y="161" |
| width="260" |
| height="34" |
| rx="8" |
| fill="#24364f" |
| stroke="#ffffff2c" |
| stroke-width="1.5" |
| id="rect146" /> |
| <text |
| x="1002" |
| y="176" |
| class="body" |
| font-size="12" |
| id="text147">local byte context c[t]</text> |
| <text |
| x="1002" |
| y="189" |
| class="micro" |
| id="text148">bytes ≤ t only · dim 2000</text> |
| <rect |
| x="990" |
| y="205" |
| width="260" |
| height="34" |
| rx="8" |
| fill="#0f766e" |
| stroke="#ffffff2c" |
| stroke-width="1.5" |
| id="rect148" /> |
| <text |
| x="1002" |
| y="220" |
| class="body" |
| font-size="12" |
| id="text149">latest fine readout: S8</text> |
| <text |
| x="1002" |
| y="233" |
| class="micro" |
| id="text150">available at byte t · dim 2000</text> |
| <rect |
| x="990" |
| y="249" |
| width="260" |
| height="34" |
| rx="8" |
| fill="#115e59" |
| stroke="#ffffff2c" |
| stroke-width="1.5" |
| id="rect150" /> |
| <text |
| x="1002" |
| y="264" |
| class="body" |
| font-size="12" |
| id="text151">previous-byte fine readout: S8</text> |
| <text |
| x="1002" |
| y="277" |
| class="micro" |
| id="text152">readout used at byte t−1 · dim 2000</text> |
| <rect |
| x="990" |
| y="293" |
| width="260" |
| height="34" |
| rx="8" |
| fill="#92400e" |
| stroke="#ffffff2c" |
| stroke-width="1.5" |
| id="rect152" /> |
| <text |
| x="1002" |
| y="308" |
| class="body" |
| font-size="12" |
| id="text153">level-2 state G2</text> |
| <text |
| x="1002" |
| y="321" |
| class="micro" |
| id="text154">last completed L2 pool · dim 2000</text> |
| <rect |
| x="990" |
| y="337" |
| width="260" |
| height="34" |
| rx="8" |
| fill="#5b21b6" |
| stroke="#ffffff2c" |
| stroke-width="1.5" |
| id="rect154" /> |
| <text |
| x="1002" |
| y="352" |
| class="body" |
| font-size="12" |
| id="text155">level-3 state H1</text> |
| <text |
| x="1002" |
| y="365" |
| class="micro" |
| id="text156">last completed L3 pool · dim 2000</text> |
| <path |
| d="M1120 371 V390" |
| class="flow" |
| id="path156" /> |
| <rect |
| x="990" |
| y="390" |
| width="260" |
| height="72" |
| rx="12" |
| fill="url(#dec)" |
| class="box" |
| id="rect156" /> |
| <text |
| x="1120" |
| y="412" |
| text-anchor="middle" |
| class="headw" |
| id="text157">Fusion decoder</text> |
| <text |
| x="1120" |
| y="432" |
| text-anchor="middle" |
| class="body" |
| font-size="12" |
| id="text158">LayerNorm(10,000) → Linear(10,000 → 8,000) → GELU</text> |
| <text |
| x="1120" |
| y="449" |
| text-anchor="middle" |
| class="body" |
| font-size="12" |
| id="text159">Linear(8,000 → 2,000) → GELU</text> |
| <path |
| d="m 1299.358,420.81534 53.2045,-60.55682" |
| class="flow" |
| id="path159" /> |
| <rect |
| x="1268.5646" |
| y="286.39615" |
| width="178.21445" |
| height="71.941719" |
| rx="12.961051" |
| fill="#164e63" |
| stroke="#67e8f9" |
| stroke-width="1.55828" |
| id="rect159" /> |
| <text |
| x="1357.1718" |
| y="304.367" |
| text-anchor="middle" |
| class="headw" |
| font-size="12" |
| id="text160" |
| style="font-style:normal;font-variant:normal;font-weight:700;font-stretch:normal;font-size:15px;line-height:normal;font-family:system-ui, sans-serif">pool controller</text> |
| <text |
| x="1357.1718" |
| y="320.367" |
| text-anchor="middle" |
| class="micro" |
| id="text161" |
| style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;font-size:10.5px;line-height:normal;font-family:system-ui, sans-serif">Linear(2,000 → 9) · causal scores</text> |
| <text |
| x="1357.1718" |
| y="334.367" |
| text-anchor="middle" |
| class="micro" |
| id="text162" |
| style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;font-size:10.5px;line-height:normal;font-family:system-ui, sans-serif">per level: hold · refresh ·</text> |
| <text |
| x="1357.1718" |
| y="348.367" |
| text-anchor="middle" |
| class="micro" |
| id="text163" |
| style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;font-size:10.5px;line-height:normal;font-family:system-ui, sans-serif">compress · acts at t+1</text> |
| <path |
| d="M1120 462 V482" |
| class="flow" |
| id="path163" /> |
| <rect |
| x="990" |
| y="482" |
| width="260" |
| height="40" |
| rx="10" |
| fill="#1e3a5f" |
| stroke="#ffffff2c" |
| stroke-width="1.5" |
| id="rect163" /> |
| <text |
| x="1120" |
| y="507" |
| text-anchor="middle" |
| class="body" |
| id="text164">LM head — Linear(2,000 → 260)</text> |
| <path |
| d="M1120 522 V542" |
| class="flow" |
| id="path164" /> |
| <rect |
| x="990" |
| y="542" |
| width="260" |
| height="44" |
| rx="10" |
| fill="#0f2744" |
| stroke="#38bdf8" |
| stroke-width="1.5" |
| id="rect164" /> |
| <text |
| x="1120" |
| y="562" |
| text-anchor="middle" |
| class="headw" |
| id="text165">next-byte logits (260)</text> |
| <text |
| x="1120" |
| y="578" |
| text-anchor="middle" |
| class="micro" |
| id="text166">sample → byte t+1</text> |
| |
| <path |
| d="M 1357.358,284.70736 V 117.80185 H 779.64957 v 38.65827" |
| class="feedback" |
| id="path166" /> |
| <text |
| x="1060" |
| y="112" |
| text-anchor="middle" |
| class="micro" |
| fill="#67e8f9" |
| id="text167">delayed pool controller: scores emitted at t−1 adjust the close decision at t</text> |
| |
| <rect |
| x="40" |
| y="600" |
| width="440" |
| height="250" |
| rx="14" |
| fill="#101d33" |
| stroke="#ffffff22" |
| id="rect167" /> |
| <text |
| x="60" |
| y="630" |
| class="head" |
| id="text168">What byte t can see (causal alignment)</text> |
| <text |
| x="60" |
| y="658" |
| class="body" |
| font-size="12" |
| id="text169">byte t sits in fine pool 9, L2 pool 3, L3 pool 2 —</text> |
| <text |
| x="60" |
| y="676" |
| class="body" |
| font-size="12" |
| id="text170">all three groups are still open at decode time</text> |
| <text |
| x="60" |
| y="700" |
| class="body" |
| font-size="12" |
| id="text171">• local context — bytes ≤ t only</text> |
| <text |
| x="60" |
| y="720" |
| class="body" |
| font-size="12" |
| id="text172">• fine S8 — latest completed readout available at byte t</text> |
| <text |
| x="60" |
| y="740" |
| class="body" |
| font-size="12" |
| id="text173">• fine S8 — previous-byte readout (same until closure)</text> |
| <text |
| x="60" |
| y="760" |
| class="body" |
| font-size="12" |
| id="text174">• level 2 G2 — last completed L2 state</text> |
| <text |
| x="60" |
| y="780" |
| class="body" |
| font-size="12" |
| id="text175">• level 3 H1 — last completed L3 state</text> |
| <text |
| x="60" |
| y="808" |
| class="tiny" |
| fill="#7dd3fc" |
| id="text176">the two fine inputs differ only on the first byte after closure</text> |
| <rect |
| x="500" |
| y="600" |
| width="450" |
| height="250" |
| rx="14" |
| fill="#101d33" |
| stroke="#ffffff22" |
| id="rect176" /> |
| <text |
| x="520" |
| y="630" |
| class="head" |
| id="text177">Learned pool close policy</text> |
| <text |
| x="520" |
| y="658" |
| class="body" |
| font-size="12" |
| id="text178">fine: 1–96 bytes per pool</text> |
| <text |
| x="520" |
| y="676" |
| class="body" |
| font-size="12" |
| id="text179"> close when p + 0.05·midpoint ≥ 0.98, forced at 96</text> |
| <text |
| x="520" |
| y="700" |
| class="body" |
| font-size="12" |
| id="text180">quota: ≤ 3,000 pools < 6 bytes in a rolling 6,000-pool</text> |
| <text |
| x="520" |
| y="718" |
| class="body" |
| font-size="12" |
| id="text181"> window, else the pool minimum rises to 6 bytes</text> |
| <text |
| x="520" |
| y="742" |
| class="body" |
| font-size="12" |
| id="text182">level 2: groups of 4–16 fine pools · threshold 0.98</text> |
| <text |
| x="520" |
| y="762" |
| class="body" |
| font-size="12" |
| id="text183">level 3: groups of 2–16 level-2 pools · threshold 0.98</text> |
| <text |
| x="520" |
| y="786" |
| class="tiny" |
| fill="#7dd3fc" |
| id="text184">all close heads are learned sigmoids; boundary gradients</text> |
| <text |
| x="520" |
| y="802" |
| class="tiny" |
| fill="#7dd3fc" |
| id="text185">are detached from the content path</text> |
| <rect |
| x="970" |
| y="600" |
| width="470" |
| height="250" |
| rx="14" |
| fill="#101d33" |
| stroke="#ffffff22" |
| id="rect185" /> |
| <text |
| x="990" |
| y="630" |
| class="head" |
| id="text186">Parameters — 2.478B total</text> |
| <rect |
| x="990" |
| y="650" |
| width="65" |
| height="24" |
| rx="3" |
| fill="#14b8a6" |
| id="rect186" /> |
| <rect |
| x="1055" |
| y="650" |
| width="131" |
| height="24" |
| fill="#f59e0b" |
| id="rect187" /> |
| <rect |
| x="1186" |
| y="650" |
| width="194" |
| height="24" |
| rx="3" |
| fill="#8b5cf6" |
| id="rect188" /> |
| <rect |
| x="990" |
| y="700" |
| width="10" |
| height="10" |
| fill="#14b8a6" |
| id="rect189" /> |
| <text |
| x="1008" |
| y="709" |
| class="tiny" |
| id="text189">fine + shared decoder + LM head — 411.95M (16.6%)</text> |
| <rect |
| x="990" |
| y="724" |
| width="10" |
| height="10" |
| fill="#f59e0b" |
| id="rect190" /> |
| <text |
| x="1008" |
| y="733" |
| class="tiny" |
| id="text190">level 2 pooler + 20 Mamba-2 blocks — 831.56M (33.5%)</text> |
| <rect |
| x="990" |
| y="748" |
| width="10" |
| height="10" |
| fill="#8b5cf6" |
| id="rect191" /> |
| <text |
| x="1008" |
| y="757" |
| class="tiny" |
| id="text191">level 3 pooler + 30 Mamba-2 blocks — 1,235.31M (49.8%)</text> |
| <text |
| x="990" |
| y="786" |
| class="tiny" |
| fill="#7dd3fc" |
| id="text192">the deepest level holds the most parameters even</text> |
| <text |
| x="990" |
| y="802" |
| class="tiny" |
| fill="#7dd3fc" |
| id="text193">though it updates least frequently</text> |
| |
| <text |
| x="40" |
| y="880" |
| class="micro" |
| id="text194">8192-bin modular positions + per-level length embeddings · BOE at every level · solid = completed/readable state · dashed = open/unreadable pool</text> |
| </svg> |
|
|
