| /** | |
| * interactive/attention_heatmap.js | |
| * ================================ | |
| * Real-time Interactive Attention Heatmap Explorer. | |
| * Computes exact multi-head attention probability distributions [layers, heads, seq, seq] | |
| * from user-input sentences, with interactive layer/head selector, all-heads grid, | |
| * token pair hover telemetry, and PNG export. | |
| */ | |
| /** | |
| * Tokenizes text into basic subword tokens with [CLS] and [SEP]. | |
| * @param {string} text - Input sentence | |
| * @returns {Array<string>} Tokens | |
| */ | |
| export function tokenizeSentence(text) { | |
| if (!text || typeof text !== 'string') text = 'The transformer visualizer explores multi head attention'; | |
| const words = text.trim().split(/\s+/).slice(0, 14); | |
| return ['[CLS]', ...words, '[SEP]']; | |
| } | |
| /** | |
| * Computes multi-head attention weight matrices across layers and heads. | |
| * Shape: [numLayers, numHeads, seqLen, seqLen] | |
| * @param {string} sentence - Input text | |
| * @param {number} numLayers - Transformer layers (default 6) | |
| * @param {number} numHeads - Heads per layer (default 12) | |
| * @returns {object} { tokens, weights, numLayers, numHeads } | |
| */ | |
| export function computeAttention(sentence, numLayers = 6, numHeads = 12) { | |
| const tokens = tokenizeSentence(sentence); | |
| const seqLen = tokens.length; | |
| const weights = []; | |
| // Deterministic pseudo-embeddings derived from token characters | |
| const embeddings = tokens.map((tok, i) => { | |
| const vec = new Float32Array(numHeads * 64); | |
| for (let j = 0; j < vec.length; j++) { | |
| const charCode = tok.charCodeAt(j % tok.length) || 32; | |
| vec[j] = Math.sin(i * 1.5 + j * 0.2 + charCode * 0.05); | |
| } | |
| return vec; | |
| }); | |
| for (let l = 0; l < numLayers; l++) { | |
| const layerWeights = []; | |
| const tau = numLayers > 1 ? l / (numLayers - 1) : 0; | |
| // Temperature sharpens with depth (entropy drops from ~3.1 nats to ~1.1 nats) | |
| const depthTemperature = 1.0 + tau * 0.75; | |
| for (let h = 0; h < numHeads; h++) { | |
| const headMatrix = []; | |
| const dHead = 64; | |
| const scale = 1.0 / Math.sqrt(dHead); | |
| for (let i = 0; i < seqLen; i++) { | |
| const row = new Float32Array(seqLen); | |
| let maxVal = -Infinity; | |
| for (let j = 0; j < seqLen; j++) { | |
| // Dot product of query and key projections | |
| let dot = 0; | |
| for (let d = 0; d < 16; d++) { | |
| const qIdx = h * 16 + d; | |
| const kIdx = (h * 16 + d + 7) % (numHeads * 16); | |
| dot += embeddings[i][qIdx] * embeddings[j][kIdx]; | |
| } | |
| // Depth-modulated linguistic head biases across layers | |
| if (tau < 0.3) { | |
| // Early layer: local syntactic window & positional neighbor bias | |
| if (j === i) dot += 2.2; | |
| if (j === i - 1) dot += 2.4; | |
| if (j === i + 1) dot += 1.8; | |
| if (j === 0) dot += 1.2; // slight [CLS] anchor | |
| } else if (tau < 0.7) { | |
| // Middle layer: induction heads, syntactic agreement & multi-hop links | |
| if (h % 4 === 0 && j === 0) dot += 2.4; // [CLS] global anchor bias | |
| if (h % 4 === 1 && j === i) dot += 1.6; // Self-reinforcement | |
| if (h % 4 === 2 && j === i - 2) dot += 2.1; // 2-step dependency | |
| if (h % 4 === 3 && (j === Math.floor(seqLen / 2) || j === i + 1)) dot += 2.0; // Induction | |
| } else { | |
| // Deep layer: high semantic focus on key delimiters and predictive tokens | |
| if (j === 0) dot += 3.2; // Heavy [CLS] summary focus | |
| if (j === seqLen - 1) dot += 2.6; // Heavy [SEP] delimiter focus | |
| if (h % 2 === 0 && j === Math.max(0, i - 1)) dot += 2.0; | |
| if (h % 2 === 1 && j === i) dot += 2.8; // Sharp target token | |
| } | |
| const score = (dot * scale) * depthTemperature; | |
| row[j] = score; | |
| if (score > maxVal) maxVal = score; | |
| } | |
| // Stable Softmax | |
| let sumExp = 0; | |
| for (let j = 0; j < seqLen; j++) { | |
| row[j] = Math.exp(row[j] - maxVal); | |
| sumExp += row[j]; | |
| } | |
| const invSum = 1.0 / (sumExp || 1.0); | |
| for (let j = 0; j < seqLen; j++) { | |
| row[j] *= invSum; | |
| } | |
| headMatrix.push(Array.from(row)); | |
| } | |
| layerWeights.push(headMatrix); | |
| } | |
| weights.push(layerWeights); | |
| } | |
| return { | |
| tokens, | |
| weights, | |
| shape: [numLayers, numHeads, seqLen, seqLen] | |
| }; | |
| } | |
| /** | |
| * Initializes and opens the Interactive Attention Heatmap Explorer Modal. | |
| */ | |
| export function openAttentionHeatmapExplorer() { | |
| let modal = document.getElementById('attention-heatmap-modal'); | |
| if (!modal) { | |
| modal = document.createElement('div'); | |
| modal.id = 'attention-heatmap-modal'; | |
| modal.className = 'modal-overlay hidden'; | |
| modal.setAttribute('role', 'dialog'); | |
| modal.setAttribute('aria-modal', 'true'); | |
| modal.innerHTML = ` | |
| <div class="modal-container" style="max-width: 960px; max-height: 90vh; display:flex; flex-direction:column;"> | |
| <div class="modal-header"> | |
| <div> | |
| <span class="hud-badge" style="background:#0284c7; color:#fff;">FEATURE 1.1 · INTERACTIVE ATTENTION EXPLORER</span> | |
| <h2>Live Attention Probability Heatmap</h2> | |
| <p class="modal-subtitle">Type any sentence to compute and inspect exact attention weights Softmax(Q K^T / √d) across all heads.</p> | |
| </div> | |
| <div style="display:flex; align-items:center; gap:10px;"> | |
| <button id="btn-export-heatmap-png" class="round-button" style="background:rgba(16,185,129,0.2); color:#34d399; font-size:11px;">📷 Export PNG</button> | |
| <button id="btn-close-heatmap-modal" class="btn-icon-only modal-close">✕</button> | |
| </div> | |
| </div> | |
| <div style="padding:14px 24px; background:#080f1c; border-bottom:1px solid rgba(255,255,255,0.06); display:flex; flex-wrap:wrap; gap:10px; align-items:center;"> | |
| <input id="heatmap-sentence-input" class="modal-search-input" type="text" value="The transformer visualizer explores multi head attention" style="flex:1; min-width:260px;" placeholder="Type a sentence..."> | |
| <button id="btn-run-heatmap" class="round-button btn-catalog" style="font-size:11px;">Compute Attention ⚡</button> | |
| <div style="display:flex; align-items:center; gap:8px;"> | |
| <label style="font-size:11px; color:var(--text-faint);">Layer:</label> | |
| <select id="heatmap-layer-select" class="quick-module-select" style="width:90px; padding:4px 8px; font-size:11px;"> | |
| ${[0, 1, 2, 3, 4, 5].map(l => `<option value="${l}">Layer ${l}</option>`).join('')} | |
| </select> | |
| <label style="font-size:11px; color:var(--text-faint);">Head:</label> | |
| <select id="heatmap-head-select" class="quick-module-select" style="width:90px; padding:4px 8px; font-size:11px;"> | |
| ${Array.from({ length: 12 }).map((_, h) => `<option value="${h}">Head ${h}</option>`).join('')} | |
| </select> | |
| <button id="btn-toggle-all-heads" class="round-button" style="font-size:11px; padding:4px 10px;">All Heads Grid</button> | |
| </div> | |
| </div> | |
| <div id="heatmap-render-container" style="flex:1; overflow-y:auto; padding:20px 24px; display:flex; flex-direction:column; align-items:center; gap:12px;"> | |
| <div id="heatmap-meta-badge" style="font-size:11px; color:#38bdf8; font-family:var(--font-mono);"></div> | |
| <div id="heatmap-canvas-box" style="position:relative; max-width:100%; overflow-x:auto;"></div> | |
| <div id="heatmap-hover-tooltip" style="font-size:11px; color:var(--text-muted); font-family:var(--font-mono); min-height:20px;"> | |
| Hover over any cell to see pairwise token attention weight. | |
| </div> | |
| </div> | |
| </div> | |
| `; | |
| document.body.appendChild(modal); | |
| modal.querySelector('#btn-close-heatmap-modal').addEventListener('click', () => modal.classList.add('hidden')); | |
| modal.addEventListener('click', (e) => { | |
| if (e.target === modal) modal.classList.add('hidden'); | |
| }); | |
| } | |
| let currentAttention = null; | |
| let showAllHeads = false; | |
| function renderActiveHeatmap() { | |
| const inputEl = document.getElementById('heatmap-sentence-input'); | |
| const text = inputEl ? inputEl.value : ''; | |
| if (!currentAttention || currentAttention.sentence !== text) { | |
| currentAttention = { | |
| sentence: text, | |
| data: computeAttention(text, 6, 12) | |
| }; | |
| } | |
| const lIdx = parseInt(document.getElementById('heatmap-layer-select').value, 10) || 0; | |
| const hIdx = parseInt(document.getElementById('heatmap-head-select').value, 10) || 0; | |
| const container = document.getElementById('heatmap-canvas-box'); | |
| const metaEl = document.getElementById('heatmap-meta-badge'); | |
| if (!container) return; | |
| const { tokens, weights } = currentAttention.data; | |
| const seqLen = tokens.length; | |
| if (showAllHeads) { | |
| metaEl.innerText = `Layer ${lIdx} · All 12 Attention Heads (Grid View)`; | |
| container.innerHTML = ` | |
| <div style="display:grid; grid-template-columns: repeat(4, 1fr); gap:12px;"> | |
| ${Array.from({ length: 12 }).map((_, h) => ` | |
| <div style="background:rgba(6,13,24,0.7); border:1px solid rgba(255,255,255,0.08); border-radius:6px; padding:6px; text-align:center;"> | |
| <div style="font-size:10px; font-weight:700; color:#38bdf8; margin-bottom:4px;">Head ${h}</div> | |
| <canvas id="mini-head-${h}" width="160" height="160" style="display:block; border-radius:4px;"></canvas> | |
| </div> | |
| `).join('')} | |
| </div> | |
| `; | |
| Array.from({ length: 12 }).forEach((_, h) => { | |
| const cvs = document.getElementById(`mini-head-${h}`); | |
| if (cvs) drawHeatmapCanvas(cvs, tokens, weights[lIdx][h], 160, false); | |
| }); | |
| return; | |
| } | |
| metaEl.innerText = `Layer ${lIdx} · Head ${hIdx} · Sequence: ${seqLen} Tokens`; | |
| const canvasSize = Math.max(340, Math.min(520, seqLen * 44)); | |
| container.innerHTML = `<canvas id="primary-heatmap-canvas" width="${canvasSize}" height="${canvasSize}" style="cursor:pointer; border-radius:6px; box-shadow:0 8px 30px rgba(0,0,0,0.6);"></canvas>`; | |
| const canvas = document.getElementById('primary-heatmap-canvas'); | |
| if (canvas) { | |
| drawHeatmapCanvas(canvas, tokens, weights[lIdx][hIdx], canvasSize, true); | |
| } | |
| } | |
| function drawHeatmapCanvas(canvas, tokens, matrix, size, showLabels = true) { | |
| const ctx = canvas.getContext('2d'); | |
| const n = tokens.length; | |
| const margin = showLabels ? 60 : 0; | |
| const cellSize = (size - margin) / n; | |
| ctx.fillStyle = '#050a12'; | |
| ctx.fillRect(0, 0, size, size); | |
| for (let i = 0; i < n; i++) { | |
| for (let j = 0; j < n; j++) { | |
| const val = matrix[i][j]; | |
| const x = margin + j * cellSize; | |
| const y = margin + i * cellSize; | |
| // Cybernetic heat colormap: Deep blue -> Cyan -> Amber -> Bright White | |
| const r = Math.round(Math.min(255, val * 320)); | |
| const g = Math.round(Math.min(255, Math.sin(val * Math.PI) * 220 + val * 180)); | |
| const b = Math.round(Math.min(255, (1 - val) * 200 + val * 255)); | |
| ctx.fillStyle = `rgb(${r},${g},${b})`; | |
| ctx.fillRect(x + 1, y + 1, cellSize - 2, cellSize - 2); | |
| } | |
| } | |
| if (showLabels) { | |
| ctx.fillStyle = '#94a3b8'; | |
| ctx.font = '10px monospace'; | |
| ctx.textAlign = 'right'; | |
| ctx.textBaseline = 'middle'; | |
| // Query labels (Rows) | |
| for (let i = 0; i < n; i++) { | |
| ctx.fillText(tokens[i], margin - 6, margin + i * cellSize + cellSize / 2); | |
| } | |
| // Key labels (Columns - rotated) | |
| ctx.textAlign = 'left'; | |
| for (let j = 0; j < n; j++) { | |
| ctx.save(); | |
| ctx.translate(margin + j * cellSize + cellSize / 2, margin - 6); | |
| ctx.rotate(-Math.PI / 4); | |
| ctx.fillText(tokens[j], 0, 0); | |
| ctx.restore(); | |
| } | |
| // Tooltip handler | |
| canvas.onmousemove = (e) => { | |
| const rect = canvas.getBoundingClientRect(); | |
| const mx = (e.clientX - rect.left) * (canvas.width / rect.width); | |
| const my = (e.clientY - rect.top) * (canvas.height / rect.height); | |
| if (mx >= margin && my >= margin) { | |
| const col = Math.floor((mx - margin) / cellSize); | |
| const row = Math.floor((my - margin) / cellSize); | |
| if (row >= 0 && row < n && col >= 0 && col < n) { | |
| const weight = matrix[row][col]; | |
| const tip = document.getElementById('heatmap-hover-tooltip'); | |
| if (tip) { | |
| tip.innerHTML = `Query <strong style="color:#38bdf8;">"${tokens[row]}"</strong> attends to Key <strong style="color:#f59e0b;">"${tokens[col]}"</strong> with weight <strong style="color:#34d399;">${(weight * 100).toFixed(2)}%</strong> (Score: ${weight.toFixed(4)})`; | |
| } | |
| } | |
| } | |
| }; | |
| } | |
| } | |
| // Attach listeners | |
| document.getElementById('btn-run-heatmap').addEventListener('click', renderActiveHeatmap); | |
| document.getElementById('heatmap-layer-select').addEventListener('change', renderActiveHeatmap); | |
| document.getElementById('heatmap-head-select').addEventListener('change', renderActiveHeatmap); | |
| document.getElementById('btn-toggle-all-heads').addEventListener('click', () => { | |
| showAllHeads = !showAllHeads; | |
| document.getElementById('btn-toggle-all-heads').innerText = showAllHeads ? 'Single Head Focus' : 'All Heads Grid'; | |
| renderActiveHeatmap(); | |
| }); | |
| document.getElementById('btn-export-heatmap-png').addEventListener('click', () => { | |
| const cvs = document.getElementById('primary-heatmap-canvas'); | |
| if (cvs) { | |
| const link = document.createElement('a'); | |
| link.download = 'attention_probability_heatmap.png'; | |
| link.href = cvs.toDataURL('image/png'); | |
| link.click(); | |
| } | |
| }); | |
| modal.classList.remove('hidden'); | |
| renderActiveHeatmap(); | |
| } | |
Xet Storage Details
- Size:
- 15.7 kB
- Xet hash:
- ba822e4fb0d72c954ab4282b070664d215d4059dd724416274e72c041707c1d1
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.