import { codepoint, glyph, hex } from "./trace/unicode.js"; import { plural } from "./trace/tracer.js"; export function el(tag, className, ...children) { const node = document.createElement(tag); if (className) node.className = className; node.append(...children); return node; } /** A string as a row of char stickers, invisible chars made visible. `changed`: indices to mark. */ export function glyphs(root, text, changed) { root.replaceChildren( ...[...text].map((ch, i) => { const kind = (ch === " " ? "g sp" : ch.codePointAt(0) < 0x80 ? "g" : "g multi") + (changed?.has(i) ? " changed" : ""); // In a `.plain` row spaces stay real spaces, so the text wraps between words. const plain = root.classList.contains("plain"); const span = el("span", kind, plain && ch === " " ? " " : plain && ch === "\n" ? "↵\n" : glyph(ch)); span.title = codepoint(ch); return span; }), ); if (!text) root.append(el("span", "empty-text", "(empty)")); } /** The input as UTF-8: one sticker per char, its bytes underneath. Returns the byte cells. */ export function mountBytes(root, chars, bytes) { const cells = []; root.replaceChildren( ...chars.map(({ ch, start, len }) => { const multi = len > 1; const row = el("span", "bs"); for (let i = start; i < start + len; i++) { const cell = el("i", "b", hex(bytes[i])); cell.title = `byte ${i}: 0x${hex(bytes[i])}`; cells[i] = cell; row.append(cell); } const group = el("span", multi ? "c multi" : "c", el("b", "", glyph(ch)), row); group.title = `${codepoint(ch)} · ${len} byte${len > 1 ? "s" : ""} at ${start}`; if (ch === "\n") return [group, el("span", "br")]; return [group]; }).flat(), ); return cells; } const HL = ["scanned", "hit", "block", "ascii", "unicode", "skip", "stable", "unstable"]; export function highlight(cells, hl) { for (const cell of cells) cell.classList.remove(...HL); for (const { from, to, cls } of hl) for (let i = from; i < to; i++) cells[i]?.classList.add(cls); // Scroll the byte strip (not the page) so the interesting byte is visible. const first = hl.find((h) => h.cls === "hit" || h.cls === "block") ?? hl[0]; const cell = first && cells[first.from]; if (!cell) return; const box = cell.closest(".bytes"); const top = cell.offsetTop - box.offsetTop; if (top < box.scrollTop || top > box.scrollTop + box.clientHeight - 40) box.scrollTop = top - 40; } /** The 16 lanes of a SIMD block: bytes, the high-bit mask, and whatever the step does with them. */ export function lanes(root, l) { if (!l) return root.replaceChildren(); const row = (label, cells) => el("div", "lane-row", el("span", "lane-label", label), ...cells); const byteCell = (b, extra = "") => { const ch = b < 0x80 ? glyph(String.fromCharCode(b)) : "·"; return el("span", `lane ${b >= 0x80 ? "multi" : ""} ${extra}`, el("b", "", ch), el("small", "", hex(b))); }; const rows = [row("16 bytes", l.block.map((b, i) => byteCell(b, i === l.lane ? "hit" : "")))]; rows.push(row("≥ 0x80?", l.block.map((b, i) => el("span", `lane bit ${b >= 0x80 ? "on" : ""} ${i === l.lane ? "hit" : ""}`, b >= 0x80 ? "yes" : "–")))); if (l.kind === "lower") { rows.push(row("capital?", l.upper.map((u) => el("span", `lane bit ${u ? "up" : ""}`, u ? "+0x20" : "–")))); rows.push(row("written", l.block.map((b, i) => byteCell(l.upper[i] ? b | 0x20 : b, l.upper[i] ? "changed" : "")))); } else if (l.kind === "bert") { rows.length = 1; // pass 2 doesn't look at high bits: non-ASCII bytes just pass through // Only the rules that fire in this block get a row. const rule = (label, hits, mark) => hits.some(Boolean) && rows.push(row(label, hits.map((x) => el("span", `lane bit ${x ? "up" : ""}`, x ? mark : "–")))); rule("tab, newline", l.spaces, "→ ·"); rule("control", l.deletes, "drop"); rule("capital", l.upper, "+0x20"); if (!l.deletes.some(Boolean)) { const out = l.block.map((b, i) => (l.spaces[i] ? 0x20 : l.upper[i] ? b | 0x20 : b)); rows.push(row("written", out.map((b, i) => byteCell(b, out[i] !== l.block[i] ? "changed" : "")))); } } else if (l.kind === "controls") { rows.push(row("control?", l.controls.map((c) => el("span", `lane bit ${c ? "up" : ""}`, c ? "yes" : "–")))); } const caption = { exit: "a byte ≥ 0x80: not pure ASCII", scan: l.lane ? `${plural(l.lane, "ASCII byte")}, then a byte ≥ 0x80` : "the very first byte is ≥ 0x80", lower: "all 16 lowercased in one go", controls: "a control char: this block goes byte by byte", bert: l.deletes && l.deletes.some(Boolean) ? "a control char to drop: byte by byte" : "all 16 fixed in one go", copy: "copied as is", }[l.kind]; root.replaceChildren(el("div", "lanes", el("div", "lanes-head", `bytes ${l.start}–${l.start + 15}`, el("span", "", caption)), ...rows)); } /** A non-ASCII run (or the whole input, before) through each Unicode stage, left to right. Like * the post's "What BERT does to your text" figure, chars a stage changed are green and removals are counted. */ export function stages(root, list) { if (!list) return root.replaceChildren(); // Short spans read best as one strip with arrows; longer text gets a row per stage. const inline = list.every(({ text }) => [...text].length <= 8); const flow = el("div", inline ? "stages inline" : "stages"); list.forEach(({ name, text }, i) => { if (inline && i) flow.append(el("span", "arrow", "→")); const g = el("div", "glyphs small"); const { changed, removed } = i ? diff([...list[i - 1].text], [...text]) : { changed: null, removed: 0 }; glyphs(g, text, changed); const head = el("span", "stage-name", name); if (removed) head.append(el("span", "removed", `−${removed}`)); flow.append(el("div", "stage", head, g)); }); root.replaceChildren(flow); } /** Which chars of `b` are new relative to `a` (not on a longest common subsequence), and how many fewer chars there are. */ function diff(a, b) { const n = a.length; const m = b.length; const dp = Array.from({ length: n + 1 }, () => new Uint16Array(m + 1)); for (let i = n - 1; i >= 0; i--) for (let j = m - 1; j >= 0; j--) dp[i][j] = a[i] === b[j] ? dp[i + 1][j + 1] + 1 : Math.max(dp[i + 1][j], dp[i][j + 1]); const kept = new Set(); for (let i = 0, j = 0; i < n && j < m; ) { if (a[i] === b[j]) (kept.add(j), i++, j++); else if (dp[i + 1][j] >= dp[i][j + 1]) i++; else j++; } const changed = new Set(); for (let j = 0; j < m; j++) if (!kept.has(j)) changed.add(j); return { changed, removed: Math.max(0, n - m) }; // like the post's "−N code points" } // ── CharacterCache: the segment's chars, their 256-code-point pages, one read per char ───────── const CACHE_CHARS = 14; // chars animated; the counters still cover the whole segment const CACHE_PAGES = 6; const reduced = matchMedia("(prefers-reduced-motion: reduce)"); let cacheTimer = 0; export function stopCache(root) { clearTimeout(cacheTimer); root.replaceChildren(); } const pageName = (page) => `U+${(page << 8).toString(16).toUpperCase().padStart(4, "0").slice(0, -2)}xx`; function pageGrid(page, isUnstable) { const canvas = document.createElement("canvas"); canvas.width = canvas.height = 16; const ctx = canvas.getContext("2d"); const css = getComputedStyle(document.documentElement); for (let k = 0; k < 256; k++) { const cp = (page << 8) + k; const valid = cp < 0xd800 || cp > 0xdfff; ctx.fillStyle = valid && isUnstable(String.fromCodePoint(cp)) ? css.getPropertyValue("--red") : css.getPropertyValue("--tint-green"); ctx.fillRect(k & 15, k >> 4, 1, 1); } return canvas; } /** Plays the reads of one segment step: arrow from the char to its page, page built on a miss. */ export function cache(root, { chars, before, isUnstable }) { clearTimeout(cacheTimer); const shown = chars.slice(0, CACHE_CHARS); const touched = [...new Set([...shown.map((c) => c.page), ...chars.map((c) => c.page), ...before])]; const pagesShown = touched.slice(0, CACHE_PAGES).sort((a, b) => a - b); const extraPages = touched.length - pagesShown.length; const count = el("span", "cache-count"); const replay = el("button", "cache-replay", "↻ replay"); const tiles = shown.map(({ ch, cp }) => { const tile = el("span", "g multi", glyph(ch)); tile.title = codepoint(ch); return tile; }); const pages = new Map( pagesShown.map((page) => { const card = el("div", "page", el("div", "page-grid"), el("span", "page-name", pageName(page))); return [page, card]; }), ); const svg = document.createElementNS("http://www.w3.org/2000/svg", "svg"); svg.setAttribute("class", "cache-arrow"); const charsRow = el("div", "glyphs small cache-chars", ...tiles); if (chars.length > shown.length) charsRow.append(el("span", "meta", `+${chars.length - shown.length}`)); const pagesRow = el("div", "cache-pages", ...pages.values()); if (extraPages > 0) pagesRow.append(el("span", "meta", `+${extraPages}`)); const box = el("div", "cache", el("div", "cache-head", el("span", "", "CharacterCache"), count, replay), charsRow, pagesRow, svg); root.replaceChildren(box); const built = new Set(before); const build = (page) => { const card = pages.get(page); if (!card || card.classList.contains("built")) return; card.classList.add("built"); card.querySelector(".page-grid").append(pageGrid(page, isUnstable)); }; before.forEach(build); const show = (i) => { // i: reads done so far (the current char is i - 1); i === chars.length: final state const done = Math.min(i, chars.length); const total = new Set([...before, ...chars.slice(0, done).map((c) => c.page)]).size; count.textContent = `${plural(done, "read")} · ${plural(total, "page")} built`; tiles.forEach((t, k) => t.classList.toggle("read", k < i - 1)); tiles.forEach((t, k) => t.classList.toggle("on", k === i - 1)); for (const card of pages.values()) card.classList.remove("fresh"), card.querySelector(".cell")?.remove(); svg.replaceChildren(); const cur = i >= 1 && i <= shown.length ? shown[i - 1] : null; if (!cur) return; const card = pages.get(cur.page); if (cur.miss) (build(cur.page), card?.classList.add("fresh")); if (!card) return; const grid = card.querySelector(".page-grid"); const cell = el("i", "cell"); cell.style.left = `${((cur.cp & 15) / 16) * 100}%`; cell.style.top = `${((cur.cp >> 4) & 15) / 16 * 100}%`; grid.append(cell); // Elbow arrow, box coordinates: down from the char, across, down into its page. const b = box.getBoundingClientRect(); const a = tiles[i - 1].getBoundingClientRect(); const p = card.getBoundingClientRect(); const x1 = a.left + a.width / 2 - b.left, y1 = a.bottom - b.top + 2; const x2 = p.left + p.width / 2 - b.left, y2 = p.top - b.top - 3; const ym = (y1 + y2) / 2, r = Math.min(8, Math.abs(x2 - x1) / 2); const s = Math.sign(x2 - x1) || 1; const d = r < 1 ? `M${x1} ${y1}V${y2}` : `M${x1} ${y1}V${ym - r}Q${x1} ${ym} ${x1 + s * r} ${ym}H${x2 - s * r}Q${x2} ${ym} ${x2} ${ym + r}V${y2}`; svg.setAttribute("viewBox", `0 0 ${b.width} ${b.height}`); svg.innerHTML = ``; }; const play = () => { clearTimeout(cacheTimer); for (const [page, card] of pages) { if (before.includes(page)) continue; card.classList.remove("built"); card.querySelector(".page-grid").replaceChildren(); } let i = 0; const tick = () => { i++; if (i > shown.length) return finish(); show(i); cacheTimer = setTimeout(tick, shown[i - 1].miss ? 1300 : 650); }; cacheTimer = setTimeout(tick, 400); show(0); }; const finish = () => { chars.forEach((c) => build(c.page)); show(chars.length + 1); }; replay.onclick = play; if (reduced.matches) finish(); else play(); }