import { codepoint, glyph, hex } from "./trace/unicode.js";
import { plural } from "./trace/tracer.js";
export function el(tag, className, ...children) {
const node = document.createElement(tag);
if (className) node.className = className;
node.append(...children);
return node;
}
/** A string as a row of char stickers, invisible chars made visible. `changed`: indices to mark. */
export function glyphs(root, text, changed) {
root.replaceChildren(
...[...text].map((ch, i) => {
const kind = (ch === " " ? "g sp" : ch.codePointAt(0) < 0x80 ? "g" : "g multi") + (changed?.has(i) ? " changed" : "");
// In a `.plain` row spaces stay real spaces, so the text wraps between words.
const plain = root.classList.contains("plain");
const span = el("span", kind, plain && ch === " " ? " " : plain && ch === "\n" ? "↵\n" : glyph(ch));
span.title = codepoint(ch);
return span;
}),
);
if (!text) root.append(el("span", "empty-text", "(empty)"));
}
/** The input as UTF-8: one sticker per char, its bytes underneath. Returns the byte cells. */
export function mountBytes(root, chars, bytes) {
const cells = [];
root.replaceChildren(
...chars.map(({ ch, start, len }) => {
const multi = len > 1;
const row = el("span", "bs");
for (let i = start; i < start + len; i++) {
const cell = el("i", "b", hex(bytes[i]));
cell.title = `byte ${i}: 0x${hex(bytes[i])}`;
cells[i] = cell;
row.append(cell);
}
const group = el("span", multi ? "c multi" : "c", el("b", "", glyph(ch)), row);
group.title = `${codepoint(ch)} · ${len} byte${len > 1 ? "s" : ""} at ${start}`;
if (ch === "\n") return [group, el("span", "br")];
return [group];
}).flat(),
);
return cells;
}
const HL = ["scanned", "hit", "block", "ascii", "unicode", "skip", "stable", "unstable"];
export function highlight(cells, hl) {
for (const cell of cells) cell.classList.remove(...HL);
for (const { from, to, cls } of hl) for (let i = from; i < to; i++) cells[i]?.classList.add(cls);
// Scroll the byte strip (not the page) so the interesting byte is visible.
const first = hl.find((h) => h.cls === "hit" || h.cls === "block") ?? hl[0];
const cell = first && cells[first.from];
if (!cell) return;
const box = cell.closest(".bytes");
const top = cell.offsetTop - box.offsetTop;
if (top < box.scrollTop || top > box.scrollTop + box.clientHeight - 40) box.scrollTop = top - 40;
}
/** The 16 lanes of a SIMD block: bytes, the high-bit mask, and whatever the step does with them. */
export function lanes(root, l) {
if (!l) return root.replaceChildren();
const row = (label, cells) => el("div", "lane-row", el("span", "lane-label", label), ...cells);
const byteCell = (b, extra = "") => {
const ch = b < 0x80 ? glyph(String.fromCharCode(b)) : "·";
return el("span", `lane ${b >= 0x80 ? "multi" : ""} ${extra}`, el("b", "", ch), el("small", "", hex(b)));
};
const rows = [row("16 bytes", l.block.map((b, i) => byteCell(b, i === l.lane ? "hit" : "")))];
rows.push(row("≥ 0x80?", l.block.map((b, i) => el("span", `lane bit ${b >= 0x80 ? "on" : ""} ${i === l.lane ? "hit" : ""}`, b >= 0x80 ? "yes" : "–"))));
if (l.kind === "lower") {
rows.push(row("capital?", l.upper.map((u) => el("span", `lane bit ${u ? "up" : ""}`, u ? "+0x20" : "–"))));
rows.push(row("written", l.block.map((b, i) => byteCell(l.upper[i] ? b | 0x20 : b, l.upper[i] ? "changed" : ""))));
} else if (l.kind === "bert") {
rows.length = 1; // pass 2 doesn't look at high bits: non-ASCII bytes just pass through
// Only the rules that fire in this block get a row.
const rule = (label, hits, mark) => hits.some(Boolean) && rows.push(row(label, hits.map((x) => el("span", `lane bit ${x ? "up" : ""}`, x ? mark : "–"))));
rule("tab, newline", l.spaces, "→ ·");
rule("control", l.deletes, "drop");
rule("capital", l.upper, "+0x20");
if (!l.deletes.some(Boolean)) {
const out = l.block.map((b, i) => (l.spaces[i] ? 0x20 : l.upper[i] ? b | 0x20 : b));
rows.push(row("written", out.map((b, i) => byteCell(b, out[i] !== l.block[i] ? "changed" : ""))));
}
} else if (l.kind === "controls") {
rows.push(row("control?", l.controls.map((c) => el("span", `lane bit ${c ? "up" : ""}`, c ? "yes" : "–"))));
}
const caption = {
exit: "a byte ≥ 0x80: not pure ASCII",
scan: l.lane ? `${plural(l.lane, "ASCII byte")}, then a byte ≥ 0x80` : "the very first byte is ≥ 0x80",
lower: "all 16 lowercased in one go",
controls: "a control char: this block goes byte by byte",
bert: l.deletes && l.deletes.some(Boolean) ? "a control char to drop: byte by byte" : "all 16 fixed in one go",
copy: "copied as is",
}[l.kind];
root.replaceChildren(el("div", "lanes", el("div", "lanes-head", `bytes ${l.start}–${l.start + 15}`, el("span", "", caption)), ...rows));
}
/** A non-ASCII run (or the whole input, before) through each Unicode stage, left to right. Like
* the post's "What BERT does to your text" figure, chars a stage changed are green and removals are counted. */
export function stages(root, list) {
if (!list) return root.replaceChildren();
// Short spans read best as one strip with arrows; longer text gets a row per stage.
const inline = list.every(({ text }) => [...text].length <= 8);
const flow = el("div", inline ? "stages inline" : "stages");
list.forEach(({ name, text }, i) => {
if (inline && i) flow.append(el("span", "arrow", "→"));
const g = el("div", "glyphs small");
const { changed, removed } = i ? diff([...list[i - 1].text], [...text]) : { changed: null, removed: 0 };
glyphs(g, text, changed);
const head = el("span", "stage-name", name);
if (removed) head.append(el("span", "removed", `−${removed}`));
flow.append(el("div", "stage", head, g));
});
root.replaceChildren(flow);
}
/** Which chars of `b` are new relative to `a` (not on a longest common subsequence), and how many fewer chars there are. */
function diff(a, b) {
const n = a.length;
const m = b.length;
const dp = Array.from({ length: n + 1 }, () => new Uint16Array(m + 1));
for (let i = n - 1; i >= 0; i--)
for (let j = m - 1; j >= 0; j--) dp[i][j] = a[i] === b[j] ? dp[i + 1][j + 1] + 1 : Math.max(dp[i + 1][j], dp[i][j + 1]);
const kept = new Set();
for (let i = 0, j = 0; i < n && j < m; ) {
if (a[i] === b[j]) (kept.add(j), i++, j++);
else if (dp[i + 1][j] >= dp[i][j + 1]) i++;
else j++;
}
const changed = new Set();
for (let j = 0; j < m; j++) if (!kept.has(j)) changed.add(j);
return { changed, removed: Math.max(0, n - m) }; // like the post's "−N code points"
}
// ── CharacterCache: the segment's chars, their 256-code-point pages, one read per char ─────────
const CACHE_CHARS = 14; // chars animated; the counters still cover the whole segment
const CACHE_PAGES = 6;
const reduced = matchMedia("(prefers-reduced-motion: reduce)");
let cacheTimer = 0;
export function stopCache(root) {
clearTimeout(cacheTimer);
root.replaceChildren();
}
const pageName = (page) => `U+${(page << 8).toString(16).toUpperCase().padStart(4, "0").slice(0, -2)}xx`;
function pageGrid(page, isUnstable) {
const canvas = document.createElement("canvas");
canvas.width = canvas.height = 16;
const ctx = canvas.getContext("2d");
const css = getComputedStyle(document.documentElement);
for (let k = 0; k < 256; k++) {
const cp = (page << 8) + k;
const valid = cp < 0xd800 || cp > 0xdfff;
ctx.fillStyle = valid && isUnstable(String.fromCodePoint(cp)) ? css.getPropertyValue("--red") : css.getPropertyValue("--tint-green");
ctx.fillRect(k & 15, k >> 4, 1, 1);
}
return canvas;
}
/** Plays the reads of one segment step: arrow from the char to its page, page built on a miss. */
export function cache(root, { chars, before, isUnstable }) {
clearTimeout(cacheTimer);
const shown = chars.slice(0, CACHE_CHARS);
const touched = [...new Set([...shown.map((c) => c.page), ...chars.map((c) => c.page), ...before])];
const pagesShown = touched.slice(0, CACHE_PAGES).sort((a, b) => a - b);
const extraPages = touched.length - pagesShown.length;
const count = el("span", "cache-count");
const replay = el("button", "cache-replay", "↻ replay");
const tiles = shown.map(({ ch, cp }) => {
const tile = el("span", "g multi", glyph(ch));
tile.title = codepoint(ch);
return tile;
});
const pages = new Map(
pagesShown.map((page) => {
const card = el("div", "page", el("div", "page-grid"), el("span", "page-name", pageName(page)));
return [page, card];
}),
);
const svg = document.createElementNS("http://www.w3.org/2000/svg", "svg");
svg.setAttribute("class", "cache-arrow");
const charsRow = el("div", "glyphs small cache-chars", ...tiles);
if (chars.length > shown.length) charsRow.append(el("span", "meta", `+${chars.length - shown.length}`));
const pagesRow = el("div", "cache-pages", ...pages.values());
if (extraPages > 0) pagesRow.append(el("span", "meta", `+${extraPages}`));
const box = el("div", "cache", el("div", "cache-head", el("span", "", "CharacterCache"), count, replay), charsRow, pagesRow, svg);
root.replaceChildren(box);
const built = new Set(before);
const build = (page) => {
const card = pages.get(page);
if (!card || card.classList.contains("built")) return;
card.classList.add("built");
card.querySelector(".page-grid").append(pageGrid(page, isUnstable));
};
before.forEach(build);
const show = (i) => {
// i: reads done so far (the current char is i - 1); i === chars.length: final state
const done = Math.min(i, chars.length);
const total = new Set([...before, ...chars.slice(0, done).map((c) => c.page)]).size;
count.textContent = `${plural(done, "read")} · ${plural(total, "page")} built`;
tiles.forEach((t, k) => t.classList.toggle("read", k < i - 1));
tiles.forEach((t, k) => t.classList.toggle("on", k === i - 1));
for (const card of pages.values()) card.classList.remove("fresh"), card.querySelector(".cell")?.remove();
svg.replaceChildren();
const cur = i >= 1 && i <= shown.length ? shown[i - 1] : null;
if (!cur) return;
const card = pages.get(cur.page);
if (cur.miss) (build(cur.page), card?.classList.add("fresh"));
if (!card) return;
const grid = card.querySelector(".page-grid");
const cell = el("i", "cell");
cell.style.left = `${((cur.cp & 15) / 16) * 100}%`;
cell.style.top = `${((cur.cp >> 4) & 15) / 16 * 100}%`;
grid.append(cell);
// Elbow arrow, box coordinates: down from the char, across, down into its page.
const b = box.getBoundingClientRect();
const a = tiles[i - 1].getBoundingClientRect();
const p = card.getBoundingClientRect();
const x1 = a.left + a.width / 2 - b.left, y1 = a.bottom - b.top + 2;
const x2 = p.left + p.width / 2 - b.left, y2 = p.top - b.top - 3;
const ym = (y1 + y2) / 2, r = Math.min(8, Math.abs(x2 - x1) / 2);
const s = Math.sign(x2 - x1) || 1;
const d = r < 1 ? `M${x1} ${y1}V${y2}` : `M${x1} ${y1}V${ym - r}Q${x1} ${ym} ${x1 + s * r} ${ym}H${x2 - s * r}Q${x2} ${ym} ${x2} ${ym + r}V${y2}`;
svg.setAttribute("viewBox", `0 0 ${b.width} ${b.height}`);
svg.innerHTML = ``;
};
const play = () => {
clearTimeout(cacheTimer);
for (const [page, card] of pages) {
if (before.includes(page)) continue;
card.classList.remove("built");
card.querySelector(".page-grid").replaceChildren();
}
let i = 0;
const tick = () => {
i++;
if (i > shown.length) return finish();
show(i);
cacheTimer = setTimeout(tick, shown[i - 1].miss ? 1300 : 650);
};
cacheTimer = setTimeout(tick, 400);
show(0);
};
const finish = () => {
chars.forEach((c) => build(c.page));
show(chars.length + 1);
};
replay.onclick = play;
if (reduced.matches) finish();
else play();
}