Spaces:
Running
Running
SBrandeis
Claude Opus 5.5
Full-width input, section titles, CharacterCache animation; drop the Rust panel
32a4f48 unverified Download render.js from sbrandeis/bert-normalizer-demo: direct link, hf CLI and curl.
- Browser
- Download file 12.1 kB
-
https://huggingface.co/spaces/sbrandeis/bert-normalizer-demo/resolve/main/render.js
- Command line
-
hf download hf://spaces/sbrandeis/bert-normalizer-demo/render.js
-
curl -L -o render.js https://huggingface.co/spaces/sbrandeis/bert-normalizer-demo/resolve/main/render.js
12.1 kB
| import { codepoint, glyph, hex } from "./trace/unicode.js"; | |
| import { plural } from "./trace/tracer.js"; | |
| export function el(tag, className, ...children) { | |
| const node = document.createElement(tag); | |
| if (className) node.className = className; | |
| node.append(...children); | |
| return node; | |
| } | |
| /** A string as a row of char stickers, invisible chars made visible. `changed`: indices to mark. */ | |
| export function glyphs(root, text, changed) { | |
| root.replaceChildren( | |
| ...[...text].map((ch, i) => { | |
| const kind = (ch === " " ? "g sp" : ch.codePointAt(0) < 0x80 ? "g" : "g multi") + (changed?.has(i) ? " changed" : ""); | |
| // In a `.plain` row spaces stay real spaces, so the text wraps between words. | |
| const plain = root.classList.contains("plain"); | |
| const span = el("span", kind, plain && ch === " " ? " " : plain && ch === "\n" ? "β΅\n" : glyph(ch)); | |
| span.title = codepoint(ch); | |
| return span; | |
| }), | |
| ); | |
| if (!text) root.append(el("span", "empty-text", "(empty)")); | |
| } | |
| /** The input as UTF-8: one sticker per char, its bytes underneath. Returns the byte cells. */ | |
| export function mountBytes(root, chars, bytes) { | |
| const cells = []; | |
| root.replaceChildren( | |
| ...chars.map(({ ch, start, len }) => { | |
| const multi = len > 1; | |
| const row = el("span", "bs"); | |
| for (let i = start; i < start + len; i++) { | |
| const cell = el("i", "b", hex(bytes[i])); | |
| cell.title = `byte ${i}: 0x${hex(bytes[i])}`; | |
| cells[i] = cell; | |
| row.append(cell); | |
| } | |
| const group = el("span", multi ? "c multi" : "c", el("b", "", glyph(ch)), row); | |
| group.title = `${codepoint(ch)} Β· ${len} byte${len > 1 ? "s" : ""} at ${start}`; | |
| if (ch === "\n") return [group, el("span", "br")]; | |
| return [group]; | |
| }).flat(), | |
| ); | |
| return cells; | |
| } | |
| const HL = ["scanned", "hit", "block", "ascii", "unicode", "skip", "stable", "unstable"]; | |
| export function highlight(cells, hl) { | |
| for (const cell of cells) cell.classList.remove(...HL); | |
| for (const { from, to, cls } of hl) for (let i = from; i < to; i++) cells[i]?.classList.add(cls); | |
| // Scroll the byte strip (not the page) so the interesting byte is visible. | |
| const first = hl.find((h) => h.cls === "hit" || h.cls === "block") ?? hl[0]; | |
| const cell = first && cells[first.from]; | |
| if (!cell) return; | |
| const box = cell.closest(".bytes"); | |
| const top = cell.offsetTop - box.offsetTop; | |
| if (top < box.scrollTop || top > box.scrollTop + box.clientHeight - 40) box.scrollTop = top - 40; | |
| } | |
| /** The 16 lanes of a SIMD block: bytes, the high-bit mask, and whatever the step does with them. */ | |
| export function lanes(root, l) { | |
| if (!l) return root.replaceChildren(); | |
| const row = (label, cells) => el("div", "lane-row", el("span", "lane-label", label), ...cells); | |
| const byteCell = (b, extra = "") => { | |
| const ch = b < 0x80 ? glyph(String.fromCharCode(b)) : "Β·"; | |
| return el("span", `lane ${b >= 0x80 ? "multi" : ""} ${extra}`, el("b", "", ch), el("small", "", hex(b))); | |
| }; | |
| const rows = [row("16 bytes", l.block.map((b, i) => byteCell(b, i === l.lane ? "hit" : "")))]; | |
| rows.push(row("β₯ 0x80?", l.block.map((b, i) => el("span", `lane bit ${b >= 0x80 ? "on" : ""} ${i === l.lane ? "hit" : ""}`, b >= 0x80 ? "yes" : "β")))); | |
| if (l.kind === "lower") { | |
| rows.push(row("capital?", l.upper.map((u) => el("span", `lane bit ${u ? "up" : ""}`, u ? "+0x20" : "β")))); | |
| rows.push(row("written", l.block.map((b, i) => byteCell(l.upper[i] ? b | 0x20 : b, l.upper[i] ? "changed" : "")))); | |
| } else if (l.kind === "bert") { | |
| rows.length = 1; // pass 2 doesn't look at high bits: non-ASCII bytes just pass through | |
| // Only the rules that fire in this block get a row. | |
| const rule = (label, hits, mark) => hits.some(Boolean) && rows.push(row(label, hits.map((x) => el("span", `lane bit ${x ? "up" : ""}`, x ? mark : "β")))); | |
| rule("tab, newline", l.spaces, "β Β·"); | |
| rule("control", l.deletes, "drop"); | |
| rule("capital", l.upper, "+0x20"); | |
| if (!l.deletes.some(Boolean)) { | |
| const out = l.block.map((b, i) => (l.spaces[i] ? 0x20 : l.upper[i] ? b | 0x20 : b)); | |
| rows.push(row("written", out.map((b, i) => byteCell(b, out[i] !== l.block[i] ? "changed" : "")))); | |
| } | |
| } else if (l.kind === "controls") { | |
| rows.push(row("control?", l.controls.map((c) => el("span", `lane bit ${c ? "up" : ""}`, c ? "yes" : "β")))); | |
| } | |
| const caption = { | |
| exit: "a byte β₯ 0x80: not pure ASCII", | |
| scan: l.lane ? `${plural(l.lane, "ASCII byte")}, then a byte β₯ 0x80` : "the very first byte is β₯ 0x80", | |
| lower: "all 16 lowercased in one go", | |
| controls: "a control char: this block goes byte by byte", | |
| bert: l.deletes && l.deletes.some(Boolean) ? "a control char to drop: byte by byte" : "all 16 fixed in one go", | |
| copy: "copied as is", | |
| }[l.kind]; | |
| root.replaceChildren(el("div", "lanes", el("div", "lanes-head", `bytes ${l.start}β${l.start + 15}`, el("span", "", caption)), ...rows)); | |
| } | |
| /** A non-ASCII run (or the whole input, before) through each Unicode stage, left to right. Like | |
| * the post's "What BERT does to your text" figure, chars a stage changed are green and removals are counted. */ | |
| export function stages(root, list) { | |
| if (!list) return root.replaceChildren(); | |
| // Short spans read best as one strip with arrows; longer text gets a row per stage. | |
| const inline = list.every(({ text }) => [...text].length <= 8); | |
| const flow = el("div", inline ? "stages inline" : "stages"); | |
| list.forEach(({ name, text }, i) => { | |
| if (inline && i) flow.append(el("span", "arrow", "β")); | |
| const g = el("div", "glyphs small"); | |
| const { changed, removed } = i ? diff([...list[i - 1].text], [...text]) : { changed: null, removed: 0 }; | |
| glyphs(g, text, changed); | |
| const head = el("span", "stage-name", name); | |
| if (removed) head.append(el("span", "removed", `β${removed}`)); | |
| flow.append(el("div", "stage", head, g)); | |
| }); | |
| root.replaceChildren(flow); | |
| } | |
| /** Which chars of `b` are new relative to `a` (not on a longest common subsequence), and how many fewer chars there are. */ | |
| function diff(a, b) { | |
| const n = a.length; | |
| const m = b.length; | |
| const dp = Array.from({ length: n + 1 }, () => new Uint16Array(m + 1)); | |
| for (let i = n - 1; i >= 0; i--) | |
| for (let j = m - 1; j >= 0; j--) dp[i][j] = a[i] === b[j] ? dp[i + 1][j + 1] + 1 : Math.max(dp[i + 1][j], dp[i][j + 1]); | |
| const kept = new Set(); | |
| for (let i = 0, j = 0; i < n && j < m; ) { | |
| if (a[i] === b[j]) (kept.add(j), i++, j++); | |
| else if (dp[i + 1][j] >= dp[i][j + 1]) i++; | |
| else j++; | |
| } | |
| const changed = new Set(); | |
| for (let j = 0; j < m; j++) if (!kept.has(j)) changed.add(j); | |
| return { changed, removed: Math.max(0, n - m) }; // like the post's "βN code points" | |
| } | |
| // ββ CharacterCache: the segment's chars, their 256-code-point pages, one read per char βββββββββ | |
| const CACHE_CHARS = 14; // chars animated; the counters still cover the whole segment | |
| const CACHE_PAGES = 6; | |
| const reduced = matchMedia("(prefers-reduced-motion: reduce)"); | |
| let cacheTimer = 0; | |
| export function stopCache(root) { | |
| clearTimeout(cacheTimer); | |
| root.replaceChildren(); | |
| } | |
| const pageName = (page) => `U+${(page << 8).toString(16).toUpperCase().padStart(4, "0").slice(0, -2)}xx`; | |
| function pageGrid(page, isUnstable) { | |
| const canvas = document.createElement("canvas"); | |
| canvas.width = canvas.height = 16; | |
| const ctx = canvas.getContext("2d"); | |
| const css = getComputedStyle(document.documentElement); | |
| for (let k = 0; k < 256; k++) { | |
| const cp = (page << 8) + k; | |
| const valid = cp < 0xd800 || cp > 0xdfff; | |
| ctx.fillStyle = valid && isUnstable(String.fromCodePoint(cp)) ? css.getPropertyValue("--red") : css.getPropertyValue("--tint-green"); | |
| ctx.fillRect(k & 15, k >> 4, 1, 1); | |
| } | |
| return canvas; | |
| } | |
| /** Plays the reads of one segment step: arrow from the char to its page, page built on a miss. */ | |
| export function cache(root, { chars, before, isUnstable }) { | |
| clearTimeout(cacheTimer); | |
| const shown = chars.slice(0, CACHE_CHARS); | |
| const touched = [...new Set([...shown.map((c) => c.page), ...chars.map((c) => c.page), ...before])]; | |
| const pagesShown = touched.slice(0, CACHE_PAGES).sort((a, b) => a - b); | |
| const extraPages = touched.length - pagesShown.length; | |
| const count = el("span", "cache-count"); | |
| const replay = el("button", "cache-replay", "β» replay"); | |
| const tiles = shown.map(({ ch, cp }) => { | |
| const tile = el("span", "g multi", glyph(ch)); | |
| tile.title = codepoint(ch); | |
| return tile; | |
| }); | |
| const pages = new Map( | |
| pagesShown.map((page) => { | |
| const card = el("div", "page", el("div", "page-grid"), el("span", "page-name", pageName(page))); | |
| return [page, card]; | |
| }), | |
| ); | |
| const svg = document.createElementNS("http://www.w3.org/2000/svg", "svg"); | |
| svg.setAttribute("class", "cache-arrow"); | |
| const charsRow = el("div", "glyphs small cache-chars", ...tiles); | |
| if (chars.length > shown.length) charsRow.append(el("span", "meta", `+${chars.length - shown.length}`)); | |
| const pagesRow = el("div", "cache-pages", ...pages.values()); | |
| if (extraPages > 0) pagesRow.append(el("span", "meta", `+${extraPages}`)); | |
| const box = el("div", "cache", el("div", "cache-head", el("span", "", "CharacterCache"), count, replay), charsRow, pagesRow, svg); | |
| root.replaceChildren(box); | |
| const built = new Set(before); | |
| const build = (page) => { | |
| const card = pages.get(page); | |
| if (!card || card.classList.contains("built")) return; | |
| card.classList.add("built"); | |
| card.querySelector(".page-grid").append(pageGrid(page, isUnstable)); | |
| }; | |
| before.forEach(build); | |
| const show = (i) => { | |
| // i: reads done so far (the current char is i - 1); i === chars.length: final state | |
| const done = Math.min(i, chars.length); | |
| const total = new Set([...before, ...chars.slice(0, done).map((c) => c.page)]).size; | |
| count.textContent = `${plural(done, "read")} Β· ${plural(total, "page")} built`; | |
| tiles.forEach((t, k) => t.classList.toggle("read", k < i - 1)); | |
| tiles.forEach((t, k) => t.classList.toggle("on", k === i - 1)); | |
| for (const card of pages.values()) card.classList.remove("fresh"), card.querySelector(".cell")?.remove(); | |
| svg.replaceChildren(); | |
| const cur = i >= 1 && i <= shown.length ? shown[i - 1] : null; | |
| if (!cur) return; | |
| const card = pages.get(cur.page); | |
| if (cur.miss) (build(cur.page), card?.classList.add("fresh")); | |
| if (!card) return; | |
| const grid = card.querySelector(".page-grid"); | |
| const cell = el("i", "cell"); | |
| cell.style.left = `${((cur.cp & 15) / 16) * 100}%`; | |
| cell.style.top = `${((cur.cp >> 4) & 15) / 16 * 100}%`; | |
| grid.append(cell); | |
| // Elbow arrow, box coordinates: down from the char, across, down into its page. | |
| const b = box.getBoundingClientRect(); | |
| const a = tiles[i - 1].getBoundingClientRect(); | |
| const p = card.getBoundingClientRect(); | |
| const x1 = a.left + a.width / 2 - b.left, y1 = a.bottom - b.top + 2; | |
| const x2 = p.left + p.width / 2 - b.left, y2 = p.top - b.top - 3; | |
| const ym = (y1 + y2) / 2, r = Math.min(8, Math.abs(x2 - x1) / 2); | |
| const s = Math.sign(x2 - x1) || 1; | |
| const d = r < 1 ? `M${x1} ${y1}V${y2}` : `M${x1} ${y1}V${ym - r}Q${x1} ${ym} ${x1 + s * r} ${ym}H${x2 - s * r}Q${x2} ${ym} ${x2} ${ym + r}V${y2}`; | |
| svg.setAttribute("viewBox", `0 0 ${b.width} ${b.height}`); | |
| svg.innerHTML = `<path d="${d}"/><path class="head" d="M${x2 - 5} ${y2 - 6}L${x2 + 5} ${y2 - 6}L${x2} ${y2}Z"/>`; | |
| }; | |
| const play = () => { | |
| clearTimeout(cacheTimer); | |
| for (const [page, card] of pages) { | |
| if (before.includes(page)) continue; | |
| card.classList.remove("built"); | |
| card.querySelector(".page-grid").replaceChildren(); | |
| } | |
| let i = 0; | |
| const tick = () => { | |
| i++; | |
| if (i > shown.length) return finish(); | |
| show(i); | |
| cacheTimer = setTimeout(tick, shown[i - 1].miss ? 1300 : 650); | |
| }; | |
| cacheTimer = setTimeout(tick, 400); | |
| show(0); | |
| }; | |
| const finish = () => { | |
| chars.forEach((c) => build(c.page)); | |
| show(chars.length + 1); | |
| }; | |
| replay.onclick = play; | |
| if (reduced.matches) finish(); | |
| else play(); | |
| } | |