Hugging Face's logo

Spaces:
HuggingFaceBio
/
Carbon-X-Release
Running

App Files Community
1
Carbon-X-Release / figures /tokenizer.html
lvwerra's picture
lvwerra HF Staff
Claude Opus 5.5
Port the rest of the Notion draft with figures
f2da764
Raw History Blame Contribute Delete
8.44 kB
<!doctype html>
<html lang="en">
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>Carbon-X tokenizer</title>
<link rel="preconnect" href="https://fonts.googleapis.com">
<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
<link href="https://fonts.googleapis.com/css2?family=Inter:wght@400;500;600;700&family=JetBrains+Mono:wght@400;500;600&display=swap" rel="stylesheet">
<style>
/* Tokens from the Carbon-X release blog (release_blog/space/index.html). */
:root {
--paper: #f7f5ee; --card: #fbfaf5; --ink: #1f1f1d; --soft: #5b5b56; --muted: #8a8a83; --rule: #e3e1d6; --hairline: #d6d3c4;
--green: #317f3f; --green-soft: rgba(49, 127, 63, 0.10); --green-line: rgba(49, 127, 63, 0.35); --fill: #efece3;
--gold-soft: rgba(184, 134, 44, 0.14); --gold-line: rgba(184, 134, 44, 0.45); --gold-ink: #7a5a1c;
--mono: "JetBrains Mono", ui-monospace, Menlo, monospace;
--sans: "Inter", "Helvetica Neue", system-ui, sans-serif;
color-scheme: light;
}
* { box-sizing: border-box; }
body { margin: 0; background: var(--paper); color: var(--ink); font: 15px/1.55 var(--sans); }
.fig { max-width: 1040px; margin: 0 auto; padding: 34px 16px 24px; position: relative; }
/* input */
.input { display: flex; justify-content: center; }
.input .box { font-family: var(--mono); font-size: 14.5px; background: var(--card); border: 1px solid var(--ink); border-radius: 8px;
padding: 8px 16px; white-space: nowrap; }
.d { color: var(--green); font-weight: 500; }
/* connector */
.conn { display: block; width: 100%; height: 46px; }
.conn path { fill: none; stroke: var(--hairline); stroke-width: 1.3; vector-effect: non-scaling-stroke;
stroke-dasharray: 400; stroke-dashoffset: 400; transition: stroke-dashoffset .7s ease; }
.conn.in path { stroke-dashoffset: 0; }
.conn .head { fill: var(--muted); opacity: 0; transition: opacity .2s .6s; }
.conn.in .head { opacity: 1; }
/* boxes */
.cols { display: grid; grid-template-columns: 1fr 1fr; gap: 20px; }
.card { background: var(--card); border: 1px solid var(--rule); border-radius: 10px; padding: 18px 22px 20px; }
.card.hy { border-color: var(--green); }
.head { display: flex; justify-content: space-between; align-items: baseline; margin-bottom: 14px; }
.head h3 { margin: 0; font-size: 15.5px; font-weight: 600; letter-spacing: -.005em; }
.count { font-family: var(--mono); font-size: 11.5px; color: var(--muted); letter-spacing: .04em; opacity: 0; transition: opacity .4s; }
.count b { font: 600 20px var(--sans); color: var(--ink); margin-right: 5px; letter-spacing: 0; }
.count.in { opacity: 1; }
.text { font-family: var(--mono); font-size: 13px; line-height: 1.75; color: var(--soft); }
.text .row { display: flex; align-items: center; height: 26px; white-space: nowrap; }
.tg { display: inline-block; font-family: var(--mono); font-size: 11.5px; background: var(--gold-soft); border: 1px solid var(--gold-line);
color: var(--gold-ink); border-radius: 4px; line-height: 1.5; overflow: hidden; vertical-align: middle;
max-width: 0; padding: 0; border-width: 0; opacity: 0; transition: max-width .45s ease, padding .45s ease, opacity .45s ease, margin .45s ease; }
.tg.in { max-width: 70px; padding: 0 4px; border-width: 1px; opacity: 1; margin: 0 3px; }
.tg.l.in { margin-left: 0; }
.down { color: var(--muted); font-size: 14px; line-height: 1; margin: 10px 0 10px; text-align: center; }
.toks { display: flex; flex-direction: column; gap: 6px; }
.toks .row { display: flex; flex-wrap: wrap; gap: 3px; min-height: 28px; }
.tok { font-family: var(--mono); font-size: 12px; padding: 2px 5px; border-radius: 5px; border: 1px solid var(--hairline); background: #fff; color: var(--ink); white-space: pre;
opacity: 0; transform: translateY(-12px); transition: opacity .3s ease, transform .3s ease; }
.tok.in { opacity: 1; transform: none; }
.tok.bpe { background: var(--fill); border-color: var(--hairline); color: var(--soft); padding: 2px 5px; }
.tok.k { background: var(--green-soft); border-color: var(--green-line); color: var(--green); font-weight: 500; }
.tok.k .pad { color: var(--muted); font-weight: 400; }
.tok.tag { background: var(--gold-soft); border-color: var(--gold-line); color: var(--gold-ink); }
.replay { position: absolute; top: 6px; right: 16px; font: 11.5px var(--mono); color: var(--muted); background: transparent; border: 0; cursor: pointer; }
.replay:hover { color: var(--ink); }
@media (max-width: 820px) {
.cols { grid-template-columns: 1fr; } .conn { display: none; } .input { margin-bottom: 14px; }
.input .box { white-space: normal; font-size: 14px; }
}
@media (prefers-reduced-motion: reduce) { .tok, .tg, .count, .conn path, .conn .head { transition: none; } }
</style>
</head>
<body>
<div class="fig">
<button class="replay" id="replay" aria-label="Replay">↻ replay</button>
<div class="input"><div class="box">What does this sequence do? <span class="d">ATGGCCATTGTAATGGGCCGCTGAAAG</span></div></div>
<svg class="conn" id="conn" viewBox="0 0 1000 46" preserveAspectRatio="none" aria-hidden="true">
<path d="M500 0 V16 Q500 22 494 22 H262 Q256 22 256 28 V40"/>
<path d="M500 16 Q500 22 506 22 H738 Q744 22 744 28 V40"/>
<polygon class="head" points="251,38 261,38 256,46"/>
<polygon class="head" points="739,38 749,38 744,46"/>
</svg>
<div class="cols">
<div class="card">
<div class="head"><h3>Qwen3 tokenizer</h3><span class="count" id="cq"><b>20</b>tokens</span></div>
<div class="text">
<div class="row">What does this sequence do?</div>
<div class="row"><span class="d">ATGGCCATTGTAATGGGCCGCTGAAAG</span></div>
</div>
<div class="down">↓</div>
<div class="toks"><div class="row" id="q1"></div><div class="row" id="q2"></div></div>
</div>
<div class="card hy">
<div class="head"><h3>Carbon-X tokenizer</h3><span class="count" id="ch"><b>13</b>tokens</span></div>
<div class="text">
<div class="row">What does this sequence do?</div>
<div class="row"><span class="tg l" id="t1">&lt;dna&gt;</span><span class="d">ATGGCCATTGTAATGGGCCGCTGAAAG</span><span class="tg" id="t2">&lt;/dna&gt;</span></div>
</div>
<div class="down">↓</div>
<div class="toks"><div class="row" id="h1"></div><div class="row" id="h2"></div></div>
</div>
</div>
</div>
<script>
// Qwen3: real output of the Qwen/Qwen3-14B tokenizer on the input (leading spaces not shown).
const TEXT = ["What", "does", "this", "sequence", "do", "?"];
const Q_DNA = ["AT","GG","CC","ATT","G","TA","AT","GG","GCC","G","CT","G","AA","AG"];
// Hybrid: same text tokens; DNA inside the tags cut into 6-mers, last one padded with N.
const H_DNA = [["<dna>","tag"],["ATGGCC","k"],["ATTGTA","k"],["ATGGGC","k"],["CGCTGA","k"],["AAG|NNN","k"],["</dna>","tag"]];
const timers = [], later = (f, ms) => timers.push(setTimeout(f, ms));
const $ = id => document.getElementById(id);
function chips(id, list) {
const box = $(id); box.innerHTML = "";
return list.map(([t, cls]) => {
const s = document.createElement("span"); s.className = "tok " + (cls || "");
if (t.includes("|")) { const [a, p] = t.split("|"); s.innerHTML = a + `<span class="pad">${p}</span>`; } else s.textContent = t;
box.appendChild(s); return s;
});
}
function play() {
timers.splice(0).forEach(clearTimeout);
["conn", "t1", "t2", "cq", "ch"].forEach(id => $(id).classList.remove("in"));
const q = [...chips("q1", TEXT.map(t => [t])), ...chips("q2", Q_DNA.map(t => [t, "bpe"]))];
const h = [...chips("h1", TEXT.map(t => [t])), ...chips("h2", H_DNA)];
const all = () => { ["conn", "t1", "t2", "cq", "ch"].forEach(id => $(id).classList.add("in")); [...q, ...h].forEach(e => e.classList.add("in")); };
if (matchMedia("(prefers-reduced-motion: reduce)").matches || location.hash === "#static") return all();
later(() => $("conn").classList.add("in"), 200);
later(() => { $("t1").classList.add("in"); $("t2").classList.add("in"); }, 1100);
const t0 = 1900, step = 95;
q.forEach((e, i) => later(() => e.classList.add("in"), t0 + i * step));
h.forEach((e, i) => later(() => e.classList.add("in"), t0 + i * step));
later(() => $("ch").classList.add("in"), t0 + h.length * step + 150);
later(() => $("cq").classList.add("in"), t0 + q.length * step + 150);
}
$("replay").addEventListener("click", play);
play();
</script>
</body>
</html>