Download tokenizer.js from xmarva/perevod: direct link, hf CLI and curl.
- Browser
- Download file 2.91 kB
-
https://huggingface.co/spaces/xmarva/perevod/resolve/main/tokenizer.js
- Command line
-
hf download hf://spaces/xmarva/perevod/tokenizer.js
-
curl -L -o tokenizer.js https://huggingface.co/spaces/xmarva/perevod/resolve/main/tokenizer.js
2.91 kB
| // The model's tokenizer (tokenizer.json from Hugging Face `tokenizers`), rewritten for the browser: | |
| // the same normalizer, pre-tokenizer, BPE and decoder, checked token for token against the original. | |
| const QUOTES = [["«", '"'], ["»", '"'], ["„", '"'], ["“", '"'], ["”", '"'], ["‘", "'"], ["’", "'"], ["–", "—"]]; | |
| // Rust's \w is Unicode-aware; JavaScript's isn't, even with the u flag, so it is spelled out | |
| const WORD = "\\p{Alphabetic}\\p{M}\\p{Nd}\\p{Pc}\\p{Join_Control}"; | |
| const PIECE = new RegExp(` ?[${WORD}]+| ?[^${WORD}\\s]+`, "gu"); | |
| export class Tokenizer { | |
| constructor(spec) { | |
| this.vocab = new Map(Object.entries(spec.model.vocab)); | |
| this.tokens = []; | |
| for (const [token, id] of this.vocab) this.tokens[id] = token; | |
| this.ranks = new Map(); | |
| spec.model.merges.forEach((pair, rank) => { | |
| const [a, b] = Array.isArray(pair) ? pair : pair.split(" "); | |
| this.ranks.set(a + "\u0000" + b, rank); | |
| }); | |
| this.special = new Set(spec.added_tokens.filter((t) => t.special).map((t) => t.id)); | |
| this.unk = this.vocab.get(spec.model.unk_token); | |
| } | |
| normalize(text) { | |
| let s = text.normalize("NFKC").replace(/\p{Mn}/gu, ""); | |
| for (const [from, to] of QUOTES) s = s.split(from).join(to); | |
| s = s.replace(/\s+-\s+/gu, " — ").replace(/\s+/gu, " ").trim(); | |
| return s ? " " + s : ""; // Prepend leaves an empty text empty | |
| } | |
| pieces(normalized) { | |
| // Split, isolated: the matches, and whatever lies between them, each a piece of its own | |
| const out = []; | |
| let last = 0; | |
| for (const match of normalized.matchAll(PIECE)) { | |
| if (match.index > last) out.push(normalized.slice(last, match.index)); | |
| out.push(match[0]); | |
| last = match.index + match[0].length; | |
| } | |
| if (last < normalized.length) out.push(normalized.slice(last)); | |
| return out.map((piece) => piece.replaceAll(" ", "▁")); // Metaspace, nothing prepended | |
| } | |
| bpe(piece) { | |
| let symbols = Array.from(piece); | |
| while (symbols.length > 1) { | |
| let best = -1; | |
| let bestRank = Infinity; | |
| for (let i = 0; i + 1 < symbols.length; i++) { | |
| const rank = this.ranks.get(symbols[i] + "\u0000" + symbols[i + 1]); | |
| if (rank !== undefined && rank < bestRank) { | |
| bestRank = rank; | |
| best = i; | |
| } | |
| } | |
| if (best < 0) break; | |
| symbols = [...symbols.slice(0, best), symbols[best] + symbols[best + 1], ...symbols.slice(best + 2)]; | |
| } | |
| return symbols.map((symbol) => this.vocab.get(symbol) ?? this.unk); | |
| } | |
| encode(text) { | |
| const ids = []; | |
| for (const piece of this.pieces(this.normalize(text))) ids.push(...this.bpe(piece)); | |
| return ids; | |
| } | |
| decode(ids) { | |
| const words = ids.filter((id) => !this.special.has(id)).map((id) => this.tokens[id].replaceAll("▁", " ")); | |
| if (words.length && words[0].startsWith(" ")) words[0] = words[0].slice(1); | |
| return words.join(""); | |
| } | |
| } | |