perevod / tokenizer.js
xmarva's picture
perevod in the browser: the model in ONNX, the tokenizer and beam search in JavaScript
ce265cf verified
Raw History Blame Contribute Delete
2.91 kB
// The model's tokenizer (tokenizer.json from Hugging Face `tokenizers`), rewritten for the browser:
// the same normalizer, pre-tokenizer, BPE and decoder, checked token for token against the original.
const QUOTES = [["«", '"'], ["»", '"'], ["„", '"'], ["“", '"'], ["”", '"'], ["‘", "'"], ["’", "'"], ["–", "—"]];
// Rust's \w is Unicode-aware; JavaScript's isn't, even with the u flag, so it is spelled out
const WORD = "\\p{Alphabetic}\\p{M}\\p{Nd}\\p{Pc}\\p{Join_Control}";
const PIECE = new RegExp(` ?[${WORD}]+| ?[^${WORD}\\s]+`, "gu");
export class Tokenizer {
constructor(spec) {
this.vocab = new Map(Object.entries(spec.model.vocab));
this.tokens = [];
for (const [token, id] of this.vocab) this.tokens[id] = token;
this.ranks = new Map();
spec.model.merges.forEach((pair, rank) => {
const [a, b] = Array.isArray(pair) ? pair : pair.split(" ");
this.ranks.set(a + "\u0000" + b, rank);
});
this.special = new Set(spec.added_tokens.filter((t) => t.special).map((t) => t.id));
this.unk = this.vocab.get(spec.model.unk_token);
}
normalize(text) {
let s = text.normalize("NFKC").replace(/\p{Mn}/gu, "");
for (const [from, to] of QUOTES) s = s.split(from).join(to);
s = s.replace(/\s+-\s+/gu, " — ").replace(/\s+/gu, " ").trim();
return s ? " " + s : ""; // Prepend leaves an empty text empty
}
pieces(normalized) {
// Split, isolated: the matches, and whatever lies between them, each a piece of its own
const out = [];
let last = 0;
for (const match of normalized.matchAll(PIECE)) {
if (match.index > last) out.push(normalized.slice(last, match.index));
out.push(match[0]);
last = match.index + match[0].length;
}
if (last < normalized.length) out.push(normalized.slice(last));
return out.map((piece) => piece.replaceAll(" ", "▁")); // Metaspace, nothing prepended
}
bpe(piece) {
let symbols = Array.from(piece);
while (symbols.length > 1) {
let best = -1;
let bestRank = Infinity;
for (let i = 0; i + 1 < symbols.length; i++) {
const rank = this.ranks.get(symbols[i] + "\u0000" + symbols[i + 1]);
if (rank !== undefined && rank < bestRank) {
bestRank = rank;
best = i;
}
}
if (best < 0) break;
symbols = [...symbols.slice(0, best), symbols[best] + symbols[best + 1], ...symbols.slice(best + 2)];
}
return symbols.map((symbol) => this.vocab.get(symbol) ?? this.unk);
}
encode(text) {
const ids = [];
for (const piece of this.pieces(this.normalize(text))) ids.push(...this.bpe(piece));
return ids;
}
decode(ids) {
const words = ids.filter((id) => !this.special.has(id)).map((id) => this.tokens[id].replaceAll("▁", " "));
if (words.length && words[0].startsWith(" ")) words[0] = words[0].slice(1);
return words.join("");
}
}