import {completeIdentifiers} from "./complete.mjs"; // The released German tokenizer uses cased BERT normalization and WordPiece. // All offsets below count Unicode code points in the unchanged input. const punctuation = /\p{P}/u; const space = /\p{Z}/u; const control = /[\p{Cc}\p{Cf}]/u; function isHan(c) { return ( (c >= 0x4e00 && c <= 0x9fff) || (c >= 0x3400 && c <= 0x4dbf) || (c >= 0x20000 && c <= 0x2a6df) || (c >= 0x2a700 && c <= 0x2b73f) || (c >= 0x2b740 && c <= 0x2b81f) || (c >= 0x2b820 && c <= 0x2ceaf) || (c >= 0xf900 && c <= 0xfaff) || (c >= 0x2f800 && c <= 0x2fa1f) ); } export function tokenize(text, vocab) { const words = []; let word = []; function finish() { if (word.length) words.push(word); word = []; } const points = Array.from(text); for (let at = 0; at < points.length; at++) { const ch = points[at]; if (ch === "[") { const special = ["[UNK]", "[CLS]", "[SEP]", "[MASK]", "[PAD]"].find( (k) => points.slice(at, at + k.length).join("") === k, ); if (special) { finish(); words.push([{ ch: special, at, end: at + special.length }]); at += special.length - 1; continue; } } const n = ch.codePointAt(0); if (ch === "\n" || ch === "\t" || ch === "\r" || space.test(ch)) { finish(); continue; } if (n === 0 || n === 0xfffd || control.test(ch)) continue; if ( isHan(n) || punctuation.test(ch) || (n >= 33 && n <= 47) || (n >= 58 && n <= 64) || (n >= 91 && n <= 96) || (n >= 123 && n <= 126) ) { finish(); words.push([{ ch, at }]); continue; } word.push({ ch, at }); } finish(); const result = []; for (const chars of words) { let start = 0; const pieces = []; while (chars.length <= 100 && start < chars.length) { let end = chars.length; let id; while (end > start) { const piece = (start ? "##" : "") + chars .slice(start, end) .map((x) => x.ch) .join(""); if (Object.hasOwn(vocab, piece)) { id = vocab[piece]; break; } end--; } if (id === undefined) break; pieces.push({ id, start: chars[start].at, end: chars[end - 1].end ?? chars[end - 1].at + 1, }); start = end; } if (start === chars.length) result.push(...pieces); else result.push({ id: vocab["[UNK]"], start: chars[0].at, end: chars.at(-1).at + 1, }); } return result; } export function extendGroups(tokens, logits, labels, state) { const width = Object.keys(labels).length; for (let i = 0; i < tokens.length; i++) { const offset = (i + 1) * width; let best = 0; for (let j = 1; j < width; j++) if (logits[offset + j] > logits[offset + best]) best = j; const tag = labels[best]; if (tag === "O") { state.last = null; continue; } let sum = 0; for (let j = 0; j < width; j++) sum += Math.exp(logits[offset + j] - logits[offset + best]); const label = tag.slice(2), token = tokens[i]; if (tag.startsWith("I-") && state.last === label) { const group = state.groups.at(-1); group.end = token.end; group.total += 1 / sum; group.count += 1; } else { state.groups.push({ start: token.start, end: token.end, label, total: 1 / sum, count: 1, }); } state.last = label; } } export function filtered(text, candidates, threshold, modelId) { const points = Array.from(text); let end = 0, redacted = ""; const entities = []; const retained = candidates.flatMap(group => { if (group.total / group.count >= threshold) return [group]; return (group.original ?? []).filter(original => original.total / original.count >= threshold); }); for (const group of completeIdentifiers(text, retained)) { const confidence = group.total / group.count; redacted += points.slice(end, group.start).join("") + "[" + group.label + "]"; end = group.end; entities.push({ start: group.start, end: group.end, label: group.label, confidence: Number(confidence.toFixed(5)), }); } return { model: modelId, threshold, offset_unit: "unicode_code_point", end_exclusive: true, redacted_text: redacted + points.slice(end).join(""), entities, }; }