Download core.mjs from EasyOctopus/german-support-redactor: direct link, hf CLI and curl.
- Browser
- Download file 4.53 kB
-
https://huggingface.co/spaces/EasyOctopus/german-support-redactor/resolve/main/core.mjs
- Command line
-
hf download hf://spaces/EasyOctopus/german-support-redactor/core.mjs
-
curl -L -o core.mjs https://huggingface.co/spaces/EasyOctopus/german-support-redactor/resolve/main/core.mjs
4.53 kB
| import {completeIdentifiers} from "./complete.mjs"; | |
| // The released German tokenizer uses cased BERT normalization and WordPiece. | |
| // All offsets below count Unicode code points in the unchanged input. | |
| const punctuation = /\p{P}/u; | |
| const space = /\p{Z}/u; | |
| const control = /[\p{Cc}\p{Cf}]/u; | |
| function isHan(c) { | |
| return ( | |
| (c >= 0x4e00 && c <= 0x9fff) || | |
| (c >= 0x3400 && c <= 0x4dbf) || | |
| (c >= 0x20000 && c <= 0x2a6df) || | |
| (c >= 0x2a700 && c <= 0x2b73f) || | |
| (c >= 0x2b740 && c <= 0x2b81f) || | |
| (c >= 0x2b820 && c <= 0x2ceaf) || | |
| (c >= 0xf900 && c <= 0xfaff) || | |
| (c >= 0x2f800 && c <= 0x2fa1f) | |
| ); | |
| } | |
| export function tokenize(text, vocab) { | |
| const words = []; | |
| let word = []; | |
| function finish() { | |
| if (word.length) words.push(word); | |
| word = []; | |
| } | |
| const points = Array.from(text); | |
| for (let at = 0; at < points.length; at++) { | |
| const ch = points[at]; | |
| if (ch === "[") { | |
| const special = ["[UNK]", "[CLS]", "[SEP]", "[MASK]", "[PAD]"].find( | |
| (k) => points.slice(at, at + k.length).join("") === k, | |
| ); | |
| if (special) { | |
| finish(); | |
| words.push([{ ch: special, at, end: at + special.length }]); | |
| at += special.length - 1; | |
| continue; | |
| } | |
| } | |
| const n = ch.codePointAt(0); | |
| if (ch === "\n" || ch === "\t" || ch === "\r" || space.test(ch)) { | |
| finish(); | |
| continue; | |
| } | |
| if (n === 0 || n === 0xfffd || control.test(ch)) continue; | |
| if ( | |
| isHan(n) || | |
| punctuation.test(ch) || | |
| (n >= 33 && n <= 47) || | |
| (n >= 58 && n <= 64) || | |
| (n >= 91 && n <= 96) || | |
| (n >= 123 && n <= 126) | |
| ) { | |
| finish(); | |
| words.push([{ ch, at }]); | |
| continue; | |
| } | |
| word.push({ ch, at }); | |
| } | |
| finish(); | |
| const result = []; | |
| for (const chars of words) { | |
| let start = 0; | |
| const pieces = []; | |
| while (chars.length <= 100 && start < chars.length) { | |
| let end = chars.length; | |
| let id; | |
| while (end > start) { | |
| const piece = | |
| (start ? "##" : "") + | |
| chars | |
| .slice(start, end) | |
| .map((x) => x.ch) | |
| .join(""); | |
| if (Object.hasOwn(vocab, piece)) { | |
| id = vocab[piece]; | |
| break; | |
| } | |
| end--; | |
| } | |
| if (id === undefined) break; | |
| pieces.push({ | |
| id, | |
| start: chars[start].at, | |
| end: chars[end - 1].end ?? chars[end - 1].at + 1, | |
| }); | |
| start = end; | |
| } | |
| if (start === chars.length) result.push(...pieces); | |
| else | |
| result.push({ | |
| id: vocab["[UNK]"], | |
| start: chars[0].at, | |
| end: chars.at(-1).at + 1, | |
| }); | |
| } | |
| return result; | |
| } | |
| export function extendGroups(tokens, logits, labels, state) { | |
| const width = Object.keys(labels).length; | |
| for (let i = 0; i < tokens.length; i++) { | |
| const offset = (i + 1) * width; | |
| let best = 0; | |
| for (let j = 1; j < width; j++) | |
| if (logits[offset + j] > logits[offset + best]) best = j; | |
| const tag = labels[best]; | |
| if (tag === "O") { | |
| state.last = null; | |
| continue; | |
| } | |
| let sum = 0; | |
| for (let j = 0; j < width; j++) | |
| sum += Math.exp(logits[offset + j] - logits[offset + best]); | |
| const label = tag.slice(2), | |
| token = tokens[i]; | |
| if (tag.startsWith("I-") && state.last === label) { | |
| const group = state.groups.at(-1); | |
| group.end = token.end; | |
| group.total += 1 / sum; | |
| group.count += 1; | |
| } else { | |
| state.groups.push({ | |
| start: token.start, | |
| end: token.end, | |
| label, | |
| total: 1 / sum, | |
| count: 1, | |
| }); | |
| } | |
| state.last = label; | |
| } | |
| } | |
| export function filtered(text, candidates, threshold, modelId) { | |
| const points = Array.from(text); | |
| let end = 0, | |
| redacted = ""; | |
| const entities = []; | |
| const retained = candidates.flatMap(group => { | |
| if (group.total / group.count >= threshold) return [group]; | |
| return (group.original ?? []).filter(original => original.total / original.count >= threshold); | |
| }); | |
| for (const group of completeIdentifiers(text, retained)) { | |
| const confidence = group.total / group.count; | |
| redacted += | |
| points.slice(end, group.start).join("") + "[" + group.label + "]"; | |
| end = group.end; | |
| entities.push({ | |
| start: group.start, | |
| end: group.end, | |
| label: group.label, | |
| confidence: Number(confidence.toFixed(5)), | |
| }); | |
| } | |
| return { | |
| model: modelId, | |
| threshold, | |
| offset_unit: "unicode_code_point", | |
| end_exclusive: true, | |
| redacted_text: redacted + points.slice(end).join(""), | |
| entities, | |
| }; | |
| } | |