EasyOctopus's picture
Use the previous German support question to redact short replies on device
543abb0 verified
Raw History Blame Contribute Delete
4.53 kB
import {completeIdentifiers} from "./complete.mjs";
// The released German tokenizer uses cased BERT normalization and WordPiece.
// All offsets below count Unicode code points in the unchanged input.
const punctuation = /\p{P}/u;
const space = /\p{Z}/u;
const control = /[\p{Cc}\p{Cf}]/u;
function isHan(c) {
return (
(c >= 0x4e00 && c <= 0x9fff) ||
(c >= 0x3400 && c <= 0x4dbf) ||
(c >= 0x20000 && c <= 0x2a6df) ||
(c >= 0x2a700 && c <= 0x2b73f) ||
(c >= 0x2b740 && c <= 0x2b81f) ||
(c >= 0x2b820 && c <= 0x2ceaf) ||
(c >= 0xf900 && c <= 0xfaff) ||
(c >= 0x2f800 && c <= 0x2fa1f)
);
}
export function tokenize(text, vocab) {
const words = [];
let word = [];
function finish() {
if (word.length) words.push(word);
word = [];
}
const points = Array.from(text);
for (let at = 0; at < points.length; at++) {
const ch = points[at];
if (ch === "[") {
const special = ["[UNK]", "[CLS]", "[SEP]", "[MASK]", "[PAD]"].find(
(k) => points.slice(at, at + k.length).join("") === k,
);
if (special) {
finish();
words.push([{ ch: special, at, end: at + special.length }]);
at += special.length - 1;
continue;
}
}
const n = ch.codePointAt(0);
if (ch === "\n" || ch === "\t" || ch === "\r" || space.test(ch)) {
finish();
continue;
}
if (n === 0 || n === 0xfffd || control.test(ch)) continue;
if (
isHan(n) ||
punctuation.test(ch) ||
(n >= 33 && n <= 47) ||
(n >= 58 && n <= 64) ||
(n >= 91 && n <= 96) ||
(n >= 123 && n <= 126)
) {
finish();
words.push([{ ch, at }]);
continue;
}
word.push({ ch, at });
}
finish();
const result = [];
for (const chars of words) {
let start = 0;
const pieces = [];
while (chars.length <= 100 && start < chars.length) {
let end = chars.length;
let id;
while (end > start) {
const piece =
(start ? "##" : "") +
chars
.slice(start, end)
.map((x) => x.ch)
.join("");
if (Object.hasOwn(vocab, piece)) {
id = vocab[piece];
break;
}
end--;
}
if (id === undefined) break;
pieces.push({
id,
start: chars[start].at,
end: chars[end - 1].end ?? chars[end - 1].at + 1,
});
start = end;
}
if (start === chars.length) result.push(...pieces);
else
result.push({
id: vocab["[UNK]"],
start: chars[0].at,
end: chars.at(-1).at + 1,
});
}
return result;
}
export function extendGroups(tokens, logits, labels, state) {
const width = Object.keys(labels).length;
for (let i = 0; i < tokens.length; i++) {
const offset = (i + 1) * width;
let best = 0;
for (let j = 1; j < width; j++)
if (logits[offset + j] > logits[offset + best]) best = j;
const tag = labels[best];
if (tag === "O") {
state.last = null;
continue;
}
let sum = 0;
for (let j = 0; j < width; j++)
sum += Math.exp(logits[offset + j] - logits[offset + best]);
const label = tag.slice(2),
token = tokens[i];
if (tag.startsWith("I-") && state.last === label) {
const group = state.groups.at(-1);
group.end = token.end;
group.total += 1 / sum;
group.count += 1;
} else {
state.groups.push({
start: token.start,
end: token.end,
label,
total: 1 / sum,
count: 1,
});
}
state.last = label;
}
}
export function filtered(text, candidates, threshold, modelId) {
const points = Array.from(text);
let end = 0,
redacted = "";
const entities = [];
const retained = candidates.flatMap(group => {
if (group.total / group.count >= threshold) return [group];
return (group.original ?? []).filter(original => original.total / original.count >= threshold);
});
for (const group of completeIdentifiers(text, retained)) {
const confidence = group.total / group.count;
redacted +=
points.slice(end, group.start).join("") + "[" + group.label + "]";
end = group.end;
entities.push({
start: group.start,
end: group.end,
label: group.label,
confidence: Number(confidence.toFixed(5)),
});
}
return {
model: modelId,
threshold,
offset_unit: "unicode_code_point",
end_exclusive: true,
redacted_text: redacted + points.slice(end).join(""),
entities,
};
}