File size: 8,449 Bytes
dc5df19 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 | // The worker passes decoded v2 groups with code-point offsets and applies its existing confidence slider.
const WORD_CHAR = /^[A-Za-zÀ-ÖØ-öø-ÿĀ-žẞ]$/;
function nearestWords(points, position, step) {
const words = [];
while (position >= 0 && position < points.length && words.length < 6) {
if (!WORD_CHAR.test(points[position])) {
position += step;
continue;
}
const letters = [];
while (position >= 0 && position < points.length && WORD_CHAR.test(points[position])) {
letters.push(points[position]);
position += step;
}
words.push((step < 0 ? letters.reverse() : letters).join(''));
}
return words;
}
function fold(word) {
return Array.from(word, c => {
if ((c >= 'A' && c <= 'Z') || (c >= 'À' && c <= 'Ö') || (c >= 'Ø' && c <= 'Þ')) {
return String.fromCodePoint(c.codePointAt(0) + 32);
}
return c === 'ẞ' ? 'ß' : c;
}).join('');
}
function context(points, group) {
const {start, end, label} = group;
if (!Number.isInteger(start) || !Number.isInteger(end) || start < 0 || start >= end || end > points.length) {
throw new RangeError('invalid code-point span');
}
const output = new Set(['tag:' + label]);
const left = nearestWords(points, start - 1, -1);
const right = nearestWords(points, end, 1);
for (const [side, words] of [['L', left], ['R', right]]) {
words.forEach((raw, i) => {
const w = fold(raw);
output.add(side + ':w:' + w);
output.add(side + ':p:' + w.slice(0, 3));
output.add(side + ':s:' + w.slice(-3));
if (i === 0) output.add(side + ':n:' + w);
});
}
return [...output].sort();
}
function contextFeatures(text, group) {
return context(Array.from(text), group);
}
function applyLearned(text, groups, policy) {
if (policy.semantics.version !== 'context-latin-v1') throw new Error('unsupported policy version');
const points = Array.from(text);
return groups.filter(group => {
if (!policy.editable.includes(group.label) || policy.cutoff === 0) return true;
let logit = policy.bias;
for (const f of context(points, group)) logit += policy.weights[f] ?? 0;
return 1 / (1 + Math.exp(-logit)) >= policy.cutoff;
});
}
// Explicit public roles. Prefixes and suffixes bind one value, never a name elsewhere in the clause.
const EDITABLE = new Set(['PERSON', 'ADDRESS', 'ORDER_ID', 'CUSTOMER_ID', 'ACCOUNT_ID', 'ORGANIZATION']);
const wordEdge = String.raw`[\p{L}\p{M}\p{N}_]`;
const roles = [
'Katalog(?:artikel|nummer|kennung|referenz|titel|serie|eintrag)',
'Artikel(?:nummer|kennung|bezeichnung|titel)', 'Produkt(?:name|nummer|bezeichnung|serie|linie)',
'Modell(?:bezeichnung|reihe|serie)', 'Baureihe', 'Typen(?:kennung|bezeichnung)',
'(?:Buch|Karten|Werk|Film|Album|Serien|Kapitel)titel', 'gedruckt(?:e|en|er|em) Titel',
'Titel (?:des|der) (?:Buches|Buchs|Romans|Katalogs|Reihe|Sendung|Druckausgabe)',
'Namen? der (?:Lackfarbe|Produktserie|Radiosendung)', 'Farbtonbezeichnung',
'(?:Reparatur|Produkt|Service)leitfaden', '(?:Handbuch|Norm|Richtlinien)(?:nummer|kennung)',
'(?:FAQ|Handbuch|Katalog)[-‐‑]Referenz', 'öffentlich(?:e|en|er|em) (?:Hilfe[-‐‑])?Referenz',
'Referenz (?:im öffentlichen Katalog|der öffentlichen Anleitung)',
'Nummer (?:der technischen Norm|des Standardformulars|der Papierstärke im Katalog)',
'Bezeichnung der (?:Rohrnorm|technischen Norm)',
].join('|').replaceAll(' ', String.raw`[\t \u00a0]+`);
// Accept decomposed German vowel marks while keeping the original input indices.
const role = '(?:' + roles.replace(/[äöü]/gu, c => '(?:' + c + '|' + c.normalize('NFD') + ')') + ')';
const atom = String.raw`[\p{L}\p{N}][\p{L}\p{M}\p{N}]*(?:[./_‐‑-][\p{L}\p{N}][\p{L}\p{M}\p{N}]*)*`;
const value = `(?:„[^„“”\n]{1,96}[“”]|“[^“”\n]{1,96}”|«[^«»\n]{1,96}»|»[^«»\n]{1,96}«|‚[^‚‘\n]{1,96}‘|"[^"\n]{1,96}"|${atom})`;
const h = String.raw`[\t \u00a0]`;
const before = String.raw`(?<![\p{L}\p{M}\p{N}@+/_‐‑-])`;
const after = String.raw`(?![\p{L}\p{M}\p{N}@+/_‐‑-]|\.[\p{L}\p{N}])`;
const prefix = new RegExp(before + role + `(?:${h}*[:=]${h}*|${h}+(?:(?:lautet|heißt|ist)${h}+)?)` +
`(?<value>${value})` + after, 'dgiu');
const article = `(?:(?:der|die|das|ein|eine|einen|den|dem)${h}+)?`;
const suffix = new RegExp(before + `(?<value>${value})` + after +
`(?:${h}+(?:ist|lautet|als)${h}+${article}|${h}*,${h}+(?:der|die|das|ein|eine)${h}+|${h}*\\(${h}*${article})` +
role + `(?=${h}*(?:[,)]|$))`, 'dgiu');
const privateWords = [
'(?:mein|dein|sein|ihr|unser|eur|euer)(?:e|en|er|em|es)?', 'mir', 'mich', 'privat\\p{L}*', 'persönlich\\p{L}*',
'Herr(?:n)?', 'Frau', 'Dr', 'Hallo', 'Moin', 'Liebe(?:r|s)?', 'Sehr geehrte\\p{L}*',
'Guten (?:Tag|Morgen|Abend)', 'Mit freundlichen Grüßen', 'Viele Grüße',
'(?:Vor|Nach|Familien)name\\p{L}*', 'Anrede', 'Empfänger\\p{L}*', 'Ansprechpartner\\p{L}*',
'Mitarbeiter\\p{L}*', 'Inhaber\\p{L}*', 'Besitzer\\p{L}*', 'Familie', 'Tochter', 'Sohn', 'Kontakt(?:person|daten)?', 'Kund(?:e|in|en|innen)', 'Kunden(?:name|nummer|konto|code|kennung|karte|schlüssel|profil)',
'Bestell(?:ung|nummer|code|kennung)', 'Auftragsnummer', 'Vertrags(?:nummer|konto)', 'Aktenzeichen',
'Adresse', 'Anschrift', 'Wohnort', 'Zustellung', 'Personalakte', 'Telefon\\p{L}*', 'E[-‐‑]?Mail', 'IBAN',
'Kontoinhaber\\p{L}*', 'Kontonummer', 'Geburts\\p{L}*', 'Rückruf\\p{L}*', 'Unterschrift',
'nicht', 'kein(?:e|en|er|em|es)?', 'statt', 'angeblich', 'vertraulich',
].join('|');
const privateClause = new RegExp(`(?<!${wordEdge})(?:${privateWords})(?!${wordEdge})`, 'iu');
const filler = /^(?:von|für|mit|an|aus|im|in|am|bei|der|die|das|des|den|dem|und|ist|wird|lautet|heißt|steht|nennt|siehe|zu|zum|zur)$/iu;
const openingQuote = /^[„“«»‚"]/u;
const strongSeparator = /(?:[;|!?…\r\n]+|\.(?=\s|$))/gu;
const bankOrPhone = /^(?:[A-Z]{2}\s?\d{2}(?:\s?[A-Z\d]){10,30}|\+?[\d ()/-]{7,})$/iu;
function intervals(text) {
const result = [];
function scan(clause, origin) {
if (privateClause.test(clause.normalize('NFC'))) return;
for (const pattern of [prefix, suffix]) {
for (const match of clause.matchAll(pattern)) {
const raw = match.groups.value;
const quoted = openingQuote.test(raw);
const literal = quoted ? raw.slice(1, -1) : raw;
if (filler.test(literal) || bankOrPhone.test(literal) || literal.includes('@')) continue;
// Do not treat an unterminated quote or a token inside another quoted field as a fresh value.
const [a, b] = match.indices.groups.value;
const leading = clause.slice(0, a);
const trailing = clause.slice(b);
if (!quoted && (/[„“«»‚"'‘’]$/u.test(leading) || /^[“”«»‘’"']/u.test(trailing))) continue;
// A prefix must terminate an unquoted value. Ambiguous multiword values need quotes.
if (pattern === prefix && !quoted && !/^(?:[\t \u00a0]*(?:$|[,)]|(?:anzeigen|drucken|suchen|nachschlagen|auswählen|aufnehmen|korrigieren|ändern|ersetzen|verwenden|eintragen)\b))/iu.test(trailing)) continue;
const boundStart = origin + a + (quoted ? 1 : 0);
const boundEnd = origin + b - (quoted ? 1 : 0);
const identifier = !quoted && /\p{L}/u.test(raw) && /\p{N}/u.test(raw) && /[-‐‑/_.]/u.test(raw);
result.push({start: boundStart, end: boundEnd, identifier});
}
}
}
let first = 0;
for (const sep of text.matchAll(strongSeparator)) {
scan(text.slice(first, sep.index), first);
first = sep.index + sep[0].length;
}
scan(text.slice(first), first);
return result;
}
export function applyPolicy(text, groups, policy = null) {
const bounds = [0];
for (const cp of text) bounds.push(bounds[bounds.length - 1] + cp.length);
const licensed = intervals(text);
const retainedByLearning = policy === null ? null : new Set(applyLearned(text, groups, policy));
return groups.filter(group => {
if (!Number.isInteger(group.start) || !Number.isInteger(group.end) || group.start < 0 ||
group.end <= group.start || group.end >= bounds.length) throw new RangeError('invalid code-point span');
if (!EDITABLE.has(group.label) || retainedByLearning?.has(group)) return true;
const start = bounds[group.start], end = bounds[group.end];
return !licensed.some(span => (span.start === start && span.end === end) ||
(span.identifier && span.start <= start && end <= span.end));
});
}
|