german-support-redactor / explicit-context.mjs
EasyOctopus's picture
Retain public item names under explicit German role descriptions
dc5df19 verified
Raw History Blame Contribute Delete
8.45 kB
// The worker passes decoded v2 groups with code-point offsets and applies its existing confidence slider.
const WORD_CHAR = /^[A-Za-zÀ-ÖØ-öø-ÿĀ-žẞ]$/;
function nearestWords(points, position, step) {
const words = [];
while (position >= 0 && position < points.length && words.length < 6) {
if (!WORD_CHAR.test(points[position])) {
position += step;
continue;
}
const letters = [];
while (position >= 0 && position < points.length && WORD_CHAR.test(points[position])) {
letters.push(points[position]);
position += step;
}
words.push((step < 0 ? letters.reverse() : letters).join(''));
}
return words;
}
function fold(word) {
return Array.from(word, c => {
if ((c >= 'A' && c <= 'Z') || (c >= 'À' && c <= 'Ö') || (c >= 'Ø' && c <= 'Þ')) {
return String.fromCodePoint(c.codePointAt(0) + 32);
}
return c === 'ẞ' ? 'ß' : c;
}).join('');
}
function context(points, group) {
const {start, end, label} = group;
if (!Number.isInteger(start) || !Number.isInteger(end) || start < 0 || start >= end || end > points.length) {
throw new RangeError('invalid code-point span');
}
const output = new Set(['tag:' + label]);
const left = nearestWords(points, start - 1, -1);
const right = nearestWords(points, end, 1);
for (const [side, words] of [['L', left], ['R', right]]) {
words.forEach((raw, i) => {
const w = fold(raw);
output.add(side + ':w:' + w);
output.add(side + ':p:' + w.slice(0, 3));
output.add(side + ':s:' + w.slice(-3));
if (i === 0) output.add(side + ':n:' + w);
});
}
return [...output].sort();
}
function contextFeatures(text, group) {
return context(Array.from(text), group);
}
function applyLearned(text, groups, policy) {
if (policy.semantics.version !== 'context-latin-v1') throw new Error('unsupported policy version');
const points = Array.from(text);
return groups.filter(group => {
if (!policy.editable.includes(group.label) || policy.cutoff === 0) return true;
let logit = policy.bias;
for (const f of context(points, group)) logit += policy.weights[f] ?? 0;
return 1 / (1 + Math.exp(-logit)) >= policy.cutoff;
});
}
// Explicit public roles. Prefixes and suffixes bind one value, never a name elsewhere in the clause.
const EDITABLE = new Set(['PERSON', 'ADDRESS', 'ORDER_ID', 'CUSTOMER_ID', 'ACCOUNT_ID', 'ORGANIZATION']);
const wordEdge = String.raw`[\p{L}\p{M}\p{N}_]`;
const roles = [
'Katalog(?:artikel|nummer|kennung|referenz|titel|serie|eintrag)',
'Artikel(?:nummer|kennung|bezeichnung|titel)', 'Produkt(?:name|nummer|bezeichnung|serie|linie)',
'Modell(?:bezeichnung|reihe|serie)', 'Baureihe', 'Typen(?:kennung|bezeichnung)',
'(?:Buch|Karten|Werk|Film|Album|Serien|Kapitel)titel', 'gedruckt(?:e|en|er|em) Titel',
'Titel (?:des|der) (?:Buches|Buchs|Romans|Katalogs|Reihe|Sendung|Druckausgabe)',
'Namen? der (?:Lackfarbe|Produktserie|Radiosendung)', 'Farbtonbezeichnung',
'(?:Reparatur|Produkt|Service)leitfaden', '(?:Handbuch|Norm|Richtlinien)(?:nummer|kennung)',
'(?:FAQ|Handbuch|Katalog)[-‐‑]Referenz', 'öffentlich(?:e|en|er|em) (?:Hilfe[-‐‑])?Referenz',
'Referenz (?:im öffentlichen Katalog|der öffentlichen Anleitung)',
'Nummer (?:der technischen Norm|des Standardformulars|der Papierstärke im Katalog)',
'Bezeichnung der (?:Rohrnorm|technischen Norm)',
].join('|').replaceAll(' ', String.raw`[\t \u00a0]+`);
// Accept decomposed German vowel marks while keeping the original input indices.
const role = '(?:' + roles.replace(/[äöü]/gu, c => '(?:' + c + '|' + c.normalize('NFD') + ')') + ')';
const atom = String.raw`[\p{L}\p{N}][\p{L}\p{M}\p{N}]*(?:[./_‐‑-][\p{L}\p{N}][\p{L}\p{M}\p{N}]*)*`;
const value = `(?:„[^„“”\n]{1,96}[“”]|“[^“”\n]{1,96}”|«[^«»\n]{1,96}»|»[^«»\n]{1,96}«|‚[^‚‘\n]{1,96}‘|"[^"\n]{1,96}"|${atom})`;
const h = String.raw`[\t \u00a0]`;
const before = String.raw`(?<![\p{L}\p{M}\p{N}@+/_‐‑-])`;
const after = String.raw`(?![\p{L}\p{M}\p{N}@+/_‐‑-]|\.[\p{L}\p{N}])`;
const prefix = new RegExp(before + role + `(?:${h}*[:=]${h}*|${h}+(?:(?:lautet|heißt|ist)${h}+)?)` +
`(?<value>${value})` + after, 'dgiu');
const article = `(?:(?:der|die|das|ein|eine|einen|den|dem)${h}+)?`;
const suffix = new RegExp(before + `(?<value>${value})` + after +
`(?:${h}+(?:ist|lautet|als)${h}+${article}|${h}*,${h}+(?:der|die|das|ein|eine)${h}+|${h}*\\(${h}*${article})` +
role + `(?=${h}*(?:[,)]|$))`, 'dgiu');
const privateWords = [
'(?:mein|dein|sein|ihr|unser|eur|euer)(?:e|en|er|em|es)?', 'mir', 'mich', 'privat\\p{L}*', 'persönlich\\p{L}*',
'Herr(?:n)?', 'Frau', 'Dr', 'Hallo', 'Moin', 'Liebe(?:r|s)?', 'Sehr geehrte\\p{L}*',
'Guten (?:Tag|Morgen|Abend)', 'Mit freundlichen Grüßen', 'Viele Grüße',
'(?:Vor|Nach|Familien)name\\p{L}*', 'Anrede', 'Empfänger\\p{L}*', 'Ansprechpartner\\p{L}*',
'Mitarbeiter\\p{L}*', 'Inhaber\\p{L}*', 'Besitzer\\p{L}*', 'Familie', 'Tochter', 'Sohn', 'Kontakt(?:person|daten)?', 'Kund(?:e|in|en|innen)', 'Kunden(?:name|nummer|konto|code|kennung|karte|schlüssel|profil)',
'Bestell(?:ung|nummer|code|kennung)', 'Auftragsnummer', 'Vertrags(?:nummer|konto)', 'Aktenzeichen',
'Adresse', 'Anschrift', 'Wohnort', 'Zustellung', 'Personalakte', 'Telefon\\p{L}*', 'E[-‐‑]?Mail', 'IBAN',
'Kontoinhaber\\p{L}*', 'Kontonummer', 'Geburts\\p{L}*', 'Rückruf\\p{L}*', 'Unterschrift',
'nicht', 'kein(?:e|en|er|em|es)?', 'statt', 'angeblich', 'vertraulich',
].join('|');
const privateClause = new RegExp(`(?<!${wordEdge})(?:${privateWords})(?!${wordEdge})`, 'iu');
const filler = /^(?:von|für|mit|an|aus|im|in|am|bei|der|die|das|des|den|dem|und|ist|wird|lautet|heißt|steht|nennt|siehe|zu|zum|zur)$/iu;
const openingQuote = /^[„“«»‚"]/u;
const strongSeparator = /(?:[;|!?…\r\n]+|\.(?=\s|$))/gu;
const bankOrPhone = /^(?:[A-Z]{2}\s?\d{2}(?:\s?[A-Z\d]){10,30}|\+?[\d ()/-]{7,})$/iu;
function intervals(text) {
const result = [];
function scan(clause, origin) {
if (privateClause.test(clause.normalize('NFC'))) return;
for (const pattern of [prefix, suffix]) {
for (const match of clause.matchAll(pattern)) {
const raw = match.groups.value;
const quoted = openingQuote.test(raw);
const literal = quoted ? raw.slice(1, -1) : raw;
if (filler.test(literal) || bankOrPhone.test(literal) || literal.includes('@')) continue;
// Do not treat an unterminated quote or a token inside another quoted field as a fresh value.
const [a, b] = match.indices.groups.value;
const leading = clause.slice(0, a);
const trailing = clause.slice(b);
if (!quoted && (/[„“«»‚"'‘’]$/u.test(leading) || /^[“”«»‘’"']/u.test(trailing))) continue;
// A prefix must terminate an unquoted value. Ambiguous multiword values need quotes.
if (pattern === prefix && !quoted && !/^(?:[\t \u00a0]*(?:$|[,)]|(?:anzeigen|drucken|suchen|nachschlagen|auswählen|aufnehmen|korrigieren|ändern|ersetzen|verwenden|eintragen)\b))/iu.test(trailing)) continue;
const boundStart = origin + a + (quoted ? 1 : 0);
const boundEnd = origin + b - (quoted ? 1 : 0);
const identifier = !quoted && /\p{L}/u.test(raw) && /\p{N}/u.test(raw) && /[-‐‑/_.]/u.test(raw);
result.push({start: boundStart, end: boundEnd, identifier});
}
}
}
let first = 0;
for (const sep of text.matchAll(strongSeparator)) {
scan(text.slice(first, sep.index), first);
first = sep.index + sep[0].length;
}
scan(text.slice(first), first);
return result;
}
export function applyPolicy(text, groups, policy = null) {
const bounds = [0];
for (const cp of text) bounds.push(bounds[bounds.length - 1] + cp.length);
const licensed = intervals(text);
const retainedByLearning = policy === null ? null : new Set(applyLearned(text, groups, policy));
return groups.filter(group => {
if (!Number.isInteger(group.start) || !Number.isInteger(group.end) || group.start < 0 ||
group.end <= group.start || group.end >= bounds.length) throw new RangeError('invalid code-point span');
if (!EDITABLE.has(group.label) || retainedByLearning?.has(group)) return true;
const start = bounds[group.start], end = bounds[group.end];
return !licensed.some(span => (span.start === start && span.end === end) ||
(span.identifier && span.start <= start && end <= span.end));
});
}