// The worker passes decoded v2 groups with code-point offsets and applies its existing confidence slider. const WORD_CHAR = /^[A-Za-zÀ-ÖØ-öø-ÿĀ-žẞ]$/; function nearestWords(points, position, step) { const words = []; while (position >= 0 && position < points.length && words.length < 6) { if (!WORD_CHAR.test(points[position])) { position += step; continue; } const letters = []; while (position >= 0 && position < points.length && WORD_CHAR.test(points[position])) { letters.push(points[position]); position += step; } words.push((step < 0 ? letters.reverse() : letters).join('')); } return words; } function fold(word) { return Array.from(word, c => { if ((c >= 'A' && c <= 'Z') || (c >= 'À' && c <= 'Ö') || (c >= 'Ø' && c <= 'Þ')) { return String.fromCodePoint(c.codePointAt(0) + 32); } return c === 'ẞ' ? 'ß' : c; }).join(''); } function context(points, group) { const {start, end, label} = group; if (!Number.isInteger(start) || !Number.isInteger(end) || start < 0 || start >= end || end > points.length) { throw new RangeError('invalid code-point span'); } const output = new Set(['tag:' + label]); const left = nearestWords(points, start - 1, -1); const right = nearestWords(points, end, 1); for (const [side, words] of [['L', left], ['R', right]]) { words.forEach((raw, i) => { const w = fold(raw); output.add(side + ':w:' + w); output.add(side + ':p:' + w.slice(0, 3)); output.add(side + ':s:' + w.slice(-3)); if (i === 0) output.add(side + ':n:' + w); }); } return [...output].sort(); } function contextFeatures(text, group) { return context(Array.from(text), group); } function applyLearned(text, groups, policy) { if (policy.semantics.version !== 'context-latin-v1') throw new Error('unsupported policy version'); const points = Array.from(text); return groups.filter(group => { if (!policy.editable.includes(group.label) || policy.cutoff === 0) return true; let logit = policy.bias; for (const f of context(points, group)) logit += policy.weights[f] ?? 0; return 1 / (1 + Math.exp(-logit)) >= policy.cutoff; }); } // Explicit public roles. Prefixes and suffixes bind one value, never a name elsewhere in the clause. const EDITABLE = new Set(['PERSON', 'ADDRESS', 'ORDER_ID', 'CUSTOMER_ID', 'ACCOUNT_ID', 'ORGANIZATION']); const wordEdge = String.raw`[\p{L}\p{M}\p{N}_]`; const roles = [ 'Katalog(?:artikel|nummer|kennung|referenz|titel|serie|eintrag)', 'Artikel(?:nummer|kennung|bezeichnung|titel)', 'Produkt(?:name|nummer|bezeichnung|serie|linie)', 'Modell(?:bezeichnung|reihe|serie)', 'Baureihe', 'Typen(?:kennung|bezeichnung)', '(?:Buch|Karten|Werk|Film|Album|Serien|Kapitel)titel', 'gedruckt(?:e|en|er|em) Titel', 'Titel (?:des|der) (?:Buches|Buchs|Romans|Katalogs|Reihe|Sendung|Druckausgabe)', 'Namen? der (?:Lackfarbe|Produktserie|Radiosendung)', 'Farbtonbezeichnung', '(?:Reparatur|Produkt|Service)leitfaden', '(?:Handbuch|Norm|Richtlinien)(?:nummer|kennung)', '(?:FAQ|Handbuch|Katalog)[-‐‑]Referenz', 'öffentlich(?:e|en|er|em) (?:Hilfe[-‐‑])?Referenz', 'Referenz (?:im öffentlichen Katalog|der öffentlichen Anleitung)', 'Nummer (?:der technischen Norm|des Standardformulars|der Papierstärke im Katalog)', 'Bezeichnung der (?:Rohrnorm|technischen Norm)', ].join('|').replaceAll(' ', String.raw`[\t \u00a0]+`); // Accept decomposed German vowel marks while keeping the original input indices. const role = '(?:' + roles.replace(/[äöü]/gu, c => '(?:' + c + '|' + c.normalize('NFD') + ')') + ')'; const atom = String.raw`[\p{L}\p{N}][\p{L}\p{M}\p{N}]*(?:[./_‐‑-][\p{L}\p{N}][\p{L}\p{M}\p{N}]*)*`; const value = `(?:„[^„“”\n]{1,96}[“”]|“[^“”\n]{1,96}”|«[^«»\n]{1,96}»|»[^«»\n]{1,96}«|‚[^‚‘\n]{1,96}‘|"[^"\n]{1,96}"|${atom})`; const h = String.raw`[\t \u00a0]`; const before = String.raw`(?${value})` + after, 'dgiu'); const article = `(?:(?:der|die|das|ein|eine|einen|den|dem)${h}+)?`; const suffix = new RegExp(before + `(?${value})` + after + `(?:${h}+(?:ist|lautet|als)${h}+${article}|${h}*,${h}+(?:der|die|das|ein|eine)${h}+|${h}*\\(${h}*${article})` + role + `(?=${h}*(?:[,)]|$))`, 'dgiu'); const privateWords = [ '(?:mein|dein|sein|ihr|unser|eur|euer)(?:e|en|er|em|es)?', 'mir', 'mich', 'privat\\p{L}*', 'persönlich\\p{L}*', 'Herr(?:n)?', 'Frau', 'Dr', 'Hallo', 'Moin', 'Liebe(?:r|s)?', 'Sehr geehrte\\p{L}*', 'Guten (?:Tag|Morgen|Abend)', 'Mit freundlichen Grüßen', 'Viele Grüße', '(?:Vor|Nach|Familien)name\\p{L}*', 'Anrede', 'Empfänger\\p{L}*', 'Ansprechpartner\\p{L}*', 'Mitarbeiter\\p{L}*', 'Inhaber\\p{L}*', 'Besitzer\\p{L}*', 'Familie', 'Tochter', 'Sohn', 'Kontakt(?:person|daten)?', 'Kund(?:e|in|en|innen)', 'Kunden(?:name|nummer|konto|code|kennung|karte|schlüssel|profil)', 'Bestell(?:ung|nummer|code|kennung)', 'Auftragsnummer', 'Vertrags(?:nummer|konto)', 'Aktenzeichen', 'Adresse', 'Anschrift', 'Wohnort', 'Zustellung', 'Personalakte', 'Telefon\\p{L}*', 'E[-‐‑]?Mail', 'IBAN', 'Kontoinhaber\\p{L}*', 'Kontonummer', 'Geburts\\p{L}*', 'Rückruf\\p{L}*', 'Unterschrift', 'nicht', 'kein(?:e|en|er|em|es)?', 'statt', 'angeblich', 'vertraulich', ].join('|'); const privateClause = new RegExp(`(? { if (!Number.isInteger(group.start) || !Number.isInteger(group.end) || group.start < 0 || group.end <= group.start || group.end >= bounds.length) throw new RangeError('invalid code-point span'); if (!EDITABLE.has(group.label) || retainedByLearning?.has(group)) return true; const start = bounds[group.start], end = bounds[group.end]; return !licensed.some(span => (span.start === start && span.end === end) || (span.identifier && span.start <= start && end <= span.end)); }); }