Download explicit-context.mjs from EasyOctopus/german-support-redactor: direct link, hf CLI and curl.
- Browser
- Download file 8.45 kB
-
https://huggingface.co/spaces/EasyOctopus/german-support-redactor/resolve/main/explicit-context.mjs
- Command line
-
hf download hf://spaces/EasyOctopus/german-support-redactor/explicit-context.mjs
-
curl -L -o explicit-context.mjs https://huggingface.co/spaces/EasyOctopus/german-support-redactor/resolve/main/explicit-context.mjs
8.45 kB
| // The worker passes decoded v2 groups with code-point offsets and applies its existing confidence slider. | |
| const WORD_CHAR = /^[A-Za-zÀ-ÖØ-öø-ÿĀ-žẞ]$/; | |
| function nearestWords(points, position, step) { | |
| const words = []; | |
| while (position >= 0 && position < points.length && words.length < 6) { | |
| if (!WORD_CHAR.test(points[position])) { | |
| position += step; | |
| continue; | |
| } | |
| const letters = []; | |
| while (position >= 0 && position < points.length && WORD_CHAR.test(points[position])) { | |
| letters.push(points[position]); | |
| position += step; | |
| } | |
| words.push((step < 0 ? letters.reverse() : letters).join('')); | |
| } | |
| return words; | |
| } | |
| function fold(word) { | |
| return Array.from(word, c => { | |
| if ((c >= 'A' && c <= 'Z') || (c >= 'À' && c <= 'Ö') || (c >= 'Ø' && c <= 'Þ')) { | |
| return String.fromCodePoint(c.codePointAt(0) + 32); | |
| } | |
| return c === 'ẞ' ? 'ß' : c; | |
| }).join(''); | |
| } | |
| function context(points, group) { | |
| const {start, end, label} = group; | |
| if (!Number.isInteger(start) || !Number.isInteger(end) || start < 0 || start >= end || end > points.length) { | |
| throw new RangeError('invalid code-point span'); | |
| } | |
| const output = new Set(['tag:' + label]); | |
| const left = nearestWords(points, start - 1, -1); | |
| const right = nearestWords(points, end, 1); | |
| for (const [side, words] of [['L', left], ['R', right]]) { | |
| words.forEach((raw, i) => { | |
| const w = fold(raw); | |
| output.add(side + ':w:' + w); | |
| output.add(side + ':p:' + w.slice(0, 3)); | |
| output.add(side + ':s:' + w.slice(-3)); | |
| if (i === 0) output.add(side + ':n:' + w); | |
| }); | |
| } | |
| return [...output].sort(); | |
| } | |
| function contextFeatures(text, group) { | |
| return context(Array.from(text), group); | |
| } | |
| function applyLearned(text, groups, policy) { | |
| if (policy.semantics.version !== 'context-latin-v1') throw new Error('unsupported policy version'); | |
| const points = Array.from(text); | |
| return groups.filter(group => { | |
| if (!policy.editable.includes(group.label) || policy.cutoff === 0) return true; | |
| let logit = policy.bias; | |
| for (const f of context(points, group)) logit += policy.weights[f] ?? 0; | |
| return 1 / (1 + Math.exp(-logit)) >= policy.cutoff; | |
| }); | |
| } | |
| // Explicit public roles. Prefixes and suffixes bind one value, never a name elsewhere in the clause. | |
| const EDITABLE = new Set(['PERSON', 'ADDRESS', 'ORDER_ID', 'CUSTOMER_ID', 'ACCOUNT_ID', 'ORGANIZATION']); | |
| const wordEdge = String.raw`[\p{L}\p{M}\p{N}_]`; | |
| const roles = [ | |
| 'Katalog(?:artikel|nummer|kennung|referenz|titel|serie|eintrag)', | |
| 'Artikel(?:nummer|kennung|bezeichnung|titel)', 'Produkt(?:name|nummer|bezeichnung|serie|linie)', | |
| 'Modell(?:bezeichnung|reihe|serie)', 'Baureihe', 'Typen(?:kennung|bezeichnung)', | |
| '(?:Buch|Karten|Werk|Film|Album|Serien|Kapitel)titel', 'gedruckt(?:e|en|er|em) Titel', | |
| 'Titel (?:des|der) (?:Buches|Buchs|Romans|Katalogs|Reihe|Sendung|Druckausgabe)', | |
| 'Namen? der (?:Lackfarbe|Produktserie|Radiosendung)', 'Farbtonbezeichnung', | |
| '(?:Reparatur|Produkt|Service)leitfaden', '(?:Handbuch|Norm|Richtlinien)(?:nummer|kennung)', | |
| '(?:FAQ|Handbuch|Katalog)[-‐‑]Referenz', 'öffentlich(?:e|en|er|em) (?:Hilfe[-‐‑])?Referenz', | |
| 'Referenz (?:im öffentlichen Katalog|der öffentlichen Anleitung)', | |
| 'Nummer (?:der technischen Norm|des Standardformulars|der Papierstärke im Katalog)', | |
| 'Bezeichnung der (?:Rohrnorm|technischen Norm)', | |
| ].join('|').replaceAll(' ', String.raw`[\t \u00a0]+`); | |
| // Accept decomposed German vowel marks while keeping the original input indices. | |
| const role = '(?:' + roles.replace(/[äöü]/gu, c => '(?:' + c + '|' + c.normalize('NFD') + ')') + ')'; | |
| const atom = String.raw`[\p{L}\p{N}][\p{L}\p{M}\p{N}]*(?:[./_‐‑-][\p{L}\p{N}][\p{L}\p{M}\p{N}]*)*`; | |
| const value = `(?:„[^„“”\n]{1,96}[“”]|“[^“”\n]{1,96}”|«[^«»\n]{1,96}»|»[^«»\n]{1,96}«|‚[^‚‘\n]{1,96}‘|"[^"\n]{1,96}"|${atom})`; | |
| const h = String.raw`[\t \u00a0]`; | |
| const before = String.raw`(?<![\p{L}\p{M}\p{N}@+/_‐‑-])`; | |
| const after = String.raw`(?![\p{L}\p{M}\p{N}@+/_‐‑-]|\.[\p{L}\p{N}])`; | |
| const prefix = new RegExp(before + role + `(?:${h}*[:=]${h}*|${h}+(?:(?:lautet|heißt|ist)${h}+)?)` + | |
| `(?<value>${value})` + after, 'dgiu'); | |
| const article = `(?:(?:der|die|das|ein|eine|einen|den|dem)${h}+)?`; | |
| const suffix = new RegExp(before + `(?<value>${value})` + after + | |
| `(?:${h}+(?:ist|lautet|als)${h}+${article}|${h}*,${h}+(?:der|die|das|ein|eine)${h}+|${h}*\\(${h}*${article})` + | |
| role + `(?=${h}*(?:[,)]|$))`, 'dgiu'); | |
| const privateWords = [ | |
| '(?:mein|dein|sein|ihr|unser|eur|euer)(?:e|en|er|em|es)?', 'mir', 'mich', 'privat\\p{L}*', 'persönlich\\p{L}*', | |
| 'Herr(?:n)?', 'Frau', 'Dr', 'Hallo', 'Moin', 'Liebe(?:r|s)?', 'Sehr geehrte\\p{L}*', | |
| 'Guten (?:Tag|Morgen|Abend)', 'Mit freundlichen Grüßen', 'Viele Grüße', | |
| '(?:Vor|Nach|Familien)name\\p{L}*', 'Anrede', 'Empfänger\\p{L}*', 'Ansprechpartner\\p{L}*', | |
| 'Mitarbeiter\\p{L}*', 'Inhaber\\p{L}*', 'Besitzer\\p{L}*', 'Familie', 'Tochter', 'Sohn', 'Kontakt(?:person|daten)?', 'Kund(?:e|in|en|innen)', 'Kunden(?:name|nummer|konto|code|kennung|karte|schlüssel|profil)', | |
| 'Bestell(?:ung|nummer|code|kennung)', 'Auftragsnummer', 'Vertrags(?:nummer|konto)', 'Aktenzeichen', | |
| 'Adresse', 'Anschrift', 'Wohnort', 'Zustellung', 'Personalakte', 'Telefon\\p{L}*', 'E[-‐‑]?Mail', 'IBAN', | |
| 'Kontoinhaber\\p{L}*', 'Kontonummer', 'Geburts\\p{L}*', 'Rückruf\\p{L}*', 'Unterschrift', | |
| 'nicht', 'kein(?:e|en|er|em|es)?', 'statt', 'angeblich', 'vertraulich', | |
| ].join('|'); | |
| const privateClause = new RegExp(`(?<!${wordEdge})(?:${privateWords})(?!${wordEdge})`, 'iu'); | |
| const filler = /^(?:von|für|mit|an|aus|im|in|am|bei|der|die|das|des|den|dem|und|ist|wird|lautet|heißt|steht|nennt|siehe|zu|zum|zur)$/iu; | |
| const openingQuote = /^[„“«»‚"]/u; | |
| const strongSeparator = /(?:[;|!?…\r\n]+|\.(?=\s|$))/gu; | |
| const bankOrPhone = /^(?:[A-Z]{2}\s?\d{2}(?:\s?[A-Z\d]){10,30}|\+?[\d ()/-]{7,})$/iu; | |
| function intervals(text) { | |
| const result = []; | |
| function scan(clause, origin) { | |
| if (privateClause.test(clause.normalize('NFC'))) return; | |
| for (const pattern of [prefix, suffix]) { | |
| for (const match of clause.matchAll(pattern)) { | |
| const raw = match.groups.value; | |
| const quoted = openingQuote.test(raw); | |
| const literal = quoted ? raw.slice(1, -1) : raw; | |
| if (filler.test(literal) || bankOrPhone.test(literal) || literal.includes('@')) continue; | |
| // Do not treat an unterminated quote or a token inside another quoted field as a fresh value. | |
| const [a, b] = match.indices.groups.value; | |
| const leading = clause.slice(0, a); | |
| const trailing = clause.slice(b); | |
| if (!quoted && (/[„“«»‚"'‘’]$/u.test(leading) || /^[“”«»‘’"']/u.test(trailing))) continue; | |
| // A prefix must terminate an unquoted value. Ambiguous multiword values need quotes. | |
| if (pattern === prefix && !quoted && !/^(?:[\t \u00a0]*(?:$|[,)]|(?:anzeigen|drucken|suchen|nachschlagen|auswählen|aufnehmen|korrigieren|ändern|ersetzen|verwenden|eintragen)\b))/iu.test(trailing)) continue; | |
| const boundStart = origin + a + (quoted ? 1 : 0); | |
| const boundEnd = origin + b - (quoted ? 1 : 0); | |
| const identifier = !quoted && /\p{L}/u.test(raw) && /\p{N}/u.test(raw) && /[-‐‑/_.]/u.test(raw); | |
| result.push({start: boundStart, end: boundEnd, identifier}); | |
| } | |
| } | |
| } | |
| let first = 0; | |
| for (const sep of text.matchAll(strongSeparator)) { | |
| scan(text.slice(first, sep.index), first); | |
| first = sep.index + sep[0].length; | |
| } | |
| scan(text.slice(first), first); | |
| return result; | |
| } | |
| export function applyPolicy(text, groups, policy = null) { | |
| const bounds = [0]; | |
| for (const cp of text) bounds.push(bounds[bounds.length - 1] + cp.length); | |
| const licensed = intervals(text); | |
| const retainedByLearning = policy === null ? null : new Set(applyLearned(text, groups, policy)); | |
| return groups.filter(group => { | |
| if (!Number.isInteger(group.start) || !Number.isInteger(group.end) || group.start < 0 || | |
| group.end <= group.start || group.end >= bounds.length) throw new RangeError('invalid code-point span'); | |
| if (!EDITABLE.has(group.label) || retainedByLearning?.has(group)) return true; | |
| const start = bounds[group.start], end = bounds[group.end]; | |
| return !licensed.some(span => (span.start === start && span.end === end) || | |
| (span.identifier && span.start <= start && end <= span.end)); | |
| }); | |
| } | |