File size: 8,449 Bytes
dc5df19
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
// The worker passes decoded v2 groups with code-point offsets and applies its existing confidence slider.
const WORD_CHAR = /^[A-Za-zÀ-ÖØ-öø-ÿĀ-žẞ]$/;
function nearestWords(points, position, step) {
  const words = [];
  while (position >= 0 && position < points.length && words.length < 6) {
    if (!WORD_CHAR.test(points[position])) {
      position += step;
      continue;
    }
    const letters = [];
    while (position >= 0 && position < points.length && WORD_CHAR.test(points[position])) {
      letters.push(points[position]);
      position += step;
    }
    words.push((step < 0 ? letters.reverse() : letters).join(''));
  }
  return words;
}
function fold(word) {
  return Array.from(word, c => {
    if ((c >= 'A' && c <= 'Z') || (c >= 'À' && c <= 'Ö') || (c >= 'Ø' && c <= 'Þ')) {
      return String.fromCodePoint(c.codePointAt(0) + 32);
    }
    return c === 'ẞ' ? 'ß' : c;
  }).join('');
}
function context(points, group) {
  const {start, end, label} = group;
  if (!Number.isInteger(start) || !Number.isInteger(end) || start < 0 || start >= end || end > points.length) {
    throw new RangeError('invalid code-point span');
  }
  const output = new Set(['tag:' + label]);
  const left = nearestWords(points, start - 1, -1);
  const right = nearestWords(points, end, 1);
  for (const [side, words] of [['L', left], ['R', right]]) {
    words.forEach((raw, i) => {
      const w = fold(raw);
      output.add(side + ':w:' + w);
      output.add(side + ':p:' + w.slice(0, 3));
      output.add(side + ':s:' + w.slice(-3));
      if (i === 0) output.add(side + ':n:' + w);
    });
  }
  return [...output].sort();
}
function contextFeatures(text, group) {
  return context(Array.from(text), group);
}
function applyLearned(text, groups, policy) {
  if (policy.semantics.version !== 'context-latin-v1') throw new Error('unsupported policy version');
  const points = Array.from(text);
  return groups.filter(group => {
    if (!policy.editable.includes(group.label) || policy.cutoff === 0) return true;
    let logit = policy.bias;
    for (const f of context(points, group)) logit += policy.weights[f] ?? 0;
    return 1 / (1 + Math.exp(-logit)) >= policy.cutoff;
  });
}
// Explicit public roles. Prefixes and suffixes bind one value, never a name elsewhere in the clause.
const EDITABLE = new Set(['PERSON', 'ADDRESS', 'ORDER_ID', 'CUSTOMER_ID', 'ACCOUNT_ID', 'ORGANIZATION']);
const wordEdge = String.raw`[\p{L}\p{M}\p{N}_]`;
const roles = [
  'Katalog(?:artikel|nummer|kennung|referenz|titel|serie|eintrag)',
  'Artikel(?:nummer|kennung|bezeichnung|titel)', 'Produkt(?:name|nummer|bezeichnung|serie|linie)',
  'Modell(?:bezeichnung|reihe|serie)', 'Baureihe', 'Typen(?:kennung|bezeichnung)',
  '(?:Buch|Karten|Werk|Film|Album|Serien|Kapitel)titel', 'gedruckt(?:e|en|er|em) Titel',
  'Titel (?:des|der) (?:Buches|Buchs|Romans|Katalogs|Reihe|Sendung|Druckausgabe)',
  'Namen? der (?:Lackfarbe|Produktserie|Radiosendung)', 'Farbtonbezeichnung',
  '(?:Reparatur|Produkt|Service)leitfaden', '(?:Handbuch|Norm|Richtlinien)(?:nummer|kennung)',
  '(?:FAQ|Handbuch|Katalog)[-‐‑]Referenz', 'öffentlich(?:e|en|er|em) (?:Hilfe[-‐‑])?Referenz',
  'Referenz (?:im öffentlichen Katalog|der öffentlichen Anleitung)',
  'Nummer (?:der technischen Norm|des Standardformulars|der Papierstärke im Katalog)',
  'Bezeichnung der (?:Rohrnorm|technischen Norm)',
].join('|').replaceAll(' ', String.raw`[\t \u00a0]+`);
// Accept decomposed German vowel marks while keeping the original input indices.
const role = '(?:' + roles.replace(/[äöü]/gu, c => '(?:' + c + '|' + c.normalize('NFD') + ')') + ')';
const atom = String.raw`[\p{L}\p{N}][\p{L}\p{M}\p{N}]*(?:[./_‐‑-][\p{L}\p{N}][\p{L}\p{M}\p{N}]*)*`;
const value = `(?:„[^„“”\n]{1,96}[“”]|“[^“”\n]{1,96}”|«[^«»\n]{1,96}»|»[^«»\n]{1,96}«|‚[^‚‘\n]{1,96}‘|"[^"\n]{1,96}"|${atom})`;
const h = String.raw`[\t \u00a0]`;
const before = String.raw`(?<![\p{L}\p{M}\p{N}@+/_‐‑-])`;
const after = String.raw`(?![\p{L}\p{M}\p{N}@+/_‐‑-]|\.[\p{L}\p{N}])`;
const prefix = new RegExp(before + role + `(?:${h}*[:=]${h}*|${h}+(?:(?:lautet|heißt|ist)${h}+)?)` +
  `(?<value>${value})` + after, 'dgiu');
const article = `(?:(?:der|die|das|ein|eine|einen|den|dem)${h}+)?`;
const suffix = new RegExp(before + `(?<value>${value})` + after +
  `(?:${h}+(?:ist|lautet|als)${h}+${article}|${h}*,${h}+(?:der|die|das|ein|eine)${h}+|${h}*\\(${h}*${article})` +
  role + `(?=${h}*(?:[,)]|$))`, 'dgiu');
const privateWords = [
  '(?:mein|dein|sein|ihr|unser|eur|euer)(?:e|en|er|em|es)?', 'mir', 'mich', 'privat\\p{L}*', 'persönlich\\p{L}*',
  'Herr(?:n)?', 'Frau', 'Dr', 'Hallo', 'Moin', 'Liebe(?:r|s)?', 'Sehr geehrte\\p{L}*',
  'Guten (?:Tag|Morgen|Abend)', 'Mit freundlichen Grüßen', 'Viele Grüße',
  '(?:Vor|Nach|Familien)name\\p{L}*', 'Anrede', 'Empfänger\\p{L}*', 'Ansprechpartner\\p{L}*',
  'Mitarbeiter\\p{L}*', 'Inhaber\\p{L}*', 'Besitzer\\p{L}*', 'Familie', 'Tochter', 'Sohn', 'Kontakt(?:person|daten)?', 'Kund(?:e|in|en|innen)', 'Kunden(?:name|nummer|konto|code|kennung|karte|schlüssel|profil)',
  'Bestell(?:ung|nummer|code|kennung)', 'Auftragsnummer', 'Vertrags(?:nummer|konto)', 'Aktenzeichen',
  'Adresse', 'Anschrift', 'Wohnort', 'Zustellung', 'Personalakte', 'Telefon\\p{L}*', 'E[-‐‑]?Mail', 'IBAN',
  'Kontoinhaber\\p{L}*', 'Kontonummer', 'Geburts\\p{L}*', 'Rückruf\\p{L}*', 'Unterschrift',
  'nicht', 'kein(?:e|en|er|em|es)?', 'statt', 'angeblich', 'vertraulich',
].join('|');
const privateClause = new RegExp(`(?<!${wordEdge})(?:${privateWords})(?!${wordEdge})`, 'iu');
const filler = /^(?:von|für|mit|an|aus|im|in|am|bei|der|die|das|des|den|dem|und|ist|wird|lautet|heißt|steht|nennt|siehe|zu|zum|zur)$/iu;
const openingQuote = /^[„“«»‚"]/u;
const strongSeparator = /(?:[;|!?…\r\n]+|\.(?=\s|$))/gu;
const bankOrPhone = /^(?:[A-Z]{2}\s?\d{2}(?:\s?[A-Z\d]){10,30}|\+?[\d ()/-]{7,})$/iu;

function intervals(text) {
  const result = [];
  function scan(clause, origin) {
    if (privateClause.test(clause.normalize('NFC'))) return;
    for (const pattern of [prefix, suffix]) {
      for (const match of clause.matchAll(pattern)) {
        const raw = match.groups.value;
        const quoted = openingQuote.test(raw);
        const literal = quoted ? raw.slice(1, -1) : raw;
        if (filler.test(literal) || bankOrPhone.test(literal) || literal.includes('@')) continue;
        // Do not treat an unterminated quote or a token inside another quoted field as a fresh value.
        const [a, b] = match.indices.groups.value;
        const leading = clause.slice(0, a);
        const trailing = clause.slice(b);
        if (!quoted && (/[„“«»‚"'‘’]$/u.test(leading) || /^[“”«»‘’"']/u.test(trailing))) continue;
        // A prefix must terminate an unquoted value. Ambiguous multiword values need quotes.
        if (pattern === prefix && !quoted && !/^(?:[\t \u00a0]*(?:$|[,)]|(?:anzeigen|drucken|suchen|nachschlagen|auswählen|aufnehmen|korrigieren|ändern|ersetzen|verwenden|eintragen)\b))/iu.test(trailing)) continue;
        const boundStart = origin + a + (quoted ? 1 : 0);
        const boundEnd = origin + b - (quoted ? 1 : 0);
        const identifier = !quoted && /\p{L}/u.test(raw) && /\p{N}/u.test(raw) && /[-‐‑/_.]/u.test(raw);
        result.push({start: boundStart, end: boundEnd, identifier});
      }
    }
  }
  let first = 0;
  for (const sep of text.matchAll(strongSeparator)) {
    scan(text.slice(first, sep.index), first);
    first = sep.index + sep[0].length;
  }
  scan(text.slice(first), first);
  return result;
}

export function applyPolicy(text, groups, policy = null) {
  const bounds = [0];
  for (const cp of text) bounds.push(bounds[bounds.length - 1] + cp.length);
  const licensed = intervals(text);
  const retainedByLearning = policy === null ? null : new Set(applyLearned(text, groups, policy));
  return groups.filter(group => {
    if (!Number.isInteger(group.start) || !Number.isInteger(group.end) || group.start < 0 ||
        group.end <= group.start || group.end >= bounds.length) throw new RangeError('invalid code-point span');
    if (!EDITABLE.has(group.label) || retainedByLearning?.has(group)) return true;
    const start = bounds[group.start], end = bounds[group.end];
    return !licensed.some(span => (span.start === start && span.end === end) ||
      (span.identifier && span.start <= start && end <= span.end));
  });
}