EasyOctopus commited on
Commit
543abb0
·
verified ·
1 Parent(s): 00f8d86

Use the previous German support question to redact short replies on device

Browse files
Files changed (10) hide show
  1. README.md +9 -0
  2. app.mjs +9 -3
  3. core.mjs +4 -1
  4. evaluation.json +51 -0
  5. index.html +6 -0
  6. previous-question.mjs +71 -0
  7. release.json +3 -2
  8. role-model.json +0 -0
  9. style.css +4 -0
  10. worker.mjs +18 -5
README.md CHANGED
@@ -45,3 +45,12 @@ The shipped browser runtime reproduced the FP32 model’s labelled spans and off
45
  A new supervised fine-tune ran on one rented A100. It did not meet the predeclared development rule for retaining sensitive-span coverage while reducing false positives. The Space uses the previously released checkpoint. An 8-bit export also lost coverage on the support holdout (187 of 256 spans). The half-precision storage export preserves the tested span outputs. See [evaluation.json](./evaluation.json) for per-suite test counts and the CPU benchmark.
46
 
47
  [Read the controlled training results, external model comparison and browser checks](./experiment-results.md).
 
 
 
 
 
 
 
 
 
 
45
  A new supervised fine-tune ran on one rented A100. It did not meet the predeclared development rule for retaining sensitive-span coverage while reducing false positives. The Space uses the previously released checkpoint. An 8-bit export also lost coverage on the support holdout (187 of 256 spans). The half-precision storage export preserves the tested span outputs. See [evaluation.json](./evaluation.json) for per-suite test counts and the CPU benchmark.
46
 
47
  [Read the controlled training results, external model comparison and browser checks](./experiment-results.md).
48
+
49
+
50
+ ## Optional previous support question
51
+
52
+ Visitors can enter the immediately preceding support question above the current reply. The German DistilBERT detector processes the reply. A separate six-class logistic regression reads the question and recognises requests for a name, customer identifier, order identifier, public title or catalogue reference. The remaining class covers other questions. The classifier was trained on 771 fictional German questions on a CPU. Its additional download is 119,078 bytes and occurs when a question is supplied.
53
+
54
+ Only short, direct answers are eligible. The rule checks the requested field, the form of the answer and existing detections. An explicit public title can stay visible. A private answer can gain a new mask. The confidence slider applies to private candidates and the previous question is excluded from copied text and JSON. All inference runs on the visitor's device.
55
+
56
+ At the default threshold, 90 fictional German replies included 44 private mentions and 49 clean replies. The standalone detector fully masked 29 mentions and flagged 18 clean replies. The two-model release masked 31 and flagged 17. It handled both sides correctly in 3 of 25 pairs with identical replies and different questions. No previously fully masked mention was lost. A single previous question may be insufficient or misleading. Performance on real tickets is unknown.
app.mjs CHANGED
@@ -3,7 +3,11 @@ const by = (id) => document.getElementById(id);
3
  const input = by("input"),
4
  slider = by("threshold"),
5
  status = by("status");
 
 
6
  const presets = {
 
 
7
  book: "Bitte den Buchtitel „Martha Atlas“ in der Leseliste korrigieren. Das zweite A ist groß.",
8
  mail: "Bitte schicken Sie die Bestätigung an lea.beispiel@example.org. Vielen Dank!",
9
  city: "Wir fahren morgen nach Essen. Ist das Museum am Montag geöffnet?",
@@ -86,6 +90,7 @@ by("run").onclick = () => {
86
  const request = {
87
  id: ++nextId,
88
  text: input.value,
 
89
  start: performance.now(),
90
  first,
91
  };
@@ -102,7 +107,7 @@ by("run").onclick = () => {
102
  fail(data.error);
103
  return;
104
  }
105
- if (request.text !== input.value) {
106
  reset();
107
  status.textContent = "Der Text wurde geändert. Bitte erneut prüfen.";
108
  return;
@@ -114,9 +119,9 @@ by("run").onclick = () => {
114
  };
115
  render();
116
  };
117
- worker.postMessage({ id: request.id, text: request.text });
118
  };
119
- input.oninput = () => {
120
  reset();
121
  if (pending === null) status.textContent = "Bereit zur Prüfung.";
122
  };
@@ -124,6 +129,7 @@ slider.oninput = render;
124
  document.querySelectorAll("[data-example]").forEach(
125
  (button) =>
126
  (button.onclick = () => {
 
127
  input.value = presets[button.dataset.example];
128
  input.oninput();
129
  }),
 
3
  const input = by("input"),
4
  slider = by("threshold"),
5
  status = by("status");
6
+ const previousQuestion = by("context");
7
+ const questions = {surname: "Wie lautet Ihr Nachname?", bookreply: "Wie lautet der Buchtitel?"};
8
  const presets = {
9
+ surname: "Zinnfalter",
10
+ bookreply: "Zinnfalter",
11
  book: "Bitte den Buchtitel „Martha Atlas“ in der Leseliste korrigieren. Das zweite A ist groß.",
12
  mail: "Bitte schicken Sie die Bestätigung an lea.beispiel@example.org. Vielen Dank!",
13
  city: "Wir fahren morgen nach Essen. Ist das Museum am Montag geöffnet?",
 
90
  const request = {
91
  id: ++nextId,
92
  text: input.value,
93
+ context: previousQuestion.value,
94
  start: performance.now(),
95
  first,
96
  };
 
107
  fail(data.error);
108
  return;
109
  }
110
+ if (request.text !== input.value || request.context !== previousQuestion.value) {
111
  reset();
112
  status.textContent = "Der Text wurde geändert. Bitte erneut prüfen.";
113
  return;
 
119
  };
120
  render();
121
  };
122
+ worker.postMessage({ id: request.id, text: request.text, context: request.context });
123
  };
124
+ previousQuestion.oninput = input.oninput = () => {
125
  reset();
126
  if (pending === null) status.textContent = "Bereit zur Prüfung.";
127
  };
 
129
  document.querySelectorAll("[data-example]").forEach(
130
  (button) =>
131
  (button.onclick = () => {
132
+ previousQuestion.value = questions[button.dataset.example] ?? "";
133
  input.value = presets[button.dataset.example];
134
  input.oninput();
135
  }),
core.mjs CHANGED
@@ -136,7 +136,10 @@ export function filtered(text, candidates, threshold, modelId) {
136
  let end = 0,
137
  redacted = "";
138
  const entities = [];
139
- const retained = candidates.filter(group => group.total / group.count >= threshold);
 
 
 
140
  for (const group of completeIdentifiers(text, retained)) {
141
  const confidence = group.total / group.count;
142
  redacted +=
 
136
  let end = 0,
137
  redacted = "";
138
  const entities = [];
139
+ const retained = candidates.flatMap(group => {
140
+ if (group.total / group.count >= threshold) return [group];
141
+ return (group.original ?? []).filter(original => original.total / original.count >= threshold);
142
+ });
143
  for (const group of completeIdentifiers(text, retained)) {
144
  const confidence = group.total / group.count;
145
  redacted +=
evaluation.json CHANGED
@@ -1327,5 +1327,56 @@
1327
  }
1328
  }
1329
  }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1330
  }
1331
  }
 
1327
  }
1328
  }
1329
  }
1330
+ },
1331
+ "previous_question": {
1332
+ "default_threshold": 0,
1333
+ "weights_sha256": "5b9daa09e7a2758ba900aa4b8e8423dee95b1d4809e800172529a6cc9c25f70e",
1334
+ "training_questions": 771,
1335
+ "additional_download_bytes": 119078,
1336
+ "blind_sha256": "1cee523fc0aa144174e6ef80c24c3e3945986b39444c73a72d2f2c1b90afec9f",
1337
+ "methods": {
1338
+ "baseline": {
1339
+ "documents": 90,
1340
+ "spans": 44,
1341
+ "fully_hidden": 29,
1342
+ "clean_docs": 49,
1343
+ "clean_docs_flagged": 18,
1344
+ "pairs": 25,
1345
+ "pairs_both_correct": 0,
1346
+ "unnecessary_chars": 93
1347
+ },
1348
+ "concat": {
1349
+ "documents": 90,
1350
+ "spans": 44,
1351
+ "fully_hidden": 28,
1352
+ "clean_docs": 49,
1353
+ "clean_docs_flagged": 13,
1354
+ "pairs": 25,
1355
+ "pairs_both_correct": 2,
1356
+ "unnecessary_chars": 85
1357
+ },
1358
+ "context": {
1359
+ "documents": 90,
1360
+ "spans": 44,
1361
+ "fully_hidden": 31,
1362
+ "clean_docs": 49,
1363
+ "clean_docs_flagged": 17,
1364
+ "pairs": 25,
1365
+ "pairs_both_correct": 3,
1366
+ "unnecessary_chars": 89
1367
+ }
1368
+ },
1369
+ "add_only": {
1370
+ "documents": 90,
1371
+ "spans": 44,
1372
+ "fully_hidden": 31,
1373
+ "clean_docs": 49,
1374
+ "clean_docs_flagged": 18,
1375
+ "pairs": 25,
1376
+ "pairs_both_correct": 2,
1377
+ "unnecessary_chars": 93
1378
+ },
1379
+ "new_clean_flags": 0,
1380
+ "previously_hidden_mentions_lost": 0
1381
  }
1382
  }
index.html CHANGED
@@ -39,7 +39,13 @@
39
  ><button type="button" data-example="book">Buchtitel</button>
40
  <button type="button" data-example="guest">Faust als Gast</button>
41
  <button type="button" data-example="play">Faust als Stück</button>
 
 
42
  </section>
 
 
 
 
43
  <div class="editors">
44
  <section class="panel">
45
  <label for="input">Original</label
 
39
  ><button type="button" data-example="book">Buchtitel</button>
40
  <button type="button" data-example="guest">Faust als Gast</button>
41
  <button type="button" data-example="play">Faust als Stück</button>
42
+ <button type="button" data-example="surname">Antwort auf Namensfrage</button>
43
+ <button type="button" data-example="bookreply">Antwort auf Titelfrage</button>
44
  </section>
45
+ <div class="previous">
46
+ <label for="context">Vorherige Supportfrage (optional)</label>
47
+ <input id="context" type="text" maxlength="512" autocomplete="off" spellcheck="false" placeholder="Wie lautet Ihr Nachname?" />
48
+ </div>
49
  <div class="editors">
50
  <section class="panel">
51
  <label for="input">Original</label
previous-question.mjs ADDED
@@ -0,0 +1,71 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ // The role model sees only the previous support question. All span offsets belong to the current reply.
2
+ export function classify(question, model) {
3
+ const words = question.normalize("NFC").toLowerCase().match(/[a-zäöüß0-9]+/gu) ?? [];
4
+ const features = new Set();
5
+ for (const word of words) {
6
+ features.add("w:" + word);
7
+ const padded = "^" + word + "$";
8
+ for (const width of [3, 4]) {
9
+ for (let i = 0; i + width <= padded.length; i++)
10
+ features.add("g:" + padded.slice(i, i + width));
11
+ }
12
+ }
13
+ const scores = [...model.bias];
14
+ const active = [...features].filter(key => Object.hasOwn(model.weights, key));
15
+ const norm = Math.sqrt(active.length);
16
+ for (const key of active)
17
+ for (let i = 0; i < scores.length; i++) scores[i] += model.weights[key][i] / norm;
18
+ const peak = Math.max(...scores);
19
+ const sum = scores.reduce((n,v) => n + Math.exp(v - peak), 0);
20
+ return {role: model.classes[scores.indexOf(peak)], confidence: 1 / sum};
21
+ }
22
+
23
+ const personal = /(?:nachnam|vornam|familiennam|personennam|kundennam|kontoinhaber|empfänger|antragsteller|ansprechpartner|auf wen|wem gehört|wie (?:heiß|heiss)|\bnamen?\b)/iu;
24
+ const customer = /(?:kunden(?:nummer|kennung|referenz|[- ]?id|konto|karte)|mitglieds(?:nummer|kennung|karte))/iu;
25
+ const order = /(?:bestell|buchungs(?:nummer|code|kennung)|auftrags(?:kennung|nummer|referenz))/iu;
26
+ const title = /(?:buchtitel|produktname|produktbezeichnung|modellbezeichnung|werktitel|filmtitel|(?:titel|heißt|heisst|name).{0,30}(?:roman|buch|werk|stück|produkt|theater|spiel|film)|farbton)/iu;
27
+ const catalogue = /(?:artikel(?:nummer|code|[- ]?id)|katalog(?:nummer|referenz)|modellnummer|teilenummer|produktnummer|farbmusterkennung)/iu;
28
+ const roleCues = {person:personal, customer, order, "public-title":title, "public-item":catalogue};
29
+ const ambiguous = /(?:\b(?:kein(?:e|en|er|es)?|nicht|ohne|niemals|statt|entweder|oder|zuvor|vorher|bereits|schon|beispiel|angenommen)\b|\b(?:telefon|e-?mail|anschrift|adresse|iban|passwort)\w*)/iu;
30
+ const ordinary = /^(?:ja|nein|klar|danke|vielen dank|okay|ok|passt|später|gleich|moment|einen moment|sekunde|bitte|unbekannt|keine ahnung|weiß ich nicht|weiss ich nicht|nicht bekannt|noch offen|offen|folgt|keine angabe|nichts|alles klar|wunderbar|super|perfekt|gerne|leider|guten (?:tag|morgen|abend)|hallo|bis bald|einverstanden|erledigt|bestätigt|verstanden|stimmt|richtig|abbrechen|weiter|zurück|warte|nachher|morgen|irgendwann|vielleicht|natürlich|selbstverständlich|auf wiedersehen|kein problem)$/iu;
31
+ const wordValue = /^\p{Lu}[\p{L}\p{M}'’\-]*(?:[ \t]+(?:von |van |de |zu )?\p{Lu}[\p{L}\p{M}'’\-]*){0,2}$/u;
32
+ const idValue = /^(?=[A-Z0-9./_\-]*[0-9])[A-Z0-9][A-Z0-9./_\-]{2,23}$/iu;
33
+ const eligibleNames = new Set(["PERSON","ORGANIZATION"]);
34
+ const eligibleIds = new Set(["CUSTOMER_ID","ORDER_ID","ACCOUNT_ID"]);
35
+ const privateLabel = {person:"PERSON",customer:"CUSTOMER_ID",order:"ORDER_ID"};
36
+
37
+ function valueSpan(text) {
38
+ // Only accept one short direct answer. Mixed records stay with the first detector.
39
+ if (!text || text.length > 96 || /[\r\n;,:@!?=]/u.test(text)) return null;
40
+ const match = /^\s*(?:[„“«"]([^„“”«»"]+)[“”»"]|([^„“”«»"]+?))(?:[.!])?\s*$/du.exec(text);
41
+ if (!match) return null;
42
+ const [a,b]=match.indices[match[1]===undefined ? 2 : 1];
43
+ const value=text.slice(a,b);
44
+ if (value.trim()!==value || ordinary.test(value.normalize("NFC"))) return null;
45
+ return {value,start:[...text.slice(0,a)].length,end:[...text.slice(0,b)].length};
46
+ }
47
+
48
+ export function contextualGroups(question, text, baseline, model, allowPublic = true) {
49
+ if (!question || question.length > 512 || /[\r\n]/u.test(question) ||
50
+ (question.match(/\?/gu)?.length ?? 0) > 1 || ambiguous.test(question)) return baseline;
51
+ const answer = valueSpan(text);
52
+ if (!answer) return baseline;
53
+ const {role, confidence} = classify(question, model);
54
+ if (confidence < model.cutoff || !roleCues[role]?.test(question)) return baseline;
55
+ const publicRole = role.startsWith("public-");
56
+ if (publicRole && !allowPublic) return baseline;
57
+ // The word "Name" inside a clear title question can describe the public item.
58
+ // A second explicit private field or a personal addressee makes that interpretation uncertain.
59
+ const otherRole = Object.entries(roleCues).some(([key, pattern]) => key !== role &&
60
+ !(publicRole && key === "person") && pattern.test(question));
61
+ if (otherRole || (publicRole && /(?:nachname|vorname|familienname|empfänger|kontoinhaber|persönlich|privat|mitarbeiter|ansprechpartner|kunde\b)/iu.test(question))) return baseline;
62
+ const nameLike = role === "person" || role === "public-title";
63
+ if (!(nameLike ? wordValue : idValue).test(answer.value.normalize("NFC"))) return baseline;
64
+ const eligible = nameLike ? eligibleNames : eligibleIds;
65
+ const affected = baseline.filter(g => g.start < answer.end && g.end > answer.start);
66
+ if (affected.some(g => !eligible.has(g.label) || g.start < answer.start || g.end > answer.end)) return baseline;
67
+ if (publicRole) return baseline.filter(g => !affected.includes(g));
68
+ if (affected.some(g => g.start === answer.start && g.end === answer.end)) return baseline;
69
+ const added = {start:answer.start,end:answer.end,label:privateLabel[role],total:confidence,count:1,original:affected};
70
+ return [...baseline.filter(g => !affected.includes(g)),added].sort((a,b)=>a.start-b.start);
71
+ }
release.json CHANGED
@@ -1,5 +1,6 @@
1
  {
2
- "name": "German Support PII · Kontextprüfung",
3
  "size": "134 MB Modell",
4
- "threshold": 0
 
5
  }
 
1
  {
2
+ "name": "German Support PII · Supportfragen",
3
  "size": "134 MB Modell",
4
+ "threshold": 0,
5
+ "context_bytes": 119078
6
  }
role-model.json ADDED
The diff for this file is too large to render. See raw diff
 
style.css CHANGED
@@ -285,3 +285,7 @@ footer a {
285
  gap: 4px;
286
  }
287
  }
 
 
 
 
 
285
  gap: 4px;
286
  }
287
  }
288
+
289
+ .previous {margin: 14px 0;display: grid;gap: 5px;}
290
+ .previous label {font-weight: 600;font-size: .9rem;}
291
+ .previous input {box-sizing: border-box;width: 100%;font: inherit;border: 1px solid var(--border);border-radius: 7px;padding: 8px 12px;background: white;color: inherit;}
worker.mjs CHANGED
@@ -5,6 +5,8 @@ ort.env.wasm.numThreads = 1;
5
  ort.env.wasm.proxy = false;
6
  ort.env.wasm.wasmPaths = new URL("./vendor/", import.meta.url).href;
7
  let ready;
 
 
8
  async function asset(path, kind) {
9
  const res = await fetch(new URL(path, import.meta.url));
10
  if (!res.ok) throw new Error("Download failed (" + res.status + ")");
@@ -55,11 +57,22 @@ self.onmessage = async ({ data }) => {
55
  b.dispose();
56
  out.logits.dispose();
57
  }
58
- self.postMessage({
59
- id: data.id,
60
- groups: filterExplicitContext(data.text, state.groups, null),
61
- model: meta.onnx_sha256 + "/boundaries-5b77093267e8/guard-641065f26fb7",
62
- });
 
 
 
 
 
 
 
 
 
 
 
63
  } catch (error) {
64
  self.postMessage({ id: data.id, error: String(error.message ?? error) });
65
  }
 
5
  ort.env.wasm.proxy = false;
6
  ort.env.wasm.wasmPaths = new URL("./vendor/", import.meta.url).href;
7
  let ready;
8
+ let roleModel;
9
+ import {contextualGroups} from "./previous-question.mjs";
10
  async function asset(path, kind) {
11
  const res = await fetch(new URL(path, import.meta.url));
12
  if (!res.ok) throw new Error("Download failed (" + res.status + ")");
 
57
  b.dispose();
58
  out.logits.dispose();
59
  }
60
+ let groups = filterExplicitContext(data.text, state.groups, null);
61
+ let model = meta.onnx_sha256 + "/boundaries-5b77093267e8/guard-641065f26fb7";
62
+ if (data.context?.trim()) {
63
+ roleModel ??= asset("role-model.json", "buffer").then(async bytes => {
64
+ const hash = await crypto.subtle.digest("SHA-256", bytes);
65
+ const signature = Array.from(new Uint8Array(hash), byte => byte.toString(16).padStart(2, "0")).join("");
66
+ if (signature !== "5b9daa09e7a2758ba900aa4b8e8423dee95b1d4809e800172529a6cc9c25f70e") throw new Error("Die Datei für die Supportfrage ist unvollständig.");
67
+ return JSON.parse(new TextDecoder().decode(bytes));
68
+ }).catch(error => {
69
+ roleModel = null;
70
+ throw error;
71
+ });
72
+ groups = contextualGroups(data.context, data.text, groups, await roleModel, true);
73
+ model += "/question-5b9daa09e7a2-60bdc2d65eeb-public";
74
+ }
75
+ self.postMessage({id:data.id,groups,model});
76
  } catch (error) {
77
  self.postMessage({ id: data.id, error: String(error.message ?? error) });
78
  }