Improve German redaction and document the matched reinforcement learning trial
Browse files- README.md +8 -1
- app.mjs +7 -4
- complete.mjs +46 -0
- core.mjs +3 -2
- evaluation.json +612 -1
- experiment-methods.md +25 -0
- release.json +5 -1
- worker.mjs +1 -1
README.md
CHANGED
|
@@ -26,7 +26,14 @@ The JSON includes the redacted text and predicted entity type, score and half-op
|
|
| 26 |
|
| 27 |
The FP32 source is [EasyOctopus/german-support-pii-model](https://huggingface.co/EasyOctopus/german-support-pii-model/tree/fcdc784888d75b9c6559d6550f21620d13ee9651). The accompanying [synthetic support dataset](https://huggingface.co/datasets/EasyOctopus/german-support-pii-data) includes generation code and labelled splits. See the model card and notices for attribution. The browser runtime is ONNX Runtime Web 1.22.0, MIT. The original model and this demo are Apache-2.0.
|
| 28 |
|
| 29 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 30 |
|
| 31 |
The shipped browser runtime reproduced the FP32 model’s labelled spans and offsets on all 598 fictional messages at the default cutoff. On the 176-message published support holdout it fully covered 255 of 256 sensitive spans and flagged 2 of 48 clean messages. Fresh contrast pairs exposed more errors. At the default cutoff, it covered 64 of 80 sensitive spans and flagged 62 of 98 clean messages. At 0.95, it covered 30 spans and flagged 15 clean messages on the same set. Neither suite measures accuracy on customer data.
|
| 32 |
|
|
|
|
| 26 |
|
| 27 |
The FP32 source is [EasyOctopus/german-support-pii-model](https://huggingface.co/EasyOctopus/german-support-pii-model/tree/fcdc784888d75b9c6559d6550f21620d13ee9651). The accompanying [synthetic support dataset](https://huggingface.co/datasets/EasyOctopus/german-support-pii-data) includes generation code and labelled splits. See the model card and notices for attribution. The browser runtime is ONNX Runtime Web 1.22.0, MIT. The original model and this demo are Apache-2.0.
|
| 28 |
|
| 29 |
+
|
| 30 |
+
## Current browser release
|
| 31 |
+
|
| 32 |
+
Base detector. Complete validated identifier boundaries: True. On 120 fresh personally authored fictional German support messages, the previous release flagged 29/52 clean documents and fully masked 61/69 sensitive spans. The deployed pipeline flagged 29/52 and fully masked 61/69. On the 453-document German Gretel financial test split, it fully masked 663/1,025 in-scope annotated spans. Unknown annotations make false-positive reporting on that financial set unreliable. All policies still depend on candidate spans from the base model.
|
| 33 |
+
|
| 34 |
+
[Full evaluation](./evaluation.json) includes matched SFT and reinforcement learning results, release checks and the earlier tests. The weights stay on the client. The user can raise the existing cutoff to keep fewer masks.
|
| 35 |
+
|
| 36 |
+
## Earlier checkpoint verification
|
| 37 |
|
| 38 |
The shipped browser runtime reproduced the FP32 model’s labelled spans and offsets on all 598 fictional messages at the default cutoff. On the 176-message published support holdout it fully covered 255 of 256 sensitive spans and flagged 2 of 48 clean messages. Fresh contrast pairs exposed more errors. At the default cutoff, it covered 64 of 80 sensitive spans and flagged 62 of 98 clean messages. At 0.95, it covered 30 spans and flagged 15 clean messages on the same set. Neither suite measures accuracy on customer data.
|
| 39 |
|
app.mjs
CHANGED
|
@@ -42,14 +42,17 @@ function render() {
|
|
| 42 |
out.append(document.createTextNode(points.slice(end).join("")));
|
| 43 |
const tbody = document.querySelector("tbody");
|
| 44 |
tbody.replaceChildren();
|
| 45 |
-
|
| 46 |
-
|
| 47 |
-
|
|
|
|
|
|
|
|
|
|
| 48 |
for (const value of [
|
| 49 |
g.label,
|
| 50 |
g.start + " bis " + g.end,
|
| 51 |
confidence.toFixed(3).replace(".", ","),
|
| 52 |
-
|
| 53 |
])
|
| 54 |
tr.insertCell().textContent = value;
|
| 55 |
}
|
|
|
|
| 42 |
out.append(document.createTextNode(points.slice(end).join("")));
|
| 43 |
const tbody = document.querySelector("tbody");
|
| 44 |
tbody.replaceChildren();
|
| 45 |
+
const visible = last.groups.filter(g => g.total / g.count < threshold &&
|
| 46 |
+
!payload.entities.some(e => e.start < g.end && g.start < e.end));
|
| 47 |
+
const rows = [...payload.entities.map(g => ({...g, masked:true})),
|
| 48 |
+
...visible.map(g => ({...g, confidence:g.total/g.count, masked:false}))].sort((a,b)=>a.start-b.start);
|
| 49 |
+
for (const g of rows) {
|
| 50 |
+
const tr = tbody.insertRow(), confidence = g.confidence;
|
| 51 |
for (const value of [
|
| 52 |
g.label,
|
| 53 |
g.start + " bis " + g.end,
|
| 54 |
confidence.toFixed(3).replace(".", ","),
|
| 55 |
+
g.masked ? "geschwärzt" : "sichtbar",
|
| 56 |
])
|
| 57 |
tr.insertCell().textContent = value;
|
| 58 |
}
|
complete.mjs
ADDED
|
@@ -0,0 +1,46 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
// Complete an existing partial identifier at the user's selected threshold.
|
| 2 |
+
// Positions returned by the detector are Unicode code points, not UTF-16 units.
|
| 3 |
+
const formats = [
|
| 4 |
+
['EMAIL', /(?<![\p{L}\p{N}._%+@-])[a-z0-9_%+-]+(?:\.[a-z0-9_%+-]+)*@(?:[a-z0-9](?:[a-z0-9-]*[a-z0-9])?\.)+[a-z]{2,24}(?![\p{L}\p{N}_@-])/giu],
|
| 5 |
+
['IP', /(?<![\w.:])(?:\d{1,3}\.){3}\d{1,3}(?![\w:]|\.\d)/gu],
|
| 6 |
+
['IBAN', /(?<![\p{L}\p{N}])DE\d{2}(?:[ \u00a0]?\d){18}(?![\p{L}\p{N}])/giu]
|
| 7 |
+
];
|
| 8 |
+
function valid(label,text) {
|
| 9 |
+
if(label==='IP') return text.split('.').every(s=>Number(s)<=255 && (s==='0' || !s.startsWith('0')));
|
| 10 |
+
if(label==='IBAN'){
|
| 11 |
+
const compact=text.toUpperCase().replace(/[ \u00a0]/g,'');
|
| 12 |
+
let remainder=0;
|
| 13 |
+
for(const digit of compact.slice(4)+'1314'+compact.slice(2,4)) remainder=(remainder*10+Number(digit))%97;
|
| 14 |
+
return remainder===1;
|
| 15 |
+
}
|
| 16 |
+
return text.length<=254 && text.indexOf('@')<=64 && text.split('@')[1].split('.').every(x=>x.length<=63);
|
| 17 |
+
}
|
| 18 |
+
export function completeIdentifiers(text,groups){
|
| 19 |
+
if(!groups.length) return groups;
|
| 20 |
+
const points=Array.from(text);
|
| 21 |
+
const offsets=new Map([[0,0]]);let units=0;
|
| 22 |
+
for(let i=0;i<points.length;i++){units+=points[i].length;offsets.set(units,i+1);}
|
| 23 |
+
const candidates=[];
|
| 24 |
+
for(const [label,regex] of formats){
|
| 25 |
+
for(const m of text.matchAll(regex)){
|
| 26 |
+
if(valid(label,m[0])) candidates.push({start:offsets.get(m.index),end:offsets.get(m.index+m[0].length),label});
|
| 27 |
+
}
|
| 28 |
+
}
|
| 29 |
+
if(!candidates.length)return groups;
|
| 30 |
+
candidates.sort((a,b)=>a.start-b.start||b.end-a.end);
|
| 31 |
+
const out=[]; let i=0;
|
| 32 |
+
for(const c of candidates){
|
| 33 |
+
if(i>=groups.length)break;
|
| 34 |
+
while(i<groups.length&&groups[i].end<=c.start)out.push(groups[i++]);
|
| 35 |
+
let j=i;
|
| 36 |
+
while(j<groups.length&&groups[j].start<c.end)j++;
|
| 37 |
+
if(j===i)continue;
|
| 38 |
+
const overlap=groups.slice(i,j);
|
| 39 |
+
if(overlap.some(g=>g.start<c.start||g.end>c.end))continue;
|
| 40 |
+
const matches=overlap.filter(g=>g.label===c.label && points.slice(g.start,g.end).filter(x=>/[\p{L}\p{N}]/u.test(x)).length>=2);
|
| 41 |
+
if(!matches.length)continue;
|
| 42 |
+
const best=matches.reduce((a,b)=>a.total/a.count>=b.total/b.count?a:b);
|
| 43 |
+
out.push({...best,start:c.start,end:c.end,label:c.label});i=j;
|
| 44 |
+
}
|
| 45 |
+
return out.concat(groups.slice(i));
|
| 46 |
+
}
|
core.mjs
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
|
|
| 1 |
// The released German tokenizer uses cased BERT normalization and WordPiece.
|
| 2 |
// All offsets below count Unicode code points in the unchanged input.
|
| 3 |
const punctuation = /\p{P}/u;
|
|
@@ -135,9 +136,9 @@ export function filtered(text, candidates, threshold, modelId) {
|
|
| 135 |
let end = 0,
|
| 136 |
redacted = "";
|
| 137 |
const entities = [];
|
| 138 |
-
|
|
|
|
| 139 |
const confidence = group.total / group.count;
|
| 140 |
-
if (confidence < threshold) continue;
|
| 141 |
redacted +=
|
| 142 |
points.slice(end, group.start).join("") + "[" + group.label + "]";
|
| 143 |
end = group.end;
|
|
|
|
| 1 |
+
import {completeIdentifiers} from "./complete.mjs";
|
| 2 |
// The released German tokenizer uses cased BERT normalization and WordPiece.
|
| 3 |
// All offsets below count Unicode code points in the unchanged input.
|
| 4 |
const punctuation = /\p{P}/u;
|
|
|
|
| 136 |
let end = 0,
|
| 137 |
redacted = "";
|
| 138 |
const entities = [];
|
| 139 |
+
const retained = candidates.filter(group => group.total / group.count >= threshold);
|
| 140 |
+
for (const group of completeIdentifiers(text, retained)) {
|
| 141 |
const confidence = group.total / group.count;
|
|
|
|
| 142 |
redacted +=
|
| 143 |
points.slice(end, group.start).join("") + "[" + group.label + "]";
|
| 144 |
end = group.end;
|
evaluation.json
CHANGED
|
@@ -24,7 +24,7 @@
|
|
| 24 |
"final": "Fresh synthetic private/public contrast pairs and clean texts from unseen families",
|
| 25 |
"stress": "Separately hand-authored synthetic edge cases"
|
| 26 |
},
|
| 27 |
-
"
|
| 28 |
"v2-test/default": {
|
| 29 |
"documents": 176,
|
| 30 |
"spans": 256,
|
|
@@ -325,5 +325,616 @@
|
|
| 325 |
"ordinary_chars": 3330,
|
| 326 |
"identical_to_source": 23
|
| 327 |
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 328 |
}
|
| 329 |
}
|
|
|
|
| 24 |
"final": "Fresh synthetic private/public contrast pairs and clean texts from unseen families",
|
| 25 |
"stress": "Separately hand-authored synthetic edge cases"
|
| 26 |
},
|
| 27 |
+
"previous_release_metrics": {
|
| 28 |
"v2-test/default": {
|
| 29 |
"documents": 176,
|
| 30 |
"spans": 256,
|
|
|
|
| 325 |
"ordinary_chars": 3330,
|
| 326 |
"identical_to_source": 23
|
| 327 |
}
|
| 328 |
+
},
|
| 329 |
+
"current_release": {
|
| 330 |
+
"method": "Base detector",
|
| 331 |
+
"validated_boundaries": true,
|
| 332 |
+
"decoder": [
|
| 333 |
+
"boundaries-5b77093267e8"
|
| 334 |
+
],
|
| 335 |
+
"threshold": 0,
|
| 336 |
+
"matched_methods": {
|
| 337 |
+
"blind-support": {
|
| 338 |
+
"v2": {
|
| 339 |
+
"docs": 120,
|
| 340 |
+
"spans": 69,
|
| 341 |
+
"covered": 61,
|
| 342 |
+
"clean": 52,
|
| 343 |
+
"clean_flags": 29,
|
| 344 |
+
"collateral": 259,
|
| 345 |
+
"pairs": 36,
|
| 346 |
+
"pairs_both_correct": 5,
|
| 347 |
+
"ordinary_chars": 5766,
|
| 348 |
+
"newly_lost": 0,
|
| 349 |
+
"newly_covered": 0
|
| 350 |
+
},
|
| 351 |
+
"sft": {
|
| 352 |
+
"docs": 120,
|
| 353 |
+
"spans": 69,
|
| 354 |
+
"covered": 61,
|
| 355 |
+
"clean": 52,
|
| 356 |
+
"clean_flags": 23,
|
| 357 |
+
"collateral": 213,
|
| 358 |
+
"pairs": 36,
|
| 359 |
+
"pairs_both_correct": 10,
|
| 360 |
+
"ordinary_chars": 5766,
|
| 361 |
+
"newly_lost": 0,
|
| 362 |
+
"newly_covered": 0
|
| 363 |
+
},
|
| 364 |
+
"rl": {
|
| 365 |
+
"docs": 120,
|
| 366 |
+
"spans": 69,
|
| 367 |
+
"covered": 61,
|
| 368 |
+
"clean": 52,
|
| 369 |
+
"clean_flags": 24,
|
| 370 |
+
"collateral": 225,
|
| 371 |
+
"pairs": 36,
|
| 372 |
+
"pairs_both_correct": 9,
|
| 373 |
+
"ordinary_chars": 5766,
|
| 374 |
+
"newly_lost": 0,
|
| 375 |
+
"newly_covered": 0
|
| 376 |
+
},
|
| 377 |
+
"v2_complete": {
|
| 378 |
+
"docs": 120,
|
| 379 |
+
"spans": 69,
|
| 380 |
+
"covered": 61,
|
| 381 |
+
"clean": 52,
|
| 382 |
+
"clean_flags": 29,
|
| 383 |
+
"collateral": 259,
|
| 384 |
+
"pairs": 36,
|
| 385 |
+
"pairs_both_correct": 5,
|
| 386 |
+
"ordinary_chars": 5766,
|
| 387 |
+
"newly_lost": 0,
|
| 388 |
+
"newly_covered": 0
|
| 389 |
+
},
|
| 390 |
+
"sft_complete": {
|
| 391 |
+
"docs": 120,
|
| 392 |
+
"spans": 69,
|
| 393 |
+
"covered": 61,
|
| 394 |
+
"clean": 52,
|
| 395 |
+
"clean_flags": 23,
|
| 396 |
+
"collateral": 213,
|
| 397 |
+
"pairs": 36,
|
| 398 |
+
"pairs_both_correct": 10,
|
| 399 |
+
"ordinary_chars": 5766,
|
| 400 |
+
"newly_lost": 0,
|
| 401 |
+
"newly_covered": 0
|
| 402 |
+
},
|
| 403 |
+
"rl_complete": {
|
| 404 |
+
"docs": 120,
|
| 405 |
+
"spans": 69,
|
| 406 |
+
"covered": 61,
|
| 407 |
+
"clean": 52,
|
| 408 |
+
"clean_flags": 24,
|
| 409 |
+
"collateral": 225,
|
| 410 |
+
"pairs": 36,
|
| 411 |
+
"pairs_both_correct": 9,
|
| 412 |
+
"ordinary_chars": 5766,
|
| 413 |
+
"newly_lost": 0,
|
| 414 |
+
"newly_covered": 0
|
| 415 |
+
}
|
| 416 |
+
},
|
| 417 |
+
"external": {
|
| 418 |
+
"v2": {
|
| 419 |
+
"docs": 453,
|
| 420 |
+
"spans": 1025,
|
| 421 |
+
"covered": 661,
|
| 422 |
+
"newly_lost": 0,
|
| 423 |
+
"newly_covered": 0
|
| 424 |
+
},
|
| 425 |
+
"sft": {
|
| 426 |
+
"docs": 453,
|
| 427 |
+
"spans": 1025,
|
| 428 |
+
"covered": 659,
|
| 429 |
+
"newly_lost": 2,
|
| 430 |
+
"newly_covered": 0
|
| 431 |
+
},
|
| 432 |
+
"rl": {
|
| 433 |
+
"docs": 453,
|
| 434 |
+
"spans": 1025,
|
| 435 |
+
"covered": 660,
|
| 436 |
+
"newly_lost": 1,
|
| 437 |
+
"newly_covered": 0
|
| 438 |
+
},
|
| 439 |
+
"v2_complete": {
|
| 440 |
+
"docs": 453,
|
| 441 |
+
"spans": 1025,
|
| 442 |
+
"covered": 663,
|
| 443 |
+
"newly_lost": 0,
|
| 444 |
+
"newly_covered": 2
|
| 445 |
+
},
|
| 446 |
+
"sft_complete": {
|
| 447 |
+
"docs": 453,
|
| 448 |
+
"spans": 1025,
|
| 449 |
+
"covered": 661,
|
| 450 |
+
"newly_lost": 2,
|
| 451 |
+
"newly_covered": 2
|
| 452 |
+
},
|
| 453 |
+
"rl_complete": {
|
| 454 |
+
"docs": 453,
|
| 455 |
+
"spans": 1025,
|
| 456 |
+
"covered": 662,
|
| 457 |
+
"newly_lost": 1,
|
| 458 |
+
"newly_covered": 2
|
| 459 |
+
}
|
| 460 |
+
},
|
| 461 |
+
"prior-v2-test": {
|
| 462 |
+
"v2": {
|
| 463 |
+
"docs": 176,
|
| 464 |
+
"spans": 256,
|
| 465 |
+
"covered": 255,
|
| 466 |
+
"clean": 48,
|
| 467 |
+
"clean_flags": 2,
|
| 468 |
+
"collateral": 55,
|
| 469 |
+
"pairs": 0,
|
| 470 |
+
"pairs_both_correct": 0,
|
| 471 |
+
"ordinary_chars": 10805,
|
| 472 |
+
"newly_lost": 0,
|
| 473 |
+
"newly_covered": 0
|
| 474 |
+
},
|
| 475 |
+
"sft": {
|
| 476 |
+
"docs": 176,
|
| 477 |
+
"spans": 256,
|
| 478 |
+
"covered": 255,
|
| 479 |
+
"clean": 48,
|
| 480 |
+
"clean_flags": 2,
|
| 481 |
+
"collateral": 55,
|
| 482 |
+
"pairs": 0,
|
| 483 |
+
"pairs_both_correct": 0,
|
| 484 |
+
"ordinary_chars": 10805,
|
| 485 |
+
"newly_lost": 0,
|
| 486 |
+
"newly_covered": 0
|
| 487 |
+
},
|
| 488 |
+
"rl": {
|
| 489 |
+
"docs": 176,
|
| 490 |
+
"spans": 256,
|
| 491 |
+
"covered": 255,
|
| 492 |
+
"clean": 48,
|
| 493 |
+
"clean_flags": 2,
|
| 494 |
+
"collateral": 55,
|
| 495 |
+
"pairs": 0,
|
| 496 |
+
"pairs_both_correct": 0,
|
| 497 |
+
"ordinary_chars": 10805,
|
| 498 |
+
"newly_lost": 0,
|
| 499 |
+
"newly_covered": 0
|
| 500 |
+
},
|
| 501 |
+
"v2_complete": {
|
| 502 |
+
"docs": 176,
|
| 503 |
+
"spans": 256,
|
| 504 |
+
"covered": 255,
|
| 505 |
+
"clean": 48,
|
| 506 |
+
"clean_flags": 2,
|
| 507 |
+
"collateral": 55,
|
| 508 |
+
"pairs": 0,
|
| 509 |
+
"pairs_both_correct": 0,
|
| 510 |
+
"ordinary_chars": 10805,
|
| 511 |
+
"newly_lost": 0,
|
| 512 |
+
"newly_covered": 0
|
| 513 |
+
},
|
| 514 |
+
"sft_complete": {
|
| 515 |
+
"docs": 176,
|
| 516 |
+
"spans": 256,
|
| 517 |
+
"covered": 255,
|
| 518 |
+
"clean": 48,
|
| 519 |
+
"clean_flags": 2,
|
| 520 |
+
"collateral": 55,
|
| 521 |
+
"pairs": 0,
|
| 522 |
+
"pairs_both_correct": 0,
|
| 523 |
+
"ordinary_chars": 10805,
|
| 524 |
+
"newly_lost": 0,
|
| 525 |
+
"newly_covered": 0
|
| 526 |
+
},
|
| 527 |
+
"rl_complete": {
|
| 528 |
+
"docs": 176,
|
| 529 |
+
"spans": 256,
|
| 530 |
+
"covered": 255,
|
| 531 |
+
"clean": 48,
|
| 532 |
+
"clean_flags": 2,
|
| 533 |
+
"collateral": 55,
|
| 534 |
+
"pairs": 0,
|
| 535 |
+
"pairs_both_correct": 0,
|
| 536 |
+
"ordinary_chars": 10805,
|
| 537 |
+
"newly_lost": 0,
|
| 538 |
+
"newly_covered": 0
|
| 539 |
+
}
|
| 540 |
+
},
|
| 541 |
+
"prior-independent": {
|
| 542 |
+
"v2": {
|
| 543 |
+
"docs": 120,
|
| 544 |
+
"spans": 108,
|
| 545 |
+
"covered": 94,
|
| 546 |
+
"clean": 24,
|
| 547 |
+
"clean_flags": 3,
|
| 548 |
+
"collateral": 36,
|
| 549 |
+
"pairs": 0,
|
| 550 |
+
"pairs_both_correct": 0,
|
| 551 |
+
"ordinary_chars": 7617,
|
| 552 |
+
"newly_lost": 0,
|
| 553 |
+
"newly_covered": 0
|
| 554 |
+
},
|
| 555 |
+
"sft": {
|
| 556 |
+
"docs": 120,
|
| 557 |
+
"spans": 108,
|
| 558 |
+
"covered": 92,
|
| 559 |
+
"clean": 24,
|
| 560 |
+
"clean_flags": 2,
|
| 561 |
+
"collateral": 28,
|
| 562 |
+
"pairs": 0,
|
| 563 |
+
"pairs_both_correct": 0,
|
| 564 |
+
"ordinary_chars": 7617,
|
| 565 |
+
"newly_lost": 2,
|
| 566 |
+
"newly_covered": 0
|
| 567 |
+
},
|
| 568 |
+
"rl": {
|
| 569 |
+
"docs": 120,
|
| 570 |
+
"spans": 108,
|
| 571 |
+
"covered": 92,
|
| 572 |
+
"clean": 24,
|
| 573 |
+
"clean_flags": 3,
|
| 574 |
+
"collateral": 36,
|
| 575 |
+
"pairs": 0,
|
| 576 |
+
"pairs_both_correct": 0,
|
| 577 |
+
"ordinary_chars": 7617,
|
| 578 |
+
"newly_lost": 2,
|
| 579 |
+
"newly_covered": 0
|
| 580 |
+
},
|
| 581 |
+
"v2_complete": {
|
| 582 |
+
"docs": 120,
|
| 583 |
+
"spans": 108,
|
| 584 |
+
"covered": 95,
|
| 585 |
+
"clean": 24,
|
| 586 |
+
"clean_flags": 3,
|
| 587 |
+
"collateral": 36,
|
| 588 |
+
"pairs": 0,
|
| 589 |
+
"pairs_both_correct": 0,
|
| 590 |
+
"ordinary_chars": 7617,
|
| 591 |
+
"newly_lost": 0,
|
| 592 |
+
"newly_covered": 1
|
| 593 |
+
},
|
| 594 |
+
"sft_complete": {
|
| 595 |
+
"docs": 120,
|
| 596 |
+
"spans": 108,
|
| 597 |
+
"covered": 93,
|
| 598 |
+
"clean": 24,
|
| 599 |
+
"clean_flags": 2,
|
| 600 |
+
"collateral": 28,
|
| 601 |
+
"pairs": 0,
|
| 602 |
+
"pairs_both_correct": 0,
|
| 603 |
+
"ordinary_chars": 7617,
|
| 604 |
+
"newly_lost": 2,
|
| 605 |
+
"newly_covered": 1
|
| 606 |
+
},
|
| 607 |
+
"rl_complete": {
|
| 608 |
+
"docs": 120,
|
| 609 |
+
"spans": 108,
|
| 610 |
+
"covered": 93,
|
| 611 |
+
"clean": 24,
|
| 612 |
+
"clean_flags": 3,
|
| 613 |
+
"collateral": 36,
|
| 614 |
+
"pairs": 0,
|
| 615 |
+
"pairs_both_correct": 0,
|
| 616 |
+
"ordinary_chars": 7617,
|
| 617 |
+
"newly_lost": 2,
|
| 618 |
+
"newly_covered": 1
|
| 619 |
+
}
|
| 620 |
+
},
|
| 621 |
+
"prior-confirmation": {
|
| 622 |
+
"v2": {
|
| 623 |
+
"docs": 84,
|
| 624 |
+
"spans": 70,
|
| 625 |
+
"covered": 64,
|
| 626 |
+
"clean": 24,
|
| 627 |
+
"clean_flags": 1,
|
| 628 |
+
"collateral": 30,
|
| 629 |
+
"pairs": 0,
|
| 630 |
+
"pairs_both_correct": 0,
|
| 631 |
+
"ordinary_chars": 5759,
|
| 632 |
+
"newly_lost": 0,
|
| 633 |
+
"newly_covered": 0
|
| 634 |
+
},
|
| 635 |
+
"sft": {
|
| 636 |
+
"docs": 84,
|
| 637 |
+
"spans": 70,
|
| 638 |
+
"covered": 61,
|
| 639 |
+
"clean": 24,
|
| 640 |
+
"clean_flags": 1,
|
| 641 |
+
"collateral": 27,
|
| 642 |
+
"pairs": 0,
|
| 643 |
+
"pairs_both_correct": 0,
|
| 644 |
+
"ordinary_chars": 5759,
|
| 645 |
+
"newly_lost": 3,
|
| 646 |
+
"newly_covered": 0
|
| 647 |
+
},
|
| 648 |
+
"rl": {
|
| 649 |
+
"docs": 84,
|
| 650 |
+
"spans": 70,
|
| 651 |
+
"covered": 63,
|
| 652 |
+
"clean": 24,
|
| 653 |
+
"clean_flags": 1,
|
| 654 |
+
"collateral": 27,
|
| 655 |
+
"pairs": 0,
|
| 656 |
+
"pairs_both_correct": 0,
|
| 657 |
+
"ordinary_chars": 5759,
|
| 658 |
+
"newly_lost": 1,
|
| 659 |
+
"newly_covered": 0
|
| 660 |
+
},
|
| 661 |
+
"v2_complete": {
|
| 662 |
+
"docs": 84,
|
| 663 |
+
"spans": 70,
|
| 664 |
+
"covered": 64,
|
| 665 |
+
"clean": 24,
|
| 666 |
+
"clean_flags": 1,
|
| 667 |
+
"collateral": 30,
|
| 668 |
+
"pairs": 0,
|
| 669 |
+
"pairs_both_correct": 0,
|
| 670 |
+
"ordinary_chars": 5759,
|
| 671 |
+
"newly_lost": 0,
|
| 672 |
+
"newly_covered": 0
|
| 673 |
+
},
|
| 674 |
+
"sft_complete": {
|
| 675 |
+
"docs": 84,
|
| 676 |
+
"spans": 70,
|
| 677 |
+
"covered": 61,
|
| 678 |
+
"clean": 24,
|
| 679 |
+
"clean_flags": 1,
|
| 680 |
+
"collateral": 27,
|
| 681 |
+
"pairs": 0,
|
| 682 |
+
"pairs_both_correct": 0,
|
| 683 |
+
"ordinary_chars": 5759,
|
| 684 |
+
"newly_lost": 3,
|
| 685 |
+
"newly_covered": 0
|
| 686 |
+
},
|
| 687 |
+
"rl_complete": {
|
| 688 |
+
"docs": 84,
|
| 689 |
+
"spans": 70,
|
| 690 |
+
"covered": 63,
|
| 691 |
+
"clean": 24,
|
| 692 |
+
"clean_flags": 1,
|
| 693 |
+
"collateral": 27,
|
| 694 |
+
"pairs": 0,
|
| 695 |
+
"pairs_both_correct": 0,
|
| 696 |
+
"ordinary_chars": 5759,
|
| 697 |
+
"newly_lost": 1,
|
| 698 |
+
"newly_covered": 0
|
| 699 |
+
}
|
| 700 |
+
},
|
| 701 |
+
"prior-final": {
|
| 702 |
+
"v2": {
|
| 703 |
+
"docs": 178,
|
| 704 |
+
"spans": 80,
|
| 705 |
+
"covered": 64,
|
| 706 |
+
"clean": 98,
|
| 707 |
+
"clean_flags": 62,
|
| 708 |
+
"collateral": 671,
|
| 709 |
+
"pairs": 0,
|
| 710 |
+
"pairs_both_correct": 0,
|
| 711 |
+
"ordinary_chars": 8993,
|
| 712 |
+
"newly_lost": 0,
|
| 713 |
+
"newly_covered": 0
|
| 714 |
+
},
|
| 715 |
+
"sft": {
|
| 716 |
+
"docs": 178,
|
| 717 |
+
"spans": 80,
|
| 718 |
+
"covered": 64,
|
| 719 |
+
"clean": 98,
|
| 720 |
+
"clean_flags": 35,
|
| 721 |
+
"collateral": 452,
|
| 722 |
+
"pairs": 0,
|
| 723 |
+
"pairs_both_correct": 0,
|
| 724 |
+
"ordinary_chars": 8993,
|
| 725 |
+
"newly_lost": 0,
|
| 726 |
+
"newly_covered": 0
|
| 727 |
+
},
|
| 728 |
+
"rl": {
|
| 729 |
+
"docs": 178,
|
| 730 |
+
"spans": 80,
|
| 731 |
+
"covered": 64,
|
| 732 |
+
"clean": 98,
|
| 733 |
+
"clean_flags": 36,
|
| 734 |
+
"collateral": 455,
|
| 735 |
+
"pairs": 0,
|
| 736 |
+
"pairs_both_correct": 0,
|
| 737 |
+
"ordinary_chars": 8993,
|
| 738 |
+
"newly_lost": 0,
|
| 739 |
+
"newly_covered": 0
|
| 740 |
+
},
|
| 741 |
+
"v2_complete": {
|
| 742 |
+
"docs": 178,
|
| 743 |
+
"spans": 80,
|
| 744 |
+
"covered": 64,
|
| 745 |
+
"clean": 98,
|
| 746 |
+
"clean_flags": 62,
|
| 747 |
+
"collateral": 671,
|
| 748 |
+
"pairs": 0,
|
| 749 |
+
"pairs_both_correct": 0,
|
| 750 |
+
"ordinary_chars": 8993,
|
| 751 |
+
"newly_lost": 0,
|
| 752 |
+
"newly_covered": 0
|
| 753 |
+
},
|
| 754 |
+
"sft_complete": {
|
| 755 |
+
"docs": 178,
|
| 756 |
+
"spans": 80,
|
| 757 |
+
"covered": 64,
|
| 758 |
+
"clean": 98,
|
| 759 |
+
"clean_flags": 35,
|
| 760 |
+
"collateral": 452,
|
| 761 |
+
"pairs": 0,
|
| 762 |
+
"pairs_both_correct": 0,
|
| 763 |
+
"ordinary_chars": 8993,
|
| 764 |
+
"newly_lost": 0,
|
| 765 |
+
"newly_covered": 0
|
| 766 |
+
},
|
| 767 |
+
"rl_complete": {
|
| 768 |
+
"docs": 178,
|
| 769 |
+
"spans": 80,
|
| 770 |
+
"covered": 64,
|
| 771 |
+
"clean": 98,
|
| 772 |
+
"clean_flags": 36,
|
| 773 |
+
"collateral": 455,
|
| 774 |
+
"pairs": 0,
|
| 775 |
+
"pairs_both_correct": 0,
|
| 776 |
+
"ordinary_chars": 8993,
|
| 777 |
+
"newly_lost": 0,
|
| 778 |
+
"newly_covered": 0
|
| 779 |
+
}
|
| 780 |
+
},
|
| 781 |
+
"prior-stress": {
|
| 782 |
+
"v2": {
|
| 783 |
+
"docs": 40,
|
| 784 |
+
"spans": 21,
|
| 785 |
+
"covered": 13,
|
| 786 |
+
"clean": 20,
|
| 787 |
+
"clean_flags": 6,
|
| 788 |
+
"collateral": 59,
|
| 789 |
+
"pairs": 0,
|
| 790 |
+
"pairs_both_correct": 0,
|
| 791 |
+
"ordinary_chars": 3330,
|
| 792 |
+
"newly_lost": 0,
|
| 793 |
+
"newly_covered": 0
|
| 794 |
+
},
|
| 795 |
+
"sft": {
|
| 796 |
+
"docs": 40,
|
| 797 |
+
"spans": 21,
|
| 798 |
+
"covered": 13,
|
| 799 |
+
"clean": 20,
|
| 800 |
+
"clean_flags": 6,
|
| 801 |
+
"collateral": 59,
|
| 802 |
+
"pairs": 0,
|
| 803 |
+
"pairs_both_correct": 0,
|
| 804 |
+
"ordinary_chars": 3330,
|
| 805 |
+
"newly_lost": 0,
|
| 806 |
+
"newly_covered": 0
|
| 807 |
+
},
|
| 808 |
+
"rl": {
|
| 809 |
+
"docs": 40,
|
| 810 |
+
"spans": 21,
|
| 811 |
+
"covered": 13,
|
| 812 |
+
"clean": 20,
|
| 813 |
+
"clean_flags": 6,
|
| 814 |
+
"collateral": 59,
|
| 815 |
+
"pairs": 0,
|
| 816 |
+
"pairs_both_correct": 0,
|
| 817 |
+
"ordinary_chars": 3330,
|
| 818 |
+
"newly_lost": 0,
|
| 819 |
+
"newly_covered": 0
|
| 820 |
+
},
|
| 821 |
+
"v2_complete": {
|
| 822 |
+
"docs": 40,
|
| 823 |
+
"spans": 21,
|
| 824 |
+
"covered": 13,
|
| 825 |
+
"clean": 20,
|
| 826 |
+
"clean_flags": 6,
|
| 827 |
+
"collateral": 59,
|
| 828 |
+
"pairs": 0,
|
| 829 |
+
"pairs_both_correct": 0,
|
| 830 |
+
"ordinary_chars": 3330,
|
| 831 |
+
"newly_lost": 0,
|
| 832 |
+
"newly_covered": 0
|
| 833 |
+
},
|
| 834 |
+
"sft_complete": {
|
| 835 |
+
"docs": 40,
|
| 836 |
+
"spans": 21,
|
| 837 |
+
"covered": 13,
|
| 838 |
+
"clean": 20,
|
| 839 |
+
"clean_flags": 6,
|
| 840 |
+
"collateral": 59,
|
| 841 |
+
"pairs": 0,
|
| 842 |
+
"pairs_both_correct": 0,
|
| 843 |
+
"ordinary_chars": 3330,
|
| 844 |
+
"newly_lost": 0,
|
| 845 |
+
"newly_covered": 0
|
| 846 |
+
},
|
| 847 |
+
"rl_complete": {
|
| 848 |
+
"docs": 40,
|
| 849 |
+
"spans": 21,
|
| 850 |
+
"covered": 13,
|
| 851 |
+
"clean": 20,
|
| 852 |
+
"clean_flags": 6,
|
| 853 |
+
"collateral": 59,
|
| 854 |
+
"pairs": 0,
|
| 855 |
+
"pairs_both_correct": 0,
|
| 856 |
+
"ordinary_chars": 3330,
|
| 857 |
+
"newly_lost": 0,
|
| 858 |
+
"newly_covered": 0
|
| 859 |
+
}
|
| 860 |
+
}
|
| 861 |
+
},
|
| 862 |
+
"external_source": {
|
| 863 |
+
"dataset": "gretelai/synthetic_pii_finance_multilingual",
|
| 864 |
+
"revision": "7b844d16738527a04264f50214cb426a4cea0897",
|
| 865 |
+
"file": "data/German_test-00000-of-00001.parquet",
|
| 866 |
+
"license": "Apache-2.0",
|
| 867 |
+
"rows": 453,
|
| 868 |
+
"mapped_labels": {
|
| 869 |
+
"name": "PERSON",
|
| 870 |
+
"first_name": "PERSON",
|
| 871 |
+
"last_name": "PERSON",
|
| 872 |
+
"street_address": "ADDRESS",
|
| 873 |
+
"phone_number": "PHONE",
|
| 874 |
+
"email": "EMAIL",
|
| 875 |
+
"customer_id": "CUSTOMER_ID",
|
| 876 |
+
"swift_bic_code": "BIC",
|
| 877 |
+
"iban": "IBAN",
|
| 878 |
+
"date_of_birth": "DATE_OF_BIRTH",
|
| 879 |
+
"ipv4": "IP",
|
| 880 |
+
"ipv6": "IP"
|
| 881 |
+
},
|
| 882 |
+
"out_of_scope": {
|
| 883 |
+
"date": 118,
|
| 884 |
+
"user_name": 6,
|
| 885 |
+
"company": 436,
|
| 886 |
+
"password": 5,
|
| 887 |
+
"account_pin": 9,
|
| 888 |
+
"employee_id": 24,
|
| 889 |
+
"credit_card_number": 5,
|
| 890 |
+
"bank_routing_number": 15,
|
| 891 |
+
"driver_license_number": 9,
|
| 892 |
+
"time": 36,
|
| 893 |
+
"ssn": 9,
|
| 894 |
+
"bban": 17,
|
| 895 |
+
"passport_number": 12,
|
| 896 |
+
"date_time": 2,
|
| 897 |
+
"local_latlng": 5,
|
| 898 |
+
"credit_card_security_code": 3,
|
| 899 |
+
"api_key": 8
|
| 900 |
+
},
|
| 901 |
+
"mapped_spans": 1025,
|
| 902 |
+
"note": "The upstream annotations can omit other private text. Use span coverage for the mapped classes only. Do not infer false positives from unmatched text in this dataset."
|
| 903 |
+
},
|
| 904 |
+
"release_checks": {
|
| 905 |
+
"development_winner": "sft",
|
| 906 |
+
"learned_checks": {
|
| 907 |
+
"selected_on_development": true,
|
| 908 |
+
"historical_gate": false,
|
| 909 |
+
"blind_fewer_clean_flags": true,
|
| 910 |
+
"blind_no_newly_lost": true,
|
| 911 |
+
"external_within_one_point": true
|
| 912 |
+
},
|
| 913 |
+
"learned_approved": false,
|
| 914 |
+
"boundary_checks": {
|
| 915 |
+
"new_protection": true,
|
| 916 |
+
"blind-support_monotone": true,
|
| 917 |
+
"blind-support_clean": true,
|
| 918 |
+
"blind-support_collateral": true,
|
| 919 |
+
"external_monotone": true,
|
| 920 |
+
"prior-v2-test_monotone": true,
|
| 921 |
+
"prior-v2-test_clean": true,
|
| 922 |
+
"prior-v2-test_collateral": true,
|
| 923 |
+
"prior-independent_monotone": true,
|
| 924 |
+
"prior-independent_clean": true,
|
| 925 |
+
"prior-independent_collateral": true,
|
| 926 |
+
"prior-confirmation_monotone": true,
|
| 927 |
+
"prior-confirmation_clean": true,
|
| 928 |
+
"prior-confirmation_collateral": true,
|
| 929 |
+
"prior-final_monotone": true,
|
| 930 |
+
"prior-final_clean": true,
|
| 931 |
+
"prior-final_collateral": true,
|
| 932 |
+
"prior-stress_monotone": true,
|
| 933 |
+
"prior-stress_clean": true,
|
| 934 |
+
"prior-stress_collateral": true
|
| 935 |
+
},
|
| 936 |
+
"boundary_approved": true
|
| 937 |
+
},
|
| 938 |
+
"scope": "Synthetic support and third-party synthetic finance. Older confidence-sweep and FP32 parity data describe the prior release."
|
| 939 |
}
|
| 940 |
}
|
experiment-methods.md
ADDED
|
@@ -0,0 +1,25 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Training a German redaction policy
|
| 2 |
+
|
| 3 |
+
## What the experiment changes
|
| 4 |
+
|
| 5 |
+
The released German DistilBERT model proposes private spans. A small second model decides whether each proposal should stay masked in its surrounding German sentence. The policy sees context outside the candidate and its predicted type. The browser still applies the user’s confidence cutoff. It cannot discover an identifier the first model missed. That limitation makes full-span recall and previously covered individual spans important release checks.
|
| 6 |
+
|
| 7 |
+
The supervised and reinforcement learning policies receive the same examples and start with supervised training. The supervised comparator continues to learn from annotated actions. The RL branch samples keep/mask decisions and updates the policy using the measured reward of each outcome relative to the other sampled outcomes. A contextual bandit fits this one-step decision. A long-running agent harness would change the task.
|
| 8 |
+
|
| 9 |
+
## Methods and sources
|
| 10 |
+
|
| 11 |
+
| Source | What we used |
|
| 12 |
+
|---|---|
|
| 13 |
+
| [Ahmadian et al., ACL 2024](https://aclanthology.org/2024.acl-long.662/) and [Hugging Face TRL implementation](https://github.com/huggingface/trl/blob/main/trl/trainer/rloo_trainer.py) | Sample several decisions for a document. Estimate a policy gradient with a leave-one-out reward baseline. Our binary span policy has its own implementation. Results from preference training of language models do not establish gains on redaction. |
|
| 14 |
+
| [Minimum Risk Training, Shen et al.](https://aclanthology.org/P16-1159/) | An output-level task measure can differ from token accuracy. Count whether a whole sensitive span remains protected and charge for masking ordinary text. |
|
| 15 |
+
| [Text Anonymization Benchmark paper](https://aclanthology.org/2022.cl-4.19/) and [authors' evaluator](https://github.com/NorskRegnesentral/text-anonymisation-benchmark) | Score full coverage of a sensitive mention. Track loss of information from unnecessary masking separately. Our German task uses a narrower definition than full document anonymisation. |
|
| 16 |
+
| [RedactionBench](https://arxiv.org/abs/2606.18782) | Hold the name or reference constant and change its role. A product title and a customer's name can have the same spelling. |
|
| 17 |
+
| [CheckList](https://aclanthology.org/2020.acl-main.442/) and [companion code](https://github.com/marcotcr/checklist) | Evaluate clean controls, paired directional changes and mixed cases. Include ordinary German, line breaks, emoji and combining characters. |
|
| 18 |
+
| [Counterfactually augmented data](https://github.com/acmi-lab/counterfactually-augmented-data) | Use closely related examples with different intended decisions. Reserve unseen scenario families and phrases for evaluation. |
|
| 19 |
+
| [Presidio recognizer architecture](https://microsoft.github.io/presidio/analyzer/) and [source repository](https://github.com/data-privacy-stack/presidio) | Evaluate a separate decoder intervention that completes the boundaries of an already detected, syntactically valid identifier. Only a matching detector proposal can trigger it. |
|
| 20 |
+
| [OpenAI Privacy Filter model card](https://cdn.openai.com/pdf/c66281ed-b638-456a-8ce1-97e9f5264a90/OpenAI-Privacy-Filter-Model-Card.pdf), [reference implementation](https://github.com/openai/privacy-filter) and [Digitflow German fine-tune](https://huggingface.co/digitflow/privacy-filter-de-ft) | Reference alternatives for bidirectional classification and multilingual redaction. They use different workloads and weights. This experiment measures our changes against the shipping browser detector and a matched supervised control. |
|
| 21 |
+
| [Gretel synthetic financial documents](https://huggingface.co/datasets/gretelai/synthetic_pii_finance_multilingual) | Use the entire German test split as an external transfer check. Count annotated names, addresses, linked customer IDs, contact details, birth dates, bank identifiers and IP addresses. The dataset can omit other private text. An unmatched prediction alone is insufficient evidence of a false alarm. |
|
| 22 |
+
|
| 23 |
+
## Evidence boundaries
|
| 24 |
+
|
| 25 |
+
Contabo owns training. The Mac holds the sealed 120-message support set and the separately authored German financial test split. The blind support set includes 36 pairs, mixed messages, ordinary negatives, structured identifiers and Unicode edge cases. The frozen release rule checks complete mentions and false alarms on the same paired predictions. In the external check we use all 453 documents and count only annotations in the mapped categories. Neither synthetic test establishes customer performance.
|
release.json
CHANGED
|
@@ -1 +1,5 @@
|
|
| 1 |
-
{
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"name": "German Support PII \u00b7 gepr\u00fcfte Kennungen",
|
| 3 |
+
"size": "134 MB Modell",
|
| 4 |
+
"threshold": 0
|
| 5 |
+
}
|
worker.mjs
CHANGED
|
@@ -57,7 +57,7 @@ self.onmessage = async ({ data }) => {
|
|
| 57 |
self.postMessage({
|
| 58 |
id: data.id,
|
| 59 |
groups: state.groups,
|
| 60 |
-
model: meta.onnx_sha256,
|
| 61 |
});
|
| 62 |
} catch (error) {
|
| 63 |
self.postMessage({ id: data.id, error: String(error.message ?? error) });
|
|
|
|
| 57 |
self.postMessage({
|
| 58 |
id: data.id,
|
| 59 |
groups: state.groups,
|
| 60 |
+
model: meta.onnx_sha256 + "/boundaries-5b77093267e8",
|
| 61 |
});
|
| 62 |
} catch (error) {
|
| 63 |
self.postMessage({ id: data.id, error: String(error.message ?? error) });
|