german-support-redactor / complete.mjs
EasyOctopus's picture
Improve German redaction and document the matched reinforcement learning trial
efa548e verified
Raw History Blame Contribute Delete
2.23 kB
// Complete an existing partial identifier at the user's selected threshold.
// Positions returned by the detector are Unicode code points, not UTF-16 units.
const formats = [
['EMAIL', /(?<![\p{L}\p{N}._%+@-])[a-z0-9_%+-]+(?:\.[a-z0-9_%+-]+)*@(?:[a-z0-9](?:[a-z0-9-]*[a-z0-9])?\.)+[a-z]{2,24}(?![\p{L}\p{N}_@-])/giu],
['IP', /(?<![\w.:])(?:\d{1,3}\.){3}\d{1,3}(?![\w:]|\.\d)/gu],
['IBAN', /(?<![\p{L}\p{N}])DE\d{2}(?:[ \u00a0]?\d){18}(?![\p{L}\p{N}])/giu]
];
function valid(label,text) {
if(label==='IP') return text.split('.').every(s=>Number(s)<=255 && (s==='0' || !s.startsWith('0')));
if(label==='IBAN'){
const compact=text.toUpperCase().replace(/[ \u00a0]/g,'');
let remainder=0;
for(const digit of compact.slice(4)+'1314'+compact.slice(2,4)) remainder=(remainder*10+Number(digit))%97;
return remainder===1;
}
return text.length<=254 && text.indexOf('@')<=64 && text.split('@')[1].split('.').every(x=>x.length<=63);
}
export function completeIdentifiers(text,groups){
if(!groups.length) return groups;
const points=Array.from(text);
const offsets=new Map([[0,0]]);let units=0;
for(let i=0;i<points.length;i++){units+=points[i].length;offsets.set(units,i+1);}
const candidates=[];
for(const [label,regex] of formats){
for(const m of text.matchAll(regex)){
if(valid(label,m[0])) candidates.push({start:offsets.get(m.index),end:offsets.get(m.index+m[0].length),label});
}
}
if(!candidates.length)return groups;
candidates.sort((a,b)=>a.start-b.start||b.end-a.end);
const out=[]; let i=0;
for(const c of candidates){
if(i>=groups.length)break;
while(i<groups.length&&groups[i].end<=c.start)out.push(groups[i++]);
let j=i;
while(j<groups.length&&groups[j].start<c.end)j++;
if(j===i)continue;
const overlap=groups.slice(i,j);
if(overlap.some(g=>g.start<c.start||g.end>c.end))continue;
const matches=overlap.filter(g=>g.label===c.label && points.slice(g.start,g.end).filter(x=>/[\p{L}\p{N}]/u.test(x)).length>=2);
if(!matches.length)continue;
const best=matches.reduce((a,b)=>a.total/a.count>=b.total/b.count?a:b);
out.push({...best,start:c.start,end:c.end,label:c.label});i=j;
}
return out.concat(groups.slice(i));
}