Token Classification
Transformers
Safetensors
German
distilbert
privacy
pii
german
File size: 5,895 Bytes
cc39b33
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
"""German support encoder. Offsets refer only to the Unicode reply, never to its context."""
from pathlib import Path
import json
import torch
from transformers import AutoTokenizer,AutoModelForTokenClassification

MAX_CONTEXT = 80
MAX_WINDOW = 256
OVERLAP = 40
def windows(tokenizer, context, text, gold=None, labels=None):
    # Tokenize each side independently so their offsets belong to their respective strings.
    ctx=tokenizer(context,add_special_tokens=False)['input_ids'][-MAX_CONTEXT:] if context else []
    reply=tokenizer(text,return_offsets_mapping=True,add_special_tokens=False)
    ids=reply['input_ids'];offsets=reply['offset_mapping']
    prefix=[tokenizer.cls_token_id,*ctx,tokenizer.sep_token_id] if ctx else [tokenizer.cls_token_id]
    capacity=MAX_WINDOW-len(prefix)-1
    annotations=None
    if gold is not None:
        annotations=[labels['O']]*len(ids)
        for g in gold:
            positions=[i for i,(s,e) in enumerate(offsets) if s<g['end'] and e>g['start']]
            for j,i in enumerate(positions):
                annotations[i]=labels[('B-' if j==0 else 'I-')+g['label']]
    chunks=[]
    for offset in range(0,len(ids),capacity-OVERLAP):
        end=min(len(ids),offset+capacity);sequence=prefix+ids[offset:end]+[tokenizer.sep_token_id]
        chunks.append(dict(ids=sequence,base=offset,head=len(prefix),length=end-offset,
                           targets=None if annotations is None else [-100]*len(prefix)+annotations[offset:end]+[-100]))
        if end==len(ids):break
    return chunks, offsets

def batch(items,pad,device,training=False):
    size=max(len(i['ids']) for i in items)
    inputs=torch.tensor([v['ids']+[pad]*(size-len(v['ids'])) for v in items],device=device)
    attention=torch.tensor([[1]*len(v['ids'])+[0]*(size-len(v['ids'])) for v in items],device=device)
    ret={'input_ids':inputs,'attention_mask':attention}
    if training:ret['labels']=torch.tensor([v['targets']+[-100]*(size-len(v['ids'])) for v in items],device=device)
    return ret

def token_results(model,tokenizer,rows,device,batch_size=24):
    encoded=[windows(tokenizer,r.get('context',''),r['text']) for r in rows]
    selected=[[None]*len(offsets) for _,offsets in encoded]
    flat=[(k,w) for k,(chunks,_) in enumerate(encoded) for w in chunks]
    flat.sort(key=lambda v:len(v[1]['ids']))
    model.eval()
    with torch.inference_mode():
        for i in range(0,len(flat),batch_size):
            group=flat[i:i+batch_size]
            logits=model(**batch([w for _,w in group],tokenizer.pad_token_id,device)).logits.float()
            scores,tags=logits.softmax(-1).max(-1)
            scores=scores.cpu().tolist();tags=tags.cpu().tolist()
            for b,(k,w) in enumerate(group):
                for a in range(w['length']):
                    idx=w['base']+a; pos=w['head']+a
                    # Give edge tokens to the overlapping window with more surrounding context.
                    margin=min(a,w['length']-1-a)
                    prev=selected[k][idx]
                    if prev is None or margin>prev[0]:
                        selected[k][idx]=(margin,model.config.id2label[tags[b][pos]],scores[b][pos])
    return [list(zip(offsets,pred)) for (_,offsets),pred in zip(encoded,selected)]

def decode(text,tokens,cutoff,whole_word):
    raw=[]
    last=None
    for (s,e),(_,tag,score) in tokens:
        if tag=='O':
            last=None
            continue
        label=tag[2:]
        if tag.startswith('I-') and last==label:
            raw[-1]['end']=e;raw[-1]['total']+=score;raw[-1]['n']+=1
        else:raw.append(dict(start=s,end=e,label=label,total=score,n=1))
        last=label
    kept=[]
    for g in raw:
        score=g['total']/g['n']
        if score<cutoff:continue
        s,e=g['start'],g['end']
        if whole_word and (g['label']=='PERSON' or g['label'].endswith('_ID')):
            while s>0 and text[s-1].isalnum():s-=1
            while e<len(text) and text[e].isalnum():e+=1
        if kept and s<kept[-1]['end']:
            kept[-1]['end']=max(kept[-1]['end'],e)
        else:kept.append(dict(start=s,end=e,label=g['label'],score=round(score,6)))
    return kept

class TicketEncoder:
    def __init__(self,checkpoint,device='cpu'):
        self.tokenizer=AutoTokenizer.from_pretrained(checkpoint,local_files_only=True,use_fast=True)
        self.model=AutoModelForTokenClassification.from_pretrained(checkpoint,local_files_only=True).to(device)
        self.device=device
        self.policy=json.loads((Path(checkpoint)/'ticket_policy.json').read_text())
    def predict_many(self,rows):
        items=token_results(self.model,self.tokenizer,rows,self.device)
        return [decode(r['text'],v,self.policy['cutoff'],self.policy['whole_word']) for r,v in zip(rows,items)]
    def redact(self,text,context=''):
        if not isinstance(text,str) or not isinstance(context,str) or len(text)>12000 or len(context)>512:
            raise ValueError('Expected a reply of up to 12000 characters and a question of up to 512')
        groups=self.predict_many([{'text':text,'context':context}])[0]
        end=0;chunks=[]
        for g in groups:
            chunks.extend([text[end:g['start']],'['+g['label']+']']);end=g['end']
        return {'redacted_text':''.join(chunks)+text[end:],'entities':groups,'offset_unit':'unicode_code_point','end_exclusive':True}

if __name__=='__main__':
    import sys,argparse
    ap=argparse.ArgumentParser();ap.add_argument('--model',required=True);args=ap.parse_args()
    torch.set_num_threads(2)
    encoder=TicketEncoder(args.model)
    for line in sys.stdin:
        if not line.strip():continue
        try:
            r=json.loads(line);result=encoder.redact(r['text'],r.get('context',''))
        except (TypeError,KeyError,ValueError):
            result={'error':'Invalid ticket'}
        print(json.dumps(result,ensure_ascii=False),flush=True)