Token Classification
Transformers
Safetensors
German
distilbert
privacy
pii
german
EasyOctopus's picture
Release the contextual German support ticket experiment
cc39b33 verified
Raw History Blame Contribute Delete
5.9 kB
"""German support encoder. Offsets refer only to the Unicode reply, never to its context."""
from pathlib import Path
import json
import torch
from transformers import AutoTokenizer,AutoModelForTokenClassification
MAX_CONTEXT = 80
MAX_WINDOW = 256
OVERLAP = 40
def windows(tokenizer, context, text, gold=None, labels=None):
# Tokenize each side independently so their offsets belong to their respective strings.
ctx=tokenizer(context,add_special_tokens=False)['input_ids'][-MAX_CONTEXT:] if context else []
reply=tokenizer(text,return_offsets_mapping=True,add_special_tokens=False)
ids=reply['input_ids'];offsets=reply['offset_mapping']
prefix=[tokenizer.cls_token_id,*ctx,tokenizer.sep_token_id] if ctx else [tokenizer.cls_token_id]
capacity=MAX_WINDOW-len(prefix)-1
annotations=None
if gold is not None:
annotations=[labels['O']]*len(ids)
for g in gold:
positions=[i for i,(s,e) in enumerate(offsets) if s<g['end'] and e>g['start']]
for j,i in enumerate(positions):
annotations[i]=labels[('B-' if j==0 else 'I-')+g['label']]
chunks=[]
for offset in range(0,len(ids),capacity-OVERLAP):
end=min(len(ids),offset+capacity);sequence=prefix+ids[offset:end]+[tokenizer.sep_token_id]
chunks.append(dict(ids=sequence,base=offset,head=len(prefix),length=end-offset,
targets=None if annotations is None else [-100]*len(prefix)+annotations[offset:end]+[-100]))
if end==len(ids):break
return chunks, offsets
def batch(items,pad,device,training=False):
size=max(len(i['ids']) for i in items)
inputs=torch.tensor([v['ids']+[pad]*(size-len(v['ids'])) for v in items],device=device)
attention=torch.tensor([[1]*len(v['ids'])+[0]*(size-len(v['ids'])) for v in items],device=device)
ret={'input_ids':inputs,'attention_mask':attention}
if training:ret['labels']=torch.tensor([v['targets']+[-100]*(size-len(v['ids'])) for v in items],device=device)
return ret
def token_results(model,tokenizer,rows,device,batch_size=24):
encoded=[windows(tokenizer,r.get('context',''),r['text']) for r in rows]
selected=[[None]*len(offsets) for _,offsets in encoded]
flat=[(k,w) for k,(chunks,_) in enumerate(encoded) for w in chunks]
flat.sort(key=lambda v:len(v[1]['ids']))
model.eval()
with torch.inference_mode():
for i in range(0,len(flat),batch_size):
group=flat[i:i+batch_size]
logits=model(**batch([w for _,w in group],tokenizer.pad_token_id,device)).logits.float()
scores,tags=logits.softmax(-1).max(-1)
scores=scores.cpu().tolist();tags=tags.cpu().tolist()
for b,(k,w) in enumerate(group):
for a in range(w['length']):
idx=w['base']+a; pos=w['head']+a
# Give edge tokens to the overlapping window with more surrounding context.
margin=min(a,w['length']-1-a)
prev=selected[k][idx]
if prev is None or margin>prev[0]:
selected[k][idx]=(margin,model.config.id2label[tags[b][pos]],scores[b][pos])
return [list(zip(offsets,pred)) for (_,offsets),pred in zip(encoded,selected)]
def decode(text,tokens,cutoff,whole_word):
raw=[]
last=None
for (s,e),(_,tag,score) in tokens:
if tag=='O':
last=None
continue
label=tag[2:]
if tag.startswith('I-') and last==label:
raw[-1]['end']=e;raw[-1]['total']+=score;raw[-1]['n']+=1
else:raw.append(dict(start=s,end=e,label=label,total=score,n=1))
last=label
kept=[]
for g in raw:
score=g['total']/g['n']
if score<cutoff:continue
s,e=g['start'],g['end']
if whole_word and (g['label']=='PERSON' or g['label'].endswith('_ID')):
while s>0 and text[s-1].isalnum():s-=1
while e<len(text) and text[e].isalnum():e+=1
if kept and s<kept[-1]['end']:
kept[-1]['end']=max(kept[-1]['end'],e)
else:kept.append(dict(start=s,end=e,label=g['label'],score=round(score,6)))
return kept
class TicketEncoder:
def __init__(self,checkpoint,device='cpu'):
self.tokenizer=AutoTokenizer.from_pretrained(checkpoint,local_files_only=True,use_fast=True)
self.model=AutoModelForTokenClassification.from_pretrained(checkpoint,local_files_only=True).to(device)
self.device=device
self.policy=json.loads((Path(checkpoint)/'ticket_policy.json').read_text())
def predict_many(self,rows):
items=token_results(self.model,self.tokenizer,rows,self.device)
return [decode(r['text'],v,self.policy['cutoff'],self.policy['whole_word']) for r,v in zip(rows,items)]
def redact(self,text,context=''):
if not isinstance(text,str) or not isinstance(context,str) or len(text)>12000 or len(context)>512:
raise ValueError('Expected a reply of up to 12000 characters and a question of up to 512')
groups=self.predict_many([{'text':text,'context':context}])[0]
end=0;chunks=[]
for g in groups:
chunks.extend([text[end:g['start']],'['+g['label']+']']);end=g['end']
return {'redacted_text':''.join(chunks)+text[end:],'entities':groups,'offset_unit':'unicode_code_point','end_exclusive':True}
if __name__=='__main__':
import sys,argparse
ap=argparse.ArgumentParser();ap.add_argument('--model',required=True);args=ap.parse_args()
torch.set_num_threads(2)
encoder=TicketEncoder(args.model)
for line in sys.stdin:
if not line.strip():continue
try:
r=json.loads(line);result=encoder.redact(r['text'],r.get('context',''))
except (TypeError,KeyError,ValueError):
result={'error':'Invalid ticket'}
print(json.dumps(result,ensure_ascii=False),flush=True)