Instructions to use EasyOctopus/german-ticket-privacy-filter with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- Transformers
How to use EasyOctopus/german-ticket-privacy-filter with Transformers:
# Use a pipeline as a high-level helper from transformers import pipeline pipe = pipeline("token-classification", model="EasyOctopus/german-ticket-privacy-filter")# pip install -U transformers accelerate # Load model directly from transformers import AutoTokenizer, AutoModelForTokenClassification tokenizer = AutoTokenizer.from_pretrained("EasyOctopus/german-ticket-privacy-filter") model = AutoModelForTokenClassification.from_pretrained("EasyOctopus/german-ticket-privacy-filter", device_map="auto") - Notebooks
- Google Colab
- Kaggle
File size: 5,895 Bytes
cc39b33 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 | """German support encoder. Offsets refer only to the Unicode reply, never to its context."""
from pathlib import Path
import json
import torch
from transformers import AutoTokenizer,AutoModelForTokenClassification
MAX_CONTEXT = 80
MAX_WINDOW = 256
OVERLAP = 40
def windows(tokenizer, context, text, gold=None, labels=None):
# Tokenize each side independently so their offsets belong to their respective strings.
ctx=tokenizer(context,add_special_tokens=False)['input_ids'][-MAX_CONTEXT:] if context else []
reply=tokenizer(text,return_offsets_mapping=True,add_special_tokens=False)
ids=reply['input_ids'];offsets=reply['offset_mapping']
prefix=[tokenizer.cls_token_id,*ctx,tokenizer.sep_token_id] if ctx else [tokenizer.cls_token_id]
capacity=MAX_WINDOW-len(prefix)-1
annotations=None
if gold is not None:
annotations=[labels['O']]*len(ids)
for g in gold:
positions=[i for i,(s,e) in enumerate(offsets) if s<g['end'] and e>g['start']]
for j,i in enumerate(positions):
annotations[i]=labels[('B-' if j==0 else 'I-')+g['label']]
chunks=[]
for offset in range(0,len(ids),capacity-OVERLAP):
end=min(len(ids),offset+capacity);sequence=prefix+ids[offset:end]+[tokenizer.sep_token_id]
chunks.append(dict(ids=sequence,base=offset,head=len(prefix),length=end-offset,
targets=None if annotations is None else [-100]*len(prefix)+annotations[offset:end]+[-100]))
if end==len(ids):break
return chunks, offsets
def batch(items,pad,device,training=False):
size=max(len(i['ids']) for i in items)
inputs=torch.tensor([v['ids']+[pad]*(size-len(v['ids'])) for v in items],device=device)
attention=torch.tensor([[1]*len(v['ids'])+[0]*(size-len(v['ids'])) for v in items],device=device)
ret={'input_ids':inputs,'attention_mask':attention}
if training:ret['labels']=torch.tensor([v['targets']+[-100]*(size-len(v['ids'])) for v in items],device=device)
return ret
def token_results(model,tokenizer,rows,device,batch_size=24):
encoded=[windows(tokenizer,r.get('context',''),r['text']) for r in rows]
selected=[[None]*len(offsets) for _,offsets in encoded]
flat=[(k,w) for k,(chunks,_) in enumerate(encoded) for w in chunks]
flat.sort(key=lambda v:len(v[1]['ids']))
model.eval()
with torch.inference_mode():
for i in range(0,len(flat),batch_size):
group=flat[i:i+batch_size]
logits=model(**batch([w for _,w in group],tokenizer.pad_token_id,device)).logits.float()
scores,tags=logits.softmax(-1).max(-1)
scores=scores.cpu().tolist();tags=tags.cpu().tolist()
for b,(k,w) in enumerate(group):
for a in range(w['length']):
idx=w['base']+a; pos=w['head']+a
# Give edge tokens to the overlapping window with more surrounding context.
margin=min(a,w['length']-1-a)
prev=selected[k][idx]
if prev is None or margin>prev[0]:
selected[k][idx]=(margin,model.config.id2label[tags[b][pos]],scores[b][pos])
return [list(zip(offsets,pred)) for (_,offsets),pred in zip(encoded,selected)]
def decode(text,tokens,cutoff,whole_word):
raw=[]
last=None
for (s,e),(_,tag,score) in tokens:
if tag=='O':
last=None
continue
label=tag[2:]
if tag.startswith('I-') and last==label:
raw[-1]['end']=e;raw[-1]['total']+=score;raw[-1]['n']+=1
else:raw.append(dict(start=s,end=e,label=label,total=score,n=1))
last=label
kept=[]
for g in raw:
score=g['total']/g['n']
if score<cutoff:continue
s,e=g['start'],g['end']
if whole_word and (g['label']=='PERSON' or g['label'].endswith('_ID')):
while s>0 and text[s-1].isalnum():s-=1
while e<len(text) and text[e].isalnum():e+=1
if kept and s<kept[-1]['end']:
kept[-1]['end']=max(kept[-1]['end'],e)
else:kept.append(dict(start=s,end=e,label=g['label'],score=round(score,6)))
return kept
class TicketEncoder:
def __init__(self,checkpoint,device='cpu'):
self.tokenizer=AutoTokenizer.from_pretrained(checkpoint,local_files_only=True,use_fast=True)
self.model=AutoModelForTokenClassification.from_pretrained(checkpoint,local_files_only=True).to(device)
self.device=device
self.policy=json.loads((Path(checkpoint)/'ticket_policy.json').read_text())
def predict_many(self,rows):
items=token_results(self.model,self.tokenizer,rows,self.device)
return [decode(r['text'],v,self.policy['cutoff'],self.policy['whole_word']) for r,v in zip(rows,items)]
def redact(self,text,context=''):
if not isinstance(text,str) or not isinstance(context,str) or len(text)>12000 or len(context)>512:
raise ValueError('Expected a reply of up to 12000 characters and a question of up to 512')
groups=self.predict_many([{'text':text,'context':context}])[0]
end=0;chunks=[]
for g in groups:
chunks.extend([text[end:g['start']],'['+g['label']+']']);end=g['end']
return {'redacted_text':''.join(chunks)+text[end:],'entities':groups,'offset_unit':'unicode_code_point','end_exclusive':True}
if __name__=='__main__':
import sys,argparse
ap=argparse.ArgumentParser();ap.add_argument('--model',required=True);args=ap.parse_args()
torch.set_num_threads(2)
encoder=TicketEncoder(args.model)
for line in sys.stdin:
if not line.strip():continue
try:
r=json.loads(line);result=encoder.redact(r['text'],r.get('context',''))
except (TypeError,KeyError,ValueError):
result={'error':'Invalid ticket'}
print(json.dumps(result,ensure_ascii=False),flush=True)
|