"""German support encoder. Offsets refer only to the Unicode reply, never to its context.""" from pathlib import Path import json import torch from transformers import AutoTokenizer,AutoModelForTokenClassification MAX_CONTEXT = 80 MAX_WINDOW = 256 OVERLAP = 40 def windows(tokenizer, context, text, gold=None, labels=None): # Tokenize each side independently so their offsets belong to their respective strings. ctx=tokenizer(context,add_special_tokens=False)['input_ids'][-MAX_CONTEXT:] if context else [] reply=tokenizer(text,return_offsets_mapping=True,add_special_tokens=False) ids=reply['input_ids'];offsets=reply['offset_mapping'] prefix=[tokenizer.cls_token_id,*ctx,tokenizer.sep_token_id] if ctx else [tokenizer.cls_token_id] capacity=MAX_WINDOW-len(prefix)-1 annotations=None if gold is not None: annotations=[labels['O']]*len(ids) for g in gold: positions=[i for i,(s,e) in enumerate(offsets) if sg['start']] for j,i in enumerate(positions): annotations[i]=labels[('B-' if j==0 else 'I-')+g['label']] chunks=[] for offset in range(0,len(ids),capacity-OVERLAP): end=min(len(ids),offset+capacity);sequence=prefix+ids[offset:end]+[tokenizer.sep_token_id] chunks.append(dict(ids=sequence,base=offset,head=len(prefix),length=end-offset, targets=None if annotations is None else [-100]*len(prefix)+annotations[offset:end]+[-100])) if end==len(ids):break return chunks, offsets def batch(items,pad,device,training=False): size=max(len(i['ids']) for i in items) inputs=torch.tensor([v['ids']+[pad]*(size-len(v['ids'])) for v in items],device=device) attention=torch.tensor([[1]*len(v['ids'])+[0]*(size-len(v['ids'])) for v in items],device=device) ret={'input_ids':inputs,'attention_mask':attention} if training:ret['labels']=torch.tensor([v['targets']+[-100]*(size-len(v['ids'])) for v in items],device=device) return ret def token_results(model,tokenizer,rows,device,batch_size=24): encoded=[windows(tokenizer,r.get('context',''),r['text']) for r in rows] selected=[[None]*len(offsets) for _,offsets in encoded] flat=[(k,w) for k,(chunks,_) in enumerate(encoded) for w in chunks] flat.sort(key=lambda v:len(v[1]['ids'])) model.eval() with torch.inference_mode(): for i in range(0,len(flat),batch_size): group=flat[i:i+batch_size] logits=model(**batch([w for _,w in group],tokenizer.pad_token_id,device)).logits.float() scores,tags=logits.softmax(-1).max(-1) scores=scores.cpu().tolist();tags=tags.cpu().tolist() for b,(k,w) in enumerate(group): for a in range(w['length']): idx=w['base']+a; pos=w['head']+a # Give edge tokens to the overlapping window with more surrounding context. margin=min(a,w['length']-1-a) prev=selected[k][idx] if prev is None or margin>prev[0]: selected[k][idx]=(margin,model.config.id2label[tags[b][pos]],scores[b][pos]) return [list(zip(offsets,pred)) for (_,offsets),pred in zip(encoded,selected)] def decode(text,tokens,cutoff,whole_word): raw=[] last=None for (s,e),(_,tag,score) in tokens: if tag=='O': last=None continue label=tag[2:] if tag.startswith('I-') and last==label: raw[-1]['end']=e;raw[-1]['total']+=score;raw[-1]['n']+=1 else:raw.append(dict(start=s,end=e,label=label,total=score,n=1)) last=label kept=[] for g in raw: score=g['total']/g['n'] if score0 and text[s-1].isalnum():s-=1 while e12000 or len(context)>512: raise ValueError('Expected a reply of up to 12000 characters and a question of up to 512') groups=self.predict_many([{'text':text,'context':context}])[0] end=0;chunks=[] for g in groups: chunks.extend([text[end:g['start']],'['+g['label']+']']);end=g['end'] return {'redacted_text':''.join(chunks)+text[end:],'entities':groups,'offset_unit':'unicode_code_point','end_exclusive':True} if __name__=='__main__': import sys,argparse ap=argparse.ArgumentParser();ap.add_argument('--model',required=True);args=ap.parse_args() torch.set_num_threads(2) encoder=TicketEncoder(args.model) for line in sys.stdin: if not line.strip():continue try: r=json.loads(line);result=encoder.redact(r['text'],r.get('context','')) except (TypeError,KeyError,ValueError): result={'error':'Invalid ticket'} print(json.dumps(result,ensure_ascii=False),flush=True)