EasyOctopus's picture
Release German support experiment and held-out evaluation
14e36fb verified
Raw History Blame Contribute Delete
2.34 kB
"""Shared offline decoding for the standalone releases."""
import re
import torch
from classifier import apply_threshold, batch, predict
MARK_LABELS=["O","PERIOD","COMMA","QUESTION","COLON"]
SYMBOLS=dict(zip(MARK_LABELS,["",".",",","?",":"]))
def punct_encode(tokenizer,row,final_only=False):
text=row["text"]
words=list(re.finditer(r"\S+",text))
encoded=tokenizer(text,return_offsets_mapping=True,truncation=False,verbose=False)
if len(encoded["input_ids"])>256:
raise ValueError("Input exceeds 256 tokens including special tokens")
ends={end:i for i,(start,end) in enumerate(encoded["offset_mapping"]) if start<end}
gold={s["index"]:s["mark"] for s in row.get("gold",[])}
positions=[]
labels=[-100]*len(encoded["input_ids"])
for i,w in enumerate(words):
if w.end() not in ends:
raise ValueError("Tokenizer cannot align this word to its original end")
pos=ends[w.end()]
symbol=gold.get(i,"")
if final_only and i!=len(words)-1:
symbol=""
labels[pos]=list(SYMBOLS.values()).index(symbol)
positions.append((pos,w.end()))
return {"input_ids":encoded["input_ids"],"labels":labels,"positions":positions}
@torch.inference_mode()
def punctuation(model,tokenizer,rows):
output={}
model.eval()
for start in range(0,len(rows),24):
group=rows[start:start+24]
items=[punct_encode(tokenizer,{"text":r["text"]}) for r in group]
features=batch(items,tokenizer.pad_token_id,model.device)
del features["labels"]
indices=model(**features).logits.argmax(-1).cpu().tolist()
for row,item,ids in zip(group,items,indices):
text=row["text"]
pos=0
parts=[]
for index,end in item["positions"]:
symbol=SYMBOLS[model.config.id2label[ids[index]]]
parts.extend([text[pos:end],symbol if text[end-1] not in ".,?!:;" else ""])
pos=end
output[row["id"]]="".join(parts)+text[pos:]
return output
def pii(model,tokenizer,rows):
return apply_threshold(predict(model,tokenizer,rows),0.0)
def render_pii(text,spans):
parts=[]
end=0
for s in spans:
parts.extend([text[end:s["start"]],"["+s["label"]+"]"])
end=s["end"]
return "".join(parts)+text[end:]