File size: 2,343 Bytes
14e36fb
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
"""Shared offline decoding for the standalone releases."""
import re
import torch
from classifier import apply_threshold, batch, predict

MARK_LABELS=["O","PERIOD","COMMA","QUESTION","COLON"]
SYMBOLS=dict(zip(MARK_LABELS,["",".",",","?",":"]))

def punct_encode(tokenizer,row,final_only=False):
    text=row["text"]
    words=list(re.finditer(r"\S+",text))
    encoded=tokenizer(text,return_offsets_mapping=True,truncation=False,verbose=False)
    if len(encoded["input_ids"])>256:
        raise ValueError("Input exceeds 256 tokens including special tokens")
    ends={end:i for i,(start,end) in enumerate(encoded["offset_mapping"]) if start<end}
    gold={s["index"]:s["mark"] for s in row.get("gold",[])}
    positions=[]
    labels=[-100]*len(encoded["input_ids"])
    for i,w in enumerate(words):
        if w.end() not in ends:
            raise ValueError("Tokenizer cannot align this word to its original end")
        pos=ends[w.end()]
        symbol=gold.get(i,"")
        if final_only and i!=len(words)-1:
            symbol=""
        labels[pos]=list(SYMBOLS.values()).index(symbol)
        positions.append((pos,w.end()))
    return {"input_ids":encoded["input_ids"],"labels":labels,"positions":positions}

@torch.inference_mode()
def punctuation(model,tokenizer,rows):
    output={}
    model.eval()
    for start in range(0,len(rows),24):
        group=rows[start:start+24]
        items=[punct_encode(tokenizer,{"text":r["text"]}) for r in group]
        features=batch(items,tokenizer.pad_token_id,model.device)
        del features["labels"]
        indices=model(**features).logits.argmax(-1).cpu().tolist()
        for row,item,ids in zip(group,items,indices):
            text=row["text"]
            pos=0
            parts=[]
            for index,end in item["positions"]:
                symbol=SYMBOLS[model.config.id2label[ids[index]]]
                parts.extend([text[pos:end],symbol if text[end-1] not in ".,?!:;" else ""])
                pos=end
            output[row["id"]]="".join(parts)+text[pos:]
    return output

def pii(model,tokenizer,rows):
    return apply_threshold(predict(model,tokenizer,rows),0.0)

def render_pii(text,spans):
    parts=[]
    end=0
    for s in spans:
        parts.extend([text[end:s["start"]],"["+s["label"]+"]"])
        end=s["end"]
    return "".join(parts)+text[end:]