german-support-pii-model / classifier.py
EasyOctopus's picture
Release German support experiment and held-out evaluation
a75faed verified
Raw History Blame Contribute Delete
3.41 kB
"""Subword BIO alignment and decoding with original Unicode offsets."""
import json
from pathlib import Path
import torch
ROOT = Path(__file__).resolve().parent
def rows(name):
return [json.loads(line) for line in (ROOT/"data"/(name+".jsonl")).read_text().splitlines()]
def encode(tokenizer, row, label_ids=None):
data = tokenizer(row["text"], return_offsets_mapping=True, truncation=True, max_length=256,
return_overflowing_tokens=True, stride=0)
gold = sorted(row.get("gold", []), key=lambda e:e["start"])
index = 0
seen = set()
for ids, offsets in zip(data["input_ids"],data["offset_mapping"]):
item = {"input_ids":ids,"offsets":offsets,"id":row["id"]}
if label_ids is not None:
labels = []
for start, end in offsets:
if start == end:
labels.append(-100)
continue
while index < len(gold) and gold[index]["end"] <= start:
index += 1
if index < len(gold) and gold[index]["start"] < end:
tag = ("I-" if index in seen else "B-")+gold[index]["label"]
seen.add(index)
else:
tag = "O"
labels.append(label_ids[tag])
item["labels"] = labels
yield item
def batch(items, pad, device):
length = ((max(len(i["input_ids"]) for i in items)+7)//8)*8
def field(key, fill):
values = [i[key]+[fill]*(length-len(i[key])) for i in items]
return torch.tensor(values,device=device)
ids = field("input_ids",pad)
inputs = {"input_ids":ids, "attention_mask":torch.tensor([
[1]*len(i["input_ids"])+[0]*(length-len(i["input_ids"])) for i in items],device=device)}
if "labels" in items[0]:
inputs["labels"] = field("labels",-100)
return inputs
@torch.inference_mode()
def predict(model, tokenizer, documents, batch_size=24):
windows = [w for r in documents for w in encode(tokenizer,r)]
groups = {r["id"]:[] for r in documents}
previous = {}
model.eval()
for begin in range(0,len(windows),batch_size):
items = windows[begin:begin+batch_size]
logits = model(**batch(items,tokenizer.pad_token_id,model.device)).logits
probs, indices = logits.float().softmax(dim=-1).max(dim=-1)
for w, confidence, tags in zip(items,probs.cpu().tolist(),indices.cpu().tolist()):
rid = w["id"]
for (start,end),score,idx in zip(w["offsets"],confidence,tags):
if start == end:
continue
tag = model.config.id2label[idx]
if tag == "O":
previous[rid] = None
continue
boundary,label = tag.split("-",1)
if boundary == "I" and previous.get(rid) == label:
group = groups[rid][-1]
group["end"] = end
group["scores"].append(score)
else:
groups[rid].append({"start":start,"end":end,"label":label,"scores":[score]})
previous[rid] = label
return groups
def apply_threshold(groups, threshold):
return {rid:[{k:g[k] for k in ("start","end","label")} for g in spans
if sum(g["scores"])/len(g["scores"]) >= threshold] for rid,spans in groups.items()}