Instructions to use EasyOctopus/german-support-pii-model with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- Transformers
How to use EasyOctopus/german-support-pii-model with Transformers:
# Use a pipeline as a high-level helper from transformers import pipeline pipe = pipeline("token-classification", model="EasyOctopus/german-support-pii-model")# pip install -U transformers accelerate # Load model directly from transformers import AutoTokenizer, AutoModelForTokenClassification tokenizer = AutoTokenizer.from_pretrained("EasyOctopus/german-support-pii-model") model = AutoModelForTokenClassification.from_pretrained("EasyOctopus/german-support-pii-model", device_map="auto") - Notebooks
- Google Colab
- Kaggle
File size: 2,343 Bytes
a75faed | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 | """Shared offline decoding for the standalone releases."""
import re
import torch
from classifier import apply_threshold, batch, predict
MARK_LABELS=["O","PERIOD","COMMA","QUESTION","COLON"]
SYMBOLS=dict(zip(MARK_LABELS,["",".",",","?",":"]))
def punct_encode(tokenizer,row,final_only=False):
text=row["text"]
words=list(re.finditer(r"\S+",text))
encoded=tokenizer(text,return_offsets_mapping=True,truncation=False,verbose=False)
if len(encoded["input_ids"])>256:
raise ValueError("Input exceeds 256 tokens including special tokens")
ends={end:i for i,(start,end) in enumerate(encoded["offset_mapping"]) if start<end}
gold={s["index"]:s["mark"] for s in row.get("gold",[])}
positions=[]
labels=[-100]*len(encoded["input_ids"])
for i,w in enumerate(words):
if w.end() not in ends:
raise ValueError("Tokenizer cannot align this word to its original end")
pos=ends[w.end()]
symbol=gold.get(i,"")
if final_only and i!=len(words)-1:
symbol=""
labels[pos]=list(SYMBOLS.values()).index(symbol)
positions.append((pos,w.end()))
return {"input_ids":encoded["input_ids"],"labels":labels,"positions":positions}
@torch.inference_mode()
def punctuation(model,tokenizer,rows):
output={}
model.eval()
for start in range(0,len(rows),24):
group=rows[start:start+24]
items=[punct_encode(tokenizer,{"text":r["text"]}) for r in group]
features=batch(items,tokenizer.pad_token_id,model.device)
del features["labels"]
indices=model(**features).logits.argmax(-1).cpu().tolist()
for row,item,ids in zip(group,items,indices):
text=row["text"]
pos=0
parts=[]
for index,end in item["positions"]:
symbol=SYMBOLS[model.config.id2label[ids[index]]]
parts.extend([text[pos:end],symbol if text[end-1] not in ".,?!:;" else ""])
pos=end
output[row["id"]]="".join(parts)+text[pos:]
return output
def pii(model,tokenizer,rows):
return apply_threshold(predict(model,tokenizer,rows),0.0)
def render_pii(text,spans):
parts=[]
end=0
for s in spans:
parts.extend([text[end:s["start"]],"["+s["label"]+"]"])
end=s["end"]
return "".join(parts)+text[end:]
|