Instructions to use EasyOctopus/german-support-pii-model with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- Transformers
How to use EasyOctopus/german-support-pii-model with Transformers:
# Use a pipeline as a high-level helper from transformers import pipeline pipe = pipeline("token-classification", model="EasyOctopus/german-support-pii-model")# pip install -U transformers accelerate # Load model directly from transformers import AutoTokenizer, AutoModelForTokenClassification tokenizer = AutoTokenizer.from_pretrained("EasyOctopus/german-support-pii-model") model = AutoModelForTokenClassification.from_pretrained("EasyOctopus/german-support-pii-model", device_map="auto") - Notebooks
- Google Colab
- Kaggle
Download runtime.py from EasyOctopus/german-support-pii-model: direct link, hf CLI and curl.
- Browser
- Download file 2.34 kB
-
https://huggingface.co/EasyOctopus/german-support-pii-model/resolve/main/runtime.py
- Command line
-
hf download hf://EasyOctopus/german-support-pii-model/runtime.py
-
curl -L -o runtime.py https://huggingface.co/EasyOctopus/german-support-pii-model/resolve/main/runtime.py
2.34 kB
| """Shared offline decoding for the standalone releases.""" | |
| import re | |
| import torch | |
| from classifier import apply_threshold, batch, predict | |
| MARK_LABELS=["O","PERIOD","COMMA","QUESTION","COLON"] | |
| SYMBOLS=dict(zip(MARK_LABELS,["",".",",","?",":"])) | |
| def punct_encode(tokenizer,row,final_only=False): | |
| text=row["text"] | |
| words=list(re.finditer(r"\S+",text)) | |
| encoded=tokenizer(text,return_offsets_mapping=True,truncation=False,verbose=False) | |
| if len(encoded["input_ids"])>256: | |
| raise ValueError("Input exceeds 256 tokens including special tokens") | |
| ends={end:i for i,(start,end) in enumerate(encoded["offset_mapping"]) if start<end} | |
| gold={s["index"]:s["mark"] for s in row.get("gold",[])} | |
| positions=[] | |
| labels=[-100]*len(encoded["input_ids"]) | |
| for i,w in enumerate(words): | |
| if w.end() not in ends: | |
| raise ValueError("Tokenizer cannot align this word to its original end") | |
| pos=ends[w.end()] | |
| symbol=gold.get(i,"") | |
| if final_only and i!=len(words)-1: | |
| symbol="" | |
| labels[pos]=list(SYMBOLS.values()).index(symbol) | |
| positions.append((pos,w.end())) | |
| return {"input_ids":encoded["input_ids"],"labels":labels,"positions":positions} | |
| def punctuation(model,tokenizer,rows): | |
| output={} | |
| model.eval() | |
| for start in range(0,len(rows),24): | |
| group=rows[start:start+24] | |
| items=[punct_encode(tokenizer,{"text":r["text"]}) for r in group] | |
| features=batch(items,tokenizer.pad_token_id,model.device) | |
| del features["labels"] | |
| indices=model(**features).logits.argmax(-1).cpu().tolist() | |
| for row,item,ids in zip(group,items,indices): | |
| text=row["text"] | |
| pos=0 | |
| parts=[] | |
| for index,end in item["positions"]: | |
| symbol=SYMBOLS[model.config.id2label[ids[index]]] | |
| parts.extend([text[pos:end],symbol if text[end-1] not in ".,?!:;" else ""]) | |
| pos=end | |
| output[row["id"]]="".join(parts)+text[pos:] | |
| return output | |
| def pii(model,tokenizer,rows): | |
| return apply_threshold(predict(model,tokenizer,rows),0.0) | |
| def render_pii(text,spans): | |
| parts=[] | |
| end=0 | |
| for s in spans: | |
| parts.extend([text[end:s["start"]],"["+s["label"]+"]"]) | |
| end=s["end"] | |
| return "".join(parts)+text[end:] | |