Instructions to use EasyOctopus/german-dictation-punctuation-model with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- Transformers
How to use EasyOctopus/german-dictation-punctuation-model with Transformers:
# Use a pipeline as a high-level helper from transformers import pipeline pipe = pipeline("token-classification", model="EasyOctopus/german-dictation-punctuation-model")# pip install -U transformers accelerate # Load model directly from transformers import AutoTokenizer, AutoModelForTokenClassification tokenizer = AutoTokenizer.from_pretrained("EasyOctopus/german-dictation-punctuation-model") model = AutoModelForTokenClassification.from_pretrained("EasyOctopus/german-dictation-punctuation-model", device_map="auto") - Notebooks
- Google Colab
- Kaggle
Download classifier.py from EasyOctopus/german-dictation-punctuation-model: direct link, hf CLI and curl.
- Browser
- Download file 3.41 kB
-
https://huggingface.co/EasyOctopus/german-dictation-punctuation-model/resolve/main/classifier.py
- Command line
-
hf download hf://EasyOctopus/german-dictation-punctuation-model/classifier.py
-
curl -L -o classifier.py https://huggingface.co/EasyOctopus/german-dictation-punctuation-model/resolve/main/classifier.py
3.41 kB
| """Subword BIO alignment and decoding with original Unicode offsets.""" | |
| import json | |
| from pathlib import Path | |
| import torch | |
| ROOT = Path(__file__).resolve().parent | |
| def rows(name): | |
| return [json.loads(line) for line in (ROOT/"data"/(name+".jsonl")).read_text().splitlines()] | |
| def encode(tokenizer, row, label_ids=None): | |
| data = tokenizer(row["text"], return_offsets_mapping=True, truncation=True, max_length=256, | |
| return_overflowing_tokens=True, stride=0) | |
| gold = sorted(row.get("gold", []), key=lambda e:e["start"]) | |
| index = 0 | |
| seen = set() | |
| for ids, offsets in zip(data["input_ids"],data["offset_mapping"]): | |
| item = {"input_ids":ids,"offsets":offsets,"id":row["id"]} | |
| if label_ids is not None: | |
| labels = [] | |
| for start, end in offsets: | |
| if start == end: | |
| labels.append(-100) | |
| continue | |
| while index < len(gold) and gold[index]["end"] <= start: | |
| index += 1 | |
| if index < len(gold) and gold[index]["start"] < end: | |
| tag = ("I-" if index in seen else "B-")+gold[index]["label"] | |
| seen.add(index) | |
| else: | |
| tag = "O" | |
| labels.append(label_ids[tag]) | |
| item["labels"] = labels | |
| yield item | |
| def batch(items, pad, device): | |
| length = ((max(len(i["input_ids"]) for i in items)+7)//8)*8 | |
| def field(key, fill): | |
| values = [i[key]+[fill]*(length-len(i[key])) for i in items] | |
| return torch.tensor(values,device=device) | |
| ids = field("input_ids",pad) | |
| inputs = {"input_ids":ids, "attention_mask":torch.tensor([ | |
| [1]*len(i["input_ids"])+[0]*(length-len(i["input_ids"])) for i in items],device=device)} | |
| if "labels" in items[0]: | |
| inputs["labels"] = field("labels",-100) | |
| return inputs | |
| def predict(model, tokenizer, documents, batch_size=24): | |
| windows = [w for r in documents for w in encode(tokenizer,r)] | |
| groups = {r["id"]:[] for r in documents} | |
| previous = {} | |
| model.eval() | |
| for begin in range(0,len(windows),batch_size): | |
| items = windows[begin:begin+batch_size] | |
| logits = model(**batch(items,tokenizer.pad_token_id,model.device)).logits | |
| probs, indices = logits.float().softmax(dim=-1).max(dim=-1) | |
| for w, confidence, tags in zip(items,probs.cpu().tolist(),indices.cpu().tolist()): | |
| rid = w["id"] | |
| for (start,end),score,idx in zip(w["offsets"],confidence,tags): | |
| if start == end: | |
| continue | |
| tag = model.config.id2label[idx] | |
| if tag == "O": | |
| previous[rid] = None | |
| continue | |
| boundary,label = tag.split("-",1) | |
| if boundary == "I" and previous.get(rid) == label: | |
| group = groups[rid][-1] | |
| group["end"] = end | |
| group["scores"].append(score) | |
| else: | |
| groups[rid].append({"start":start,"end":end,"label":label,"scores":[score]}) | |
| previous[rid] = label | |
| return groups | |
| def apply_threshold(groups, threshold): | |
| return {rid:[{k:g[k] for k in ("start","end","label")} for g in spans | |
| if sum(g["scores"])/len(g["scores"]) >= threshold] for rid,spans in groups.items()} | |