File size: 4,626 Bytes
4079ffe a858237 4079ffe | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 | """Reproduce the tables in this model card.
pip install transformers datasets
python benchmark.py # both tables
python benchmark.py paradigms # UniMorph only
python benchmark.py fertility # corpora only
Everything it reads is public and pinned: a UniMorph commit, five Hugging Face datasets at a fixed
document count, and five published tokenizers. No part of the pipeline that produced this
vocabulary is involved, which is the point -- the numbers are meant to be checkable by someone who
has never seen it.
"""
import re
import sys
import unicodedata
import urllib.request
from collections import defaultdict
from datasets import load_dataset
from transformers import AutoTokenizer
TOKENIZERS = (
"picur/picur-tokenizer",
"NYTK/PULI-GPT-3SX",
"SZTAKI-HLT/hubert-base-cc",
"Qwen/Qwen3-8B",
"FacebookAI/xlm-roberta-base",
)
UNIMORPH = ("https://raw.githubusercontent.com/unimorph/hun/"
"20d5e04211bbab3dd689c99637356a8eab7c5fd7/hun")
LEMMAS = 20_000
CORPORA = (
("lazos/gutenberg-hu", None, "data", "text"),
("lazos/gazette-hu", None, "data", "text"),
("SZTAKI-HLT/HunSum-2-abstractive", None, "data", "article"),
("open-web-math/open-web-math", None, "data", "text"),
("bigcode/starcoderdata", None, "python", "content"),
)
DOCUMENTS = 2_000
MIN_CHARS = 200
WORD = re.compile(r"\w+", re.UNICODE)
def load():
found = {}
for name in TOKENIZERS:
try:
found[name] = AutoTokenizer.from_pretrained(name)
except Exception as error: # a gated model is skipped, not fatal
print(f"skip {name}: {type(error).__name__}", file=sys.stderr)
return found
def paradigms():
"""Does an inflected form begin with its lemma, and is that lemma a single row?"""
with urllib.request.urlopen(UNIMORPH, timeout=120) as handle:
raw = handle.read().decode("utf-8")
table = defaultdict(list)
for line in raw.splitlines():
parts = line.split("\t")
if len(parts) < 3 or not parts[0] or not parts[1]:
continue
if len(table) >= LEMMAS and parts[0] not in table:
continue
if parts[1] != parts[0]:
table[parts[0]].append(parts[1])
pairs = sum(len(v) for v in table.values())
print(f"\nUniMorph hun: {len(table):,} lemmas, {pairs:,} lemma/form pairs")
print(f"{'tokenizer':32s} {'stem row':>9s} {'prefix':>8s} {'lemma=1':>8s} {'tok/form':>9s}")
for name, tok in load().items():
cache = {}
def cut(word, tok=tok, cache=cache):
if word not in cache:
cache[word] = tuple(tok.tokenize(" " + word))
return cache[word]
rows = shared = whole = cost = 0
for lemma, forms in table.items():
head = cut(lemma)
entire = len(head) == 1
whole += entire
for form in forms:
piece = cut(form)
keeps = len(piece) >= len(head) and piece[:len(head)] == head
shared += keeps
rows += keeps and entire
cost += len(piece)
print(f"{name:32s} {rows/pairs:8.1%} {shared/pairs:8.1%} "
f"{whole/len(table):8.1%} {cost/pairs:9.2f}")
def fertility():
"""Characters a token buys, on a fixed prefix of each dataset."""
loaded = load()
for repo, config, folder, column in CORPORA:
stream = load_dataset(repo, config, data_dir=folder, split="train", streaming=True)
texts = []
for row in stream:
text = (row.get(column) or "").strip()
if len(text) >= MIN_CHARS:
texts.append(unicodedata.normalize("NFC", text))
if len(texts) >= DOCUMENTS:
break
chars = sum(map(len, texts))
words = [w for t in texts for w in WORD.findall(t)]
print(f"\n{repo} {len(texts):,} documents, {chars/1e6:.2f} Mchar")
for name, tok in loaded.items():
total = sum(len(tok.encode(t, add_special_tokens=False)) for t in texts)
cost = {w: len(tok.encode(" " + w, add_special_tokens=False)) for w in set(words)}
counts = [cost[w] for w in words]
print(f" {name:32s} {chars/total:6.3f} chars/token "
f"{sum(counts)/len(counts):4.2f} pieces/word "
f"{sum(n >= 4 for n in counts)/len(counts):5.1%} need 4+")
if __name__ == "__main__":
wanted = sys.argv[1] if len(sys.argv) > 1 else "all"
if wanted in ("all", "paradigms"):
paradigms()
if wanted in ("all", "fertility"):
fertility()
|