jvonrad commited on
Commit
4642d81
·
verified ·
1 Parent(s): d18b2fd

Upload src/xscript/byte_premium.py with huggingface_hub

Browse files
Files changed (1) hide show
  1. src/xscript/byte_premium.py +108 -0
src/xscript/byte_premium.py ADDED
@@ -0,0 +1,108 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Byte-premium calibration on FLORES+ (thesis-plan step 3).
2
+
3
+ premium(L) = total UTF-8 bytes of L / total UTF-8 bytes of English, summed
4
+ over the *same* parallel sentences. Computed on the exact FLORES+ split(s) we
5
+ use everywhere else, so the calibration is self-consistent and reproducible
6
+ (plan caveat #1). Text is used exactly as distributed (no Unicode
7
+ normalization), matching how the corpora are consumed downstream.
8
+
9
+ We also fetch Arnett/Chang/Bergen's precomputed premiums
10
+ (github.com/catherinearnett/byte-premium-tool) as an external sanity check
11
+ (plan caveat #2); ours drive the pipeline.
12
+ """
13
+ import json
14
+ import urllib.request
15
+ from pathlib import Path
16
+
17
+ from . import flores
18
+ from .langs import LANGS, ANCHOR
19
+ from .paths import RESULTS, ensure
20
+
21
+ ARNETT_REPO = "catherinearnett/byte-premium-tool"
22
+ # ISO 639-3 codes used by the Arnett tool for our languages.
23
+ ARNETT_CODES = {"en": "eng", "de": "deu", "fr": "fra", "ar": "arb", "zh": "cmn"}
24
+
25
+
26
+ def compute(splits=("dev", "devtest")) -> dict:
27
+ langs = list(LANGS)
28
+ per_split = {}
29
+ tot = {l: 0 for l in langs}
30
+ for split in splits:
31
+ par = flores.load_parallel(langs, split)
32
+ n = len(par[ANCHOR])
33
+ b = {l: sum(len(t.encode("utf-8")) for t in par[l]) for l in langs}
34
+ per_split[split] = {
35
+ "n_sentences": n,
36
+ "bytes": b,
37
+ "premium": {l: b[l] / b[ANCHOR] for l in langs},
38
+ }
39
+ for l in langs:
40
+ tot[l] += b[l]
41
+ return {
42
+ "per_split": per_split,
43
+ "bytes_total": tot,
44
+ "premium": {l: tot[l] / tot[ANCHOR] for l in langs},
45
+ }
46
+
47
+
48
+ def fetch_arnett() -> dict | None:
49
+ """Best-effort fetch of published premiums for comparison (not pipeline-critical)."""
50
+ api = f"https://api.github.com/repos/{ARNETT_REPO}/contents/"
51
+ try:
52
+ with urllib.request.urlopen(api, timeout=30) as r:
53
+ listing = json.load(r)
54
+ tables = [e for e in listing if e["name"].endswith((".csv", ".tsv"))]
55
+ for e in tables:
56
+ with urllib.request.urlopen(e["download_url"], timeout=60) as r:
57
+ text = r.read().decode("utf-8", errors="replace")
58
+ sep = "\t" if e["name"].endswith(".tsv") else ","
59
+ rows = [ln.split(sep) for ln in text.splitlines() if ln.strip()]
60
+ header = [h.strip().lower() for h in rows[0]]
61
+ # find a language-code column and a premium/ratio column
62
+ code_col = next((i for i, h in enumerate(header)
63
+ if h in ("language", "lang", "iso", "code", "language_code")), 0)
64
+ val_cols = [i for i, h in enumerate(header) if "premium" in h or "ratio" in h]
65
+ if not val_cols:
66
+ continue
67
+ found = {}
68
+ for row in rows[1:]:
69
+ cell = row[code_col].strip().lower()
70
+ for ours, iso in ARNETT_CODES.items():
71
+ if cell == iso or cell.startswith(iso + "_"):
72
+ try:
73
+ found.setdefault(ours, float(row[val_cols[0]]))
74
+ except (ValueError, IndexError):
75
+ pass
76
+ if len(found) >= 4:
77
+ return {"source_file": e["name"], "premium": found}
78
+ except Exception as exc: # network/GitHub issues must not block calibration
79
+ print(f"[byte_premium] Arnett tool fetch failed ({exc}); skipping comparison")
80
+ return None
81
+
82
+
83
+ def run(out_dir: Path | None = None) -> dict:
84
+ out_dir = ensure(Path(out_dir) if out_dir else RESULTS / "byte_premium")
85
+ res = compute()
86
+ res["arnett_comparison"] = fetch_arnett()
87
+ with open(out_dir / "byte_premiums.json", "w") as f:
88
+ json.dump(res, f, indent=2)
89
+
90
+ lines = ["| lang | premium (dev) | premium (devtest) | premium (combined) | Arnett et al. |",
91
+ "|---|---|---|---|---|"]
92
+ arn = (res["arnett_comparison"] or {}).get("premium", {})
93
+ for l in LANGS:
94
+ lines.append(
95
+ f"| {l} | {res['per_split']['dev']['premium'][l]:.4f} "
96
+ f"| {res['per_split']['devtest']['premium'][l]:.4f} "
97
+ f"| {res['premium'][l]:.4f} "
98
+ f"| {arn.get(l, float('nan')):.4f} |")
99
+ (out_dir / "byte_premiums.md").write_text("\n".join(lines) + "\n")
100
+ print("\n".join(lines))
101
+ return res
102
+
103
+
104
+ def load_premiums() -> dict[str, float]:
105
+ path = RESULTS / "byte_premium" / "byte_premiums.json"
106
+ if not path.exists():
107
+ raise FileNotFoundError("run `xscript byte-premium` first")
108
+ return json.loads(path.read_text())["premium"]