File size: 2,316 Bytes
fef161a | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 | """FLORES+ (openlanguagedata/flores_plus) loading.
FLORES+ is gated ("auto"): accept the terms on Hugging Face once and export
HF_TOKEN. Files are per-language jsonl: dev/<code>.jsonl, devtest/<code>.jsonl.
We key sentences by their FLORES `id` and align across languages on the
intersection of ids, so byte premiums and retrieval eval always compare the
same parallel content.
"""
import json
from pathlib import Path
from .langs import LANGS
from .paths import FLORES_DIR, ensure
REPO_ID = "openlanguagedata/flores_plus"
SPLITS = ("dev", "devtest")
_TEXT_KEYS = ("text", "sentence")
_ID_KEYS = ("id", "sentence_id")
def download(langs=None, splits=SPLITS, token=None) -> None:
from huggingface_hub import hf_hub_download
langs = langs or list(LANGS)
ensure(FLORES_DIR)
for lc in langs:
code = LANGS[lc].flores_code
for split in splits:
hf_hub_download(
repo_id=REPO_ID, repo_type="dataset",
filename=f"{split}/{code}.jsonl",
local_dir=FLORES_DIR, token=token,
)
def _pick(d: dict, keys):
for k in keys:
if k in d:
return d[k]
raise KeyError(f"none of {keys} in FLORES+ record with keys {sorted(d)}")
def load(lang: str, split: str = "dev") -> dict[int, str]:
"""Return {sentence_id: text} for one language/split."""
path = FLORES_DIR / split / f"{LANGS[lang].flores_code}.jsonl"
if not path.exists():
raise FileNotFoundError(
f"{path} missing - run `xscript flores-download` (requires HF_TOKEN "
f"with accepted terms for {REPO_ID})")
out = {}
with open(path, encoding="utf-8") as f:
for line in f:
if not line.strip():
continue
rec = json.loads(line)
out[int(_pick(rec, _ID_KEYS))] = str(_pick(rec, _TEXT_KEYS))
return out
def load_parallel(langs, split: str = "dev") -> dict[str, list[str]]:
"""Aligned parallel sentences: same order, intersection of ids."""
per_lang = {l: load(l, split) for l in langs}
common = sorted(set.intersection(*(set(v) for v in per_lang.values())))
if not common:
raise RuntimeError(f"no common sentence ids across {langs} ({split})")
return {l: [per_lang[l][i] for i in common] for l in langs}
|