File size: 3,767 Bytes
971b9e0 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 | """Language metadata: the single source of truth for the 5 study languages.
Design (thesis-plan.txt): EN anchor; DE, FR same-script (Latin); AR, ZH
cross-script. FLORES+ uses `cmn_Hans` for Simplified-Mandarin while
FineWeb2-HQ labels its Chinese subset `cmn_Hani` (Han script umbrella,
overwhelmingly Simplified web text) -- both refer to the same language here.
"""
from dataclasses import dataclass
from itertools import combinations
@dataclass(frozen=True)
class Lang:
code: str # our short code
name: str
script: str # ISO 15924 of the training data
same_script_as_en: bool
flores_code: str # file stem in openlanguagedata/flores_plus (dev/<code>.jsonl)
fineweb_repo: str # HF dataset repo for model-training text (quality-filtered -HQ)
fineweb_subdir: str # language-script config/subdir; also the raw FineWeb2 config
# name used for the (unfiltered) tokenizer-training corpus
LANGS: dict[str, Lang] = {
"en": Lang("en", "English", "Latn", True, "eng_Latn",
"epfml/FineWeb-HQ", "data"),
"de": Lang("de", "German", "Latn", True, "deu_Latn",
"epfml/FineWeb2-HQ", "deu_Latn"),
"fr": Lang("fr", "French", "Latn", True, "fra_Latn",
"epfml/FineWeb2-HQ", "fra_Latn"),
"ar": Lang("ar", "Arabic", "Arab", False, "arb_Arab",
"epfml/FineWeb2-HQ", "arb_Arab"),
"zh": Lang("zh", "Chinese", "Hans", False, "cmn_Hans",
"epfml/FineWeb2-HQ", "cmn_Hani"),
}
ANCHOR = "en"
PARTNERS = ["de", "fr", "ar", "zh"]
# Run matrix: 5 monolingual + all 10 pairwise bilingual mixtures, x 2
# tokenizer conditions. PARTNERS remains the EN-centric BTS comparison set.
MONOLINGUAL_RUNS = [(l,) for l in LANGS]
BILINGUAL_RUNS = list(combinations(LANGS, 2))
# Tokenizers (see xscript.tok.train):
# flavor: "unigram" = SentencePiece Unigram + byte fallback (ATLAS's actual
# algorithm; the faithful replication point)
# "bpe" = byte-level BPE, classical (swiss-ai/parity-aware-bpe
# learn_bpe.py) -- the baseline parity-aware modifies
# "pa" = parity-aware byte-level BPE (same repo), fertility-
# equalized over a multi-parallel FLORES+ dev set
# condition: "starved" = raw FineWeb/FineWeb2, T=100 sampling over ~419
# languages (matching MADLAD-400/ATLAS's scale)
# "destarved" = raw FineWeb/FineWeb2, our 5 languages, byte-
# premium-adjusted
#
# Tokenizer corpus is FineWeb-family (not MADLAD-400) so that it is in the same
# corpus family as the model-training pools (also FineWeb-family) under both
# conditions -- avoiding a tokenizer-corpus-vs-model-corpus domain mismatch
# that could otherwise hit AR/ZH differently than DE/FR. This is a deliberate
# deviation from ATLAS's literal MADLAD-trained tokenizer; see data/tokcorpus.py.
#
# Parity-aware balances a fixed multi-parallel dev set (our 5 languages), so it
# only has a destarved form; a 420-language "starved" version is undefined. It
# is therefore an analysis-only fidelity reference, NOT a model-training flavor
# (the starved-vs-destarved contrast needs a flavor with both conditions).
TOK_FLAVORS = ["unigram", "bpe", "pa"]
TOK_CONDITIONS = ["starved", "destarved"]
MODEL_FLAVORS = ["unigram", "bpe"] # eligible as the model-training tokenizer
def tok_name(flavor: str, condition: str) -> str:
return f"{flavor}_{condition}"
def tok_conditions(flavor: str) -> list[str]:
return ["destarved"] if flavor == "pa" else TOK_CONDITIONS
def all_tok_names() -> list[str]:
return [tok_name(f, c) for f in TOK_FLAVORS for c in tok_conditions(f)]
|