# Reproduce v0.1 (CPU, <3 GB RAM per step, ~2.5 h wall on 8 cores; everything seeded) ```bash uv venv .venv && uv pip install -e ".[train,test]" "pyarrow==19.0.1" "numpy<2" # 1. data (network): Wikipedia paragraphs, FineWeb-2 samples, Tatoeba per-language exports (data/raw/tatoeba/.tsv.bz2) python scripts/fetch_wiki.py pag,cbk-zam,tl,ilo,war,pam,bcl,ceb,en,id,ms,es,jv,su,min,ban,ace,bug,mad,vi,pt,fr,de,ja,zh,ko,th,ar,hi,ru,la,it,nl,pl,tr,sw,mg,haw,sm,mi,fj,to,tet 30000000 python scripts/fetch_fw2.py hil_Latn,war_Latn,ilo_Latn,bcl_Latn,pam_Latn,pag_Latn,cbk_Latn,tsg_Latn,krj_Latn,ceb_Latn,fil_Latn,ind_Latn,zsm_Latn,spa_Latn,eng_Latn 60000000 # 2. corpus (tiers R/W/G, domain-held-out splits), filters, subsets, helper lists python scripts/build_corpus.py && python scripts/filter_corpus.py && python scripts/prep_train_subset.py python scripts/make_names.py && python scripts/make_eng_pure.py && python scripts/induce_lexicon.py # 3. token model (final config: no context pooling, English purification) ~32 min python scripts/train_token.py --n 800000 --epochs 2 --out models/tok_v2 --workers 4 --ctx none --purify python scripts/export_model.py models/tok_v2 && cp data/work/induced_lexicon.json models/tok_v2/lexicon.json # 4. head + conformal (synthetic docs from dev/test splits) for s in "dev 24000 31 h3_train" "dev 8000 32 h3_calib" "test 8000 33 h3_test"; do set -- $s; python scripts/make_head_data.py models/tok_v2/model_int8.npz $1 $2 $3 data/work/$4.npz; done python scripts/train_head.py data/work/h3_train.npz data/work/h3_calib.npz data/work/h3_test.npz models/ourlid-0.1 0 cp models/tok_v2/model_int8.npz models/tok_v2/lexicon.json models/ourlid-0.1/ # 5. benchmark + evaluation python scripts/build_benchmark.py # eval/phlid_hard_v0.jsonl (seed 20261010) python eval/run_predictor.py models/ourlid-0.1 eval/phlid_hard_v0.jsonl hard ourlid python eval/run_baselines.py eval/phlid_hard_v0.jsonl hard glotlid halolid openlid3 lid176 lid201 lingua langid # needs baseline models (their own licences) python eval/final_table.py && python eval/abstain_report.py pytest -q tests ``` Ablations: `scripts/run_ablation.sh ` (320k items, 1 epoch) -> `eval/ablations.tsv`, `eval/compare.py` for paired bootstrap CIs. Private sets (`eval/private/*`) are NOT part of the package; the public equivalents are the benchmark and the Wikipedia held-out set (`eval/wiki_test_v0.jsonl`, built in the session from the Tier-R test split). Known reproducibility gaps: Tatoeba download step is manual; FineWeb-2 sample order depends on the Hub's shard order; PyTorch SparseAdam is deterministic per seed on CPU but thread count (3) is part of the config.