| """Central path layout. |
| |
| Everything heavy lives on Lustre scratch (override with XSCRIPT_SCRATCH). |
| Small, committable outputs (tables, metrics) go to <repo>/results. |
| """ |
| import os |
| from pathlib import Path |
|
|
| REPO = Path(__file__).resolve().parents[2] |
|
|
| |
| _user = os.environ.get("USER", "user") |
| _project = _user.rsplit(".", 1)[-1] |
| SCRATCH = Path(os.environ.get("XSCRIPT_SCRATCH", f"/scratch/{_project}/{_user}/xscript")) |
|
|
| RESULTS = Path(os.environ.get("XSCRIPT_RESULTS", REPO / "results")) |
|
|
| |
| FLORES_DIR = SCRATCH / "flores_plus" |
| MANIFEST_CACHE = SCRATCH / "manifests" |
| TOK_CORPORA = SCRATCH / "tok_corpora" |
| TOKENIZERS = SCRATCH / "tokenizers" |
| POOLS = SCRATCH / "pools" |
| SHARDS = SCRATCH / "shards" |
| RUNS = SCRATCH / "runs" |
| HOLDOUT = SCRATCH / "holdout" |
|
|
|
|
| def ensure(p: Path) -> Path: |
| p.mkdir(parents=True, exist_ok=True) |
| return p |
|
|
|
|
| def tokenizer_dir(name: str) -> Path: |
| return TOKENIZERS / name |
|
|
|
|
| def pool_dir(lang: str) -> Path: |
| return POOLS / lang |
|
|
|
|
| def shard_dir(lang: str, tok: str) -> Path: |
| return SHARDS / f"{lang}__{tok}" |
|
|
|
|
| def run_dir(run_name: str) -> Path: |
| return RUNS / run_name |
|
|
