Delete hf_team_repo

hf_team_repo/README.md

@@ -1,17 +0,0 @@
-# team-ape-llama: SinhalaMMLU shared task (ICTer 2026)
-
-- **Code repo (this):** https://huggingface.co/SinhalaMMLU-SharedTask/team-ape-llama
-- **Adapter repo:** https://huggingface.co/SinhalaMMLU-SharedTask/team-ape-llama-gemma4-e4b-lora
-- **Run:** `pip install -r requirements.txt && python inference.py --input <questions folder or .jsonl> --output predictions.jsonl` (`--perm 1` = single pass)
-
-- **Base model:** `google/gemma-4-e4b-it` (text-only load; per-layer embeddings counted): 7.480 B parameters, loaded in bf16.
-- **Adapter (single LoRA):** 17,440,768 parameters (r=8, all attention + MLP projections). Repo: https://huggingface.co/SinhalaMMLU-SharedTask/team-ape-llama-gemma4-e4b-lora.
-- **Total inference parameters:** about 7.498 B (<= 8 B). One base model + one adapter. No ensemble, no router, no retrieval, no internet, no closed API at inference.
-- **Prompt:** Sinhala instruction (`si_basic`), zero-shot, logit scoring over the option labels 1-4.
-- **perm4 (what we submitted):** deterministic option rotation: the same question is scored 4 times with the options in a fixed rotated order and the scores are summed. Same model, same adapter, no extra parameters; no sampling (`do_sample=False`), fixed seed.
-- **Codabench:** Easy 52.0249 / Medium 49.2349 / Hard 57.7320 / Macro 52.9973 (`predictions.jsonl` in this repo is the exact file that was uploaded).
-- **Hardware / runtime:** 1x NVIDIA A100 (Google Colab); test set (2,238 questions) scored in about 285 s with perm4, plus model download.
-- **Extra data used:** see `training/README_training.md`.
-- **Training:** `training/`.
-- **Full code and MANIFEST:** private GitHub repo, tag `final-oct28` (branch `final/oct28`), configs `configs/sub_run2_bf16_perm4.yaml` (and `..._perm1.yaml`).
-- **Citation:** Pramodya et al., *SinhalaMMLU: A Comprehensive Benchmark for Evaluating Multitask Language Understanding in Sinhala*, EMNLP 2025, pp. 32943-32961.

hf_team_repo/configs/final_bf16_perm4.yaml

@@ -1,16 +0,0 @@
-# Oct 28 submission probe: Gemma-4-E4B-it + Run 2 adapter (ape-llama-sinmmlu/ft_gemma4_e4b_real_plus_exp), text-only, si_basic, fewshot 0.
-# NO retrieval / router / ensemble. Single model + single adapter. Only dtype (bf16) and permutations (4) differ between probes.
-model:
-  name_or_path: google/gemma-4-e4b-it
-  load_in_4bit: false
-  dtype: bfloat16
-  text_only: true
-  adapter_path: __ADAPTER_DIR__  # replaced by inference.py with the downloaded adapter repo
-prompt:
-  template: si_basic
-  fewshot_k: 0
-  fewshot_file: null
-inference:
-  batch_size: 8
-  max_length: 1024
-  permutations: 4

hf_team_repo/inference.py

@@ -1,59 +0,0 @@
-"""Reproduces our final Codabench submission (Gemma-4-E4B-it + one LoRA adapter, bf16, deterministic).
-
-Everything is downloaded from Hugging Face inside this script (base model, adapter, optional test data).
-No local paths, no retrieval, no internet other than Hugging Face, no closed API.
-
-Usage (fresh Colab / Kaggle, 1 GPU with >= 24 GB):
-    pip install -r requirements.txt
-    python inference.py --input <folder or .jsonl with the questions> --output predictions.jsonl
-    # or, if the organisers host the questions on Hugging Face: set DATA_REPO below (or pass --data_repo)
-
-perm4 = deterministic option rotation (same model + same adapter, 4 scoring passes, scores summed; no extra parameters,
-no sampling, fixed seed). Use --perm 1 for the single-pass variant.
-"""
-import argparse
-import subprocess
-import sys
-import tempfile
-from pathlib import Path
-
-from huggingface_hub import snapshot_download
-
-# ---- team settings -------------------------------------------------------------------------------------------
-BASE_MODEL = "google/gemma-4-e4b-it"                                    # downloaded by transformers from the Hub
-MODEL_ID = "SinhalaMMLU-SharedTask/team-ape-llama-gemma4-e4b-lora"      # our adapter repo (weights + adapter_config.json)
-DATA_REPO = None                                                         # organisers: HF dataset repo with the questions
-# --------------------------------------------------------------------------------------------------------------
-
-HERE = Path(__file__).resolve().parent
-
-
-def main() -> int:
-    ap = argparse.ArgumentParser()
-    ap.add_argument("--input", help="folder (easy/medium/hard/*.json) or .jsonl with the questions")
-    ap.add_argument("--data_repo", default=DATA_REPO, help="HF dataset repo with the questions (used when --input is not given)")
-    ap.add_argument("--output", default="predictions.jsonl")
-    ap.add_argument("--perm", type=int, default=4, choices=[1, 4], help="4 = option rotation (submitted); 1 = single pass")
-    args = ap.parse_args()
-
-    if args.input:
-        questions = args.input
-    elif args.data_repo:
-        questions = snapshot_download(args.data_repo, repo_type="dataset")
-    else:
-        sys.exit("give --input or set DATA_REPO")
-
-    adapter_dir = snapshot_download(MODEL_ID)  # adapter_config.json + adapter_model.safetensors
-    cfg = (HERE / "configs" / "final_bf16_perm4.yaml").read_text(encoding="utf-8")
-    cfg = cfg.replace("__ADAPTER_DIR__", adapter_dir).replace("permutations: 4", f"permutations: {args.perm}")
-    with tempfile.NamedTemporaryFile("w", suffix=".yaml", delete=False, encoding="utf-8") as f:
-        f.write(cfg)
-        cfg_path = f.name
-
-    cmd = [sys.executable, str(HERE / "predict.py"), "--config", cfg_path, "--input", questions, "--output", args.output]
-    print("running:", " ".join(cmd))
-    return subprocess.call(cmd, cwd=str(HERE))
-
-
-if __name__ == "__main__":
-    raise SystemExit(main())

hf_team_repo/predict.py

@@ -1,98 +0,0 @@
-"""Answer every question in a file and write predictions.jsonl.
-
-Examples
---------
-# baseline on the dev held-out split, with scores logged to results/results.csv
-python predict.py --config configs/qwen25_7b.yaml --input data/dev_heldout.jsonl --evaluate --who Chamani
-
-# test predictions + the zip Codabench expects
-python predict.py --config configs/final.yaml --input data/test.jsonl --zip
-
-# quick check that everything runs (first 5 questions only)
-python predict.py --config configs/smoke_test.yaml --input data/sample/sample_dev.jsonl --limit 5
-"""
-
-from __future__ import annotations
-
-import argparse
-import sys
-import time
-from pathlib import Path
-
-from sinmmlu.config import load_config
-from sinmmlu.data import has_gold, load_questions, prediction_row, write_jsonl
-from sinmmlu.metrics import accuracy_by_level, format_report
-from sinmmlu.modeling import count_parameters, describe_budget, load_model, load_tokenizer, set_determinism
-from sinmmlu.runinfo import append_result, write_run_info
-from sinmmlu.scoring import Scorer
-
-
-def main(argv=None) -> int:
-    ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
-    ap.add_argument("--config", required=True)
-    ap.add_argument("--input", required=True, help="jsonl of questions (dev, held-out or test)")
-    ap.add_argument("--output", help="default: outputs/<config name>/<input name>.predictions.jsonl")
-    ap.add_argument("--limit", type=int, help="only the first N questions (debugging)")
-    ap.add_argument("--evaluate", action="store_true", help="input has gold answers: print + log scores")
-    ap.add_argument("--zip", action="store_true", help="also write predictions.zip for Codabench")
-    ap.add_argument("--who", default="", help="your name, for the results log")
-    ap.add_argument("--note", default="", help="free text for the results log")
-    args = ap.parse_args(argv)
-
-    cfg = load_config(args.config)
-    set_determinism(cfg["seed"])
-
-    records = load_questions(args.input, cfg["data"]["answer_base"])
-    if args.limit:
-        records = records[: args.limit]
-    print(f"[data] {len(records)} questions from {args.input}")
-
-    pool = []
-    if cfg["prompt"]["fewshot_k"] > 0:
-        if not cfg["prompt"]["fewshot_file"]:
-            sys.exit("prompt.fewshot_k > 0 needs prompt.fewshot_file (a dev TRAIN split)")
-        if Path(cfg["prompt"]["fewshot_file"]).resolve() == Path(args.input).resolve():
-            sys.exit("few-shot file must not be the file being predicted")
-        pool = load_questions(cfg["prompt"]["fewshot_file"], cfg["data"]["answer_base"])
-        if not has_gold(pool):
-            sys.exit("few-shot file has no gold answers")
-
-    t0 = time.time()
-    tok = load_tokenizer(cfg["model"])
-    model = load_model(cfg["model"])
-    n_params = count_parameters(model)
-    print(f"[model] {cfg['model']['name_or_path']} : {describe_budget(n_params)}")
-    if cfg["model"]["load_in_4bit"]:
-        print("[model] 4-bit: count above is understated; use tools/count_params.py for the manifest")
-
-    scorer = Scorer(model, tok, cfg, pool)
-    preds = scorer.predict(records)
-    elapsed = time.time() - t0
-
-    base = int(cfg["output"]["answer_base"])
-    rows = [prediction_row(r, str(p + base)) for r, p in zip(records, preds)]
-    out = Path(args.output) if args.output else Path("outputs") / cfg["name"] / f"{Path(args.input).stem}.predictions.jsonl"
-    write_jsonl(rows, out)
-    print(f"[out] {len(rows)} predictions -> {out}  ({elapsed:.0f}s)")
-
-    extra = {"input": args.input, "n": len(records), "seconds": round(elapsed, 1), "loaded_params": n_params}
-    if args.evaluate:
-        if not has_gold(records):
-            sys.exit("--evaluate: input file has no gold answers")
-        result = accuracy_by_level([r["level"] for r in records], preds, [r["gold"] for r in records])
-        print(format_report(result))
-        if not args.limit:
-            append_result(cfg, args.input, result, args.who, args.note)
-            print("[log] appended to results/results.csv")
-        extra["result"] = result
-    write_run_info(out.with_suffix(".run.json"), cfg, extra)
-
-    if args.zip:
-        from tools.make_submission import make_zip
-
-        make_zip(out, out.parent / "predictions.zip", expected=records)
-    return 0
-
-
-if __name__ == "__main__":
-    raise SystemExit(main())

hf_team_repo/requirements.txt

@@ -1,10 +0,0 @@
-transformers==5.18.0
-peft==0.21.2
-accelerate==1.15.0
-huggingface_hub==1.33.0
-safetensors==0.8.0
-pyyaml==6.0.3
-tqdm
-numpy
-# torch: use the CUDA build that matches your machine (tested with torch 2.x on an A100).
-# If peft raises "ImportError: incompatible version of torchao" run: pip uninstall -y torchao

hf_team_repo/sinmmlu/__init__.py

@@ -1,8 +0,0 @@
-"""Sinhala MMLU shared task (ICTer 2026) - shared library code.
-
-Every entry script (predict.py, evaluate.py, train.py, tools/*) imports from
-here, so training and inference always use exactly the same data loading and
-prompt building.
-"""
-
-__version__ = "0.1.0"

hf_team_repo/sinmmlu/config.py

@@ -1,89 +0,0 @@
-"""Load experiment configs (YAML) with defaults.
-
-One YAML file = one experiment. To try something new, copy a config and
-change it; don't edit the Python code. That keeps every leaderboard
-submission reproducible from (git commit + config file).
-"""
-
-from __future__ import annotations
-
-import copy
-from pathlib import Path
-from typing import Any
-
-import yaml
-
-DEFAULTS: dict[str, Any] = {
-    "name": None,  # defaults to the config file name
-    "seed": 42,
-    "model": {
-        "name_or_path": None,  # Hugging Face id or local folder
-        "revision": None,  # pin a HF commit hash for the final run
-        "dtype": "auto",  # auto | bfloat16 | float16 | float32
-        "load_in_4bit": False,  # needs bitsandbytes + GPU
-        "adapter_path": None,  # LoRA adapter folder (from train.py)
-        "trust_remote_code": False,
-        # multimodal checkpoints (Gemma 4): build only the text model, so vision/audio encoders are never loaded
-        # (organisers: unused encoders do not count towards 8B). Fails if any text weight is missing.
-        "text_only": False,
-    },
-    "prompt": {
-        "template": "si_basic",  # see sinmmlu/prompts.py TEMPLATES
-        "fewshot_k": 0,  # number of solved examples in the prompt
-        "fewshot_file": None,  # jsonl with gold answers (dev TRAIN split only)
-        "use_chat_template": "auto",  # auto | true | false
-        "chat_template_kwargs": {},  # e.g. {enable_thinking: false} for Qwen3
-    },
-    "inference": {
-        "batch_size": 4,
-        "max_length": 2048,  # prompts longer than this are left-truncated
-        "permutations": 1,  # >1 = average over cyclic option orders (debias)
-        "option_bias": None,  # optional [b1..b4] added to the option log-probs (position-bias correction, tools/calibrate.py)
-    },
-    "data": {
-        "answer_base": "auto",  # gold labels: auto | 0 | 1
-    },
-    "output": {
-        "answer_base": 1,  # predictions.jsonl uses "1".."4" (check with organisers)
-    },
-    "train": {
-        "train_file": None,
-        "eval_file": None,
-        "output_dir": "outputs/adapter",
-        "epochs": 2,
-        "learning_rate": 2.0e-4,
-        "batch_size": 2,
-        "grad_accum": 8,
-        "lora_r": 16,
-        "lora_alpha": 32,
-        "lora_dropout": 0.05,
-        "target_modules": ["q_proj", "k_proj", "v_proj", "o_proj"],
-        "gradient_checkpointing": True,
-        "rotate_options": True,  # move the right answer to every position equally
-        "logging_steps": 10,
-    },
-}
-
-
-def _merge(base: dict, override: dict) -> dict:
-    out = copy.deepcopy(base)
-    for key, value in (override or {}).items():
-        if isinstance(value, dict) and isinstance(out.get(key), dict):
-            out[key] = _merge(out[key], value)
-        else:
-            out[key] = value
-    return out
-
-
-def load_config(path: str | Path, overrides: dict | None = None) -> dict:
-    path = Path(path)
-    with path.open(encoding="utf-8") as f:
-        user_cfg = yaml.safe_load(f) or {}
-    cfg = _merge(DEFAULTS, user_cfg)
-    if overrides:
-        cfg = _merge(cfg, overrides)
-    if not cfg["name"]:
-        cfg["name"] = path.stem
-    if not cfg["model"]["name_or_path"]:
-        raise ValueError(f"{path}: model.name_or_path is required")
-    return cfg

hf_team_repo/sinmmlu/data.py

@@ -1,221 +0,0 @@
-"""Read question files and normalise them to one internal format.
-
-The official dev/test format is released on Oct 5. Until then we accept the
-common field names below, so only this file needs adjusting if the real
-format differs.
-
-Internal record (a dict):
-    id        str
-    level     "easy" | "medium" | "hard"   (lower-cased)
-    question  str
-    options   list[str]
-    gold      int | None    0-based index of the correct option (None for test)
-"""
-
-from __future__ import annotations
-
-import json
-from pathlib import Path
-
-QUESTION_KEYS = ("question", "q", "prompt", "stem")
-OPTIONS_KEYS = ("options", "choices", "answers", "candidates")
-GOLD_KEYS = ("answer", "label", "gold", "correct", "answer_idx")
-LEVEL_KEYS = ("level", "difficulty")
-LETTERS = "ABCDEFGH"
-
-
-def read_jsonl(path: str | Path) -> list[dict]:
-    """Read JSON Lines, or a single JSON array (the official dev file is an array).
-
-    A folder is also accepted: the official TEST release is a folder tree
-    (easy/ medium/ hard/, one JSON array per subject). All *.json / *.jsonl
-    files under it are read in sorted path order and concatenated.
-    """
-    if Path(path).is_dir():
-        files = sorted(
-            f for f in Path(path).rglob("*")
-            if f.suffix in (".json", ".jsonl") and not any(part.startswith(".") for part in f.parts)
-        )
-        if not files:
-            raise ValueError(f"{path}: folder has no .json/.jsonl files")
-        rows = []
-        for f in files:
-            rows.extend(read_jsonl(f))
-        return rows
-    text = Path(path).read_text(encoding="utf-8-sig")
-    if text.lstrip().startswith("["):
-        try:
-            data = json.loads(text)
-        except json.JSONDecodeError as e:
-            raise ValueError(f"{path}: invalid JSON array ({e})") from e
-        if not all(isinstance(r, dict) for r in data):
-            raise ValueError(f"{path}: expected a JSON array of objects")
-        return data
-    rows = []
-    with Path(path).open(encoding="utf-8-sig") as f:
-        for line_no, line in enumerate(f, 1):
-            line = line.strip()
-            if not line:
-                continue
-            try:
-                rows.append(json.loads(line))
-            except json.JSONDecodeError as e:
-                raise ValueError(f"{path}:{line_no}: invalid JSON ({e})") from e
-    return rows
-
-
-def write_jsonl(rows: list[dict], path: str | Path) -> None:
-    path = Path(path)
-    path.parent.mkdir(parents=True, exist_ok=True)
-    with path.open("w", encoding="utf-8", newline="\n") as f:
-        for row in rows:
-            f.write(json.dumps(row, ensure_ascii=False) + "\n")
-
-
-def level_of(row: dict):
-    """Difficulty label: top-level level/difficulty, or metadata.difficulty (official format)."""
-    value = _first(row, LEVEL_KEYS)
-    if value is None and isinstance(row.get("metadata"), dict):
-        value = _first(row["metadata"], LEVEL_KEYS)
-    return value
-
-
-def row_ids(rows: list[dict]) -> list[str]:
-    """Stable ids: 'id' if present, else 'q_no' when unique, else the row position."""
-    if all("id" in r for r in rows):
-        return [str(r["id"]) for r in rows]
-    qn = [r.get("q_no") for r in rows]
-    if all(q is not None for q in qn) and len(set(qn)) == len(qn):
-        return [str(q) for q in qn]
-    return [str(r["id"]) if "id" in r else f"row_{i:05d}" for i, r in enumerate(rows)]
-
-
-def _first(row: dict, keys: tuple[str, ...]):
-    for key in keys:
-        if key in row and row[key] is not None:
-            return row[key]
-    return None
-
-
-def _options(row: dict) -> list[str]:
-    opts = _first(row, OPTIONS_KEYS)
-    if isinstance(opts, dict):  # {"1": "...", "2": "..."} or {"A": ...}
-        opts = [opts[k] for k in sorted(opts)]
-    if opts is None:  # option1..option4 / option_a.. / A..D columns
-        for pattern in ("option{}", "option_{}", "opt{}", "choice{}"):
-            found = [row.get(pattern.format(i)) for i in range(1, 9)]
-            found = [o for o in found if o is not None]
-            if found:
-                opts = found
-                break
-    if opts is None:
-        found = [row.get(c) for c in LETTERS if row.get(c) is not None]
-        opts = found or None
-    if not opts:
-        raise ValueError(f"no options found in record: {list(row)}")
-    return [str(o).strip() for o in opts]
-
-
-def _raw_gold(value):
-    """Return gold as an int (base not yet known) or a letter index marker."""
-    if value is None:
-        return None
-    if isinstance(value, bool):
-        raise ValueError(f"unexpected gold value {value!r}")
-    if isinstance(value, int):
-        return value
-    text = str(value).strip()
-    if text.isdigit():
-        return int(text)
-    if len(text) == 1 and text.upper() in LETTERS:
-        return ("letter", LETTERS.index(text.upper()))
-    raise ValueError(f"cannot parse gold answer {value!r}")
-
-
-def load_questions(path: str | Path, answer_base="auto") -> list[dict]:
-    """Load a dev/test/train file into internal records.
-
-    answer_base: "auto" guesses from the data (any 0 -> 0-based, otherwise
-    1-based). After Oct 5, confirm this once and set it explicitly in the
-    config - a wrong base shifts every answer and silently ruins the score.
-    """
-    raw = read_jsonl(path)
-    ids = row_ids(raw)
-    records = []
-    for i, row in enumerate(raw):
-        rid = ids[i]
-        question = _first(row, QUESTION_KEYS)
-        if question is None:
-            raise ValueError(f"{path}: record {rid} has no question field")
-        level_raw = level_of(row)
-        level = str(level_raw or "unknown").strip().lower()
-        records.append(
-            {
-                "id": str(rid),
-                "level": level,
-                "level_raw": level_raw,  # written back unchanged in predictions
-                "question": str(question).strip(),
-                "options": _options(row),
-                "subject": row.get("subject"),
-                "category": row.get("category"),
-                "q_no": row.get("q_no"),
-                "_raw_gold": _raw_gold(_first(row, GOLD_KEYS)),
-            }
-        )
-
-    numeric = [r["_raw_gold"] for r in records if isinstance(r["_raw_gold"], int)]
-    if answer_base == "auto":
-        base = 0 if (numeric and min(numeric) == 0) else 1
-    else:
-        base = int(answer_base)
-
-    for r in records:
-        g = r.pop("_raw_gold")
-        if g is None:
-            r["gold"] = None
-        elif isinstance(g, tuple):  # letter
-            r["gold"] = g[1]
-        else:
-            r["gold"] = g - base
-        if r["gold"] is not None and not 0 <= r["gold"] < len(r["options"]):
-            # data quirk: some rows store the option TEXT as the answer (e.g. answer 24, choices [..,"24",..])
-            raw = str(r["gold"] + base)
-            hits = [i for i, o in enumerate(r["options"]) if str(o).strip().lstrip("0") == raw.lstrip("0")]
-            if len(hits) == 1:
-                print(f"[data] {path}: record {r['id']} answer {raw} matched option text -> option {hits[0] + 1}")
-                r["gold"] = hits[0]
-                continue
-            raise ValueError(
-                f"{path}: record {r['id']} gold {r['gold'] + base} out of range "
-                f"for {len(r['options'])} options (answer_base={base})"
-            )
-    return records
-
-
-def prediction_row(record: dict, answer: str) -> dict:
-    """One line of predictions.jsonl in the official Codabench format:
-    {"q_no":1,"subject":"Arts","category":"humanities","level":"easy","answer":"2"}
-    Files without q_no (our own sample/train data) fall back to an "id" key."""
-    level = record["level_raw"] if record.get("level_raw") is not None else record["level"]
-    # Codabench example: {"q_no":1,"subject":"Arts","category":"humanities","level":"easy","answer":"2"}
-    # -> q_no is a number, every other value a string.
-    s = lambda v: "" if v is None else str(v)
-    if record.get("q_no") is not None:
-        q = record["q_no"]
-        q = int(q) if str(q).strip().isdigit() else s(q)
-        return {"q_no": q, "subject": s(record.get("subject")), "category": s(record.get("category")),
-                "level": s(level), "answer": s(answer)}
-    return {"id": s(record["id"]), "level": s(level), "answer": s(answer)}
-
-
-def match_key(d: dict):
-    """Key that links a prediction line to its question: (subject, level, q_no) in
-    the official format (q_no restarts per subject file, and the TEST set has the
-    same subject in several levels), else the id."""
-    if d.get("q_no") is not None:
-        return ("q", str(d.get("subject")), str(d.get("level") or "").strip().lower(), str(d["q_no"]))
-    return ("id", str(d["id"]))
-
-
-def has_gold(records: list[dict]) -> bool:
-    return bool(records) and all(r["gold"] is not None for r in records)

hf_team_repo/sinmmlu/metrics.py

@@ -1,33 +0,0 @@
-"""The official metric: accuracy per difficulty level and their macro average."""
-
-from __future__ import annotations
-
-LEVELS = ("easy", "medium", "hard")
-
-
-def accuracy_by_level(levels: list[str], preds: list[int], golds: list[int]) -> dict:
-    hit: dict[str, int] = {}
-    tot: dict[str, int] = {}
-    for level, p, g in zip(levels, preds, golds):
-        tot[level] = tot.get(level, 0) + 1
-        hit[level] = hit.get(level, 0) + int(p == g)
-    acc = {lv: hit[lv] / tot[lv] for lv in tot}
-    present = [lv for lv in LEVELS if lv in acc] or list(acc)
-    return {
-        "acc": acc,
-        "count": tot,
-        # Macro = (E + M + H) / 3: each level weighs the same, however many
-        # questions it has. Hard accuracy counts for a full third.
-        "macro": sum(acc[lv] for lv in present) / len(present) if present else 0.0,
-        "overall": sum(hit.values()) / max(1, sum(tot.values())),
-    }
-
-
-def format_report(result: dict) -> str:
-    lines = [f"{'level':<8} {'n':>6} {'acc':>8}"]
-    for lv in list(LEVELS) + sorted(set(result["acc"]) - set(LEVELS)):
-        if lv in result["acc"]:
-            lines.append(f"{lv:<8} {result['count'][lv]:>6} {result['acc'][lv] * 100:>7.2f}%")
-    lines.append(f"{'MACRO':<8} {'':>6} {result['macro'] * 100:>7.2f}%")
-    lines.append(f"{'overall':<8} {sum(result['count'].values()):>6} {result['overall'] * 100:>7.2f}%")
-    return "\n".join(lines)

hf_team_repo/sinmmlu/modeling.py

@@ -1,126 +0,0 @@
-"""Model and tokenizer loading + parameter counting (the 8B rule)."""
-
-from __future__ import annotations
-
-import os
-import random
-
-import numpy as np
-import torch
-
-LIMIT = 8_000_000_000
-
-
-def set_determinism(seed: int) -> None:
-    """Same input -> same predictions on every run (needed for verification)."""
-    random.seed(seed)
-    np.random.seed(seed)
-    torch.manual_seed(seed)
-    torch.cuda.manual_seed_all(seed)
-    os.environ.setdefault("CUBLAS_WORKSPACE_CONFIG", ":4096:8")
-    torch.backends.cuda.matmul.allow_tf32 = False
-    torch.backends.cudnn.allow_tf32 = False
-    torch.backends.cudnn.benchmark = False
-    torch.backends.cudnn.deterministic = True
-
-
-def pick_dtype(name: str):
-    if name == "auto":
-        if torch.cuda.is_available():
-            # T4/P100 (Kaggle, free Colab) have no fast bf16 -> float16
-            return torch.bfloat16 if torch.cuda.is_bf16_supported() else torch.float16
-        return torch.float32
-    return getattr(torch, name)
-
-
-def load_tokenizer(model_cfg: dict):
-    from transformers import AutoTokenizer
-
-    tok = AutoTokenizer.from_pretrained(
-        model_cfg["name_or_path"],
-        revision=model_cfg["revision"],
-        trust_remote_code=model_cfg["trust_remote_code"],
-    )
-    if tok.pad_token is None:
-        tok.pad_token = tok.eos_token
-    tok.padding_side = "left"  # last position = next-token prediction for every row
-    return tok
-
-
-def load_model(model_cfg: dict, for_training: bool = False):
-    from transformers import AutoModelForCausalLM
-
-    kwargs = {
-        "revision": model_cfg["revision"],
-        "trust_remote_code": model_cfg["trust_remote_code"],
-        "dtype": pick_dtype(model_cfg["dtype"]),
-    }
-    if torch.cuda.is_available():
-        kwargs["device_map"] = "auto"
-    if model_cfg["load_in_4bit"]:
-        from transformers import BitsAndBytesConfig
-
-        kwargs["quantization_config"] = BitsAndBytesConfig(
-            load_in_4bit=True,
-            bnb_4bit_quant_type="nf4",
-            bnb_4bit_use_double_quant=True,
-            bnb_4bit_compute_dtype=kwargs["dtype"] if kwargs["dtype"] != torch.float32 else torch.float16,
-        )
-
-    if model_cfg.get("text_only"):
-        from transformers import AutoConfig
-
-        full = AutoConfig.from_pretrained(
-            model_cfg["name_or_path"], revision=model_cfg["revision"], trust_remote_code=model_cfg["trust_remote_code"]
-        )
-        kwargs["config"] = getattr(full, "text_config", full)
-        # multimodal checkpoints store the text weights as model.language_model.* ; the text-only class wants model.*
-        # (vision/audio tensors have no counterpart in the text-only model, so they are not loaded)
-        kwargs["key_mapping"] = {r"^model\.language_model\.": "model."}
-        model, info = AutoModelForCausalLM.from_pretrained(model_cfg["name_or_path"], output_loading_info=True, **kwargs)
-        tied = getattr(kwargs["config"], "tie_word_embeddings", False)
-        missing = [k for k in info["missing_keys"] if not (tied and k == "lm_head.weight")]
-        if missing:  # a name mismatch would leave random weights and silently ruin the score
-            raise RuntimeError(f"text_only load: {len(missing)} weights missing, e.g. {missing[:5]}")
-        print(f"[model] text_only: {type(model).__name__}, ignored {len(info['unexpected_keys'])} non-text tensors")
-    else:
-        model = AutoModelForCausalLM.from_pretrained(model_cfg["name_or_path"], **kwargs)
-
-    if model_cfg.get("adapter_path") and not for_training:
-        from peft import PeftModel
-
-        model = PeftModel.from_pretrained(model, model_cfg["adapter_path"])
-    if not for_training:
-        model.eval()
-    return model
-
-
-def prepare_for_kbit_training(model, use_gradient_checkpointing: bool, fp32_below: int = 100_000_000):
-    """peft.prepare_model_for_kbit_training, except that only SMALL fp16/bf16 parameters (norms) become float32.
-
-    peft casts every non-quantised parameter. Gemma 4 E4B has a 2.8 B-parameter per-layer embedding table
-    (10.5 GiB in float32), which does not fit on a 15 GB T4 (OOM on Oct 17). Big tables stay in half precision.
-    """
-    for p in model.parameters():
-        p.requires_grad = False
-        if p.dtype in (torch.float16, torch.bfloat16) and p.numel() < fp32_below:
-            p.data = p.data.to(torch.float32)
-    if use_gradient_checkpointing:
-        model.enable_input_require_grads()
-        model.gradient_checkpointing_enable(gradient_checkpointing_kwargs={"use_reentrant": True})
-    return model
-
-
-def count_parameters(model) -> int:
-    """Parameters of the loaded model (base + any adapter).
-
-    Note: 4-bit loading packs two weights per stored element, so for a
-    4-bit model this UNDER-counts. Use tools/count_params.py (reads the
-    original checkpoint) for the number that goes in the manifest.
-    """
-    return sum(p.numel() for p in model.parameters())
-
-
-def describe_budget(n_params: int) -> str:
-    status = "OK" if n_params <= LIMIT else "OVER THE 8B LIMIT"
-    return f"{n_params:,} params ({n_params / 1e9:.3f}B) -> {status}"

hf_team_repo/sinmmlu/prompts.py

@@ -1,140 +0,0 @@
-"""Prompt templates.
-
-The SAME functions build prompts for inference (predict.py) and fine-tuning
-(train.py). If training and inference prompts differ, fine-tuning gains
-mostly disappear, so never build prompts anywhere else.
-
-To add a prompt experiment: add an entry to TEMPLATES, then point a config's
-prompt.template at it. Few-shot examples must come from the dev TRAIN split,
-never from the test set (rule 04).
-"""
-
-from __future__ import annotations
-
-TEMPLATES = {
-    # Sinhala instruction
-    "si_basic": {
-        "system": None,
-        "instruction": "පහත බහුවරණ ප්‍රශ්නයට නිවැරදි පිළිතුරේ අංකය පමණක් ලබා දෙන්න.",
-        "question_label": "ප්‍රශ්නය",
-        "answer_label": "පිළිතුර",
-    },
-    # English instruction, Sinhala question (some models follow English
-    # instructions better)
-    "en_basic": {
-        "system": None,
-        "instruction": (
-            "Answer the following multiple-choice question written in Sinhala. "
-            "Reply with the number of the correct option only."
-        ),
-        "question_label": "Question",
-        "answer_label": "Answer",
-    },
-    # Sinhala instruction + teacher role in the system prompt
-    "si_teacher": {
-        "system": "ඔබ ශ්‍රී ලංකාවේ පාසල් විෂයයන් පිළිබඳ ගැඹුරු දැනුමක් ඇති ගුරුවරයෙකි.",
-        "instruction": "පහත බහුවරණ ප්‍රශ්නයට නිවැරදි පිළිතුරේ අංකය පමණක් ලබා දෙන්න.",
-        "question_label": "ප්‍රශ්නය",
-        "answer_label": "පිළිතුර",
-    },
-}
-
-
-def option_labels(n: int) -> list[str]:
-    return [str(i) for i in range(1, n + 1)]
-
-
-def format_question(record: dict, template: dict, options: list[str] | None = None) -> str:
-    opts = record["options"] if options is None else options
-    lines = [f"{template['question_label']}: {record['question']}"]
-    lines += [f"{label}. {text}" for label, text in zip(option_labels(len(opts)), opts)]
-    return "\n".join(lines)
-
-
-def build_messages(
-    record: dict,
-    template_name: str,
-    fewshot: list[dict] | None = None,
-    options: list[str] | None = None,
-) -> list[dict]:
-    """Chat messages for one question. Few-shot examples become earlier
-    user/assistant turns, so the model sees exactly how to answer."""
-    if template_name not in TEMPLATES:
-        raise KeyError(f"unknown prompt template {template_name!r}; options: {list(TEMPLATES)}")
-    t = TEMPLATES[template_name]
-    messages = []
-    if t["system"]:
-        messages.append({"role": "system", "content": t["system"]})
-    for i, ex in enumerate(fewshot or []):
-        content = format_question(ex, t)
-        if i == 0:
-            content = f"{t['instruction']}\n\n{content}"
-        messages.append({"role": "user", "content": content})
-        messages.append({"role": "assistant", "content": str(ex["gold"] + 1)})
-    content = format_question(record, t, options)
-    if not fewshot:
-        content = f"{t['instruction']}\n\n{content}"
-    messages.append({"role": "user", "content": content})
-    return messages
-
-
-def render_prompt(
-    tokenizer, messages: list[dict], use_chat_template="auto", chat_template_kwargs: dict | None = None
-) -> tuple[str, str]:
-    """Return (prompt_text, label_prefix).
-
-    The answer label is scored right after prompt_text: for chat models the
-    answer is the assistant's first token ("2"); for base models the prompt
-    ends with "Answer:" and the label is " 2".
-    """
-    has_template = getattr(tokenizer, "chat_template", None) is not None
-    use_chat = has_template if use_chat_template == "auto" else bool(use_chat_template)
-
-    if use_chat:
-        msgs = messages
-        if msgs[0]["role"] == "system" and not _supports_system(tokenizer):
-            msgs = [{"role": "user", "content": f"{msgs[0]['content']}\n\n{msgs[1]['content']}"}] + msgs[2:]
-        text = tokenizer.apply_chat_template(
-            msgs, tokenize=False, add_generation_prompt=True, **(chat_template_kwargs or {})
-        )
-        return text, ""
-
-    # Plain-text fallback for base (non-chat) models
-    answer_label = _answer_label(messages)
-    parts = []
-    for m in messages:
-        if m["role"] == "assistant":
-            parts[-1] += f"\n{answer_label}: {m['content']}"
-        else:
-            parts.append(m["content"])
-    text = "\n\n".join(parts) + f"\n{answer_label}:"
-    return text, " "
-
-
-def _answer_label(messages: list[dict]) -> str:
-    for t in TEMPLATES.values():
-        if any(t["question_label"] + ":" in m["content"] for m in messages):
-            return t["answer_label"]
-    return "Answer"
-
-
-def _supports_system(tokenizer) -> bool:
-    try:
-        tokenizer.apply_chat_template(
-            [{"role": "system", "content": "s"}, {"role": "user", "content": "u"}],
-            tokenize=False,
-            add_generation_prompt=True,
-        )
-        return True
-    except Exception:
-        return False
-
-
-def select_fewshot(record: dict, pool: list[dict], k: int) -> list[dict]:
-    """Deterministic few-shot choice: same difficulty level first, in file
-    order, never the question itself."""
-    if k <= 0 or not pool:
-        return []
-    same = [p for p in pool if p["level"] == record["level"] and p["id"] != record["id"]]
-    other = [p for p in pool if p["level"] != record["level"] and p["id"] != record["id"]]
-    return (same + other)[:k]

hf_team_repo/sinmmlu/runinfo.py

@@ -1,85 +0,0 @@
-"""Record what produced each result, so any score can be reproduced later."""
-
-from __future__ import annotations
-
-import csv
-import datetime as dt
-import json
-import platform
-import subprocess
-from pathlib import Path
-
-RESULTS_CSV = Path("results/results.csv")
-FIELDS = [
-    "time", "who", "config", "model", "adapter", "template", "fewshot_k",
-    "permutations", "data_file", "n", "easy", "medium", "hard", "macro", "git_commit", "note",
-]
-
-
-def git_commit() -> str:
-    try:
-        sha = subprocess.check_output(["git", "rev-parse", "--short", "HEAD"], stderr=subprocess.DEVNULL, text=True).strip()
-        dirty = subprocess.call(["git", "diff", "--quiet", "HEAD"], stderr=subprocess.DEVNULL) != 0
-        return sha + ("-dirty" if dirty else "")
-    except Exception:
-        return "no-git"
-
-
-def env_info() -> dict:
-    info = {"python": platform.python_version(), "platform": platform.platform()}
-    for mod in ("torch", "transformers", "peft", "accelerate", "bitsandbytes"):
-        try:
-            info[mod] = __import__(mod).__version__
-        except Exception:
-            info[mod] = None
-    try:
-        import torch
-
-        info["cuda"] = torch.version.cuda
-        info["gpu"] = torch.cuda.get_device_name(0) if torch.cuda.is_available() else None
-    except Exception:
-        pass
-    return info
-
-
-def write_run_info(path: Path, cfg: dict, extra: dict) -> None:
-    payload = {
-        "time": dt.datetime.now().isoformat(timespec="seconds"),
-        "git_commit": git_commit(),
-        "config": cfg,
-        "environment": env_info(),
-        **extra,
-    }
-    path.parent.mkdir(parents=True, exist_ok=True)
-    path.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8")
-
-
-def append_result(cfg: dict, data_file: str, result: dict, who: str = "", note: str = "") -> None:
-    """Add one row to results/results.csv (the team's experiment log and,
-    later, the report's ablation table)."""
-    RESULTS_CSV.parent.mkdir(parents=True, exist_ok=True)
-    new = not RESULTS_CSV.exists()
-    acc = result["acc"]
-    row = {
-        "time": dt.datetime.now().isoformat(timespec="minutes"),
-        "who": who,
-        "config": cfg["name"],
-        "model": cfg["model"]["name_or_path"],
-        "adapter": cfg["model"]["adapter_path"] or "",
-        "template": cfg["prompt"]["template"],
-        "fewshot_k": cfg["prompt"]["fewshot_k"],
-        "permutations": cfg["inference"]["permutations"],
-        "data_file": data_file,
-        "n": sum(result["count"].values()),
-        "easy": f"{acc.get('easy', float('nan')) * 100:.2f}",
-        "medium": f"{acc.get('medium', float('nan')) * 100:.2f}",
-        "hard": f"{acc.get('hard', float('nan')) * 100:.2f}",
-        "macro": f"{result['macro'] * 100:.2f}",
-        "git_commit": git_commit(),
-        "note": note,
-    }
-    with RESULTS_CSV.open("a", encoding="utf-8", newline="") as f:
-        w = csv.DictWriter(f, fieldnames=FIELDS)
-        if new:
-            w.writeheader()
-        w.writerow(row)

hf_team_repo/sinmmlu/scoring.py

@@ -1,135 +0,0 @@
-"""Logit scoring: pick the option whose label ("1".."4") the model rates
-most likely as its next token.
-
-Why not let the model generate text?
-  * always returns a valid answer (no parsing of free text)
-  * deterministic: same scores every run, which the verification phase needs
-  * one forward pass per question: much faster than generation
-"""
-
-from __future__ import annotations
-
-import torch
-
-from .prompts import build_messages, option_labels, render_prompt, select_fewshot
-
-
-def label_token_id(tokenizer, prompt: str, label: str) -> int:
-    """Token id that starts `label` when it is written right after `prompt`.
-
-    Computed from the full string so tokenizers that merge characters across
-    the boundary (or add a leading-space marker) are handled correctly.
-    """
-    p_ids = tokenizer(prompt, add_special_tokens=False)["input_ids"]
-    f_ids = tokenizer(prompt + label, add_special_tokens=False)["input_ids"]
-    k = 0
-    while k < len(p_ids) and k < len(f_ids) and p_ids[k] == f_ids[k]:
-        k += 1
-    if k < len(p_ids) or k >= len(f_ids):
-        # Boundary merged into the prompt's last token: fall back to the
-        # label tokenised alone.
-        return tokenizer(label, add_special_tokens=False)["input_ids"][-1]
-    return f_ids[k]
-
-
-# Codabench Data page: "Each question has four options (1-4) and exactly one correct
-# answer." A few test items carry extra (often empty) choices, so only options 1-4 are
-# shown, scored and rotated, so the answer is always "1".."4" (also avoids the "1"/"10" token clash).
-MAX_SCORED = 4
-
-
-def _rotations(n: int, count: int) -> list[int]:
-    count = max(1, min(count, n))
-    return [round(i * n / count) % n for i in range(count)]
-
-
-class Scorer:
-    def __init__(self, model, tokenizer, cfg: dict, fewshot_pool: list[dict] | None = None):
-        self.model = model
-        self.tok = tokenizer
-        self.cfg = cfg
-        self.pool = fewshot_pool or []
-        self._label_cache: dict[tuple, list[int]] = {}
-        self.device = next(model.parameters()).device
-
-    def _label_ids(self, prompt: str, prefix: str, n: int) -> list[int]:
-        # Labels depend only on the end of the prompt, so cache by its tail.
-        key = (prompt[-64:], prefix, n)
-        if key not in self._label_cache:
-            ids = [label_token_id(self.tok, prompt, prefix + l) for l in option_labels(n)]
-            if len(set(ids)) != n:
-                raise RuntimeError(
-                    f"option labels map to non-unique tokens {ids}; "
-                    "use a different prompt ending or label style"
-                )
-            self._label_cache[key] = ids
-        return self._label_cache[key]
-
-    def _build(self, record: dict, shift: int) -> tuple[str, list[int]]:
-        p = self.cfg["prompt"]
-        opts = record["options"][:MAX_SCORED]
-        n = len(opts)
-        options = opts[shift:] + opts[:shift]
-        fewshot = select_fewshot(record, self.pool, p["fewshot_k"])
-        msgs = build_messages(record, p["template"], fewshot, options)
-        text, prefix = render_prompt(self.tok, msgs, p["use_chat_template"], p["chat_template_kwargs"])
-        return text, self._label_ids(text, prefix, n)
-
-    @torch.no_grad()
-    def _logprobs(self, prompts: list[str], label_ids: list[list[int]]) -> list[torch.Tensor]:
-        enc = self.tok(
-            prompts,
-            return_tensors="pt",
-            padding=True,
-            truncation=True,
-            max_length=self.cfg["inference"]["max_length"],
-            add_special_tokens=False,  # chat templates already add BOS
-        ).to(self.device)
-        logits = self.model(**enc).logits[:, -1, :].float()
-        out = []
-        for row, ids in zip(logits, label_ids):
-            out.append(torch.log_softmax(row[ids], dim=-1).cpu())
-        return out
-
-    def score(self, records: list[dict], progress: bool = True) -> list[list[float]]:
-        """Return per-option log-probabilities for every record (original order)."""
-        n_perm = self.cfg["inference"]["permutations"]
-        bs = self.cfg["inference"]["batch_size"]
-
-        jobs = []  # (record index, shift, prompt, label ids)
-        for i, r in enumerate(records):
-            for shift in _rotations(min(len(r["options"]), MAX_SCORED), n_perm):
-                text, ids = self._build(r, shift)
-                jobs.append((i, shift, text, ids))
-
-        totals = [torch.zeros(min(len(r["options"]), MAX_SCORED)) for r in records]
-        counts = [0] * len(records)
-
-        iterator = range(0, len(jobs), bs)
-        if progress:
-            try:
-                from tqdm import tqdm
-
-                iterator = tqdm(iterator, desc="scoring", unit="batch")
-            except ImportError:
-                pass
-        for start in iterator:
-            batch = jobs[start : start + bs]
-            lps = self._logprobs([j[2] for j in batch], [j[3] for j in batch])
-            for (i, shift, _, _), lp in zip(batch, lps):
-                n = len(lp)
-                # position j in the rotated list = original option (j+shift) % n
-                original = torch.empty(n)
-                for j in range(n):
-                    original[(j + shift) % n] = lp[j]
-                totals[i] += original
-                counts[i] += 1
-        return [(t / c).tolist() for t, c in zip(totals, counts)]
-
-    def predict(self, records: list[dict], progress: bool = True) -> list[int]:
-        """0-based predicted option index for every record."""
-        scores = self.score(records, progress)
-        bias = self.cfg["inference"].get("option_bias")
-        if bias:
-            scores = [[v + bias[j] for j, v in enumerate(s)] for s in scores]
-        return [max(range(len(s)), key=lambda j: s[j]) for s in scores]

hf_team_repo/training/README_training.md

@@ -1,7 +0,0 @@
-# Training of the adapter (Run 2)
-
-- Config: `ft_gemma4_e4b_real_plus_exp.yaml` (QLoRA, base 4-bit NF4, LoRA r=8, all attention + MLP projections, 1 epoch, LR 5e-5, batch 2 x grad-accum 8, options rotated during training).
-- Code: `train.py` (from our repository, `sinmmlu/` package; see the code link in the main README).
-- Data: `data/train_real_plus_explanations_clean.jsonl` (7,999 rows): real exam questions + textbook explanations. TODO (Hashini): exact sources and links, whether any rows were generated with a closed API (model name, number of rows), and the overlap check against the test set (counts only).
-- Note: the saved `adapter_config.json` says `lora_alpha: 8`; the config above says 16. The released adapter is the one in the adapter repo.
-- No test question was used for training, tuning or prompt design.

hf_team_repo/training/ft_gemma4_e4b_real_plus_exp.yaml

@@ -1,33 +0,0 @@
-# Hashini Run 2: Gentle QLoRA on Real Exam + Textbook Explanations (7,999 rows)
-# Adapter on HF: ape-llama-sinmmlu/ft_gemma4_e4b_real_plus_exp
-model:
-  name_or_path: google/gemma-4-e4b-it
-  adapter_path: ape-llama-sinmmlu/ft_gemma4_e4b_real_plus_exp
-  load_in_4bit: true
-  text_only: true
-prompt:
-  template: si_basic
-train:
-  train_file: data/train_real_plus_explanations_clean.jsonl
-  eval_file: data/eval_v2.jsonl
-  output_dir: outputs/ft_gemma4_e4b_real_plus_exp
-  epochs: 1
-  learning_rate: 5.0e-5
-  batch_size: 2
-  grad_accum: 8
-  lora_r: 8
-  lora_alpha: 16
-  lora_dropout: 0.05
-  target_modules:
-    - q_proj
-    - k_proj
-    - v_proj
-    - o_proj
-    - gate_proj
-    - up_proj
-    - down_proj
-  rotate_options: true
-inference:
-  batch_size: 8
-  max_length: 1024
-  permutations: 4

hf_team_repo/training/train.py

@@ -1,181 +0,0 @@
-"""LoRA / QLoRA fine-tuning on Sinhala MCQs.
-
-Training data = jsonl in the same format as the dev set (question, options,
-answer, level). The prompt is built by the SAME code as inference, and the
-loss is only on the answer token - exactly what predict.py scores.
-
-    python train.py --config configs/train_qlora_qwen25_7b.yaml
-
-After training, predict with the adapter:
-    model.adapter_path: outputs/<name>/adapter   (in a predict config)
-
-Rules to remember:
-  * never train on test questions (rule 04)
-  * LoRA parameters count towards the 8B limit (base + adapter)
-  * data generated with closed APIs (GPT/Claude/Gemini) must be disclosed
-"""
-
-from __future__ import annotations
-
-import argparse
-import json
-import math
-import shutil
-import sys
-from pathlib import Path
-
-import torch
-
-from sinmmlu.config import load_config
-from sinmmlu.data import has_gold, load_questions
-from sinmmlu.metrics import accuracy_by_level, format_report
-from sinmmlu.modeling import describe_budget, load_model, load_tokenizer, pick_dtype, prepare_for_kbit_training, set_determinism
-from sinmmlu.prompts import build_messages, render_prompt
-from sinmmlu.runinfo import append_result, write_run_info
-from sinmmlu.scoring import Scorer, label_token_id
-
-
-def build_examples(records: list[dict], tok, cfg: dict) -> list[dict]:
-    """One example = prompt tokens + the correct label token.
-
-    Options are rotated (deterministically) so the right answer appears in
-    every position equally often - otherwise the model can learn "the answer
-    is usually 2" instead of learning the content.
-    """
-    p = cfg["prompt"]
-    max_len = cfg["inference"]["max_length"]
-    rotate = cfg["train"].get("rotate_options", True)
-    examples, skipped = [], 0
-    for i, r in enumerate(records):
-        n = len(r["options"])
-        shift = (i % n) if rotate else 0
-        options = r["options"][shift:] + r["options"][:shift]
-        gold_pos = (r["gold"] - shift) % n  # where the correct option ends up
-        msgs = build_messages(r, p["template"], None, options)
-        text, prefix = render_prompt(tok, msgs, p["use_chat_template"], p["chat_template_kwargs"])
-        ids = tok(text, add_special_tokens=False)["input_ids"]
-        if len(ids) + 1 > max_len:
-            skipped += 1
-            continue
-        target = label_token_id(tok, text, prefix + str(gold_pos + 1))
-        examples.append({"input_ids": ids + [target], "labels": [-100] * len(ids) + [target]})
-    if skipped:
-        print(f"[train] skipped {skipped} examples longer than {max_len} tokens")
-    return examples
-
-
-class Collator:
-    def __init__(self, pad_id: int):
-        self.pad_id = pad_id
-
-    def __call__(self, batch: list[dict]) -> dict:
-        width = max(len(b["input_ids"]) for b in batch)
-        ids, labels, mask = [], [], []
-        for b in batch:
-            pad = width - len(b["input_ids"])
-            ids.append([self.pad_id] * pad + b["input_ids"])
-            labels.append([-100] * pad + b["labels"])
-            mask.append([0] * pad + [1] * len(b["input_ids"]))
-        return {
-            "input_ids": torch.tensor(ids),
-            "labels": torch.tensor(labels),
-            "attention_mask": torch.tensor(mask),
-        }
-
-
-def main(argv=None) -> int:
-    from peft import LoraConfig, get_peft_model, prepare_model_for_kbit_training
-    from transformers import Trainer, TrainingArguments
-
-    ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
-    ap.add_argument("--config", required=True)
-    ap.add_argument("--who", default="")
-    ap.add_argument("--max-steps", type=int, default=-1, help="stop early (debugging)")
-    args = ap.parse_args(argv)
-
-    cfg = load_config(args.config)
-    tc = cfg["train"]
-    set_determinism(cfg["seed"])
-    if not tc["train_file"]:
-        sys.exit("train.train_file is required")
-
-    train_records = load_questions(tc["train_file"], cfg["data"]["answer_base"])
-    if not has_gold(train_records):
-        sys.exit("training file has no gold answers")
-    print(f"[train] {len(train_records)} training questions from {tc['train_file']}")
-
-    tok = load_tokenizer(cfg["model"])
-    model = load_model(cfg["model"], for_training=True)
-    if cfg["model"]["load_in_4bit"] and cfg["model"].get("text_only"):
-        # peft's version would cast Gemma 4's 2.8 B-parameter embedding table to float32 (10.5 GiB -> OOM on a T4)
-        model = prepare_for_kbit_training(model, use_gradient_checkpointing=tc["gradient_checkpointing"])
-    elif cfg["model"]["load_in_4bit"]:
-        model = prepare_model_for_kbit_training(model, use_gradient_checkpointing=tc["gradient_checkpointing"])
-    elif tc["gradient_checkpointing"]:
-        model.gradient_checkpointing_enable()
-        model.enable_input_require_grads()
-
-    lora = LoraConfig(
-        r=tc["lora_r"],
-        lora_alpha=tc["lora_alpha"],
-        lora_dropout=tc["lora_dropout"],
-        target_modules=tc["target_modules"],
-        task_type="CAUSAL_LM",
-    )
-    model = get_peft_model(model, lora)
-    trainable = sum(p.numel() for p in model.parameters() if p.requires_grad)
-    print(f"[lora] adapter parameters: {trainable:,}")
-
-    examples = build_examples(train_records, tok, cfg)
-    steps_per_epoch = math.ceil(len(examples) / (tc["batch_size"] * tc["grad_accum"]))
-    total_steps = args.max_steps if args.max_steps > 0 else steps_per_epoch * tc["epochs"]
-    out_dir = Path(tc["output_dir"])
-    dtype = pick_dtype(cfg["model"]["dtype"])
-    on_gpu = torch.cuda.is_available()
-    targs = TrainingArguments(
-        output_dir=str(out_dir / "checkpoints"),
-        num_train_epochs=tc["epochs"],
-        max_steps=args.max_steps,
-        learning_rate=tc["learning_rate"],
-        per_device_train_batch_size=tc["batch_size"],
-        gradient_accumulation_steps=tc["grad_accum"],
-        lr_scheduler_type="cosine",
-        warmup_steps=math.ceil(0.03 * total_steps),  # 3% warm-up
-        logging_steps=tc["logging_steps"],
-        save_strategy="no",
-        report_to=[],
-        seed=cfg["seed"],
-        data_seed=cfg["seed"],
-        bf16=on_gpu and dtype == torch.bfloat16,
-        fp16=on_gpu and dtype == torch.float16,
-        remove_unused_columns=False,
-        dataloader_num_workers=0,
-    )
-    trainer = Trainer(model=model, args=targs, train_dataset=examples, data_collator=Collator(tok.pad_token_id))
-    trainer.train()
-
-    adapter_dir = out_dir / "adapter"
-    model.save_pretrained(adapter_dir)
-    tok.save_pretrained(adapter_dir)
-    shutil.copy(args.config, out_dir / "train_config.yaml")
-    total = sum(p.numel() for p in model.parameters())
-    print(f"[save] adapter -> {adapter_dir}")
-    print(f"[budget] base + adapter as loaded: {describe_budget(total)}")
-
-    extra = {"train_file": tc["train_file"], "n_train": len(examples), "adapter_params": trainable}
-    if tc["eval_file"]:
-        model.eval()
-        eval_records = load_questions(tc["eval_file"], cfg["data"]["answer_base"])
-        preds = Scorer(model, tok, cfg).predict(eval_records)
-        result = accuracy_by_level([r["level"] for r in eval_records], preds, [r["gold"] for r in eval_records])
-        print(f"[eval] {tc['eval_file']}\n" + format_report(result))
-        cfg_logged = json.loads(json.dumps(cfg))
-        cfg_logged["model"]["adapter_path"] = str(adapter_dir)
-        append_result(cfg_logged, tc["eval_file"], result, args.who, "after training")
-        extra["eval_result"] = result
-    write_run_info(out_dir / "train.run.json", cfg, extra)
-    return 0
-
-
-if __name__ == "__main__":
-    raise SystemExit(main())