Instructions to use SinhalaMMLU-SharedTask/team-sinbotics with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- PEFT
How to use SinhalaMMLU-SharedTask/team-sinbotics with PEFT:
Task type is invalid.
- Notebooks
- Google Colab
- Kaggle
Download training/sinmmlu/rationales.py from SinhalaMMLU-SharedTask/team-sinbotics: direct link, hf CLI and curl.
- Browser
- Download file 1.8 kB
-
https://huggingface.co/SinhalaMMLU-SharedTask/team-sinbotics/resolve/main/training/sinmmlu/rationales.py
- Command line
-
hf download hf://SinhalaMMLU-SharedTask/team-sinbotics/training/sinmmlu/rationales.py
-
curl -L -o rationales.py https://huggingface.co/SinhalaMMLU-SharedTask/team-sinbotics/resolve/main/training/sinmmlu/rationales.py
1.8 kB
| """Worked examples for reasoning prompts: the model's own English reasoning on the few-shot pool questions, kept only | |
| when its stated answer is the gold answer (dev data only; the pool is disjoint from every evaluation set). | |
| python -m sinmmlu.rationales --model configs/models/gemma3_4b_it.yaml | |
| Writes dev/archive/fewshot_cot_en.jsonl (pool rows + "rationale", the reasoning up to and including "Answer: N"), in the pool | |
| order, so the first examples of a subject are used first. | |
| """ | |
| import argparse | |
| from . import prompts as P | |
| from .data import load_jsonl, write_jsonl | |
| from .evaluate import ANSWER_RE, load_model_cfg | |
| def main(): | |
| ap = argparse.ArgumentParser() | |
| ap.add_argument("--model", required=True) | |
| ap.add_argument("--prompt", default="cot_en") | |
| a = ap.parse_args() | |
| from .model import LM, load_lm, seed_everything | |
| cfg = load_model_cfg(a.model) | |
| seed_everything(cfg["seed"]) | |
| model, tok, info = load_lm(cfg["model"], cfg["revision"], cfg["dtype"], cfg["quant"]) | |
| lm = LM(model, tok, chat=True, context=info["context_length"], gen_batch_size=cfg["gen_batch_size"]) | |
| pool = load_jsonl(P.FEWSHOT_POOL) | |
| p = P.PROMPTS[a.prompt] | |
| gens = lm.generate([P.build(r, a.prompt) for r in pool], p.max_new_tokens) | |
| kept = [] | |
| for r, g in zip(pool, gens): | |
| m = list(ANSWER_RE.finditer(g)) | |
| if m and int(m[-1].group(1)) == int(r["answer"]): | |
| kept.append(dict(r, rationale=g[:m[-1].end()].strip())) | |
| write_jsonl(P.RATIONALE_POOL, kept) | |
| subjects = {r["file"] for r in pool} | |
| covered = {r["file"] for r in kept} | |
| print(f"{len(kept)}/{len(pool)} pool questions with a correct rationale; subjects without one: " | |
| f"{sorted(subjects - covered) or 'none'} -> {P.RATIONALE_POOL}") | |
| if __name__ == "__main__": | |
| main() | |