team-sinbotics / training /sinmmlu /rationales.py
JaniRaviRash-523's picture
Upload folder using huggingface_hub
7a2335a verified
Raw History Blame Contribute Delete
1.8 kB
"""Worked examples for reasoning prompts: the model's own English reasoning on the few-shot pool questions, kept only
when its stated answer is the gold answer (dev data only; the pool is disjoint from every evaluation set).
python -m sinmmlu.rationales --model configs/models/gemma3_4b_it.yaml
Writes dev/archive/fewshot_cot_en.jsonl (pool rows + "rationale", the reasoning up to and including "Answer: N"), in the pool
order, so the first examples of a subject are used first.
"""
import argparse
from . import prompts as P
from .data import load_jsonl, write_jsonl
from .evaluate import ANSWER_RE, load_model_cfg
def main():
ap = argparse.ArgumentParser()
ap.add_argument("--model", required=True)
ap.add_argument("--prompt", default="cot_en")
a = ap.parse_args()
from .model import LM, load_lm, seed_everything
cfg = load_model_cfg(a.model)
seed_everything(cfg["seed"])
model, tok, info = load_lm(cfg["model"], cfg["revision"], cfg["dtype"], cfg["quant"])
lm = LM(model, tok, chat=True, context=info["context_length"], gen_batch_size=cfg["gen_batch_size"])
pool = load_jsonl(P.FEWSHOT_POOL)
p = P.PROMPTS[a.prompt]
gens = lm.generate([P.build(r, a.prompt) for r in pool], p.max_new_tokens)
kept = []
for r, g in zip(pool, gens):
m = list(ANSWER_RE.finditer(g))
if m and int(m[-1].group(1)) == int(r["answer"]):
kept.append(dict(r, rationale=g[:m[-1].end()].strip()))
write_jsonl(P.RATIONALE_POOL, kept)
subjects = {r["file"] for r in pool}
covered = {r["file"] for r in kept}
print(f"{len(kept)}/{len(pool)} pool questions with a correct rationale; subjects without one: "
f"{sorted(subjects - covered) or 'none'} -> {P.RATIONALE_POOL}")
if __name__ == "__main__":
main()