Download src/smmlu.py from SinhalaMMLU-SharedTask/team-mission-x: direct link, hf CLI and curl.
- Browser
- Download file 30.6 kB
-
https://huggingface.co/SinhalaMMLU-SharedTask/team-mission-x/resolve/main/src/smmlu.py
- Command line
-
hf download hf://SinhalaMMLU-SharedTask/team-mission-x/src/smmlu.py
-
curl -L -o smmlu.py https://huggingface.co/SinhalaMMLU-SharedTask/team-mission-x/resolve/main/src/smmlu.py
30.6 kB
| """Zero-shot multiple-choice answering for Sinhala MMLU-style data. | |
| Each question is scored by the model's next-token log-probability of the | |
| option numbers 1-4. Scores can be averaged over cyclic reorderings of the | |
| options to remove position bias. | |
| python smmlu.py inspect --data dev.jsonl | |
| python smmlu.py run --model Qwen/Qwen3-4B --data dev.jsonl --tag qwen3_4b_dev | |
| python smmlu.py generate --model Qwen/Qwen3-4B --data dev.jsonl --tag think_dev --shard 0 --num-shards 2 | |
| python smmlu.py combine --data dev.jsonl --tag think_dev --parts outputs/think_dev_s0_gen.jsonl outputs/think_dev_s1_gen.jsonl | |
| `generate` lets the model reason before answering and falls back to option | |
| scoring when no answer can be extracted. `inspect` prints schema and counts | |
| only, never question text. | |
| """ | |
| import argparse | |
| import csv | |
| import hashlib | |
| import json | |
| import os | |
| import re | |
| import time | |
| import zipfile | |
| from collections import Counter, defaultdict | |
| from pathlib import Path | |
| LEVELS = ("easy", "medium", "hard") | |
| PARAM_LIMIT = 8_000_000_000 | |
| QUESTION_KEYS = ("question", "Question", "question_text", "query", "text", "q") | |
| GOLD_KEYS = ("answer", "Answer", "label", "gold", "correct", "correct_answer", "answer_key") | |
| OPTION_PREFIXES = ("option_", "option", "Option ", "Option", "opt", "choice_", "choice", "") | |
| INSTRUCTION = ( | |
| "The following is a multiple-choice question in Sinhala. " | |
| "Exactly one option is correct. " | |
| "Reply with only the number of the correct option: 1, 2, 3 or 4." | |
| ) | |
| GEN_INSTRUCTION = ( | |
| "The following is a multiple-choice question in Sinhala. " | |
| "Exactly one option is correct. " | |
| "Think it through, then end with a final line of the form 'Answer: N', where N is 1, 2, 3 or 4." | |
| ) | |
| INSTRUCTIONS = { | |
| "en": {"score": INSTRUCTION, "gen": GEN_INSTRUCTION}, | |
| "si": {"score": "පහත දැක්වෙන්නේ සිංහල බහුවරණ ප්රශ්නයකි. එක් පිළිතුරක් පමණක් නිවැරදි ය. " | |
| "නිවැරදි පිළිතුරේ අංකය (1, 2, 3 හෝ 4) පමණක් ලියන්න.", | |
| "gen": "පහත දැක්වෙන්නේ සිංහල බහුවරණ ප්රශ්නයකි. එක් පිළිතුරක් පමණක් නිවැරදි ය. " | |
| "ප්රශ්නය හොඳින් සිතා බලා, අවසානයේ 'Answer: N' ලෙස පිළිතුර දෙන්න (N යනු 1, 2, 3 හෝ 4)."}, | |
| } | |
| THINK_END_MARKERS = ("</think>", "<channel|>") | |
| ANSWER_RE = re.compile(r"(?:answer|පිළිතුර)\s*(?:is)?\s*[::]?\s*\**\s*\(?\s*([1-4])\b", re.IGNORECASE) | |
| # ---------------------------------------------------------------- data | |
| def load_records(path): | |
| path = Path(path) | |
| if path.suffix == ".jsonl": | |
| with open(path, encoding="utf-8-sig") as f: | |
| return [json.loads(line) for line in f if line.strip()] | |
| if path.suffix == ".json": | |
| with open(path, encoding="utf-8-sig") as f: | |
| data = json.load(f) | |
| if isinstance(data, dict): | |
| data = next(v for v in data.values() if isinstance(v, list)) | |
| return data | |
| if path.suffix == ".csv": | |
| with open(path, encoding="utf-8-sig", newline="") as f: | |
| return list(csv.DictReader(f)) | |
| raise ValueError(f"Unsupported file type: {path.suffix}") | |
| def get_question(rec): | |
| for k in QUESTION_KEYS: | |
| if k in rec and rec[k] not in (None, ""): | |
| return str(rec[k]) | |
| raise KeyError(f"No question field among keys {sorted(rec)}") | |
| def get_options(rec): | |
| opts = rec.get("options") or rec.get("choices") | |
| if isinstance(opts, str): | |
| opts = json.loads(opts) | |
| if isinstance(opts, dict): | |
| opts = [opts[k] for k in sorted(opts, key=str)] | |
| if isinstance(opts, list): | |
| return [str(o) for o in opts] | |
| for prefix in OPTION_PREFIXES: | |
| for labels in ("1234", "abcd", "ABCD"): | |
| keys = [f"{prefix}{c}" for c in labels] | |
| if all(k in rec for k in keys): | |
| return [str(rec[k]) for k in keys] | |
| raise KeyError(f"No option fields among keys {sorted(rec)}") | |
| def get_gold(rec, options): | |
| """Return the gold option as 1-4, or None when the record has no label.""" | |
| for k in GOLD_KEYS: | |
| if k not in rec or rec[k] in (None, ""): | |
| continue | |
| v = str(rec[k]).strip() | |
| if v in ("1", "2", "3", "4"): | |
| return int(v) | |
| if v.upper() in ("A", "B", "C", "D"): | |
| return "ABCD".index(v.upper()) + 1 | |
| if v in options: | |
| return options.index(v) + 1 | |
| return None | |
| def get_level(rec): | |
| return str(rec.get("level") or rec.get("difficulty") or "").strip().lower() | |
| def get_qno(rec): | |
| for k in ("q_no", "id", "qid", "question_id"): | |
| if rec.get(k) not in (None, ""): | |
| return rec[k] | |
| return None | |
| def text_hash(s): | |
| return hashlib.sha1(" ".join(s.split()).encode("utf-8")).hexdigest() | |
| def loose_hash(s): | |
| """Hash that ignores whitespace, punctuation and case, for overlap checks.""" | |
| kept = "".join(ch for ch in s.lower() if ch.isalnum() or "" <= ch <= "") | |
| return hashlib.sha1(kept.encode("utf-8")).hexdigest() | |
| def inspect(args): | |
| for path in args.data: | |
| recs = load_records(path) | |
| keys = Counter(k for r in recs for k in r) | |
| print(f"\n=== {path}: {len(recs)} records") | |
| print("fields:", dict(keys)) | |
| n_opts, golds, problems = Counter(), Counter(), Counter() | |
| q_hashes, full_hashes = Counter(), Counter() | |
| for r in recs: | |
| try: | |
| q, opts = get_question(r), get_options(r) | |
| except KeyError as e: | |
| problems[str(e)[:80]] += 1 | |
| continue | |
| n_opts[len(opts)] += 1 | |
| golds[get_gold(r, opts)] += 1 | |
| q_hashes[text_hash(q)] += 1 | |
| full_hashes[text_hash(q + "||" + "||".join(opts))] += 1 | |
| if any(not o.strip() for o in opts): | |
| problems["empty option"] += 1 | |
| print("level:", dict(Counter(get_level(r) for r in recs))) | |
| print("category:", dict(Counter(r.get("category") for r in recs))) | |
| print("subjects:", len(Counter(r.get("subject") for r in recs))) | |
| print("options per question:", dict(n_opts)) | |
| print("gold answer distribution (1-4):", dict(golds)) | |
| for k in GOLD_KEYS: | |
| raw = Counter(str(r[k]) for r in recs if k in r) | |
| if raw: | |
| print(f"raw '{k}' values (top 8):", dict(raw.most_common(8))) | |
| print("duplicate questions (same text):", sum(c - 1 for c in q_hashes.values() if c > 1)) | |
| print("duplicate questions (text + options):", sum(c - 1 for c in full_hashes.values() if c > 1)) | |
| q_nos = [get_qno(r) for r in recs] | |
| keys = {(get_level(r), str(r.get("subject")), str(get_qno(r))) for r in recs} | |
| print("q_no unique alone:", len(set(map(str, q_nos))) == len(q_nos), | |
| "| (level, subject, q_no) unique:", len(keys) == len(recs), | |
| "| missing q_no:", q_nos.count(None), "| q_no type:", dict(Counter(type(q).__name__ for q in q_nos))) | |
| if problems: | |
| print("problems:", dict(problems)) | |
| # ---------------------------------------------------------------- model | |
| def build_prompt(tokenizer, question, options, subject=None, instruction=INSTRUCTION, thinking=False): | |
| lines = [instruction, ""] | |
| if subject: | |
| lines.append(f"Subject: {subject}") | |
| lines += [question, ""] | |
| lines += [f"{i}. {o}" for i, o in enumerate(options, 1)] | |
| user = "\n".join(lines) | |
| if tokenizer.chat_template: | |
| messages = [{"role": "user", "content": user}] | |
| try: | |
| return tokenizer.apply_chat_template( | |
| messages, tokenize=False, add_generation_prompt=True, enable_thinking=thinking) | |
| except TypeError: | |
| return tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True) | |
| return user + "\n\nAnswer:" | |
| def extract_answer(text, thinking): | |
| """Final option (1-4) from a generated reply, or None if it cannot be read reliably. | |
| Reasoning ends with `</think>` (Qwen) or `<channel|>` (Gemma 4); only the text after it is read. | |
| """ | |
| ends = [text.rfind(m) + len(m) for m in THINK_END_MARKERS if m in text] | |
| if thinking and not ends: | |
| return None # reasoning was cut off by the token limit | |
| tail = text[max(ends):] if ends else text | |
| found = ANSWER_RE.findall(tail) | |
| if found: | |
| return int(found[-1]) | |
| digits = re.findall(r"(?<![0-9])([1-4])(?![0-9])", tail.strip()[-20:]) | |
| return int(digits[-1]) if digits else None | |
| def digit_token_ids(tokenizer): | |
| """Token ids that can start an answer of 1-4, with and without a leading space.""" | |
| ids = [] | |
| for d in "1234": | |
| cands = set() | |
| for s in (d, " " + d): | |
| toks = tokenizer.encode(s, add_special_tokens=False) | |
| if toks: | |
| cands.add(toks[-1]) | |
| ids.append(sorted(cands)) | |
| return ids | |
| def load_model(name, dtype, adapter=None, device_map="auto", param_limit=PARAM_LIMIT): | |
| import torch | |
| import transformers | |
| from transformers import AutoModelForCausalLM, AutoTokenizer | |
| revision = os.environ.get("SMMLU_REVISION") or None # pinned commit for reproduction | |
| tokenizer = AutoTokenizer.from_pretrained(name, revision=revision) | |
| tokenizer.padding_side = "left" | |
| tokenizer.truncation_side = "left" # keep the options and answer cue when a prompt is too long | |
| if tokenizer.pad_token is None: | |
| tokenizer.pad_token = tokenizer.eos_token | |
| # transformers >= 4.56 takes `dtype`; older versions only know `torch_dtype` | |
| major, minor = (int(x) for x in transformers.__version__.split(".")[:2]) | |
| dtype_key = "dtype" if (major, minor) >= (4, 56) else "torch_dtype" | |
| kwargs = {dtype_key: getattr(torch, dtype), "device_map": device_map, "revision": revision} | |
| try: | |
| model = AutoModelForCausalLM.from_pretrained(name, **kwargs) | |
| except (ValueError, KeyError) as e: | |
| # Multimodal checkpoints (Qwen3.5, Gemma 4) register under other auto classes | |
| print("AutoModelForCausalLM failed, trying multimodal classes:", str(e)[:120]) | |
| model = None | |
| for cls_name in ("AutoModelForImageTextToText", "AutoModelForMultimodalLM"): | |
| cls = getattr(transformers, cls_name, None) | |
| if cls is None: | |
| continue | |
| try: | |
| model = cls.from_pretrained(name, **kwargs) | |
| break | |
| except (ValueError, KeyError) as e2: | |
| print(f"{cls_name} failed:", str(e2)[:120]) | |
| if model is None: | |
| raise SystemExit(f"No auto class could load {name}") | |
| if adapter: | |
| from peft import PeftModel | |
| model = PeftModel.from_pretrained(model, adapter) | |
| model.eval() | |
| n_params = sum(p.numel() for p in model.parameters()) | |
| loaded = Counter(str(p.dtype).replace("torch.", "") for p in model.parameters()) | |
| print(f"Parameters: {n_params:,} | dtypes: {dict(loaded)} | device: {model.device}") | |
| if dtype not in loaded: | |
| raise SystemExit(f"Model did not load in {dtype} (got {dict(loaded)}); check the dtype argument") | |
| if n_params > param_limit: | |
| raise SystemExit(f"{name} has {n_params:,} parameters, above the {param_limit:,.0f} limit") | |
| return tokenizer, model, n_params | |
| def score(tokenizer, model, prompts, batch_size, max_len): | |
| """Return a list of 4 log-probabilities (options 1-4) per prompt.""" | |
| import torch | |
| cand_ids = digit_token_ids(tokenizer) | |
| order = sorted(range(len(prompts)), key=lambda i: len(prompts[i]), reverse=True) | |
| out = [None] * len(prompts) | |
| start = time.time() | |
| for b in range(0, len(order), batch_size): | |
| idx = order[b:b + batch_size] | |
| enc = tokenizer([prompts[i] for i in idx], return_tensors="pt", padding=True, | |
| truncation=True, max_length=max_len, add_special_tokens=False) | |
| enc = {k: v.to(model.device) for k, v in enc.items()} | |
| with torch.no_grad(): | |
| try: | |
| logits = model(**enc, logits_to_keep=1).logits[:, -1, :].float() | |
| except TypeError: | |
| logits = model(**enc).logits[:, -1, :].float() | |
| logp = torch.log_softmax(logits, dim=-1) | |
| for row, i in enumerate(idx): | |
| out[i] = [torch.logsumexp(logp[row, ids], dim=0).item() for ids in cand_ids] | |
| done = b + len(idx) | |
| if (b // batch_size) % 20 == 0: | |
| rate = done / max(time.time() - start, 1e-6) | |
| print(f" {done}/{len(prompts)} prompts {rate:.1f}/s eta {(len(prompts) - done) / rate / 60:.1f} min") | |
| return out | |
| def generate_texts(tokenizer, model, prompts, batch_size, max_len, max_new_tokens): | |
| """Greedy-decode a reply for each prompt; returns (text, n_new_tokens) pairs.""" | |
| import torch | |
| strip = [t for t in {tokenizer.pad_token, tokenizer.eos_token} if t] | |
| order = sorted(range(len(prompts)), key=lambda i: len(prompts[i]), reverse=True) | |
| out = [None] * len(prompts) | |
| start = time.time() | |
| for b in range(0, len(order), batch_size): | |
| idx = order[b:b + batch_size] | |
| enc = tokenizer([prompts[i] for i in idx], return_tensors="pt", padding=True, | |
| truncation=True, max_length=max_len, add_special_tokens=False) | |
| enc = {k: v.to(model.device) for k, v in enc.items()} | |
| with torch.no_grad(): | |
| gen = model.generate(**enc, max_new_tokens=max_new_tokens, do_sample=False, | |
| temperature=None, top_p=None, top_k=None, | |
| pad_token_id=tokenizer.pad_token_id) | |
| new = gen[:, enc["input_ids"].shape[1]:] | |
| for row, i in enumerate(idx): | |
| n = int((new[row] != tokenizer.pad_token_id).sum()) | |
| text = tokenizer.decode(new[row], skip_special_tokens=False) | |
| for t in strip: | |
| text = text.replace(t, "") | |
| out[i] = (text, n) | |
| done = b + len(idx) | |
| rate = done / max(time.time() - start, 1e-6) | |
| print(f" {done}/{len(prompts)} questions {rate * 60:.1f}/min " | |
| f"eta {(len(prompts) - done) / rate / 60:.1f} min", flush=True) | |
| return out | |
| def norm_category(c): | |
| return " ".join(str(c or "").replace("_", " ").lower().split()) | |
| def routed_to_reasoning(rec, levels, categories, rules=()): | |
| """True if a question goes to the reasoning path (no filters at all means every question does). | |
| levels / categories match on their own; each rule is "level:category" and needs both to match. | |
| """ | |
| if not levels and not categories and not rules: | |
| return True | |
| level, category = get_level(rec), norm_category(rec.get("category")) | |
| if level in {l.lower() for l in levels} or category in {norm_category(c) for c in categories}: | |
| return True | |
| for rule in rules: | |
| r_level, _, r_cat = rule.partition(":") | |
| if level == r_level.strip().lower() and category == norm_category(r_cat): | |
| return True | |
| return False | |
| def generate(args): | |
| """Reason-then-answer predictions for one shard, with option scoring as the fallback.""" | |
| import torch | |
| torch.manual_seed(args.seed) | |
| recs = load_records(args.data) | |
| if args.limit: | |
| recs = recs[:args.limit] | |
| selected = [i for i in range(len(recs)) if routed_to_reasoning(recs[i], args.gen_levels, args.gen_categories, args.gen_rules)] | |
| if args.revote_of: | |
| # Second opinion only where the first reasoning pass is uncertain: no readable answer, | |
| # or an answer that disagrees with option scoring. | |
| if not args.ll_scores: | |
| raise SystemExit("--revote-of needs --ll-scores") | |
| base = {} | |
| for path in args.revote_of: | |
| base.update({row["idx"]: row for row in load_records(path)}) | |
| ll = {row["idx"]: row["pred"] for row in load_records(args.ll_scores)} | |
| selected = [i for i in selected if i in base | |
| and (not base[i]["from_reasoning"] or base[i]["answer"] != ll[i])] | |
| mine = [i for k, i in enumerate(selected) if k % args.num_shards == args.shard] | |
| tokenizer, model, n_params = load_model(args.model, args.dtype, args.adapter, param_limit=args.param_limit) | |
| def subject_of(r): | |
| return r.get("subject") if args.with_subject else None | |
| instr = INSTRUCTIONS[args.lang] | |
| # --shift K shows the options rotated by K places; displayed slot s holds original option shown[s]. | |
| shown = [(j + args.shift) % 4 for j in range(4)] | |
| prompts = [] | |
| for i in mine: | |
| opts = get_options(recs[i]) | |
| prompts.append(build_prompt(tokenizer, get_question(recs[i]), [opts[p] for p in shown], subject_of(recs[i]), | |
| instr["gen"], args.thinking)) | |
| print(f"shard {args.shard}/{args.num_shards}: {len(mine)} questions, thinking={args.thinking}, shift={args.shift}") | |
| replies = generate_texts(tokenizer, model, prompts, args.batch_size, args.max_len, args.max_new_tokens) | |
| answers = [extract_answer(text, args.thinking) for text, _ in replies] | |
| answers = [shown[a - 1] + 1 if a else None for a in answers] | |
| # Fallback: option scoring over all 4 cyclic orderings for anything without a readable answer. | |
| missing = [k for k, a in enumerate(answers) if a is None] | |
| fallback = {} | |
| if missing: | |
| fb_prompts, owners = [], [] | |
| for k in missing: | |
| r = recs[mine[k]] | |
| opts = get_options(r) | |
| for s in range(4): | |
| perm = [(j + s) % 4 for j in range(4)] | |
| fb_prompts.append(build_prompt(tokenizer, get_question(r), [opts[p] for p in perm], subject_of(r), | |
| instr["score"])) | |
| owners.append((k, perm)) | |
| totals = {k: [0.0] * 4 for k in missing} | |
| for (k, perm), lp in zip(owners, score(tokenizer, model, fb_prompts, args.batch_size, args.max_len)): | |
| for slot, orig in enumerate(perm): | |
| totals[k][orig] += lp[slot] / 4 | |
| fallback = {k: max(range(4), key=lambda j: totals[k][j]) + 1 for k in missing} | |
| out_dir = Path(args.out_dir) | |
| out_dir.mkdir(parents=True, exist_ok=True) | |
| path = out_dir / f"{args.tag}_s{args.shard}_gen.jsonl" | |
| with open(path, "w", encoding="utf-8", newline="\n") as f: | |
| for k, i in enumerate(mine): | |
| text, n_tok = replies[k] | |
| f.write(json.dumps({"idx": i, "answer": answers[k] or fallback[k], "from_reasoning": answers[k] is not None, | |
| "shift": args.shift, "new_tokens": n_tok, "reply": text}, ensure_ascii=False) + "\n") | |
| lens = sorted(n for _, n in replies) | |
| print(f"\nModel: {args.model} params: {n_params:,} thinking: {args.thinking}") | |
| print(f"answers read from reasoning: {len(mine) - len(missing)}/{len(mine)} | fallback to scoring: {len(missing)}") | |
| print(f"new tokens: median {lens[len(lens) // 2]} max {lens[-1]} (limit {args.max_new_tokens})") | |
| print(f"Wrote {path}") | |
| def combine(args): | |
| """Merge shard outputs of `generate` into predictions.jsonl, a zip and (with gold) accuracy. | |
| With --ll-scores, questions outside --gen-levels / --gen-categories take the option-scoring | |
| answer from a `run` scores file instead (hybrid: reasoning where it helps, scoring elsewhere). | |
| """ | |
| recs = load_records(args.data) | |
| if args.limit: | |
| recs = recs[:args.limit] | |
| answers = {} | |
| for part in args.parts: | |
| for row in load_records(part): | |
| answers[row["idx"]] = row["answer"] | |
| use_gen = [routed_to_reasoning(r, args.gen_levels, args.gen_categories, args.gen_rules) for r in recs] | |
| missing = [i for i in range(len(recs)) if use_gen[i] and i not in answers] | |
| if missing: | |
| raise SystemExit(f"{len(missing)} questions have no reasoning answer; check that every shard finished " | |
| f"and that --gen-levels/--gen-categories match the generate run") | |
| if all(use_gen): | |
| preds = [answers[i] for i in range(len(recs))] | |
| else: | |
| if not args.ll_scores: | |
| raise SystemExit("--gen-levels/--gen-categories need --ll-scores for the remaining questions") | |
| ll = {row["idx"]: row["pred"] for row in load_records(args.ll_scores)} | |
| if len(ll) != len(recs): | |
| raise SystemExit(f"--ll-scores has {len(ll)} rows for {len(recs)} questions") | |
| preds = [answers[i] if use_gen[i] else ll[i] for i in range(len(recs))] | |
| print(f"hybrid: reasoning answers for {sum(use_gen)}/{len(recs)} questions " | |
| f"(levels {args.gen_levels}, categories {args.gen_categories}, rules {args.gen_rules}), " | |
| f"option scoring for the rest") | |
| if args.vote_parts: | |
| preds = vote(args, preds, use_gen) | |
| write_outputs(args, recs, preds) | |
| def vote(args, preds, use_gen): | |
| """Plurality vote over the reasoning answers of every pass. | |
| A pass votes only with an answer read from its reasoning (not its scoring fallback). Ties go to the | |
| earliest pass (first pass, then --vote-parts in the given order). Option scoring is used only when | |
| no pass has a reasoning answer: where reasoning and scoring disagree, scoring is rarely the right one. | |
| """ | |
| if not args.ll_scores: | |
| raise SystemExit("--vote-parts needs --ll-scores") | |
| ll = {row["idx"]: row["pred"] for row in load_records(args.ll_scores)} | |
| first = {row["idx"]: row for part in args.parts for row in load_records(part)} | |
| extra = [{row["idx"]: row for row in load_records(part)} for part in args.vote_parts] | |
| voted = sorted(set().union(*extra)) | |
| if any(not use_gen[i] for i in voted): | |
| raise SystemExit("--vote-parts contain questions outside the reasoning groups") | |
| preds, changed = list(preds), 0 | |
| for i in voted: | |
| ballots = [r[i]["answer"] for r in [first] + extra if i in r and r[i]["from_reasoning"]] or [ll[i]] | |
| counts = Counter(ballots) | |
| winner = max(ballots, key=lambda a: (counts[a], -ballots.index(a))) | |
| changed += winner != preds[i] | |
| preds[i] = winner | |
| print(f"vote: {len(voted)} questions re-answered by {len(extra)} extra pass(es); {changed} answers changed") | |
| return preds | |
| def write_outputs(args, recs, preds): | |
| out_dir = Path(args.out_dir) | |
| out_dir.mkdir(parents=True, exist_ok=True) | |
| pred_path = out_dir / f"{args.tag}_predictions.jsonl" | |
| with open(pred_path, "w", encoding="utf-8", newline="\n") as f: | |
| for r, p in zip(recs, preds): | |
| f.write(json.dumps({"q_no": get_qno(r), "subject": r.get("subject"), | |
| "category": r.get("category"), "level": get_level(r), | |
| "answer": str(p)}, ensure_ascii=False) + "\n") | |
| zip_path = out_dir / f"{args.tag}.zip" | |
| with zipfile.ZipFile(zip_path, "w", zipfile.ZIP_DEFLATED) as z: | |
| z.write(pred_path, arcname="predictions.jsonl") | |
| print("Predicted answer distribution:", dict(sorted(Counter(preds).items()))) | |
| by_level = defaultdict(lambda: [0, 0]) | |
| for r, p in zip(recs, preds): | |
| gold = get_gold(r, get_options(r)) | |
| if gold is not None: | |
| by_level[get_level(r)][0] += int(p == gold) | |
| by_level[get_level(r)][1] += 1 | |
| accs = {} | |
| for lvl in sorted(by_level, key=lambda l: (LEVELS.index(l) if l in LEVELS else len(LEVELS), l)): | |
| c, n = by_level[lvl] | |
| accs[lvl or "all"] = c / n | |
| print(f" {lvl or 'all':<6} acc = {c / n:.4f} ({c}/{n})") | |
| if all(lvl in accs for lvl in LEVELS): | |
| print(f" MACRO acc = {sum(accs[l] for l in LEVELS) / 3:.4f}") | |
| if accs: | |
| with open(out_dir / f"{args.tag}_summary.json", "w") as f: | |
| json.dump({"model": getattr(args, "model", None), "adapter": getattr(args, "adapter", None), | |
| "params": getattr(args, "n_params", 0), "perms": getattr(args, "perms", "gen"), | |
| "with_subject": getattr(args, "with_subject", None), "lang": getattr(args, "lang", None), | |
| "acc": accs, | |
| "macro": sum(accs[l] for l in LEVELS) / 3 if all(l in accs for l in LEVELS) else None}, | |
| f, indent=2) | |
| print(f"Wrote {zip_path}") | |
| def run(args): | |
| import torch | |
| torch.manual_seed(args.seed) | |
| recs = load_records(args.data) | |
| if args.limit: | |
| recs = recs[:args.limit] | |
| tokenizer, model, n_params = load_model(args.model, args.dtype, args.adapter, param_limit=args.param_limit) | |
| shifts = list(range(args.perms)) | |
| prompts, owners = [], [] | |
| for qi, r in enumerate(recs): | |
| opts = get_options(r) | |
| subject = r.get("subject") if args.with_subject else None | |
| for k in shifts: | |
| perm = [(j + k) % 4 for j in range(4)] # displayed slot j shows original option perm[j] | |
| prompts.append(build_prompt(tokenizer, get_question(r), [opts[p] for p in perm], subject, | |
| INSTRUCTIONS[args.lang]["score"])) | |
| owners.append((qi, perm)) | |
| print(f"{len(recs)} questions x {len(shifts)} orderings = {len(prompts)} prompts") | |
| print("Example prompt template (first 200 chars):", repr(prompts[0][:200]) if args.show_prompt else "hidden") | |
| raw = score(tokenizer, model, prompts, args.batch_size, args.max_len) | |
| totals = [[0.0] * 4 for _ in recs] | |
| for (qi, perm), lp in zip(owners, raw): | |
| for slot, orig in enumerate(perm): | |
| totals[qi][orig] += lp[slot] / len(shifts) | |
| out_dir = Path(args.out_dir) | |
| out_dir.mkdir(parents=True, exist_ok=True) | |
| preds = [max(range(4), key=lambda j: s[j]) + 1 for s in totals] | |
| with open(out_dir / f"{args.tag}_scores.jsonl", "w", encoding="utf-8", newline="\n") as f: | |
| for i, (r, s, pred) in enumerate(zip(recs, totals, preds)): | |
| f.write(json.dumps({"idx": i, "q_no": get_qno(r), "level": get_level(r), "scores": s, | |
| "pred": pred, "gold": get_gold(r, get_options(r))}) + "\n") | |
| print(f"\nModel: {args.model} params: {n_params:,} orderings: {len(shifts)}") | |
| args.n_params = n_params | |
| write_outputs(args, recs, preds) | |
| def main(): | |
| ap = argparse.ArgumentParser() | |
| sub = ap.add_subparsers(dest="cmd", required=True) | |
| p = sub.add_parser("inspect") | |
| p.add_argument("--data", nargs="+", required=True) | |
| p = sub.add_parser("run") | |
| p.add_argument("--model", required=True) | |
| p.add_argument("--adapter", default=None, help="LoRA adapter directory") | |
| p.add_argument("--data", required=True) | |
| p.add_argument("--tag", required=True) | |
| p.add_argument("--out-dir", default="outputs") | |
| p.add_argument("--perms", type=int, default=4, choices=[1, 2, 3, 4]) | |
| p.add_argument("--batch-size", type=int, default=8) | |
| p.add_argument("--max-len", type=int, default=2048) | |
| p.add_argument("--dtype", default="float16", choices=["float16", "bfloat16", "float32"]) | |
| p.add_argument("--with-subject", action="store_true") | |
| p.add_argument("--lang", default="en", choices=sorted(INSTRUCTIONS), help="instruction language") | |
| p.add_argument("--show-prompt", action="store_true") | |
| p.add_argument("--limit", type=int, default=0) | |
| p.add_argument("--param-limit", type=float, default=PARAM_LIMIT, | |
| help="raise only if the organisers confirm a model above 8e9 (e.g. Qwen3-8B, 8.2B) is accepted") | |
| p.add_argument("--seed", type=int, default=42) | |
| p = sub.add_parser("generate") | |
| p.add_argument("--model", required=True) | |
| p.add_argument("--adapter", default=None) | |
| p.add_argument("--data", required=True) | |
| p.add_argument("--tag", required=True) | |
| p.add_argument("--out-dir", default="outputs") | |
| p.add_argument("--shard", type=int, default=0) | |
| p.add_argument("--num-shards", type=int, default=1) | |
| p.add_argument("--thinking", action="store_true", help="enable the chat template's thinking mode") | |
| p.add_argument("--gen-levels", nargs="*", default=[], help="only answer these levels (hybrid)") | |
| p.add_argument("--gen-categories", nargs="*", default=[], help="only answer these categories (hybrid)") | |
| p.add_argument("--gen-rules", nargs="*", default=[], help='"level:category" pairs, e.g. "medium:social science"') | |
| p.add_argument("--shift", type=int, default=0, choices=[0, 1, 2, 3], help="show the options rotated by this many places") | |
| p.add_argument("--revote-of", nargs="*", default=[], | |
| help="earlier generate outputs; only questions with no readable answer there, or one that " | |
| "disagrees with --ll-scores, are answered again") | |
| p.add_argument("--ll-scores", default=None, help="scores.jsonl from `run` (for --revote-of)") | |
| p.add_argument("--max-new-tokens", type=int, default=2048) | |
| p.add_argument("--batch-size", type=int, default=8) | |
| p.add_argument("--max-len", type=int, default=2048) | |
| p.add_argument("--dtype", default="float16", choices=["float16", "bfloat16", "float32"]) | |
| p.add_argument("--with-subject", action="store_true") | |
| p.add_argument("--lang", default="en", choices=sorted(INSTRUCTIONS), help="instruction language") | |
| p.add_argument("--limit", type=int, default=0) | |
| p.add_argument("--param-limit", type=float, default=PARAM_LIMIT) | |
| p.add_argument("--seed", type=int, default=42) | |
| p = sub.add_parser("combine") | |
| p.add_argument("--data", required=True) | |
| p.add_argument("--tag", required=True) | |
| p.add_argument("--parts", nargs="+", required=True) | |
| p.add_argument("--out-dir", default="outputs") | |
| p.add_argument("--model", default=None, help="recorded in the summary only") | |
| p.add_argument("--ll-scores", default=None, help="scores.jsonl from `run`; enables the hybrid") | |
| p.add_argument("--gen-levels", nargs="*", default=[], help="levels answered by reasoning in the hybrid") | |
| p.add_argument("--gen-categories", nargs="*", default=[], help="categories answered by reasoning in the hybrid") | |
| p.add_argument("--gen-rules", nargs="*", default=[], help='"level:category" pairs, e.g. "medium:social science"') | |
| p.add_argument("--vote-parts", nargs="*", default=[], | |
| help="generate outputs of extra reasoning passes (e.g. --shift 1, 2 with --revote-of); " | |
| "their questions take the plurality vote of the reasoning passes") | |
| p.add_argument("--limit", type=int, default=0) | |
| args = ap.parse_args() | |
| {"inspect": inspect, "run": run, "generate": generate, "combine": combine}[args.cmd](args) | |
| if __name__ == "__main__": | |
| main() | |