"""Zero-shot multiple-choice answering for Sinhala MMLU-style data. Each question is scored by the model's next-token log-probability of the option numbers 1-4. Scores can be averaged over cyclic reorderings of the options to remove position bias. python smmlu.py inspect --data dev.jsonl python smmlu.py run --model Qwen/Qwen3-4B --data dev.jsonl --tag qwen3_4b_dev python smmlu.py generate --model Qwen/Qwen3-4B --data dev.jsonl --tag think_dev --shard 0 --num-shards 2 python smmlu.py combine --data dev.jsonl --tag think_dev --parts outputs/think_dev_s0_gen.jsonl outputs/think_dev_s1_gen.jsonl `generate` lets the model reason before answering and falls back to option scoring when no answer can be extracted. `inspect` prints schema and counts only, never question text. """ import argparse import csv import hashlib import json import os import re import time import zipfile from collections import Counter, defaultdict from pathlib import Path LEVELS = ("easy", "medium", "hard") PARAM_LIMIT = 8_000_000_000 QUESTION_KEYS = ("question", "Question", "question_text", "query", "text", "q") GOLD_KEYS = ("answer", "Answer", "label", "gold", "correct", "correct_answer", "answer_key") OPTION_PREFIXES = ("option_", "option", "Option ", "Option", "opt", "choice_", "choice", "") INSTRUCTION = ( "The following is a multiple-choice question in Sinhala. " "Exactly one option is correct. " "Reply with only the number of the correct option: 1, 2, 3 or 4." ) GEN_INSTRUCTION = ( "The following is a multiple-choice question in Sinhala. " "Exactly one option is correct. " "Think it through, then end with a final line of the form 'Answer: N', where N is 1, 2, 3 or 4." ) INSTRUCTIONS = { "en": {"score": INSTRUCTION, "gen": GEN_INSTRUCTION}, "si": {"score": "පහත දැක්වෙන්නේ සිංහල බහුවරණ ප්‍රශ්නයකි. එක් පිළිතුරක් පමණක් නිවැරදි ය. " "නිවැරදි පිළිතුරේ අංකය (1, 2, 3 හෝ 4) පමණක් ලියන්න.", "gen": "පහත දැක්වෙන්නේ සිංහල බහුවරණ ප්‍රශ්නයකි. එක් පිළිතුරක් පමණක් නිවැරදි ය. " "ප්‍රශ්නය හොඳින් සිතා බලා, අවසානයේ 'Answer: N' ලෙස පිළිතුර දෙන්න (N යනු 1, 2, 3 හෝ 4)."}, } THINK_END_MARKERS = ("", "") ANSWER_RE = re.compile(r"(?:answer|පිළිතුර)\s*(?:is)?\s*[::]?\s*\**\s*\(?\s*([1-4])\b", re.IGNORECASE) # ---------------------------------------------------------------- data def load_records(path): path = Path(path) if path.suffix == ".jsonl": with open(path, encoding="utf-8-sig") as f: return [json.loads(line) for line in f if line.strip()] if path.suffix == ".json": with open(path, encoding="utf-8-sig") as f: data = json.load(f) if isinstance(data, dict): data = next(v for v in data.values() if isinstance(v, list)) return data if path.suffix == ".csv": with open(path, encoding="utf-8-sig", newline="") as f: return list(csv.DictReader(f)) raise ValueError(f"Unsupported file type: {path.suffix}") def get_question(rec): for k in QUESTION_KEYS: if k in rec and rec[k] not in (None, ""): return str(rec[k]) raise KeyError(f"No question field among keys {sorted(rec)}") def get_options(rec): opts = rec.get("options") or rec.get("choices") if isinstance(opts, str): opts = json.loads(opts) if isinstance(opts, dict): opts = [opts[k] for k in sorted(opts, key=str)] if isinstance(opts, list): return [str(o) for o in opts] for prefix in OPTION_PREFIXES: for labels in ("1234", "abcd", "ABCD"): keys = [f"{prefix}{c}" for c in labels] if all(k in rec for k in keys): return [str(rec[k]) for k in keys] raise KeyError(f"No option fields among keys {sorted(rec)}") def get_gold(rec, options): """Return the gold option as 1-4, or None when the record has no label.""" for k in GOLD_KEYS: if k not in rec or rec[k] in (None, ""): continue v = str(rec[k]).strip() if v in ("1", "2", "3", "4"): return int(v) if v.upper() in ("A", "B", "C", "D"): return "ABCD".index(v.upper()) + 1 if v in options: return options.index(v) + 1 return None def get_level(rec): return str(rec.get("level") or rec.get("difficulty") or "").strip().lower() def get_qno(rec): for k in ("q_no", "id", "qid", "question_id"): if rec.get(k) not in (None, ""): return rec[k] return None def text_hash(s): return hashlib.sha1(" ".join(s.split()).encode("utf-8")).hexdigest() def loose_hash(s): """Hash that ignores whitespace, punctuation and case, for overlap checks.""" kept = "".join(ch for ch in s.lower() if ch.isalnum() or "඀" <= ch <= "෿") return hashlib.sha1(kept.encode("utf-8")).hexdigest() def inspect(args): for path in args.data: recs = load_records(path) keys = Counter(k for r in recs for k in r) print(f"\n=== {path}: {len(recs)} records") print("fields:", dict(keys)) n_opts, golds, problems = Counter(), Counter(), Counter() q_hashes, full_hashes = Counter(), Counter() for r in recs: try: q, opts = get_question(r), get_options(r) except KeyError as e: problems[str(e)[:80]] += 1 continue n_opts[len(opts)] += 1 golds[get_gold(r, opts)] += 1 q_hashes[text_hash(q)] += 1 full_hashes[text_hash(q + "||" + "||".join(opts))] += 1 if any(not o.strip() for o in opts): problems["empty option"] += 1 print("level:", dict(Counter(get_level(r) for r in recs))) print("category:", dict(Counter(r.get("category") for r in recs))) print("subjects:", len(Counter(r.get("subject") for r in recs))) print("options per question:", dict(n_opts)) print("gold answer distribution (1-4):", dict(golds)) for k in GOLD_KEYS: raw = Counter(str(r[k]) for r in recs if k in r) if raw: print(f"raw '{k}' values (top 8):", dict(raw.most_common(8))) print("duplicate questions (same text):", sum(c - 1 for c in q_hashes.values() if c > 1)) print("duplicate questions (text + options):", sum(c - 1 for c in full_hashes.values() if c > 1)) q_nos = [get_qno(r) for r in recs] keys = {(get_level(r), str(r.get("subject")), str(get_qno(r))) for r in recs} print("q_no unique alone:", len(set(map(str, q_nos))) == len(q_nos), "| (level, subject, q_no) unique:", len(keys) == len(recs), "| missing q_no:", q_nos.count(None), "| q_no type:", dict(Counter(type(q).__name__ for q in q_nos))) if problems: print("problems:", dict(problems)) # ---------------------------------------------------------------- model def build_prompt(tokenizer, question, options, subject=None, instruction=INSTRUCTION, thinking=False): lines = [instruction, ""] if subject: lines.append(f"Subject: {subject}") lines += [question, ""] lines += [f"{i}. {o}" for i, o in enumerate(options, 1)] user = "\n".join(lines) if tokenizer.chat_template: messages = [{"role": "user", "content": user}] try: return tokenizer.apply_chat_template( messages, tokenize=False, add_generation_prompt=True, enable_thinking=thinking) except TypeError: return tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True) return user + "\n\nAnswer:" def extract_answer(text, thinking): """Final option (1-4) from a generated reply, or None if it cannot be read reliably. Reasoning ends with `` (Qwen) or `` (Gemma 4); only the text after it is read. """ ends = [text.rfind(m) + len(m) for m in THINK_END_MARKERS if m in text] if thinking and not ends: return None # reasoning was cut off by the token limit tail = text[max(ends):] if ends else text found = ANSWER_RE.findall(tail) if found: return int(found[-1]) digits = re.findall(r"(?= 4.56 takes `dtype`; older versions only know `torch_dtype` major, minor = (int(x) for x in transformers.__version__.split(".")[:2]) dtype_key = "dtype" if (major, minor) >= (4, 56) else "torch_dtype" kwargs = {dtype_key: getattr(torch, dtype), "device_map": device_map, "revision": revision} try: model = AutoModelForCausalLM.from_pretrained(name, **kwargs) except (ValueError, KeyError) as e: # Multimodal checkpoints (Qwen3.5, Gemma 4) register under other auto classes print("AutoModelForCausalLM failed, trying multimodal classes:", str(e)[:120]) model = None for cls_name in ("AutoModelForImageTextToText", "AutoModelForMultimodalLM"): cls = getattr(transformers, cls_name, None) if cls is None: continue try: model = cls.from_pretrained(name, **kwargs) break except (ValueError, KeyError) as e2: print(f"{cls_name} failed:", str(e2)[:120]) if model is None: raise SystemExit(f"No auto class could load {name}") if adapter: from peft import PeftModel model = PeftModel.from_pretrained(model, adapter) model.eval() n_params = sum(p.numel() for p in model.parameters()) loaded = Counter(str(p.dtype).replace("torch.", "") for p in model.parameters()) print(f"Parameters: {n_params:,} | dtypes: {dict(loaded)} | device: {model.device}") if dtype not in loaded: raise SystemExit(f"Model did not load in {dtype} (got {dict(loaded)}); check the dtype argument") if n_params > param_limit: raise SystemExit(f"{name} has {n_params:,} parameters, above the {param_limit:,.0f} limit") return tokenizer, model, n_params def score(tokenizer, model, prompts, batch_size, max_len): """Return a list of 4 log-probabilities (options 1-4) per prompt.""" import torch cand_ids = digit_token_ids(tokenizer) order = sorted(range(len(prompts)), key=lambda i: len(prompts[i]), reverse=True) out = [None] * len(prompts) start = time.time() for b in range(0, len(order), batch_size): idx = order[b:b + batch_size] enc = tokenizer([prompts[i] for i in idx], return_tensors="pt", padding=True, truncation=True, max_length=max_len, add_special_tokens=False) enc = {k: v.to(model.device) for k, v in enc.items()} with torch.no_grad(): try: logits = model(**enc, logits_to_keep=1).logits[:, -1, :].float() except TypeError: logits = model(**enc).logits[:, -1, :].float() logp = torch.log_softmax(logits, dim=-1) for row, i in enumerate(idx): out[i] = [torch.logsumexp(logp[row, ids], dim=0).item() for ids in cand_ids] done = b + len(idx) if (b // batch_size) % 20 == 0: rate = done / max(time.time() - start, 1e-6) print(f" {done}/{len(prompts)} prompts {rate:.1f}/s eta {(len(prompts) - done) / rate / 60:.1f} min") return out def generate_texts(tokenizer, model, prompts, batch_size, max_len, max_new_tokens): """Greedy-decode a reply for each prompt; returns (text, n_new_tokens) pairs.""" import torch strip = [t for t in {tokenizer.pad_token, tokenizer.eos_token} if t] order = sorted(range(len(prompts)), key=lambda i: len(prompts[i]), reverse=True) out = [None] * len(prompts) start = time.time() for b in range(0, len(order), batch_size): idx = order[b:b + batch_size] enc = tokenizer([prompts[i] for i in idx], return_tensors="pt", padding=True, truncation=True, max_length=max_len, add_special_tokens=False) enc = {k: v.to(model.device) for k, v in enc.items()} with torch.no_grad(): gen = model.generate(**enc, max_new_tokens=max_new_tokens, do_sample=False, temperature=None, top_p=None, top_k=None, pad_token_id=tokenizer.pad_token_id) new = gen[:, enc["input_ids"].shape[1]:] for row, i in enumerate(idx): n = int((new[row] != tokenizer.pad_token_id).sum()) text = tokenizer.decode(new[row], skip_special_tokens=False) for t in strip: text = text.replace(t, "") out[i] = (text, n) done = b + len(idx) rate = done / max(time.time() - start, 1e-6) print(f" {done}/{len(prompts)} questions {rate * 60:.1f}/min " f"eta {(len(prompts) - done) / rate / 60:.1f} min", flush=True) return out def norm_category(c): return " ".join(str(c or "").replace("_", " ").lower().split()) def routed_to_reasoning(rec, levels, categories, rules=()): """True if a question goes to the reasoning path (no filters at all means every question does). levels / categories match on their own; each rule is "level:category" and needs both to match. """ if not levels and not categories and not rules: return True level, category = get_level(rec), norm_category(rec.get("category")) if level in {l.lower() for l in levels} or category in {norm_category(c) for c in categories}: return True for rule in rules: r_level, _, r_cat = rule.partition(":") if level == r_level.strip().lower() and category == norm_category(r_cat): return True return False def generate(args): """Reason-then-answer predictions for one shard, with option scoring as the fallback.""" import torch torch.manual_seed(args.seed) recs = load_records(args.data) if args.limit: recs = recs[:args.limit] selected = [i for i in range(len(recs)) if routed_to_reasoning(recs[i], args.gen_levels, args.gen_categories, args.gen_rules)] if args.revote_of: # Second opinion only where the first reasoning pass is uncertain: no readable answer, # or an answer that disagrees with option scoring. if not args.ll_scores: raise SystemExit("--revote-of needs --ll-scores") base = {} for path in args.revote_of: base.update({row["idx"]: row for row in load_records(path)}) ll = {row["idx"]: row["pred"] for row in load_records(args.ll_scores)} selected = [i for i in selected if i in base and (not base[i]["from_reasoning"] or base[i]["answer"] != ll[i])] mine = [i for k, i in enumerate(selected) if k % args.num_shards == args.shard] tokenizer, model, n_params = load_model(args.model, args.dtype, args.adapter, param_limit=args.param_limit) def subject_of(r): return r.get("subject") if args.with_subject else None instr = INSTRUCTIONS[args.lang] # --shift K shows the options rotated by K places; displayed slot s holds original option shown[s]. shown = [(j + args.shift) % 4 for j in range(4)] prompts = [] for i in mine: opts = get_options(recs[i]) prompts.append(build_prompt(tokenizer, get_question(recs[i]), [opts[p] for p in shown], subject_of(recs[i]), instr["gen"], args.thinking)) print(f"shard {args.shard}/{args.num_shards}: {len(mine)} questions, thinking={args.thinking}, shift={args.shift}") replies = generate_texts(tokenizer, model, prompts, args.batch_size, args.max_len, args.max_new_tokens) answers = [extract_answer(text, args.thinking) for text, _ in replies] answers = [shown[a - 1] + 1 if a else None for a in answers] # Fallback: option scoring over all 4 cyclic orderings for anything without a readable answer. missing = [k for k, a in enumerate(answers) if a is None] fallback = {} if missing: fb_prompts, owners = [], [] for k in missing: r = recs[mine[k]] opts = get_options(r) for s in range(4): perm = [(j + s) % 4 for j in range(4)] fb_prompts.append(build_prompt(tokenizer, get_question(r), [opts[p] for p in perm], subject_of(r), instr["score"])) owners.append((k, perm)) totals = {k: [0.0] * 4 for k in missing} for (k, perm), lp in zip(owners, score(tokenizer, model, fb_prompts, args.batch_size, args.max_len)): for slot, orig in enumerate(perm): totals[k][orig] += lp[slot] / 4 fallback = {k: max(range(4), key=lambda j: totals[k][j]) + 1 for k in missing} out_dir = Path(args.out_dir) out_dir.mkdir(parents=True, exist_ok=True) path = out_dir / f"{args.tag}_s{args.shard}_gen.jsonl" with open(path, "w", encoding="utf-8", newline="\n") as f: for k, i in enumerate(mine): text, n_tok = replies[k] f.write(json.dumps({"idx": i, "answer": answers[k] or fallback[k], "from_reasoning": answers[k] is not None, "shift": args.shift, "new_tokens": n_tok, "reply": text}, ensure_ascii=False) + "\n") lens = sorted(n for _, n in replies) print(f"\nModel: {args.model} params: {n_params:,} thinking: {args.thinking}") print(f"answers read from reasoning: {len(mine) - len(missing)}/{len(mine)} | fallback to scoring: {len(missing)}") print(f"new tokens: median {lens[len(lens) // 2]} max {lens[-1]} (limit {args.max_new_tokens})") print(f"Wrote {path}") def combine(args): """Merge shard outputs of `generate` into predictions.jsonl, a zip and (with gold) accuracy. With --ll-scores, questions outside --gen-levels / --gen-categories take the option-scoring answer from a `run` scores file instead (hybrid: reasoning where it helps, scoring elsewhere). """ recs = load_records(args.data) if args.limit: recs = recs[:args.limit] answers = {} for part in args.parts: for row in load_records(part): answers[row["idx"]] = row["answer"] use_gen = [routed_to_reasoning(r, args.gen_levels, args.gen_categories, args.gen_rules) for r in recs] missing = [i for i in range(len(recs)) if use_gen[i] and i not in answers] if missing: raise SystemExit(f"{len(missing)} questions have no reasoning answer; check that every shard finished " f"and that --gen-levels/--gen-categories match the generate run") if all(use_gen): preds = [answers[i] for i in range(len(recs))] else: if not args.ll_scores: raise SystemExit("--gen-levels/--gen-categories need --ll-scores for the remaining questions") ll = {row["idx"]: row["pred"] for row in load_records(args.ll_scores)} if len(ll) != len(recs): raise SystemExit(f"--ll-scores has {len(ll)} rows for {len(recs)} questions") preds = [answers[i] if use_gen[i] else ll[i] for i in range(len(recs))] print(f"hybrid: reasoning answers for {sum(use_gen)}/{len(recs)} questions " f"(levels {args.gen_levels}, categories {args.gen_categories}, rules {args.gen_rules}), " f"option scoring for the rest") if args.vote_parts: preds = vote(args, preds, use_gen) write_outputs(args, recs, preds) def vote(args, preds, use_gen): """Plurality vote over the reasoning answers of every pass. A pass votes only with an answer read from its reasoning (not its scoring fallback). Ties go to the earliest pass (first pass, then --vote-parts in the given order). Option scoring is used only when no pass has a reasoning answer: where reasoning and scoring disagree, scoring is rarely the right one. """ if not args.ll_scores: raise SystemExit("--vote-parts needs --ll-scores") ll = {row["idx"]: row["pred"] for row in load_records(args.ll_scores)} first = {row["idx"]: row for part in args.parts for row in load_records(part)} extra = [{row["idx"]: row for row in load_records(part)} for part in args.vote_parts] voted = sorted(set().union(*extra)) if any(not use_gen[i] for i in voted): raise SystemExit("--vote-parts contain questions outside the reasoning groups") preds, changed = list(preds), 0 for i in voted: ballots = [r[i]["answer"] for r in [first] + extra if i in r and r[i]["from_reasoning"]] or [ll[i]] counts = Counter(ballots) winner = max(ballots, key=lambda a: (counts[a], -ballots.index(a))) changed += winner != preds[i] preds[i] = winner print(f"vote: {len(voted)} questions re-answered by {len(extra)} extra pass(es); {changed} answers changed") return preds def write_outputs(args, recs, preds): out_dir = Path(args.out_dir) out_dir.mkdir(parents=True, exist_ok=True) pred_path = out_dir / f"{args.tag}_predictions.jsonl" with open(pred_path, "w", encoding="utf-8", newline="\n") as f: for r, p in zip(recs, preds): f.write(json.dumps({"q_no": get_qno(r), "subject": r.get("subject"), "category": r.get("category"), "level": get_level(r), "answer": str(p)}, ensure_ascii=False) + "\n") zip_path = out_dir / f"{args.tag}.zip" with zipfile.ZipFile(zip_path, "w", zipfile.ZIP_DEFLATED) as z: z.write(pred_path, arcname="predictions.jsonl") print("Predicted answer distribution:", dict(sorted(Counter(preds).items()))) by_level = defaultdict(lambda: [0, 0]) for r, p in zip(recs, preds): gold = get_gold(r, get_options(r)) if gold is not None: by_level[get_level(r)][0] += int(p == gold) by_level[get_level(r)][1] += 1 accs = {} for lvl in sorted(by_level, key=lambda l: (LEVELS.index(l) if l in LEVELS else len(LEVELS), l)): c, n = by_level[lvl] accs[lvl or "all"] = c / n print(f" {lvl or 'all':<6} acc = {c / n:.4f} ({c}/{n})") if all(lvl in accs for lvl in LEVELS): print(f" MACRO acc = {sum(accs[l] for l in LEVELS) / 3:.4f}") if accs: with open(out_dir / f"{args.tag}_summary.json", "w") as f: json.dump({"model": getattr(args, "model", None), "adapter": getattr(args, "adapter", None), "params": getattr(args, "n_params", 0), "perms": getattr(args, "perms", "gen"), "with_subject": getattr(args, "with_subject", None), "lang": getattr(args, "lang", None), "acc": accs, "macro": sum(accs[l] for l in LEVELS) / 3 if all(l in accs for l in LEVELS) else None}, f, indent=2) print(f"Wrote {zip_path}") def run(args): import torch torch.manual_seed(args.seed) recs = load_records(args.data) if args.limit: recs = recs[:args.limit] tokenizer, model, n_params = load_model(args.model, args.dtype, args.adapter, param_limit=args.param_limit) shifts = list(range(args.perms)) prompts, owners = [], [] for qi, r in enumerate(recs): opts = get_options(r) subject = r.get("subject") if args.with_subject else None for k in shifts: perm = [(j + k) % 4 for j in range(4)] # displayed slot j shows original option perm[j] prompts.append(build_prompt(tokenizer, get_question(r), [opts[p] for p in perm], subject, INSTRUCTIONS[args.lang]["score"])) owners.append((qi, perm)) print(f"{len(recs)} questions x {len(shifts)} orderings = {len(prompts)} prompts") print("Example prompt template (first 200 chars):", repr(prompts[0][:200]) if args.show_prompt else "hidden") raw = score(tokenizer, model, prompts, args.batch_size, args.max_len) totals = [[0.0] * 4 for _ in recs] for (qi, perm), lp in zip(owners, raw): for slot, orig in enumerate(perm): totals[qi][orig] += lp[slot] / len(shifts) out_dir = Path(args.out_dir) out_dir.mkdir(parents=True, exist_ok=True) preds = [max(range(4), key=lambda j: s[j]) + 1 for s in totals] with open(out_dir / f"{args.tag}_scores.jsonl", "w", encoding="utf-8", newline="\n") as f: for i, (r, s, pred) in enumerate(zip(recs, totals, preds)): f.write(json.dumps({"idx": i, "q_no": get_qno(r), "level": get_level(r), "scores": s, "pred": pred, "gold": get_gold(r, get_options(r))}) + "\n") print(f"\nModel: {args.model} params: {n_params:,} orderings: {len(shifts)}") args.n_params = n_params write_outputs(args, recs, preds) def main(): ap = argparse.ArgumentParser() sub = ap.add_subparsers(dest="cmd", required=True) p = sub.add_parser("inspect") p.add_argument("--data", nargs="+", required=True) p = sub.add_parser("run") p.add_argument("--model", required=True) p.add_argument("--adapter", default=None, help="LoRA adapter directory") p.add_argument("--data", required=True) p.add_argument("--tag", required=True) p.add_argument("--out-dir", default="outputs") p.add_argument("--perms", type=int, default=4, choices=[1, 2, 3, 4]) p.add_argument("--batch-size", type=int, default=8) p.add_argument("--max-len", type=int, default=2048) p.add_argument("--dtype", default="float16", choices=["float16", "bfloat16", "float32"]) p.add_argument("--with-subject", action="store_true") p.add_argument("--lang", default="en", choices=sorted(INSTRUCTIONS), help="instruction language") p.add_argument("--show-prompt", action="store_true") p.add_argument("--limit", type=int, default=0) p.add_argument("--param-limit", type=float, default=PARAM_LIMIT, help="raise only if the organisers confirm a model above 8e9 (e.g. Qwen3-8B, 8.2B) is accepted") p.add_argument("--seed", type=int, default=42) p = sub.add_parser("generate") p.add_argument("--model", required=True) p.add_argument("--adapter", default=None) p.add_argument("--data", required=True) p.add_argument("--tag", required=True) p.add_argument("--out-dir", default="outputs") p.add_argument("--shard", type=int, default=0) p.add_argument("--num-shards", type=int, default=1) p.add_argument("--thinking", action="store_true", help="enable the chat template's thinking mode") p.add_argument("--gen-levels", nargs="*", default=[], help="only answer these levels (hybrid)") p.add_argument("--gen-categories", nargs="*", default=[], help="only answer these categories (hybrid)") p.add_argument("--gen-rules", nargs="*", default=[], help='"level:category" pairs, e.g. "medium:social science"') p.add_argument("--shift", type=int, default=0, choices=[0, 1, 2, 3], help="show the options rotated by this many places") p.add_argument("--revote-of", nargs="*", default=[], help="earlier generate outputs; only questions with no readable answer there, or one that " "disagrees with --ll-scores, are answered again") p.add_argument("--ll-scores", default=None, help="scores.jsonl from `run` (for --revote-of)") p.add_argument("--max-new-tokens", type=int, default=2048) p.add_argument("--batch-size", type=int, default=8) p.add_argument("--max-len", type=int, default=2048) p.add_argument("--dtype", default="float16", choices=["float16", "bfloat16", "float32"]) p.add_argument("--with-subject", action="store_true") p.add_argument("--lang", default="en", choices=sorted(INSTRUCTIONS), help="instruction language") p.add_argument("--limit", type=int, default=0) p.add_argument("--param-limit", type=float, default=PARAM_LIMIT) p.add_argument("--seed", type=int, default=42) p = sub.add_parser("combine") p.add_argument("--data", required=True) p.add_argument("--tag", required=True) p.add_argument("--parts", nargs="+", required=True) p.add_argument("--out-dir", default="outputs") p.add_argument("--model", default=None, help="recorded in the summary only") p.add_argument("--ll-scores", default=None, help="scores.jsonl from `run`; enables the hybrid") p.add_argument("--gen-levels", nargs="*", default=[], help="levels answered by reasoning in the hybrid") p.add_argument("--gen-categories", nargs="*", default=[], help="categories answered by reasoning in the hybrid") p.add_argument("--gen-rules", nargs="*", default=[], help='"level:category" pairs, e.g. "medium:social science"') p.add_argument("--vote-parts", nargs="*", default=[], help="generate outputs of extra reasoning passes (e.g. --shift 1, 2 with --revote-of); " "their questions take the plurality vote of the reasoning passes") p.add_argument("--limit", type=int, default=0) args = ap.parse_args() {"inspect": inspect, "run": run, "generate": generate, "combine": combine}[args.cmd](args) if __name__ == "__main__": main()