"""Sentinel S1 Job A — evidence run. The unmodified starting model answers each frozen question 5 times; each answer is scored right/wrong by script against the known answer. No training, no model changes.""" import argparse, hashlib, json, os, re, string, sys, time MODEL, REV, SEED, N_TRIES = "allenai/Olmo-3-7B-Instruct", "6e5971d9eba42665f5bd5a0fcf047f299ce1dccc", 20261112, 5 PROMPT = {"fact": "Answer the question with a short factual answer only (a few words, no explanation).\n\nQuestion: {q}", "math": "Solve the problem. Show your working briefly, then give the final answer on the last line in the form 'Answer: '.\n\nProblem: {q}"} MAXTOK = {"fact": 48, "math": 640} def norm(s): s = s.lower(); s = "".join(c for c in s if c not in set(string.punctuation)) s = re.sub(r"\b(a|an|the)\b", " ", s); return " ".join(s.split()) def fact_ok(pred, golds): p = f" {norm(pred)} " return any(f" {norm(g)} " in p for g in golds if norm(g)) NUM = re.compile(r"-?\d[\d,]*\.?\d*") def math_val(text): m = re.findall(r"answer\s*[:=]\s*\$?\s*(-?\d[\d,]*\.?\d*)", text, flags=re.I) cand = m[-1] if m else (NUM.findall(text)[-1] if NUM.findall(text) else None) if cand is None: return None try: return float(cand.replace(",", "").rstrip(".")) except ValueError: return None def math_ok(pred, golds): v = math_val(pred) try: g = float(golds[0]) except ValueError: return False return v is not None and abs(v - g) < 1e-6 def main(): ap = argparse.ArgumentParser(); ap.add_argument("--questions"); ap.add_argument("--out"); ap.add_argument("--limit", type=int, default=0); a = ap.parse_args() raw = open(a.questions, "rb").read(); qsha = hashlib.sha256(raw).hexdigest() qs = [json.loads(l) for l in raw.decode().splitlines() if l.strip()] if a.limit: qs = qs[:a.limit] import torch, vllm from vllm import LLM, SamplingParams env = {"torch": torch.__version__, "cuda_available": torch.cuda.is_available(), "gpu": torch.cuda.get_device_name(0) if torch.cuda.is_available() else None, "vllm": vllm.__version__, "model": MODEL, "revision": REV, "questions_sha256": qsha, "n_questions": len(qs), "tries": N_TRIES, "seed": SEED} json.dump(env, open(os.path.join(a.out, "env.json"), "w"), indent=1) if not torch.cuda.is_available(): sys.exit("CUDA not available") llm = LLM(model=MODEL, revision=REV, dtype="bfloat16", max_model_len=2048, gpu_memory_utilization=0.92, seed=SEED, enable_prefix_caching=True) t0 = time.time(); recs = open(os.path.join(a.out, "records.jsonl"), "w"); summ = {"fact": [0, 0], "math": [0, 0]}; hist = {"fact": [0]*6, "math": [0]*6}; trunc = 0 CH = 500 for i in range(0, len(qs), CH): batch = qs[i:i+CH] convs = [[{"role": "user", "content": PROMPT[q["kind"]].format(q=q["question"])}] for q in batch] sps = [SamplingParams(n=N_TRIES, temperature=0.6, top_p=0.95, max_tokens=MAXTOK[q["kind"]], seed=SEED + i + j) for j, q in enumerate(batch)] outs = llm.chat(convs, sps, use_tqdm=False) for q, o in zip(batch, outs): texts = [c.text for c in o.outputs]; fin = [c.finish_reason for c in o.outputs] ok = [bool(fact_ok(t, q["gold"]) if q["kind"] == "fact" else math_ok(t, q["gold"])) for t in texts] k = sum(ok); summ[q["kind"]][0] += k; summ[q["kind"]][1] += len(ok); hist[q["kind"]][k] += 1; trunc += sum(f == "length" for f in fin) recs.write(json.dumps({"id": q["id"], "kind": q["kind"], "question": q["question"], "gold": q["gold"], "outputs": texts, "correct": ok, "finish": fin, "n_correct": k}) + "\n") recs.flush() open(os.path.join(a.out, "progress.txt"), "w").write(f"{min(i+CH, len(qs))}/{len(qs)} questions {time.time()-t0:.0f}s\n") recs.close() s = {"accuracy_per_try": {k: (v[0]/v[1] if v[1] else None) for k, v in summ.items()}, "questions_by_n_correct_of_5": hist, "truncated_answers": trunc, "seconds": round(time.time()-t0), **env} json.dump(s, open(os.path.join(a.out, "summary.json"), "w"), indent=1); print(json.dumps(s)) if __name__ == "__main__": main()