Sentinel-S1-32B / eval /job_a.py
Sentovara's picture
Sentinel S1 release: adapter, model card, evaluation code, results
4d54b76 verified
Raw History Blame Contribute Delete
4.16 kB
"""Sentinel S1 Job A — evidence run. The unmodified starting model answers each frozen question 5 times;
each answer is scored right/wrong by script against the known answer. No training, no model changes."""
import argparse, hashlib, json, os, re, string, sys, time
MODEL, REV, SEED, N_TRIES = "allenai/Olmo-3-7B-Instruct", "6e5971d9eba42665f5bd5a0fcf047f299ce1dccc", 20261112, 5
PROMPT = {"fact": "Answer the question with a short factual answer only (a few words, no explanation).\n\nQuestion: {q}",
"math": "Solve the problem. Show your working briefly, then give the final answer on the last line in the form 'Answer: <number>'.\n\nProblem: {q}"}
MAXTOK = {"fact": 48, "math": 640}
def norm(s):
s = s.lower(); s = "".join(c for c in s if c not in set(string.punctuation))
s = re.sub(r"\b(a|an|the)\b", " ", s); return " ".join(s.split())
def fact_ok(pred, golds):
p = f" {norm(pred)} "
return any(f" {norm(g)} " in p for g in golds if norm(g))
NUM = re.compile(r"-?\d[\d,]*\.?\d*")
def math_val(text):
m = re.findall(r"answer\s*[:=]\s*\$?\s*(-?\d[\d,]*\.?\d*)", text, flags=re.I)
cand = m[-1] if m else (NUM.findall(text)[-1] if NUM.findall(text) else None)
if cand is None: return None
try: return float(cand.replace(",", "").rstrip("."))
except ValueError: return None
def math_ok(pred, golds):
v = math_val(pred)
try: g = float(golds[0])
except ValueError: return False
return v is not None and abs(v - g) < 1e-6
def main():
ap = argparse.ArgumentParser(); ap.add_argument("--questions"); ap.add_argument("--out"); ap.add_argument("--limit", type=int, default=0); a = ap.parse_args()
raw = open(a.questions, "rb").read(); qsha = hashlib.sha256(raw).hexdigest()
qs = [json.loads(l) for l in raw.decode().splitlines() if l.strip()]
if a.limit: qs = qs[:a.limit]
import torch, vllm
from vllm import LLM, SamplingParams
env = {"torch": torch.__version__, "cuda_available": torch.cuda.is_available(), "gpu": torch.cuda.get_device_name(0) if torch.cuda.is_available() else None,
"vllm": vllm.__version__, "model": MODEL, "revision": REV, "questions_sha256": qsha, "n_questions": len(qs), "tries": N_TRIES, "seed": SEED}
json.dump(env, open(os.path.join(a.out, "env.json"), "w"), indent=1)
if not torch.cuda.is_available(): sys.exit("CUDA not available")
llm = LLM(model=MODEL, revision=REV, dtype="bfloat16", max_model_len=2048, gpu_memory_utilization=0.92, seed=SEED, enable_prefix_caching=True)
t0 = time.time(); recs = open(os.path.join(a.out, "records.jsonl"), "w"); summ = {"fact": [0, 0], "math": [0, 0]}; hist = {"fact": [0]*6, "math": [0]*6}; trunc = 0
CH = 500
for i in range(0, len(qs), CH):
batch = qs[i:i+CH]
convs = [[{"role": "user", "content": PROMPT[q["kind"]].format(q=q["question"])}] for q in batch]
sps = [SamplingParams(n=N_TRIES, temperature=0.6, top_p=0.95, max_tokens=MAXTOK[q["kind"]], seed=SEED + i + j) for j, q in enumerate(batch)]
outs = llm.chat(convs, sps, use_tqdm=False)
for q, o in zip(batch, outs):
texts = [c.text for c in o.outputs]; fin = [c.finish_reason for c in o.outputs]
ok = [bool(fact_ok(t, q["gold"]) if q["kind"] == "fact" else math_ok(t, q["gold"])) for t in texts]
k = sum(ok); summ[q["kind"]][0] += k; summ[q["kind"]][1] += len(ok); hist[q["kind"]][k] += 1; trunc += sum(f == "length" for f in fin)
recs.write(json.dumps({"id": q["id"], "kind": q["kind"], "question": q["question"], "gold": q["gold"], "outputs": texts, "correct": ok, "finish": fin, "n_correct": k}) + "\n")
recs.flush()
open(os.path.join(a.out, "progress.txt"), "w").write(f"{min(i+CH, len(qs))}/{len(qs)} questions {time.time()-t0:.0f}s\n")
recs.close()
s = {"accuracy_per_try": {k: (v[0]/v[1] if v[1] else None) for k, v in summ.items()},
"questions_by_n_correct_of_5": hist, "truncated_answers": trunc, "seconds": round(time.time()-t0), **env}
json.dump(s, open(os.path.join(a.out, "summary.json"), "w"), indent=1); print(json.dumps(s))
if __name__ == "__main__":
main()