File size: 4,156 Bytes
4e9bf3d
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
"""Sentinel S1 Job A — evidence run. The unmodified starting model answers each frozen question 5 times;
each answer is scored right/wrong by script against the known answer. No training, no model changes."""
import argparse, hashlib, json, os, re, string, sys, time
MODEL, REV, SEED, N_TRIES = "allenai/Olmo-3-7B-Instruct", "6e5971d9eba42665f5bd5a0fcf047f299ce1dccc", 20261112, 5
PROMPT = {"fact": "Answer the question with a short factual answer only (a few words, no explanation).\n\nQuestion: {q}",
          "math": "Solve the problem. Show your working briefly, then give the final answer on the last line in the form 'Answer: <number>'.\n\nProblem: {q}"}
MAXTOK = {"fact": 48, "math": 640}

def norm(s):
    s = s.lower(); s = "".join(c for c in s if c not in set(string.punctuation))
    s = re.sub(r"\b(a|an|the)\b", " ", s); return " ".join(s.split())

def fact_ok(pred, golds):
    p = f" {norm(pred)} "
    return any(f" {norm(g)} " in p for g in golds if norm(g))

NUM = re.compile(r"-?\d[\d,]*\.?\d*")
def math_val(text):
    m = re.findall(r"answer\s*[:=]\s*\$?\s*(-?\d[\d,]*\.?\d*)", text, flags=re.I)
    cand = m[-1] if m else (NUM.findall(text)[-1] if NUM.findall(text) else None)
    if cand is None: return None
    try: return float(cand.replace(",", "").rstrip("."))
    except ValueError: return None

def math_ok(pred, golds):
    v = math_val(pred)
    try: g = float(golds[0])
    except ValueError: return False
    return v is not None and abs(v - g) < 1e-6

def main():
    ap = argparse.ArgumentParser(); ap.add_argument("--questions"); ap.add_argument("--out"); ap.add_argument("--limit", type=int, default=0); a = ap.parse_args()
    raw = open(a.questions, "rb").read(); qsha = hashlib.sha256(raw).hexdigest()
    qs = [json.loads(l) for l in raw.decode().splitlines() if l.strip()]
    if a.limit: qs = qs[:a.limit]
    import torch, vllm
    from vllm import LLM, SamplingParams
    env = {"torch": torch.__version__, "cuda_available": torch.cuda.is_available(), "gpu": torch.cuda.get_device_name(0) if torch.cuda.is_available() else None,
           "vllm": vllm.__version__, "model": MODEL, "revision": REV, "questions_sha256": qsha, "n_questions": len(qs), "tries": N_TRIES, "seed": SEED}
    json.dump(env, open(os.path.join(a.out, "env.json"), "w"), indent=1)
    if not torch.cuda.is_available(): sys.exit("CUDA not available")
    llm = LLM(model=MODEL, revision=REV, dtype="bfloat16", max_model_len=2048, gpu_memory_utilization=0.92, seed=SEED, enable_prefix_caching=True)
    t0 = time.time(); recs = open(os.path.join(a.out, "records.jsonl"), "w"); summ = {"fact": [0, 0], "math": [0, 0]}; hist = {"fact": [0]*6, "math": [0]*6}; trunc = 0
    CH = 500
    for i in range(0, len(qs), CH):
        batch = qs[i:i+CH]
        convs = [[{"role": "user", "content": PROMPT[q["kind"]].format(q=q["question"])}] for q in batch]
        sps = [SamplingParams(n=N_TRIES, temperature=0.6, top_p=0.95, max_tokens=MAXTOK[q["kind"]], seed=SEED + i + j) for j, q in enumerate(batch)]
        outs = llm.chat(convs, sps, use_tqdm=False)
        for q, o in zip(batch, outs):
            texts = [c.text for c in o.outputs]; fin = [c.finish_reason for c in o.outputs]
            ok = [bool(fact_ok(t, q["gold"]) if q["kind"] == "fact" else math_ok(t, q["gold"])) for t in texts]
            k = sum(ok); summ[q["kind"]][0] += k; summ[q["kind"]][1] += len(ok); hist[q["kind"]][k] += 1; trunc += sum(f == "length" for f in fin)
            recs.write(json.dumps({"id": q["id"], "kind": q["kind"], "question": q["question"], "gold": q["gold"], "outputs": texts, "correct": ok, "finish": fin, "n_correct": k}) + "\n")
        recs.flush()
        open(os.path.join(a.out, "progress.txt"), "w").write(f"{min(i+CH, len(qs))}/{len(qs)} questions {time.time()-t0:.0f}s\n")
    recs.close()
    s = {"accuracy_per_try": {k: (v[0]/v[1] if v[1] else None) for k, v in summ.items()},
         "questions_by_n_correct_of_5": hist, "truncated_answers": trunc, "seconds": round(time.time()-t0), **env}
    json.dump(s, open(os.path.join(a.out, "summary.json"), "w"), indent=1); print(json.dumps(s))

if __name__ == "__main__":
    main()