Sentinel-S1-32B / eval /job_f3.py
Sentovara's picture
Sentinel S1 release: adapter, model card, evaluation code, results
4d54b76 verified
Raw History Blame Contribute Delete
19.3 kB
"""Sentinel S1 Job F3 — sign-off of the 32B Sentinel (owner GO 14 Nov 2026 11:47 UK). The 9 checks of rev 4, same methods and pass marks:
checks 1-4 exactly as Job C2/C4 (IFEval, GSM8K test, NQ-val + SimpleQA, clean endings), check 5 with the adopted stronger search (E3 method:
2018 + 2023 Wikipedia, BM25 top 50 each, bge-reranker-v2-m3 top 10, system-named source), and K1-K4 exactly as Job D/D3 on fresh
conversations. Original model = unmodified Olmo 3.1 32B Instruct; Sentinel = same weights + the Job F2 adapter; identical code, greedy."""
import argparse, hashlib, json, os, re, shutil, threading, time
from job_a import SEED, PROMPT, fact_ok, math_ok, norm
from model32 import MODEL32, REV32
WIKI_URL, WIKI_MD5 = "https://rgw.cs.uwaterloo.ca/pyserini/indexes/lucene-index.wikipedia-dpr-100w.20210120.d1b9e6.tar.gz", "7b58c08da992b2ea7e96667f0b176651"
JAR_URL, JAR_SHA256 = "https://repo1.maven.org/maven2/io/anserini/anserini/1.1.1/anserini-1.1.1-fatjar.jar", "69270ba4d160826953347411ce5d7e205ce363766a9cc72ac9da3b945341af83"
RERANKER, RERANKER_REV = "BAAI/bge-reranker-v2-m3", "953dc6f6f85a1b2dbfca4c34a2796e7dde08d41e"
PROBES = [("Eiffel Tower Paris", "eiffel"), ("Albert Einstein theory of relativity", "einstein"), ("William Shakespeare playwright Hamlet", "shakespeare"),
("Amazon river South America", "amazon"), ("Mount Everest highest mountain", "everest")]
K_EACH, K_FINAL, MAXTOK, CH, EARLY_TOK, LAST_TOK = 50, 10, 1024, 512, 512, 1024
NEW_PROMPT = "Answer the question using the Wikipedia passages below. Give a short factual answer only (a few words), then on a new line write 'Source: ' followed by the number of the passage you used.\n\n"
SEVEN_B_REV4 = {"ifeval": [0.8022, 0.8041], "gsm8k": [0.9143, 0.9128], "unflagged": 0.7506, "flags_wrong": 0.9404, "search_after": 0.2505, "conv_unflagged": 0.8228, "conv_flags_wrong": 0.9247, "conv_chat_flagged": 0.06}
def flagged(t): return t.lstrip().upper().startswith("UNSURE:")
def unflag(t):
s = t.lstrip(); return s[len("UNSURE:"):].strip() if s.upper().startswith("UNSURE:") else t
def sha_file(p):
h = hashlib.sha256()
with open(p, "rb") as f:
for b in iter(lambda: f.read(1 << 24), b""): h.update(b)
return h.hexdigest()
def main():
ap = argparse.ArgumentParser()
for k in ("signoff", "convs", "adapter", "w23", "w23meta", "out"): ap.add_argument("--" + k)
ap.add_argument("--tp", type=int, default=4); ap.add_argument("--limit", type=int, default=0); ap.add_argument("--w18_dir", default=""); ap.add_argument("--jar", default=""); ap.add_argument("--java_home", default=""); ap.add_argument("--rerank_device", default="cuda:0")
a = ap.parse_args(); t0 = time.time(); prog = lambda m: open(os.path.join(a.out, "progress.txt"), "w").write(f"{m} {time.time()-t0:.0f}s\n")
disks = {d: shutil.disk_usage(d).free for d in ("/opt/ml", "/tmp") if os.path.exists(d)}; wdir = os.path.join(max(disks, key=disks.get), "scratch_wiki"); os.makedirs(wdir, exist_ok=True)
dl = {"ok": bool(a.w18_dir), "err": None, "index_dir": a.w18_dir, "jar": a.jar, "java_home": a.java_home}
def fetch():
s0 = time.time()
try:
import urllib.request, tarfile, jdk
jar = os.path.join(wdir, "anserini-1.1.1-fatjar.jar"); urllib.request.urlretrieve(JAR_URL, jar)
if sha_file(jar) != JAR_SHA256: raise ValueError("Anserini jar hash mismatch")
dl["java_home"] = jdk.install("21", path=os.path.join(wdir, "jdk")); tgz = os.path.join(wdir, "index.tar.gz"); urllib.request.urlretrieve(WIKI_URL, tgz); h = hashlib.md5()
with open(tgz, "rb") as f:
for b in iter(lambda: f.read(1 << 24), b""): h.update(b)
if h.hexdigest() != WIKI_MD5: raise ValueError("index md5 mismatch")
with tarfile.open(tgz) as t: t.extractall(wdir, filter="data")
os.remove(tgz); dl["index_dir"] = [os.path.join(wdir, d) for d in os.listdir(wdir) if d.startswith("lucene-index")][0]; dl["jar"] = jar; dl["ok"] = True
except Exception as e: dl["err"] = repr(e)[:500]
dl["seconds"] = round(time.time() - s0)
th = None
if not a.w18_dir: th = threading.Thread(target=fetch, daemon=True); th.start()
load = lambda p: [json.loads(l) for l in open(p) if l.strip()]
sets = {k: load(os.path.join(a.signoff, f"{k}.jsonl")) for k in ("ifeval", "gsm8k_test", "nq_val", "simpleqa")}
convs = load(os.path.join(a.convs, "conversations.jsonl"))
if a.limit: sets = {k: v[:a.limit] for k, v in sets.items()}; convs = convs[:a.limit] + [c for c in convs if c["group"].startswith("D_")][:a.limit]
import torch, vllm
from vllm import LLM, SamplingParams
from vllm.lora.request import LoRARequest
env = {"model": MODEL32, "revision": REV32, "vllm": vllm.__version__, "torch": torch.__version__, "gpus": [torch.cuda.get_device_name(i) for i in range(torch.cuda.device_count())],
"signoff_sha256": {k: sha_file(os.path.join(a.signoff, f"{k}.jsonl")) for k in sets}, "conversations_sha256": sha_file(os.path.join(a.convs, "conversations.jsonl")),
"adapter_sha256": {f: sha_file(os.path.join(a.adapter, f)) for f in sorted(os.listdir(a.adapter)) if os.path.isfile(os.path.join(a.adapter, f))}, "tensor_parallel": a.tp}
json.dump(env, open(os.path.join(a.out, "env.json"), "w"), indent=1)
llm = LLM(model=MODEL32, revision=REV32, dtype="bfloat16", tensor_parallel_size=a.tp, max_model_len=8192, gpu_memory_utilization=0.85, seed=SEED, enforce_eager=True, # D72: A10G memory fix (no CUDA graphs)
enable_lora=True, max_lora_rank=16, max_loras=1, enable_prefix_caching=True)
lora = LoRARequest("sentinel", 1, a.adapter)
# ===== checks 1-4: exactly as Job C2/C4
sp = SamplingParams(temperature=0.0, max_tokens=MAXTOK, seed=SEED)
from ifeval.utils import process_results
recs = {}
for arm in ("base", "sentinel"):
out = []
jobs = [("ifeval", r, r["prompt"]) for r in sets["ifeval"]] + [(k, r, PROMPT[r["kind"]].format(q=r["question"])) for k in ("gsm8k_test", "nq_val", "simpleqa") for r in sets[k]]
for i in range(0, len(jobs), CH):
part = jobs[i:i+CH]
res = llm.chat([[{"role": "user", "content": p}] for _, _, p in part], sp, lora_request=lora if arm == "sentinel" else None, use_tqdm=False)
for (k, r, _), o in zip(part, res):
t = o.outputs[0].text; fin = o.outputs[0].finish_reason
rec = {"set": k, "id": r.get("id", r.get("key")), "reply": t, "finish": fin, "flagged": flagged(t)}
if k == "ifeval": rec["correct"] = bool(process_results(r, [t])["prompt_level_strict_acc"])
else: rec["correct"] = bool(fact_ok(unflag(t), r["gold"]) if r["kind"] == "fact" else math_ok(unflag(t), r["gold"]))
out.append(rec)
prog(f"signoff {arm} {min(i+CH, len(jobs))}/{len(jobs)}")
recs[arm] = out
with open(os.path.join(a.out, f"signoff_{arm}.jsonl"), "w") as f:
for x in out: f.write(json.dumps(x) + "\n")
def acc(rows): return sum(r["correct"] for r in rows) / len(rows) if rows else None
B, S = recs["base"], recs["sentinel"]; sel = lambda rows, ks: [r for r in rows if r["set"] in ks]
QA = ("gsm8k_test", "nq_val", "simpleqa"); FACT = ("nq_val", "simpleqa")
c1 = {"base": acc(sel(B, ("ifeval",))), "sentinel": acc(sel(S, ("ifeval",)))}; c1["pass"] = c1["sentinel"] >= c1["base"] - 0.02
c2 = {"base": acc(sel(B, ("gsm8k_test",))), "sentinel": acc(sel(S, ("gsm8k_test",)))}; c2["pass"] = c2["sentinel"] >= c2["base"] - 0.02
def c3for(ks):
b = sel(B, ks); s = sel(S, ks); un = [r for r in s if not r["flagged"]]; wrong = [r for r in s if not r["correct"]]
return {"base_accuracy": acc(b), "sentinel_unflagged_accuracy": acc(un), "sentinel_unflagged_n": len(un), "sentinel_flag_rate": sum(r["flagged"] for r in s) / len(s),
"sentinel_flag_rate_on_its_wrong_answers": (sum(r["flagged"] for r in wrong) / len(wrong)) if wrong else None, "sentinel_overall_accuracy": acc(s)}
c3 = c3for(QA); c3["pass"] = (c3["sentinel_unflagged_accuracy"] is not None and c3["sentinel_unflagged_accuracy"] >= c3["base_accuracy"] + 0.10
and c3["sentinel_flag_rate_on_its_wrong_answers"] is not None and c3["sentinel_flag_rate_on_its_wrong_answers"] >= 0.50)
c3_breakdown = {"facts_only": c3for(FACT), "maths_only": c3for(("gsm8k_test",)), "nq_val": c3for(("nq_val",)), "simpleqa": c3for(("simpleqa",))}
c4 = {"sentinel_natural_end_rate": sum(r["finish"] == "stop" for r in S) / len(S), "base_natural_end_rate": sum(r["finish"] == "stop" for r in B) / len(B)}; c4["pass"] = c4["sentinel_natural_end_rate"] >= 0.95
summary = {"check1_instructions_ifeval": c1, "check2_maths_gsm8k": c2, "check3_knows_when_unsure": c3, "check3_breakdown_D18": c3_breakdown, "check4_clean_replies": c4}
json.dump(summary, open(os.path.join(a.out, "summary.json"), "w"), indent=1)
# ===== K1-K4: exactly as Job D/D3
crec = {}
for arm in ("base", "sentinel"):
msgs = {c["id"]: [] for c in convs}; turns = {c["id"]: [] for c in convs}
for step in range(max(len(c["user_turns"]) for c in convs)):
act = [c for c in convs if len(c["user_turns"]) > step]
for c in act: msgs[c["id"]].append({"role": "user", "content": c["user_turns"][step]})
sps = [SamplingParams(temperature=0.0, seed=SEED, max_tokens=LAST_TOK if step == len(c["user_turns"]) - 1 else EARLY_TOK) for c in act]
res = llm.chat([msgs[c["id"]] for c in act], sps, lora_request=lora if arm == "sentinel" else None, use_tqdm=False)
for c, o in zip(act, res):
t = o.outputs[0].text; msgs[c["id"]].append({"role": "assistant", "content": t}); turns[c["id"]].append({"reply": t, "finish": o.outputs[0].finish_reason})
prog(f"conversations {arm} turn {step+1}")
out = []
for c in convs:
last = turns[c["id"]][-1]; t = last["reply"]
r = {"id": c["id"], "group": c["group"], "kind": c["kind"], "reply": t, "finish": last["finish"], "flagged": flagged(t), "unsure_anywhere": "UNSURE:" in t.upper(),
"earlier_turns_flagged": sum(flagged(x["reply"]) for x in turns[c["id"]][:-1])}
if c["kind"]: r["correct"] = bool(fact_ok(unflag(t), c["gold"]) if c["kind"] == "fact" else math_ok(unflag(t), c["gold"]))
out.append(r)
crec[arm] = out
with open(os.path.join(a.out, f"conv_records_{arm}.jsonl"), "w") as f:
for x in out: f.write(json.dumps(x) + "\n")
with open(os.path.join(a.out, f"conv_transcripts_{arm}.jsonl"), "w") as f:
for c in convs: f.write(json.dumps({"id": c["id"], "messages": msgs[c["id"]]}) + "\n")
CB, CS = crec["base"], crec["sentinel"]; q = lambda rows: [r for r in rows if r["kind"]]
un = [r for r in q(CS) if not r["flagged"]]; wr = [r for r in q(CS) if not r["correct"]]; D = [r for r in CS if r["group"].startswith("D_")]
K1 = {"sentinel_unflagged_accuracy": acc(un), "base_accuracy": acc(q(CB))}; K1["pass"] = K1["sentinel_unflagged_accuracy"] is not None and K1["sentinel_unflagged_accuracy"] >= K1["base_accuracy"] + 0.10
K2 = {"sentinel_flag_rate_on_its_wrong": (sum(r["flagged"] for r in wr) / len(wr)) if wr else None}; K2["pass"] = K2["sentinel_flag_rate_on_its_wrong"] is not None and K2["sentinel_flag_rate_on_its_wrong"] >= 0.50
K3 = {"sentinel_clean_end_rate": sum(r["finish"] == "stop" for r in CS) / len(CS), "base_clean_end_rate": sum(r["finish"] == "stop" for r in CB) / len(CB)}; K3["pass"] = K3["sentinel_clean_end_rate"] >= 0.95
K4 = {"sentinel_flag_rate_no_fact_chats": sum(r["flagged"] for r in D) / len(D), "n": len(D)}; K4["pass"] = K4["sentinel_flag_rate_no_fact_chats"] <= 0.10
summary.update({"K1_unflagged_beats_original_by_10pts": K1, "K2_flags_half_its_wrong": K2, "K3_clean_endings": K3, "K4_no_false_flags_on_chat": K4,
"conv_reported": {"sentinel_overall_accuracy": acc(q(CS)), "sentinel_flag_rate": sum(r["flagged"] for r in q(CS)) / len(q(CS)), "earlier_turns_flagged": sum(r["earlier_turns_flagged"] for r in CS),
"maths_flag_rate_on_wrong": (lambda w: sum(r["flagged"] for r in w) / len(w) if w else None)([r for r in q(CS) if r["kind"] == "math" and not r["correct"]])}})
json.dump(summary, open(os.path.join(a.out, "summary.json"), "w"), indent=1)
# ===== check 5: stronger search (E3 method) on the fact questions Sentinel flags
prog("waiting for 2018 index")
if th: th.join()
c5 = {"index_2018": {k: v for k, v in dl.items() if k != "java_home"}}
man = json.load(open(os.path.join(a.w23meta, "index_manifest.json"))); bad = [k for k, v in man.items() if sha_file(os.path.join(a.w23, k)) != v["sha256"]]
c5["w23_index_files_verified"] = f"{len(man) - len(bad)}/{len(man)}"
ffs = [r for r in sel(S, FACT) if r["flagged"]]; ffc = [r for r in q(CS) if r["kind"] == "fact" and r["flagged"]]; c5["flagged_fact_questions"] = len(ffs); c5["flagged_fact_conversations"] = len(ffc)
if dl["ok"] and not bad and ffs:
os.environ["JAVA_HOME"] = dl["java_home"]; os.environ["PATH"] = os.path.join(dl["java_home"], "bin") + os.pathsep + os.environ["PATH"]
import jnius_config; jnius_config.add_options("-Xmx8g"); jnius_config.set_classpath(dl["jar"])
from jnius import autoclass
JS = autoclass("io.anserini.search.SimpleSearcher"); AL = autoclass("java.util.ArrayList"); SR = {"w18": JS(dl["index_dir"]), "w23": JS(a.w23)}
def passage(s, docid):
raw = SR[s].doc(docid).get("raw") or ""
try: c = json.loads(raw).get("contents", raw)
except Exception: c = raw
first, _, rest = c.partition("\n"); return {"title": first.strip().strip('"'), "text": rest.strip(), "src": s}
def search(s, texts, k):
out = []
for i in range(0, len(texts), 512):
part = texts[i:i+512]; ql, ids = AL(), AL()
for j, t in enumerate(part): ql.add(t); ids.add(str(j))
res = SR[s].batch_search(ql, ids, int(k), 8); out += [[passage(s, h.docid) for h in res.get(str(j))] for j in range(len(part))]
return out
c5["probe_hits"] = {s: sum(any(w in d["title"].lower() for d in row) for (p, w), row in zip(PROBES, search(s, [p for p, _ in PROBES], 3))) for s in SR}
if min(c5["probe_hits"].values()) >= 4:
qtext = {r["id"]: r for k in FACT for r in sets[k]}; cmap = {c["id"]: c for c in convs}
items = [("signoff", r, qtext[r["id"]]["question"], qtext[r["id"]]["gold"]) for r in ffs] + [("conversation", r, cmap[r["id"]]["user_turns"][-1], cmap[r["id"]]["gold"]) for r in ffc]
texts = [it[2] for it in items]; prog("searching")
c18 = search("w18", texts, K_EACH); c23 = search("w23", texts, K_EACH); cands = []
for x, y in zip(c18, c23):
seen, row = set(), []
for d in x + y:
key = (d["title"], d["text"][:200])
if key not in seen: seen.add(key); row.append(d)
cands.append(row)
prog("reranking")
from transformers import AutoTokenizer, AutoModelForSequenceClassification
tok = AutoTokenizer.from_pretrained(RERANKER, revision=RERANKER_REV); rr = AutoModelForSequenceClassification.from_pretrained(RERANKER, revision=RERANKER_REV, dtype=torch.float16 if a.rerank_device.startswith("cuda") else torch.float32).to(a.rerank_device).eval()
top = []
for qi, (qq, row) in enumerate(zip(texts, cands)):
pairs = [[qq, f"{d['title']}: {d['text']}"] for d in row]; scores = []
for i in range(0, len(pairs), 32):
enc = tok(pairs[i:i+32], padding=True, truncation=True, max_length=384, return_tensors="pt").to(a.rerank_device)
with torch.no_grad(): scores += rr(**enc).logits.view(-1).float().tolist()
top.append([row[i] for i in sorted(range(len(row)), key=lambda i: -scores[i])[:K_FINAL]])
if qi % 200 == 0: prog(f"reranking {qi}/{len(texts)}")
del rr
prog("answering with search")
prompts = [NEW_PROMPT + "\n\n".join(f"[{j+1}] {d['title']}: {d['text'][:600]}" for j, d in enumerate(row)) + f"\n\nQuestion: {qq}" for qq, row in zip(texts, top)]
res = llm.chat([[{"role": "user", "content": p}] for p in prompts], SamplingParams(temperature=0.0, max_tokens=128, seed=SEED), lora_request=lora, use_tqdm=False)
def cite(ans, row):
na = norm(ans)
if not na: return None
for j, d in enumerate(row):
if f" {na} " in f" {norm(d['title'] + ' ' + d['text'][:600])} ": return j + 1
return None
srecs = []
for (kind, r, qq, gold), row, o in zip(items, top, res):
t = o.outputs[0].text; ans = unflag(t.split("Source:")[0].strip()); n = cite(ans, row)
srecs.append({"kind": kind, "id": r["id"], "guess_correct": r["correct"], "reply": t, "answer": ans, "search_correct": bool(fact_ok(ans, gold)),
"system_source": row[n - 1]["title"] if n else "no source found", "passages": [[d["src"], d["title"]] for d in row]})
with open(os.path.join(a.out, "search_records.jsonl"), "w") as f:
for x in srecs: f.write(json.dumps(x) + "\n")
ss = [x for x in srecs if x["kind"] == "signoff"]; sc = [x for x in srecs if x["kind"] == "conversation"]
c5["guess_accuracy_on_flagged"] = sum(x["guess_correct"] for x in ss) / len(ss); c5["search_accuracy_on_flagged"] = sum(x["search_correct"] for x in ss) / len(ss)
c5["answers_with_a_source"] = sum(x["system_source"] != "no source found" for x in ss) / len(ss)
c5["conversation_guess"] = (sum(x["guess_correct"] for x in sc) / len(sc)) if sc else None; c5["conversation_search"] = (sum(x["search_correct"] for x in sc) / len(sc)) if sc else None
c5["pass"] = c5["search_accuracy_on_flagged"] > c5["guess_accuracy_on_flagged"]
else: c5["pass"] = None; c5["note"] = "index probe gate failed (<4/5): search not evaluated"
else: c5["pass"] = None; c5["note"] = "2018 index download failed, 2023 index files did not verify, or nothing flagged"
summary["check5_search_when_unsure"] = c5
keys = ("check1_instructions_ifeval", "check2_maths_gsm8k", "check3_knows_when_unsure", "check4_clean_replies", "check5_search_when_unsure",
"K1_unflagged_beats_original_by_10pts", "K2_flags_half_its_wrong", "K3_clean_endings", "K4_no_false_flags_on_chat")
summary["checks_passed"] = f"{sum(summary[k].get('pass') is True for k in keys)}/9"; summary["all_pass"] = all(summary[k].get("pass") is True for k in keys)
summary["for_comparison_7b_rev4"] = SEVEN_B_REV4; summary["seconds"] = round(time.time() - t0); summary["env"] = env
json.dump(summary, open(os.path.join(a.out, "summary.json"), "w"), indent=1); prog("done"); print(json.dumps({"checks_passed": summary["checks_passed"]}))
if __name__ == "__main__": main()