Sentinel-S1-7B / eval /job_c2.py
Sentovara's picture
Sentinel S1 release: adapter, model card, evaluation code, results
4e9bf3d verified
Raw History Blame Contribute Delete
11.9 kB
"""Sentinel S1 Job C2 (rev 2) — sign-off. Identical to Job C except the search source (owner GO 13 Nov 07:26). Starting model vs Sentinel (same weights + Job B LoRA) on frozen sets never trained on,
identical code, greedy. Then search-when-unsure on the fact questions Sentinel flags. Pass/fail rules as pre-registered (D13, D18, D19)."""
import argparse, hashlib, json, os, shutil, sys, threading, time
from job_a import MODEL, REV, SEED, PROMPT, fact_ok, math_ok
WIKI = "pyserini wikipedia-dpr-100w (Lucene BM25, 21M passages, DPR Wikipedia Dec 2018)"
WIKI_URL, WIKI_MD5 = "https://rgw.cs.uwaterloo.ca/pyserini/indexes/lucene-index.wikipedia-dpr-100w.20210120.d1b9e6.tar.gz", "7b58c08da992b2ea7e96667f0b176651"
JAR_URL, JAR_SHA256 = "https://repo1.maven.org/maven2/io/anserini/anserini/1.1.1/anserini-1.1.1-fatjar.jar", "69270ba4d160826953347411ce5d7e205ce363766a9cc72ac9da3b945341af83"
WIKI_REV = WIKI_MD5
MAXTOK, CH = 1024, 512
PROBES = [("Eiffel Tower Paris", "eiffel"), ("Albert Einstein theory of relativity", "einstein"), ("William Shakespeare playwright Hamlet", "shakespeare"),
("Amazon river South America", "amazon"), ("Mount Everest highest mountain", "everest")]
def flagged(t): return t.lstrip().upper().startswith("UNSURE:")
def unflag(t):
s = t.lstrip(); return s[len("UNSURE:"):].strip() if s.upper().startswith("UNSURE:") else t
def sha(p): return hashlib.sha256(open(p, "rb").read()).hexdigest()
def load(p): return [json.loads(l) for l in open(p) if l.strip()]
def main():
ap = argparse.ArgumentParser(); ap.add_argument("--signoff"); ap.add_argument("--adapter"); ap.add_argument("--out"); a = ap.parse_args()
t0 = time.time(); prog = lambda m: open(os.path.join(a.out, "progress.txt"), "w").write(f"{m} {time.time()-t0:.0f}s\n")
# --- pick the bigger disk for the 40 GB Wikipedia index and start downloading it in the background
disks = {d: shutil.disk_usage(d).free for d in ("/opt/ml", "/tmp") if os.path.exists(d)}
wdir = os.path.join(max(disks, key=disks.get), "scratch_wiki"); os.makedirs(wdir, exist_ok=True)
dl = {"ok": False, "err": None, "seconds": None}
def fetch():
s0 = time.time()
try:
import urllib.request, tarfile, jdk
jar = os.path.join(wdir, "anserini-1.1.1-fatjar.jar"); urllib.request.urlretrieve(JAR_URL, jar)
if hashlib.sha256(open(jar, "rb").read()).hexdigest() != JAR_SHA256: raise ValueError("Anserini jar hash mismatch")
dl["java_home"] = jdk.install("21", path=os.path.join(wdir, "jdk"))
tgz = os.path.join(wdir, "index.tar.gz"); urllib.request.urlretrieve(WIKI_URL, tgz)
h = hashlib.md5()
with open(tgz, "rb") as f:
for b in iter(lambda: f.read(1 << 24), b""): h.update(b)
if h.hexdigest() != WIKI_MD5: raise ValueError("index md5 mismatch")
with tarfile.open(tgz) as t: t.extractall(wdir, filter="data")
os.remove(tgz)
dl["index_dir"] = [os.path.join(wdir, d) for d in os.listdir(wdir) if d.startswith("lucene-index")][0]; dl["jar"] = jar; dl["ok"] = True
except Exception as e: dl["err"] = repr(e)[:500]
dl["seconds"] = round(time.time() - s0)
th = threading.Thread(target=fetch, daemon=True); th.start()
sets = {k: load(os.path.join(a.signoff, f"{k}.jsonl")) for k in ("ifeval", "gsm8k_test", "nq_val", "simpleqa")}
hashes = {k: sha(os.path.join(a.signoff, f"{k}.jsonl")) for k in sets}
adapter_files = {f: sha(os.path.join(a.adapter, f)) for f in sorted(os.listdir(a.adapter)) if os.path.isfile(os.path.join(a.adapter, f))}
import torch, vllm
from vllm import LLM, SamplingParams
from vllm.lora.request import LoRARequest
env = {"model": MODEL, "revision": REV, "vllm": vllm.__version__, "torch": torch.__version__, "gpu": torch.cuda.get_device_name(0),
"signoff_sha256": hashes, "adapter_sha256": adapter_files, "wiki_index": WIKI, "wiki_revision": WIKI_REV, "disk_free_gb": {k: round(v/1e9) for k, v in disks.items()}, "wiki_dir": wdir}
json.dump(env, open(os.path.join(a.out, "env.json"), "w"), indent=1)
llm = LLM(model=MODEL, revision=REV, dtype="bfloat16", max_model_len=3072, gpu_memory_utilization=0.90, seed=SEED,
enable_lora=True, max_lora_rank=16, max_loras=1, enable_prefix_caching=True)
lora = LoRARequest("sentinel", 1, a.adapter)
sp = SamplingParams(temperature=0.0, max_tokens=MAXTOK, seed=SEED)
from ifeval.utils import process_results
recs = {}
for arm in ("base", "sentinel"):
out = []
jobs = [("ifeval", r, r["prompt"]) for r in sets["ifeval"]] + [(k, r, PROMPT[r["kind"]].format(q=r["question"])) for k in ("gsm8k_test", "nq_val", "simpleqa") for r in sets[k]]
for i in range(0, len(jobs), CH):
part = jobs[i:i+CH]
res = llm.chat([[{"role": "user", "content": p}] for _, _, p in part], sp, lora_request=lora if arm == "sentinel" else None, use_tqdm=False)
for (k, r, _), o in zip(part, res):
t = o.outputs[0].text; fin = o.outputs[0].finish_reason
rec = {"set": k, "id": r.get("id", r.get("key")), "reply": t, "finish": fin, "flagged": flagged(t)}
if k == "ifeval": rec["correct"] = bool(process_results(r, [t])["prompt_level_strict_acc"]) # scored exactly as shown (D19)
else: rec["correct"] = bool(fact_ok(unflag(t), r["gold"]) if r["kind"] == "fact" else math_ok(unflag(t), r["gold"]))
out.append(rec)
prog(f"{arm} {min(i+CH, len(jobs))}/{len(jobs)}")
recs[arm] = out
with open(os.path.join(a.out, f"signoff_{arm}.jsonl"), "w") as f:
for x in out: f.write(json.dumps(x) + "\n")
# --- checks 1-4
def acc(rows): return sum(r["correct"] for r in rows) / len(rows) if rows else None
B, S = recs["base"], recs["sentinel"]; sel = lambda rows, ks: [r for r in rows if r["set"] in ks]
QA = ("gsm8k_test", "nq_val", "simpleqa"); FACT = ("nq_val", "simpleqa")
c1 = {"base": acc(sel(B, ("ifeval",))), "sentinel": acc(sel(S, ("ifeval",)))}; c1["pass"] = c1["sentinel"] >= c1["base"] - 0.02
c2 = {"base": acc(sel(B, ("gsm8k_test",))), "sentinel": acc(sel(S, ("gsm8k_test",)))}; c2["pass"] = c2["sentinel"] >= c2["base"] - 0.02
def c3for(ks):
b = sel(B, ks); s = sel(S, ks); un = [r for r in s if not r["flagged"]]; wrong = [r for r in s if not r["correct"]]
return {"base_accuracy": acc(b), "sentinel_unflagged_accuracy": acc(un), "sentinel_unflagged_n": len(un), "sentinel_flag_rate": sum(r["flagged"] for r in s) / len(s),
"sentinel_flag_rate_on_its_wrong_answers": (sum(r["flagged"] for r in wrong) / len(wrong)) if wrong else None, "sentinel_overall_accuracy": acc(s)}
c3 = c3for(QA); c3["pass"] = (c3["sentinel_unflagged_accuracy"] is not None and c3["sentinel_unflagged_accuracy"] >= c3["base_accuracy"] + 0.10
and c3["sentinel_flag_rate_on_its_wrong_answers"] is not None and c3["sentinel_flag_rate_on_its_wrong_answers"] >= 0.50)
c3_breakdown = {"facts_only": c3for(FACT), "maths_only": c3for(("gsm8k_test",)), "nq_val": c3for(("nq_val",)), "simpleqa": c3for(("simpleqa",))} # D18 reporting
c4 = {"sentinel_natural_end_rate": sum(r["finish"] == "stop" for r in S) / len(S), "base_natural_end_rate": sum(r["finish"] == "stop" for r in B) / len(B)}; c4["pass"] = c4["sentinel_natural_end_rate"] >= 0.95
summary = {"check1_instructions_ifeval": c1, "check2_maths_gsm8k": c2, "check3_knows_when_unsure": c3, "check3_breakdown_D18": c3_breakdown, "check4_clean_replies": c4}
json.dump(summary, open(os.path.join(a.out, "signoff_summary.json"), "w"), indent=1)
# --- check 5: search when unsure (fact questions Sentinel flags)
prog("waiting for Wikipedia index"); th.join()
c5 = {"index_download": dl}
flagged_facts = [r for r in sel(S, FACT) if r["flagged"]]; c5["flagged_fact_questions"] = len(flagged_facts)
if dl["ok"] and flagged_facts:
os.environ["JAVA_HOME"] = dl["java_home"]; os.environ["PATH"] = os.path.join(dl["java_home"], "bin") + os.pathsep + os.environ["PATH"]
import jnius_config; jnius_config.add_options("-Xmx4g"); jnius_config.set_classpath(dl["jar"])
from jnius import autoclass
JS = autoclass("io.anserini.search.SimpleSearcher"); AL = autoclass("java.util.ArrayList"); jsr = JS(dl["index_dir"])
c5["index_docs"] = int(jsr.get_total_num_docs())
def passage(docid):
raw = jsr.doc(docid).get("raw") or ""
try: c = json.loads(raw).get("contents", raw)
except Exception: c = raw
first, _, rest = c.partition("\n"); return {"title": first.strip().strip('"'), "section": "", "text": rest.strip()}
def search(qs, k):
q, ids = AL(), AL()
for i, t in enumerate(qs): q.add(t); ids.add(str(i))
res = jsr.batch_search(q, ids, int(k), 4)
return [[passage(h.docid) for h in res.get(str(i))] for i in range(len(qs))]
probe = search([q for q, _ in PROBES], 3); hits = [any(w in str(d.get("title", "")).lower() for d in row) for (q, w), row in zip(PROBES, probe)]
c5["probe_hits"] = f"{sum(hits)}/{len(hits)}"; c5["probe_top_titles"] = [[d.get("title") for d in row] for row in probe]
if sum(hits) >= 4:
qtext = {r["id"]: r for k in FACT for r in sets[k]}; srecs = []
for i in range(0, len(flagged_facts), 256):
part = flagged_facts[i:i+256]; qs = [qtext[r["id"]]["question"] for r in part]; docs = search(qs, 5)
prompts = []
for q, row in zip(qs, docs):
ctx = "\n\n".join(f"[{j+1}] {d.get('title','')} — {d.get('section','')}: {str(d.get('text','')).strip()[:600]}" for j, d in enumerate(row))
prompts.append("Answer the question using the Wikipedia passages below. Give a short factual answer only (a few words), then on a new line write 'Source: ' followed by the title of the passage you used.\n\n" + ctx + f"\n\nQuestion: {q}")
res = llm.chat([[{"role": "user", "content": p}] for p in prompts], SamplingParams(temperature=0.0, max_tokens=128, seed=SEED), lora_request=lora, use_tqdm=False)
for r, row, o in zip(part, docs, res):
t = o.outputs[0].text; ans = unflag(t.split("Source:")[0].strip())
srecs.append({"id": r["id"], "set": r["set"], "guess_correct": r["correct"], "search_reply": t, "search_correct": bool(fact_ok(ans, qtext[r["id"]]["gold"])),
"passage_titles": [d.get("title") for d in row]})
prog(f"search {min(i+256, len(flagged_facts))}/{len(flagged_facts)}")
with open(os.path.join(a.out, "search_records.jsonl"), "w") as f:
for x in srecs: f.write(json.dumps(x) + "\n")
c5["guess_accuracy_on_flagged"] = sum(x["guess_correct"] for x in srecs) / len(srecs)
c5["search_accuracy_on_flagged"] = sum(x["search_correct"] for x in srecs) / len(srecs)
c5["pass"] = c5["search_accuracy_on_flagged"] > c5["guess_accuracy_on_flagged"]
else: c5["pass"] = None; c5["note"] = "index probe gate failed (<4/5): search not evaluated (D19)"
else: c5["pass"] = None; c5["note"] = "index download failed or nothing flagged"
summary["check5_search_when_unsure"] = c5
summary["all_pass"] = all(summary[k].get("pass") is True for k in ("check1_instructions_ifeval", "check2_maths_gsm8k", "check3_knows_when_unsure", "check4_clean_replies", "check5_search_when_unsure"))
summary["seconds"] = round(time.time() - t0); summary["env"] = env
json.dump(summary, open(os.path.join(a.out, "signoff_summary.json"), "w"), indent=1); prog("done"); print(json.dumps({"all_pass": summary["all_pass"]}))
if __name__ == "__main__":
main()