"""Sentinel S1 Job D — conversation test (owner GO 13 Nov 2026 13:42 UK; pre-registration D25-D30, PREREG-JOB-D-2026-11-13.md sha256 465f9a67…). Original model vs Sentinel (same weights + Job B2 LoRA), identical code, greedy, no system prompt. Each model answers every turn itself; only the last reply is scored. Pass marks K1-K4 as pre-registered (D28). Search-when-unsure reported only (D29).""" import argparse, hashlib, json, os, shutil, threading, time from job_a import MODEL, REV, SEED, fact_ok, math_ok WIKI_URL, WIKI_MD5 = "https://rgw.cs.uwaterloo.ca/pyserini/indexes/lucene-index.wikipedia-dpr-100w.20210120.d1b9e6.tar.gz", "7b58c08da992b2ea7e96667f0b176651" JAR_URL, JAR_SHA256 = "https://repo1.maven.org/maven2/io/anserini/anserini/1.1.1/anserini-1.1.1-fatjar.jar", "69270ba4d160826953347411ce5d7e205ce363766a9cc72ac9da3b945341af83" PROBES = [("Eiffel Tower Paris", "eiffel"), ("Albert Einstein theory of relativity", "einstein"), ("William Shakespeare playwright Hamlet", "shakespeare"), ("Amazon river South America", "amazon"), ("Mount Everest highest mountain", "everest")] EARLY_TOK, LAST_TOK = 512, 1024 def flagged(t): return t.lstrip().upper().startswith("UNSURE:") def unflag(t): s = t.lstrip(); return s[len("UNSURE:"):].strip() if s.upper().startswith("UNSURE:") else t def sha(p): return hashlib.sha256(open(p, "rb").read()).hexdigest() def main(): ap = argparse.ArgumentParser(); ap.add_argument("--convs"); ap.add_argument("--adapter"); ap.add_argument("--out"); a = ap.parse_args() t0 = time.time(); prog = lambda m: open(os.path.join(a.out, "progress.txt"), "w").write(f"{m} {time.time()-t0:.0f}s\n") disks = {d: shutil.disk_usage(d).free for d in ("/opt/ml", "/tmp") if os.path.exists(d)} wdir = os.path.join(max(disks, key=disks.get), "scratch_wiki"); os.makedirs(wdir, exist_ok=True); dl = {"ok": False, "err": None, "seconds": None} def fetch(): s0 = time.time() try: import urllib.request, tarfile, jdk jar = os.path.join(wdir, "anserini-1.1.1-fatjar.jar"); urllib.request.urlretrieve(JAR_URL, jar) if hashlib.sha256(open(jar, "rb").read()).hexdigest() != JAR_SHA256: raise ValueError("Anserini jar hash mismatch") dl["java_home"] = jdk.install("21", path=os.path.join(wdir, "jdk")) tgz = os.path.join(wdir, "index.tar.gz"); urllib.request.urlretrieve(WIKI_URL, tgz); h = hashlib.md5() with open(tgz, "rb") as f: for b in iter(lambda: f.read(1 << 24), b""): h.update(b) if h.hexdigest() != WIKI_MD5: raise ValueError("index md5 mismatch") with tarfile.open(tgz) as t: t.extractall(wdir, filter="data") os.remove(tgz); dl["index_dir"] = [os.path.join(wdir, d) for d in os.listdir(wdir) if d.startswith("lucene-index")][0]; dl["jar"] = jar; dl["ok"] = True except Exception as e: dl["err"] = repr(e)[:500] dl["seconds"] = round(time.time() - s0) th = threading.Thread(target=fetch, daemon=True); th.start() cpath = os.path.join(a.convs, "conversations.jsonl"); convs = [json.loads(l) for l in open(cpath) if l.strip()] adapter_files = {f: sha(os.path.join(a.adapter, f)) for f in sorted(os.listdir(a.adapter)) if os.path.isfile(os.path.join(a.adapter, f))} import torch, vllm from vllm import LLM, SamplingParams from vllm.lora.request import LoRARequest env = {"model": MODEL, "revision": REV, "vllm": vllm.__version__, "torch": torch.__version__, "gpu": torch.cuda.get_device_name(0), "conversations_sha256": sha(cpath), "n_conversations": len(convs), "adapter_sha256": adapter_files, "early_max_tokens": EARLY_TOK, "last_max_tokens": LAST_TOK} json.dump(env, open(os.path.join(a.out, "env.json"), "w"), indent=1) llm = LLM(model=MODEL, revision=REV, dtype="bfloat16", max_model_len=8192, gpu_memory_utilization=0.90, seed=SEED, enable_lora=True, max_lora_rank=16, max_loras=1, enable_prefix_caching=True) lora = LoRARequest("sentinel", 1, a.adapter) recs = {} for arm in ("base", "sentinel"): msgs = {c["id"]: [] for c in convs}; turns = {c["id"]: [] for c in convs} for step in range(max(len(c["user_turns"]) for c in convs)): act = [c for c in convs if len(c["user_turns"]) > step] for c in act: msgs[c["id"]].append({"role": "user", "content": c["user_turns"][step]}) sps = [SamplingParams(temperature=0.0, seed=SEED, max_tokens=LAST_TOK if step == len(c["user_turns"]) - 1 else EARLY_TOK) for c in act] res = llm.chat([msgs[c["id"]] for c in act], sps, lora_request=lora if arm == "sentinel" else None, use_tqdm=False) for c, o in zip(act, res): t = o.outputs[0].text; msgs[c["id"]].append({"role": "assistant", "content": t}); turns[c["id"]].append({"reply": t, "finish": o.outputs[0].finish_reason}) prog(f"{arm} turn {step+1}") out = [] for c in convs: last = turns[c["id"]][-1]; t = last["reply"] r = {"id": c["id"], "group": c["group"], "kind": c["kind"], "reply": t, "finish": last["finish"], "flagged": flagged(t), "unsure_anywhere": "UNSURE:" in t.upper(), "earlier_turns_flagged": sum(flagged(x["reply"]) for x in turns[c["id"]][:-1])} if c["kind"]: r["correct"] = bool(fact_ok(unflag(t), c["gold"]) if c["kind"] == "fact" else math_ok(unflag(t), c["gold"])) out.append(r) recs[arm] = out with open(os.path.join(a.out, f"records_{arm}.jsonl"), "w") as f: for x in out: f.write(json.dumps(x) + "\n") with open(os.path.join(a.out, f"transcripts_{arm}.jsonl"), "w") as f: for c in convs: f.write(json.dumps({"id": c["id"], "messages": msgs[c["id"]]}) + "\n") B, S = recs["base"], recs["sentinel"] acc = lambda rows: (sum(r["correct"] for r in rows) / len(rows)) if rows else None def block(b, s): un = [r for r in s if not r["flagged"]]; wrong = [r for r in s if not r["correct"]] return {"n": len(s), "base_accuracy": acc(b), "sentinel_overall_accuracy": acc(s), "sentinel_unflagged_accuracy": acc(un), "sentinel_unflagged_n": len(un), "sentinel_flag_rate": sum(r["flagged"] for r in s) / len(s), "sentinel_flag_rate_on_its_wrong": (sum(r["flagged"] for r in wrong) / len(wrong)) if wrong else None, "sentinel_unsure_anywhere_not_leading": sum(r["unsure_anywhere"] and not r["flagged"] for r in s)} q = lambda rows: [r for r in rows if r["kind"]] main_b = block(q(B), q(S)) K1 = {"sentinel_unflagged_accuracy": main_b["sentinel_unflagged_accuracy"], "base_accuracy": main_b["base_accuracy"]} K1["pass"] = K1["sentinel_unflagged_accuracy"] is not None and K1["sentinel_unflagged_accuracy"] >= K1["base_accuracy"] + 0.10 K2 = {"sentinel_flag_rate_on_its_wrong": main_b["sentinel_flag_rate_on_its_wrong"]}; K2["pass"] = K2["sentinel_flag_rate_on_its_wrong"] is not None and K2["sentinel_flag_rate_on_its_wrong"] >= 0.50 K3 = {"sentinel_clean_end_rate": sum(r["finish"] == "stop" for r in S) / len(S), "base_clean_end_rate": sum(r["finish"] == "stop" for r in B) / len(B)}; K3["pass"] = K3["sentinel_clean_end_rate"] >= 0.95 D = [r for r in S if r["group"].startswith("D_")] K4 = {"sentinel_flag_rate_no_fact_chats": sum(r["flagged"] for r in D) / len(D), "n": len(D)}; K4["pass"] = K4["sentinel_flag_rate_no_fact_chats"] <= 0.10 by = {} for g in ("A_loose_single", "B_multi_turn", "C_mixed_chat"): for k in ("fact", "math", None): sel = lambda rows: [r for r in rows if r["group"] == g and (k is None or r["kind"] == k)] by[f"{g}/{k or 'all'}"] = block(sel(B), sel(S)) for k in ("fact", "math"): by[f"all/{k}"] = block([r for r in q(B) if r["kind"] == k], [r for r in q(S) if r["kind"] == k]) summary = {"K1_unflagged_beats_original_by_10pts": K1, "K2_flags_half_its_wrong": K2, "K3_clean_endings": K3, "K4_no_false_flags_on_chat": K4, "question_conversations": main_b, "breakdown": by, "earlier_turns_flagged_sentinel": sum(r["earlier_turns_flagged"] for r in S)} summary["all_pass"] = all(summary[k]["pass"] is True for k in ("K1_unflagged_beats_original_by_10pts", "K2_flags_half_its_wrong", "K3_clean_endings", "K4_no_false_flags_on_chat")) json.dump(summary, open(os.path.join(a.out, "summary.json"), "w"), indent=1) # --- reported only: search-when-unsure on flagged fact conversations, query = user's last message prog("waiting for Wikipedia index"); th.join(); sr = {"index_download": dl} cmap = {c["id"]: c for c in convs}; ff = [r for r in S if r["kind"] == "fact" and r["flagged"]]; sr["flagged_fact_conversations"] = len(ff) if dl["ok"] and ff: os.environ["JAVA_HOME"] = dl["java_home"]; os.environ["PATH"] = os.path.join(dl["java_home"], "bin") + os.pathsep + os.environ["PATH"] import jnius_config; jnius_config.add_options("-Xmx4g"); jnius_config.set_classpath(dl["jar"]) from jnius import autoclass JS = autoclass("io.anserini.search.SimpleSearcher"); AL = autoclass("java.util.ArrayList"); jsr = JS(dl["index_dir"]) def passage(docid): raw = jsr.doc(docid).get("raw") or "" try: c = json.loads(raw).get("contents", raw) except Exception: c = raw first, _, rest = c.partition("\n"); return {"title": first.strip().strip('"'), "text": rest.strip()} def search(qs, k): ql, ids = AL(), AL() for i, t in enumerate(qs): ql.add(t); ids.add(str(i)) res = jsr.batch_search(ql, ids, int(k), 4); return [[passage(h.docid) for h in res.get(str(i))] for i in range(len(qs))] probe = search([p for p, _ in PROBES], 3); hits = [any(w in d["title"].lower() for d in row) for (p, w), row in zip(PROBES, probe)]; sr["probe_hits"] = f"{sum(hits)}/{len(hits)}" if sum(hits) >= 4: qs = [cmap[r["id"]]["user_turns"][-1] for r in ff]; docs = search(qs, 5); prompts = [] for qq, row in zip(qs, docs): ctx = "\n\n".join(f"[{j+1}] {d['title']}: {d['text'][:600]}" for j, d in enumerate(row)) prompts.append("Answer the question using the Wikipedia passages below. Give a short factual answer only (a few words), then on a new line write 'Source: ' followed by the title of the passage you used.\n\n" + ctx + f"\n\nQuestion: {qq}") res = llm.chat([[{"role": "user", "content": p}] for p in prompts], SamplingParams(temperature=0.0, max_tokens=128, seed=SEED), lora_request=lora, use_tqdm=False) srecs = [] for r, row, o in zip(ff, docs, res): t = o.outputs[0].text; ans = unflag(t.split("Source:")[0].strip()) srecs.append({"id": r["id"], "guess_correct": r["correct"], "search_reply": t, "search_correct": bool(fact_ok(ans, cmap[r["id"]]["gold"])), "passage_titles": [d["title"] for d in row]}) with open(os.path.join(a.out, "search_records.jsonl"), "w") as f: for x in srecs: f.write(json.dumps(x) + "\n") sr["guess_accuracy_on_flagged"] = sum(x["guess_correct"] for x in srecs) / len(srecs); sr["search_accuracy_on_flagged"] = sum(x["search_correct"] for x in srecs) / len(srecs) else: sr["note"] = "index probe gate failed (<4/5): search not evaluated" else: sr["note"] = "index download failed or nothing flagged" summary["search_when_unsure_reported"] = sr; summary["seconds"] = round(time.time() - t0); summary["env"] = env json.dump(summary, open(os.path.join(a.out, "summary.json"), "w"), indent=1); prog("done"); print(json.dumps({"all_pass": summary["all_pass"]})) if __name__ == "__main__": main()