File size: 11,761 Bytes
4e9bf3d
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
"""Sentinel S1 Job D — conversation test (owner GO 13 Nov 2026 13:42 UK; pre-registration D25-D30, PREREG-JOB-D-2026-11-13.md sha256 465f9a67…).
Original model vs Sentinel (same weights + Job B2 LoRA), identical code, greedy, no system prompt. Each model answers every turn itself;
only the last reply is scored. Pass marks K1-K4 as pre-registered (D28). Search-when-unsure reported only (D29)."""
import argparse, hashlib, json, os, shutil, threading, time
from job_a import MODEL, REV, SEED, fact_ok, math_ok
WIKI_URL, WIKI_MD5 = "https://rgw.cs.uwaterloo.ca/pyserini/indexes/lucene-index.wikipedia-dpr-100w.20210120.d1b9e6.tar.gz", "7b58c08da992b2ea7e96667f0b176651"
JAR_URL, JAR_SHA256 = "https://repo1.maven.org/maven2/io/anserini/anserini/1.1.1/anserini-1.1.1-fatjar.jar", "69270ba4d160826953347411ce5d7e205ce363766a9cc72ac9da3b945341af83"
PROBES = [("Eiffel Tower Paris", "eiffel"), ("Albert Einstein theory of relativity", "einstein"), ("William Shakespeare playwright Hamlet", "shakespeare"),
          ("Amazon river South America", "amazon"), ("Mount Everest highest mountain", "everest")]
EARLY_TOK, LAST_TOK = 512, 1024

def flagged(t): return t.lstrip().upper().startswith("UNSURE:")
def unflag(t):
    s = t.lstrip(); return s[len("UNSURE:"):].strip() if s.upper().startswith("UNSURE:") else t
def sha(p): return hashlib.sha256(open(p, "rb").read()).hexdigest()

def main():
    ap = argparse.ArgumentParser(); ap.add_argument("--convs"); ap.add_argument("--adapter"); ap.add_argument("--out"); a = ap.parse_args()
    t0 = time.time(); prog = lambda m: open(os.path.join(a.out, "progress.txt"), "w").write(f"{m} {time.time()-t0:.0f}s\n")
    disks = {d: shutil.disk_usage(d).free for d in ("/opt/ml", "/tmp") if os.path.exists(d)}
    wdir = os.path.join(max(disks, key=disks.get), "scratch_wiki"); os.makedirs(wdir, exist_ok=True); dl = {"ok": False, "err": None, "seconds": None}
    def fetch():
        s0 = time.time()
        try:
            import urllib.request, tarfile, jdk
            jar = os.path.join(wdir, "anserini-1.1.1-fatjar.jar"); urllib.request.urlretrieve(JAR_URL, jar)
            if hashlib.sha256(open(jar, "rb").read()).hexdigest() != JAR_SHA256: raise ValueError("Anserini jar hash mismatch")
            dl["java_home"] = jdk.install("21", path=os.path.join(wdir, "jdk"))
            tgz = os.path.join(wdir, "index.tar.gz"); urllib.request.urlretrieve(WIKI_URL, tgz); h = hashlib.md5()
            with open(tgz, "rb") as f:
                for b in iter(lambda: f.read(1 << 24), b""): h.update(b)
            if h.hexdigest() != WIKI_MD5: raise ValueError("index md5 mismatch")
            with tarfile.open(tgz) as t: t.extractall(wdir, filter="data")
            os.remove(tgz); dl["index_dir"] = [os.path.join(wdir, d) for d in os.listdir(wdir) if d.startswith("lucene-index")][0]; dl["jar"] = jar; dl["ok"] = True
        except Exception as e: dl["err"] = repr(e)[:500]
        dl["seconds"] = round(time.time() - s0)
    th = threading.Thread(target=fetch, daemon=True); th.start()
    cpath = os.path.join(a.convs, "conversations.jsonl"); convs = [json.loads(l) for l in open(cpath) if l.strip()]
    adapter_files = {f: sha(os.path.join(a.adapter, f)) for f in sorted(os.listdir(a.adapter)) if os.path.isfile(os.path.join(a.adapter, f))}
    import torch, vllm
    from vllm import LLM, SamplingParams
    from vllm.lora.request import LoRARequest
    env = {"model": MODEL, "revision": REV, "vllm": vllm.__version__, "torch": torch.__version__, "gpu": torch.cuda.get_device_name(0),
           "conversations_sha256": sha(cpath), "n_conversations": len(convs), "adapter_sha256": adapter_files, "early_max_tokens": EARLY_TOK, "last_max_tokens": LAST_TOK}
    json.dump(env, open(os.path.join(a.out, "env.json"), "w"), indent=1)
    llm = LLM(model=MODEL, revision=REV, dtype="bfloat16", max_model_len=8192, gpu_memory_utilization=0.90, seed=SEED,
              enable_lora=True, max_lora_rank=16, max_loras=1, enable_prefix_caching=True)
    lora = LoRARequest("sentinel", 1, a.adapter)
    recs = {}
    for arm in ("base", "sentinel"):
        msgs = {c["id"]: [] for c in convs}; turns = {c["id"]: [] for c in convs}
        for step in range(max(len(c["user_turns"]) for c in convs)):
            act = [c for c in convs if len(c["user_turns"]) > step]
            for c in act: msgs[c["id"]].append({"role": "user", "content": c["user_turns"][step]})
            sps = [SamplingParams(temperature=0.0, seed=SEED, max_tokens=LAST_TOK if step == len(c["user_turns"]) - 1 else EARLY_TOK) for c in act]
            res = llm.chat([msgs[c["id"]] for c in act], sps, lora_request=lora if arm == "sentinel" else None, use_tqdm=False)
            for c, o in zip(act, res):
                t = o.outputs[0].text; msgs[c["id"]].append({"role": "assistant", "content": t}); turns[c["id"]].append({"reply": t, "finish": o.outputs[0].finish_reason})
            prog(f"{arm} turn {step+1}")
        out = []
        for c in convs:
            last = turns[c["id"]][-1]; t = last["reply"]
            r = {"id": c["id"], "group": c["group"], "kind": c["kind"], "reply": t, "finish": last["finish"], "flagged": flagged(t), "unsure_anywhere": "UNSURE:" in t.upper(),
                 "earlier_turns_flagged": sum(flagged(x["reply"]) for x in turns[c["id"]][:-1])}
            if c["kind"]: r["correct"] = bool(fact_ok(unflag(t), c["gold"]) if c["kind"] == "fact" else math_ok(unflag(t), c["gold"]))
            out.append(r)
        recs[arm] = out
        with open(os.path.join(a.out, f"records_{arm}.jsonl"), "w") as f:
            for x in out: f.write(json.dumps(x) + "\n")
        with open(os.path.join(a.out, f"transcripts_{arm}.jsonl"), "w") as f:
            for c in convs: f.write(json.dumps({"id": c["id"], "messages": msgs[c["id"]]}) + "\n")
    B, S = recs["base"], recs["sentinel"]
    acc = lambda rows: (sum(r["correct"] for r in rows) / len(rows)) if rows else None
    def block(b, s):
        un = [r for r in s if not r["flagged"]]; wrong = [r for r in s if not r["correct"]]
        return {"n": len(s), "base_accuracy": acc(b), "sentinel_overall_accuracy": acc(s), "sentinel_unflagged_accuracy": acc(un), "sentinel_unflagged_n": len(un),
                "sentinel_flag_rate": sum(r["flagged"] for r in s) / len(s), "sentinel_flag_rate_on_its_wrong": (sum(r["flagged"] for r in wrong) / len(wrong)) if wrong else None,
                "sentinel_unsure_anywhere_not_leading": sum(r["unsure_anywhere"] and not r["flagged"] for r in s)}
    q = lambda rows: [r for r in rows if r["kind"]]
    main_b = block(q(B), q(S))
    K1 = {"sentinel_unflagged_accuracy": main_b["sentinel_unflagged_accuracy"], "base_accuracy": main_b["base_accuracy"]}
    K1["pass"] = K1["sentinel_unflagged_accuracy"] is not None and K1["sentinel_unflagged_accuracy"] >= K1["base_accuracy"] + 0.10
    K2 = {"sentinel_flag_rate_on_its_wrong": main_b["sentinel_flag_rate_on_its_wrong"]}; K2["pass"] = K2["sentinel_flag_rate_on_its_wrong"] is not None and K2["sentinel_flag_rate_on_its_wrong"] >= 0.50
    K3 = {"sentinel_clean_end_rate": sum(r["finish"] == "stop" for r in S) / len(S), "base_clean_end_rate": sum(r["finish"] == "stop" for r in B) / len(B)}; K3["pass"] = K3["sentinel_clean_end_rate"] >= 0.95
    D = [r for r in S if r["group"].startswith("D_")]
    K4 = {"sentinel_flag_rate_no_fact_chats": sum(r["flagged"] for r in D) / len(D), "n": len(D)}; K4["pass"] = K4["sentinel_flag_rate_no_fact_chats"] <= 0.10
    by = {}
    for g in ("A_loose_single", "B_multi_turn", "C_mixed_chat"):
        for k in ("fact", "math", None):
            sel = lambda rows: [r for r in rows if r["group"] == g and (k is None or r["kind"] == k)]
            by[f"{g}/{k or 'all'}"] = block(sel(B), sel(S))
    for k in ("fact", "math"): by[f"all/{k}"] = block([r for r in q(B) if r["kind"] == k], [r for r in q(S) if r["kind"] == k])
    summary = {"K1_unflagged_beats_original_by_10pts": K1, "K2_flags_half_its_wrong": K2, "K3_clean_endings": K3, "K4_no_false_flags_on_chat": K4,
               "question_conversations": main_b, "breakdown": by, "earlier_turns_flagged_sentinel": sum(r["earlier_turns_flagged"] for r in S)}
    summary["all_pass"] = all(summary[k]["pass"] is True for k in ("K1_unflagged_beats_original_by_10pts", "K2_flags_half_its_wrong", "K3_clean_endings", "K4_no_false_flags_on_chat"))
    json.dump(summary, open(os.path.join(a.out, "summary.json"), "w"), indent=1)
    # --- reported only: search-when-unsure on flagged fact conversations, query = user's last message
    prog("waiting for Wikipedia index"); th.join(); sr = {"index_download": dl}
    cmap = {c["id"]: c for c in convs}; ff = [r for r in S if r["kind"] == "fact" and r["flagged"]]; sr["flagged_fact_conversations"] = len(ff)
    if dl["ok"] and ff:
        os.environ["JAVA_HOME"] = dl["java_home"]; os.environ["PATH"] = os.path.join(dl["java_home"], "bin") + os.pathsep + os.environ["PATH"]
        import jnius_config; jnius_config.add_options("-Xmx4g"); jnius_config.set_classpath(dl["jar"])
        from jnius import autoclass
        JS = autoclass("io.anserini.search.SimpleSearcher"); AL = autoclass("java.util.ArrayList"); jsr = JS(dl["index_dir"])
        def passage(docid):
            raw = jsr.doc(docid).get("raw") or ""
            try: c = json.loads(raw).get("contents", raw)
            except Exception: c = raw
            first, _, rest = c.partition("\n"); return {"title": first.strip().strip('"'), "text": rest.strip()}
        def search(qs, k):
            ql, ids = AL(), AL()
            for i, t in enumerate(qs): ql.add(t); ids.add(str(i))
            res = jsr.batch_search(ql, ids, int(k), 4); return [[passage(h.docid) for h in res.get(str(i))] for i in range(len(qs))]
        probe = search([p for p, _ in PROBES], 3); hits = [any(w in d["title"].lower() for d in row) for (p, w), row in zip(PROBES, probe)]; sr["probe_hits"] = f"{sum(hits)}/{len(hits)}"
        if sum(hits) >= 4:
            qs = [cmap[r["id"]]["user_turns"][-1] for r in ff]; docs = search(qs, 5); prompts = []
            for qq, row in zip(qs, docs):
                ctx = "\n\n".join(f"[{j+1}] {d['title']}: {d['text'][:600]}" for j, d in enumerate(row))
                prompts.append("Answer the question using the Wikipedia passages below. Give a short factual answer only (a few words), then on a new line write 'Source: ' followed by the title of the passage you used.\n\n" + ctx + f"\n\nQuestion: {qq}")
            res = llm.chat([[{"role": "user", "content": p}] for p in prompts], SamplingParams(temperature=0.0, max_tokens=128, seed=SEED), lora_request=lora, use_tqdm=False)
            srecs = []
            for r, row, o in zip(ff, docs, res):
                t = o.outputs[0].text; ans = unflag(t.split("Source:")[0].strip())
                srecs.append({"id": r["id"], "guess_correct": r["correct"], "search_reply": t, "search_correct": bool(fact_ok(ans, cmap[r["id"]]["gold"])), "passage_titles": [d["title"] for d in row]})
            with open(os.path.join(a.out, "search_records.jsonl"), "w") as f:
                for x in srecs: f.write(json.dumps(x) + "\n")
            sr["guess_accuracy_on_flagged"] = sum(x["guess_correct"] for x in srecs) / len(srecs); sr["search_accuracy_on_flagged"] = sum(x["search_correct"] for x in srecs) / len(srecs)
        else: sr["note"] = "index probe gate failed (<4/5): search not evaluated"
    else: sr["note"] = "index download failed or nothing flagged"
    summary["search_when_unsure_reported"] = sr; summary["seconds"] = round(time.time() - t0); summary["env"] = env
    json.dump(summary, open(os.path.join(a.out, "summary.json"), "w"), indent=1); prog("done"); print(json.dumps({"all_pass": summary["all_pass"]}))

if __name__ == "__main__":
    main()