File size: 11,934 Bytes
4e9bf3d
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
"""Sentinel S1 Job C2 (rev 2) — sign-off. Identical to Job C except the search source (owner GO 13 Nov 07:26). Starting model vs Sentinel (same weights + Job B LoRA) on frozen sets never trained on,
identical code, greedy. Then search-when-unsure on the fact questions Sentinel flags. Pass/fail rules as pre-registered (D13, D18, D19)."""
import argparse, hashlib, json, os, shutil, sys, threading, time
from job_a import MODEL, REV, SEED, PROMPT, fact_ok, math_ok
WIKI = "pyserini wikipedia-dpr-100w (Lucene BM25, 21M passages, DPR Wikipedia Dec 2018)"
WIKI_URL, WIKI_MD5 = "https://rgw.cs.uwaterloo.ca/pyserini/indexes/lucene-index.wikipedia-dpr-100w.20210120.d1b9e6.tar.gz", "7b58c08da992b2ea7e96667f0b176651"
JAR_URL, JAR_SHA256 = "https://repo1.maven.org/maven2/io/anserini/anserini/1.1.1/anserini-1.1.1-fatjar.jar", "69270ba4d160826953347411ce5d7e205ce363766a9cc72ac9da3b945341af83"
WIKI_REV = WIKI_MD5
MAXTOK, CH = 1024, 512
PROBES = [("Eiffel Tower Paris", "eiffel"), ("Albert Einstein theory of relativity", "einstein"), ("William Shakespeare playwright Hamlet", "shakespeare"),
          ("Amazon river South America", "amazon"), ("Mount Everest highest mountain", "everest")]

def flagged(t): return t.lstrip().upper().startswith("UNSURE:")
def unflag(t):
    s = t.lstrip(); return s[len("UNSURE:"):].strip() if s.upper().startswith("UNSURE:") else t
def sha(p): return hashlib.sha256(open(p, "rb").read()).hexdigest()
def load(p): return [json.loads(l) for l in open(p) if l.strip()]

def main():
    ap = argparse.ArgumentParser(); ap.add_argument("--signoff"); ap.add_argument("--adapter"); ap.add_argument("--out"); a = ap.parse_args()
    t0 = time.time(); prog = lambda m: open(os.path.join(a.out, "progress.txt"), "w").write(f"{m} {time.time()-t0:.0f}s\n")
    # --- pick the bigger disk for the 40 GB Wikipedia index and start downloading it in the background
    disks = {d: shutil.disk_usage(d).free for d in ("/opt/ml", "/tmp") if os.path.exists(d)}
    wdir = os.path.join(max(disks, key=disks.get), "scratch_wiki"); os.makedirs(wdir, exist_ok=True)
    dl = {"ok": False, "err": None, "seconds": None}
    def fetch():
        s0 = time.time()
        try:
            import urllib.request, tarfile, jdk
            jar = os.path.join(wdir, "anserini-1.1.1-fatjar.jar"); urllib.request.urlretrieve(JAR_URL, jar)
            if hashlib.sha256(open(jar, "rb").read()).hexdigest() != JAR_SHA256: raise ValueError("Anserini jar hash mismatch")
            dl["java_home"] = jdk.install("21", path=os.path.join(wdir, "jdk"))
            tgz = os.path.join(wdir, "index.tar.gz"); urllib.request.urlretrieve(WIKI_URL, tgz)
            h = hashlib.md5()
            with open(tgz, "rb") as f:
                for b in iter(lambda: f.read(1 << 24), b""): h.update(b)
            if h.hexdigest() != WIKI_MD5: raise ValueError("index md5 mismatch")
            with tarfile.open(tgz) as t: t.extractall(wdir, filter="data")
            os.remove(tgz)
            dl["index_dir"] = [os.path.join(wdir, d) for d in os.listdir(wdir) if d.startswith("lucene-index")][0]; dl["jar"] = jar; dl["ok"] = True
        except Exception as e: dl["err"] = repr(e)[:500]
        dl["seconds"] = round(time.time() - s0)
    th = threading.Thread(target=fetch, daemon=True); th.start()
    sets = {k: load(os.path.join(a.signoff, f"{k}.jsonl")) for k in ("ifeval", "gsm8k_test", "nq_val", "simpleqa")}
    hashes = {k: sha(os.path.join(a.signoff, f"{k}.jsonl")) for k in sets}
    adapter_files = {f: sha(os.path.join(a.adapter, f)) for f in sorted(os.listdir(a.adapter)) if os.path.isfile(os.path.join(a.adapter, f))}
    import torch, vllm
    from vllm import LLM, SamplingParams
    from vllm.lora.request import LoRARequest
    env = {"model": MODEL, "revision": REV, "vllm": vllm.__version__, "torch": torch.__version__, "gpu": torch.cuda.get_device_name(0),
           "signoff_sha256": hashes, "adapter_sha256": adapter_files, "wiki_index": WIKI, "wiki_revision": WIKI_REV, "disk_free_gb": {k: round(v/1e9) for k, v in disks.items()}, "wiki_dir": wdir}
    json.dump(env, open(os.path.join(a.out, "env.json"), "w"), indent=1)
    llm = LLM(model=MODEL, revision=REV, dtype="bfloat16", max_model_len=3072, gpu_memory_utilization=0.90, seed=SEED,
              enable_lora=True, max_lora_rank=16, max_loras=1, enable_prefix_caching=True)
    lora = LoRARequest("sentinel", 1, a.adapter)
    sp = SamplingParams(temperature=0.0, max_tokens=MAXTOK, seed=SEED)
    from ifeval.utils import process_results
    recs = {}
    for arm in ("base", "sentinel"):
        out = []
        jobs = [("ifeval", r, r["prompt"]) for r in sets["ifeval"]] + [(k, r, PROMPT[r["kind"]].format(q=r["question"])) for k in ("gsm8k_test", "nq_val", "simpleqa") for r in sets[k]]
        for i in range(0, len(jobs), CH):
            part = jobs[i:i+CH]
            res = llm.chat([[{"role": "user", "content": p}] for _, _, p in part], sp, lora_request=lora if arm == "sentinel" else None, use_tqdm=False)
            for (k, r, _), o in zip(part, res):
                t = o.outputs[0].text; fin = o.outputs[0].finish_reason
                rec = {"set": k, "id": r.get("id", r.get("key")), "reply": t, "finish": fin, "flagged": flagged(t)}
                if k == "ifeval": rec["correct"] = bool(process_results(r, [t])["prompt_level_strict_acc"])   # scored exactly as shown (D19)
                else: rec["correct"] = bool(fact_ok(unflag(t), r["gold"]) if r["kind"] == "fact" else math_ok(unflag(t), r["gold"]))
                out.append(rec)
            prog(f"{arm} {min(i+CH, len(jobs))}/{len(jobs)}")
        recs[arm] = out
        with open(os.path.join(a.out, f"signoff_{arm}.jsonl"), "w") as f:
            for x in out: f.write(json.dumps(x) + "\n")
    # --- checks 1-4
    def acc(rows): return sum(r["correct"] for r in rows) / len(rows) if rows else None
    B, S = recs["base"], recs["sentinel"]; sel = lambda rows, ks: [r for r in rows if r["set"] in ks]
    QA = ("gsm8k_test", "nq_val", "simpleqa"); FACT = ("nq_val", "simpleqa")
    c1 = {"base": acc(sel(B, ("ifeval",))), "sentinel": acc(sel(S, ("ifeval",)))}; c1["pass"] = c1["sentinel"] >= c1["base"] - 0.02
    c2 = {"base": acc(sel(B, ("gsm8k_test",))), "sentinel": acc(sel(S, ("gsm8k_test",)))}; c2["pass"] = c2["sentinel"] >= c2["base"] - 0.02
    def c3for(ks):
        b = sel(B, ks); s = sel(S, ks); un = [r for r in s if not r["flagged"]]; wrong = [r for r in s if not r["correct"]]
        return {"base_accuracy": acc(b), "sentinel_unflagged_accuracy": acc(un), "sentinel_unflagged_n": len(un), "sentinel_flag_rate": sum(r["flagged"] for r in s) / len(s),
                "sentinel_flag_rate_on_its_wrong_answers": (sum(r["flagged"] for r in wrong) / len(wrong)) if wrong else None, "sentinel_overall_accuracy": acc(s)}
    c3 = c3for(QA); c3["pass"] = (c3["sentinel_unflagged_accuracy"] is not None and c3["sentinel_unflagged_accuracy"] >= c3["base_accuracy"] + 0.10
                                  and c3["sentinel_flag_rate_on_its_wrong_answers"] is not None and c3["sentinel_flag_rate_on_its_wrong_answers"] >= 0.50)
    c3_breakdown = {"facts_only": c3for(FACT), "maths_only": c3for(("gsm8k_test",)), "nq_val": c3for(("nq_val",)), "simpleqa": c3for(("simpleqa",))}   # D18 reporting
    c4 = {"sentinel_natural_end_rate": sum(r["finish"] == "stop" for r in S) / len(S), "base_natural_end_rate": sum(r["finish"] == "stop" for r in B) / len(B)}; c4["pass"] = c4["sentinel_natural_end_rate"] >= 0.95
    summary = {"check1_instructions_ifeval": c1, "check2_maths_gsm8k": c2, "check3_knows_when_unsure": c3, "check3_breakdown_D18": c3_breakdown, "check4_clean_replies": c4}
    json.dump(summary, open(os.path.join(a.out, "signoff_summary.json"), "w"), indent=1)
    # --- check 5: search when unsure (fact questions Sentinel flags)
    prog("waiting for Wikipedia index"); th.join()
    c5 = {"index_download": dl}
    flagged_facts = [r for r in sel(S, FACT) if r["flagged"]]; c5["flagged_fact_questions"] = len(flagged_facts)
    if dl["ok"] and flagged_facts:
        os.environ["JAVA_HOME"] = dl["java_home"]; os.environ["PATH"] = os.path.join(dl["java_home"], "bin") + os.pathsep + os.environ["PATH"]
        import jnius_config; jnius_config.add_options("-Xmx4g"); jnius_config.set_classpath(dl["jar"])
        from jnius import autoclass
        JS = autoclass("io.anserini.search.SimpleSearcher"); AL = autoclass("java.util.ArrayList"); jsr = JS(dl["index_dir"])
        c5["index_docs"] = int(jsr.get_total_num_docs())
        def passage(docid):
            raw = jsr.doc(docid).get("raw") or ""
            try: c = json.loads(raw).get("contents", raw)
            except Exception: c = raw
            first, _, rest = c.partition("\n"); return {"title": first.strip().strip('"'), "section": "", "text": rest.strip()}
        def search(qs, k):
            q, ids = AL(), AL()
            for i, t in enumerate(qs): q.add(t); ids.add(str(i))
            res = jsr.batch_search(q, ids, int(k), 4)
            return [[passage(h.docid) for h in res.get(str(i))] for i in range(len(qs))]
        probe = search([q for q, _ in PROBES], 3); hits = [any(w in str(d.get("title", "")).lower() for d in row) for (q, w), row in zip(PROBES, probe)]
        c5["probe_hits"] = f"{sum(hits)}/{len(hits)}"; c5["probe_top_titles"] = [[d.get("title") for d in row] for row in probe]
        if sum(hits) >= 4:
            qtext = {r["id"]: r for k in FACT for r in sets[k]}; srecs = []
            for i in range(0, len(flagged_facts), 256):
                part = flagged_facts[i:i+256]; qs = [qtext[r["id"]]["question"] for r in part]; docs = search(qs, 5)
                prompts = []
                for q, row in zip(qs, docs):
                    ctx = "\n\n".join(f"[{j+1}] {d.get('title','')} — {d.get('section','')}: {str(d.get('text','')).strip()[:600]}" for j, d in enumerate(row))
                    prompts.append("Answer the question using the Wikipedia passages below. Give a short factual answer only (a few words), then on a new line write 'Source: ' followed by the title of the passage you used.\n\n" + ctx + f"\n\nQuestion: {q}")
                res = llm.chat([[{"role": "user", "content": p}] for p in prompts], SamplingParams(temperature=0.0, max_tokens=128, seed=SEED), lora_request=lora, use_tqdm=False)
                for r, row, o in zip(part, docs, res):
                    t = o.outputs[0].text; ans = unflag(t.split("Source:")[0].strip())
                    srecs.append({"id": r["id"], "set": r["set"], "guess_correct": r["correct"], "search_reply": t, "search_correct": bool(fact_ok(ans, qtext[r["id"]]["gold"])),
                                  "passage_titles": [d.get("title") for d in row]})
                prog(f"search {min(i+256, len(flagged_facts))}/{len(flagged_facts)}")
            with open(os.path.join(a.out, "search_records.jsonl"), "w") as f:
                for x in srecs: f.write(json.dumps(x) + "\n")
            c5["guess_accuracy_on_flagged"] = sum(x["guess_correct"] for x in srecs) / len(srecs)
            c5["search_accuracy_on_flagged"] = sum(x["search_correct"] for x in srecs) / len(srecs)
            c5["pass"] = c5["search_accuracy_on_flagged"] > c5["guess_accuracy_on_flagged"]
        else: c5["pass"] = None; c5["note"] = "index probe gate failed (<4/5): search not evaluated (D19)"
    else: c5["pass"] = None; c5["note"] = "index download failed or nothing flagged"
    summary["check5_search_when_unsure"] = c5
    summary["all_pass"] = all(summary[k].get("pass") is True for k in ("check1_instructions_ifeval", "check2_maths_gsm8k", "check3_knows_when_unsure", "check4_clean_replies", "check5_search_when_unsure"))
    summary["seconds"] = round(time.time() - t0); summary["env"] = env
    json.dump(summary, open(os.path.join(a.out, "signoff_summary.json"), "w"), indent=1); prog("done"); print(json.dumps({"all_pass": summary["all_pass"]}))

if __name__ == "__main__":
    main()