File size: 12,528 Bytes
4e9bf3d
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
"""Sentinel S1 Job E3 (owner GO 14 Nov 2026 09:09 UK): stronger search only, with the system naming the source (D58-D60). Retrieval,
reranking and the answer prompt are byte-identical to Job E2; the current search is not re-run (E2 measured it). Original E2 note:
Job E2: current search vs stronger search, same model (Sentinel S1 rev 4 = Olmo 3 7B
Instruct rev 6e5971d9 + B4 adapter d1a21fd9...), same flagged questions, same marking. Current = Job C2/C4 search exactly (2018 DPR
Wikipedia, BM25 top 5, same prompt). Stronger = BM25 top 50 from the 2018 index + top 50 from the 2023-11-01 Wikipedia index (Job E1),
reranked by BAAI/bge-reranker-v2-m3 (Apache 2.0, pinned revision), best 10 passages, Source given as passage number and mapped to
the article title by the system. Pass marks P1-P3 pre-registered (decision log D51-D55)."""
import argparse, hashlib, json, os, shutil, threading, time
from job_a import MODEL, REV, SEED, fact_ok, norm
WIKI_URL, WIKI_MD5 = "https://rgw.cs.uwaterloo.ca/pyserini/indexes/lucene-index.wikipedia-dpr-100w.20210120.d1b9e6.tar.gz", "7b58c08da992b2ea7e96667f0b176651"
JAR_URL, JAR_SHA256 = "https://repo1.maven.org/maven2/io/anserini/anserini/1.1.1/anserini-1.1.1-fatjar.jar", "69270ba4d160826953347411ce5d7e205ce363766a9cc72ac9da3b945341af83"
RERANKER, RERANKER_REV = "BAAI/bge-reranker-v2-m3", "953dc6f6f85a1b2dbfca4c34a2796e7dde08d41e"
PROBES = [("Eiffel Tower Paris", "eiffel"), ("Albert Einstein theory of relativity", "einstein"), ("William Shakespeare playwright Hamlet", "shakespeare"),
          ("Amazon river South America", "amazon"), ("Mount Everest highest mountain", "everest")]
K_EACH, K_FINAL = 50, 10
OLD_PROMPT = "Answer the question using the Wikipedia passages below. Give a short factual answer only (a few words), then on a new line write 'Source: ' followed by the title of the passage you used.\n\n"
NEW_PROMPT = "Answer the question using the Wikipedia passages below. Give a short factual answer only (a few words), then on a new line write 'Source: ' followed by the number of the passage you used.\n\n"
def unflag(t):
    s = t.lstrip(); return s[len("UNSURE:"):].strip() if s.upper().startswith("UNSURE:") else t
def sha_file(p):
    h = hashlib.sha256()
    with open(p, "rb") as f:
        for b in iter(lambda: f.read(1 << 24), b""): h.update(b)
    return h.hexdigest()
def main():
    ap = argparse.ArgumentParser(); ap.add_argument("--evalset"); ap.add_argument("--adapter"); ap.add_argument("--w23"); ap.add_argument("--w23meta"); ap.add_argument("--out")
    ap.add_argument("--w18_dir", default=""); ap.add_argument("--jar", default=""); ap.add_argument("--java_home", default=""); ap.add_argument("--rerank_device", default="cuda"); ap.add_argument("--limit", type=int, default=0)
    a = ap.parse_args(); t0 = time.time(); prog = lambda m: open(os.path.join(a.out, "progress.txt"), "w").write(f"{m} {time.time()-t0:.0f}s\n")
    disks = {d: shutil.disk_usage(d).free for d in ("/opt/ml", "/tmp") if os.path.exists(d)}; wdir = os.path.join(max(disks, key=disks.get), "scratch_wiki"); os.makedirs(wdir, exist_ok=True)
    dl = {"ok": bool(a.w18_dir), "err": None, "index_dir": a.w18_dir, "jar": a.jar, "java_home": a.java_home}
    def fetch():
        s0 = time.time()
        try:
            import urllib.request, tarfile, jdk
            jar = os.path.join(wdir, "anserini-1.1.1-fatjar.jar"); urllib.request.urlretrieve(JAR_URL, jar)
            if sha_file(jar) != JAR_SHA256: raise ValueError("Anserini jar hash mismatch")
            dl["java_home"] = jdk.install("21", path=os.path.join(wdir, "jdk")); tgz = os.path.join(wdir, "index.tar.gz"); urllib.request.urlretrieve(WIKI_URL, tgz); h = hashlib.md5()
            with open(tgz, "rb") as f:
                for b in iter(lambda: f.read(1 << 24), b""): h.update(b)
            if h.hexdigest() != WIKI_MD5: raise ValueError("index md5 mismatch")
            with tarfile.open(tgz) as t: t.extractall(wdir, filter="data")
            os.remove(tgz); dl["index_dir"] = [os.path.join(wdir, d) for d in os.listdir(wdir) if d.startswith("lucene-index")][0]; dl["jar"] = jar; dl["ok"] = True
        except Exception as e: dl["err"] = repr(e)[:500]
        dl["seconds"] = round(time.time() - s0)
    th = None
    if not a.w18_dir: th = threading.Thread(target=fetch, daemon=True); th.start()
    qs = [json.loads(l) for l in open(os.path.join(a.evalset, "flagged_signoff.jsonl"))] + [json.loads(l) for l in open(os.path.join(a.evalset, "flagged_conversations.jsonl"))]
    if a.limit: qs = [q for st in ("nq_val", "simpleqa", "conversation") for q in [x for x in qs if x["set"] == st][:a.limit]]
    env = {"evalset_sha256": {f: sha_file(os.path.join(a.evalset, f)) for f in sorted(os.listdir(a.evalset))}, "adapter_sha256": {f: sha_file(os.path.join(a.adapter, f)) for f in sorted(os.listdir(a.adapter)) if os.path.isfile(os.path.join(a.adapter, f))}}
    prog("verifying 2023 index"); man = json.load(open(os.path.join(a.w23meta, "index_manifest.json"))); bad = [k for k, v in man.items() if sha_file(os.path.join(a.w23, k)) != v["sha256"]]
    env["w23_index_files_verified"] = f"{len(man) - len(bad)}/{len(man)}"
    if bad: raise SystemExit(f"2023 index files do not match their manifest: {bad[:5]}")
    prog("waiting for 2018 index")
    if th: th.join()
    if not dl["ok"]: raise SystemExit("2018 index download failed: " + str(dl["err"]))
    os.environ["JAVA_HOME"] = dl["java_home"]; os.environ["PATH"] = os.path.join(dl["java_home"], "bin") + os.pathsep + os.environ["PATH"]
    import jnius_config; jnius_config.add_options("-Xmx6g"); jnius_config.set_classpath(dl["jar"])
    from jnius import autoclass
    JS = autoclass("io.anserini.search.SimpleSearcher"); AL = autoclass("java.util.ArrayList"); S = {"w18": JS(dl["index_dir"]), "w23": JS(a.w23)}
    def passage(s, docid):
        raw = S[s].doc(docid).get("raw") or ""
        try: c = json.loads(raw).get("contents", raw)
        except Exception: c = raw
        first, _, rest = c.partition("\n"); return {"title": first.strip().strip('"'), "text": rest.strip(), "src": s, "docid": docid}
    def search(s, texts, k):
        out = []
        for i in range(0, len(texts), 512):
            part = texts[i:i+512]; ql, ids = AL(), AL()
            for j, t in enumerate(part): ql.add(t); ids.add(str(j))
            res = S[s].batch_search(ql, ids, int(k), 4); out += [[passage(s, h.docid) for h in res.get(str(j))] for j in range(len(part))]
        return out
    env["index_docs"] = {s: int(S[s].get_total_num_docs()) for s in S}
    env["probe_hits"] = {s: sum(any(w in d["title"].lower() for d in row) for (p, w), row in zip(PROBES, search(s, [p for p, _ in PROBES], 3))) for s in S}
    texts = [q["query"] for q in qs]; prog("searching")
    old = search("w18", texts, 5); c18 = search("w18", texts, K_EACH); c23 = search("w23", texts, K_EACH)
    cands = []
    for x, y in zip(c18, c23):
        seen, row = set(), []
        for d in x + y:
            key = (d["title"], d["text"][:200])
            if key not in seen: seen.add(key); row.append(d)
        cands.append(row)
    prog("reranking")
    import torch
    from transformers import AutoTokenizer, AutoModelForSequenceClassification
    tok = AutoTokenizer.from_pretrained(RERANKER, revision=RERANKER_REV); rr = AutoModelForSequenceClassification.from_pretrained(RERANKER, revision=RERANKER_REV, torch_dtype=torch.float16 if a.rerank_device == "cuda" else torch.float32).to(a.rerank_device).eval()
    new = []; r0 = time.time()
    for qi, (q, row) in enumerate(zip(texts, cands)):
        pairs = [[q, f"{d['title']}: {d['text']}"] for d in row]; scores = []
        for i in range(0, len(pairs), 64):
            enc = tok(pairs[i:i+64], padding=True, truncation=True, max_length=384, return_tensors="pt").to(a.rerank_device)
            with torch.no_grad(): scores += rr(**enc).logits.view(-1).float().tolist()
        new.append([row[i] for i in sorted(range(len(row)), key=lambda i: -scores[i])[:K_FINAL]])
        if qi % 200 == 0: prog(f"reranking {qi}/{len(texts)}")
    env["rerank_seconds"] = round(time.time() - r0); del rr; torch.cuda.empty_cache() if a.rerank_device == "cuda" else None
    prog("answering")
    from vllm import LLM, SamplingParams
    from vllm.lora.request import LoRARequest
    llm = LLM(model=MODEL, revision=REV, dtype="bfloat16", max_model_len=4096, gpu_memory_utilization=0.85, seed=SEED, enable_lora=True, max_lora_rank=16, max_loras=1, enable_prefix_caching=True)
    lora = LoRARequest("sentinel", 1, a.adapter); sp = SamplingParams(temperature=0.0, max_tokens=128, seed=SEED)
    p_old = None and [OLD_PROMPT + "\n\n".join(f"[{j+1}] {d['title']} — : {d['text'][:600]}" for j, d in enumerate(row)) + f"\n\nQuestion: {q}" for q, row in zip(texts, old)]
    p_new = [NEW_PROMPT + "\n\n".join(f"[{j+1}] {d['title']}: {d['text'][:600]}" for j, d in enumerate(row)) + f"\n\nQuestion: {q}" for q, row in zip(texts, new)]
    r_new = llm.chat([[{"role": "user", "content": p}] for p in p_new], sp, lora_request=lora, use_tqdm=False)
    import re
    def cite(ans, row):
        na = norm(ans)
        if not na: return None, None
        for j, d in enumerate(row):
            if f" {na} " in f" {norm(d['title'] + ' ' + d['text'][:600])} ": return j + 1, d["title"]
        return None, None
    recs = []
    for q, rn, row_new in zip(qs, r_new, new):
        tn = rn.outputs[0].text; ans = unflag(tn.split("Source:")[0].strip())
        m = re.search(r"Source:\s*\[?(\d+)\]?", tn); model_n = int(m.group(1)) if m else None
        sys_n, sys_title = cite(ans, row_new)
        recs.append({"id": q["id"], "set": q["set"], "guess_correct": q["guess_correct"], "new_reply": tn, "answer": ans, "new_correct": bool(fact_ok(ans, q["gold"])),
                     "system_source_n": sys_n, "system_source_title": sys_title or "no source found", "model_source_n": model_n,
                     "new_passages": [[d["src"], d["title"], d["text"][:600]] for d in row_new]})
    with open(os.path.join(a.out, "search_records.jsonl"), "w") as f:
        for x in recs: f.write(json.dumps(x) + "\n")
    acc = lambda rows, k: (sum(r[k] for r in rows) / len(rows)) if rows else None
    sel = lambda st: [r for r in recs if r["set"] in st]
    main_rows = sel(("nq_val", "simpleqa")); conv = sel(("conversation",))
    E2_CURRENT = {"all": 0.2512862171676144, "nq_val": 0.2828535669586984, "simpleqa": 0.22720763723150358}   # Job E2 current-search results (D57)
    summ = {"n_signoff_flagged": len(main_rows), "n_conversation_flagged": len(conv)}
    for name, rows in (("all", main_rows), ("nq_val", sel(("nq_val",))), ("simpleqa", sel(("simpleqa",))), ("conversation", conv)):
        summ[name] = {"n": len(rows), "guess": acc(rows, "guess_correct"), "stronger_search": acc(rows, "new_correct"), "e2_current_search": E2_CURRENT.get(name)}
    P1 = {"e2_current": E2_CURRENT["all"], "stronger": summ["all"]["stronger_search"]}; P1["pass"] = P1["stronger"] >= P1["e2_current"] + 0.05
    P2 = {st: {"e2_current": E2_CURRENT[st], "stronger": summ[st]["stronger_search"]} for st in ("nq_val", "simpleqa")}
    P2["pass"] = all(P2[st]["stronger"] is not None and P2[st]["stronger"] >= P2[st]["e2_current"] for st in ("nq_val", "simpleqa"))
    right = [r for r in main_rows if r["new_correct"]]
    P3 = {"right_answers": len(right), "right_answers_with_a_source_containing_the_answer": sum(r["system_source_n"] is not None for r in right)}
    P3["rate"] = P3["right_answers_with_a_source_containing_the_answer"] / max(1, len(right)); P3["pass"] = P3["rate"] >= 0.95
    summ.update({"P1_five_points_better": P1, "P2_not_worse_on_either_set": P2, "P3_right_answers_cite_a_passage_containing_them": P3, "all_pass": P1["pass"] and P2["pass"] and P3["pass"],
                 "all_answers_with_a_source": sum(r["system_source_n"] is not None for r in main_rows) / len(main_rows),
                 "model_gave_a_source_line": sum(r["model_source_n"] is not None for r in main_rows) / len(main_rows),
                 "share_of_new_passages_from_2023": sum(p[0] == "w23" for r in main_rows for p in r["new_passages"]) / max(1, sum(len(r["new_passages"]) for r in main_rows)),
                 "seconds": round(time.time() - t0), "env": env, "reranker": {"model": RERANKER, "revision": RERANKER_REV, "candidates_each": K_EACH, "final": K_FINAL}})
    json.dump(summ, open(os.path.join(a.out, "summary.json"), "w"), indent=1); prog("done"); print(json.dumps({"all_pass": summ["all_pass"]}))
if __name__ == "__main__": main()