Text Generation
PEFT
Safetensors
English
lora
olmo
uncertainty
calibration
abstention
retrieval-augmented-generation
Instructions to use Sentovara/Sentinel-S1-7B with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- PEFT
How to use Sentovara/Sentinel-S1-7B with PEFT:
from peft import PeftModel from transformers import AutoModelForCausalLM base_model = AutoModelForCausalLM.from_pretrained("allenai/Olmo-3-7B-Instruct") model = PeftModel.from_pretrained(base_model, "Sentovara/Sentinel-S1-7B") - Notebooks
- Google Colab
- Kaggle
File size: 12,528 Bytes
4e9bf3d | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 | """Sentinel S1 Job E3 (owner GO 14 Nov 2026 09:09 UK): stronger search only, with the system naming the source (D58-D60). Retrieval,
reranking and the answer prompt are byte-identical to Job E2; the current search is not re-run (E2 measured it). Original E2 note:
Job E2: current search vs stronger search, same model (Sentinel S1 rev 4 = Olmo 3 7B
Instruct rev 6e5971d9 + B4 adapter d1a21fd9...), same flagged questions, same marking. Current = Job C2/C4 search exactly (2018 DPR
Wikipedia, BM25 top 5, same prompt). Stronger = BM25 top 50 from the 2018 index + top 50 from the 2023-11-01 Wikipedia index (Job E1),
reranked by BAAI/bge-reranker-v2-m3 (Apache 2.0, pinned revision), best 10 passages, Source given as passage number and mapped to
the article title by the system. Pass marks P1-P3 pre-registered (decision log D51-D55)."""
import argparse, hashlib, json, os, shutil, threading, time
from job_a import MODEL, REV, SEED, fact_ok, norm
WIKI_URL, WIKI_MD5 = "https://rgw.cs.uwaterloo.ca/pyserini/indexes/lucene-index.wikipedia-dpr-100w.20210120.d1b9e6.tar.gz", "7b58c08da992b2ea7e96667f0b176651"
JAR_URL, JAR_SHA256 = "https://repo1.maven.org/maven2/io/anserini/anserini/1.1.1/anserini-1.1.1-fatjar.jar", "69270ba4d160826953347411ce5d7e205ce363766a9cc72ac9da3b945341af83"
RERANKER, RERANKER_REV = "BAAI/bge-reranker-v2-m3", "953dc6f6f85a1b2dbfca4c34a2796e7dde08d41e"
PROBES = [("Eiffel Tower Paris", "eiffel"), ("Albert Einstein theory of relativity", "einstein"), ("William Shakespeare playwright Hamlet", "shakespeare"),
("Amazon river South America", "amazon"), ("Mount Everest highest mountain", "everest")]
K_EACH, K_FINAL = 50, 10
OLD_PROMPT = "Answer the question using the Wikipedia passages below. Give a short factual answer only (a few words), then on a new line write 'Source: ' followed by the title of the passage you used.\n\n"
NEW_PROMPT = "Answer the question using the Wikipedia passages below. Give a short factual answer only (a few words), then on a new line write 'Source: ' followed by the number of the passage you used.\n\n"
def unflag(t):
s = t.lstrip(); return s[len("UNSURE:"):].strip() if s.upper().startswith("UNSURE:") else t
def sha_file(p):
h = hashlib.sha256()
with open(p, "rb") as f:
for b in iter(lambda: f.read(1 << 24), b""): h.update(b)
return h.hexdigest()
def main():
ap = argparse.ArgumentParser(); ap.add_argument("--evalset"); ap.add_argument("--adapter"); ap.add_argument("--w23"); ap.add_argument("--w23meta"); ap.add_argument("--out")
ap.add_argument("--w18_dir", default=""); ap.add_argument("--jar", default=""); ap.add_argument("--java_home", default=""); ap.add_argument("--rerank_device", default="cuda"); ap.add_argument("--limit", type=int, default=0)
a = ap.parse_args(); t0 = time.time(); prog = lambda m: open(os.path.join(a.out, "progress.txt"), "w").write(f"{m} {time.time()-t0:.0f}s\n")
disks = {d: shutil.disk_usage(d).free for d in ("/opt/ml", "/tmp") if os.path.exists(d)}; wdir = os.path.join(max(disks, key=disks.get), "scratch_wiki"); os.makedirs(wdir, exist_ok=True)
dl = {"ok": bool(a.w18_dir), "err": None, "index_dir": a.w18_dir, "jar": a.jar, "java_home": a.java_home}
def fetch():
s0 = time.time()
try:
import urllib.request, tarfile, jdk
jar = os.path.join(wdir, "anserini-1.1.1-fatjar.jar"); urllib.request.urlretrieve(JAR_URL, jar)
if sha_file(jar) != JAR_SHA256: raise ValueError("Anserini jar hash mismatch")
dl["java_home"] = jdk.install("21", path=os.path.join(wdir, "jdk")); tgz = os.path.join(wdir, "index.tar.gz"); urllib.request.urlretrieve(WIKI_URL, tgz); h = hashlib.md5()
with open(tgz, "rb") as f:
for b in iter(lambda: f.read(1 << 24), b""): h.update(b)
if h.hexdigest() != WIKI_MD5: raise ValueError("index md5 mismatch")
with tarfile.open(tgz) as t: t.extractall(wdir, filter="data")
os.remove(tgz); dl["index_dir"] = [os.path.join(wdir, d) for d in os.listdir(wdir) if d.startswith("lucene-index")][0]; dl["jar"] = jar; dl["ok"] = True
except Exception as e: dl["err"] = repr(e)[:500]
dl["seconds"] = round(time.time() - s0)
th = None
if not a.w18_dir: th = threading.Thread(target=fetch, daemon=True); th.start()
qs = [json.loads(l) for l in open(os.path.join(a.evalset, "flagged_signoff.jsonl"))] + [json.loads(l) for l in open(os.path.join(a.evalset, "flagged_conversations.jsonl"))]
if a.limit: qs = [q for st in ("nq_val", "simpleqa", "conversation") for q in [x for x in qs if x["set"] == st][:a.limit]]
env = {"evalset_sha256": {f: sha_file(os.path.join(a.evalset, f)) for f in sorted(os.listdir(a.evalset))}, "adapter_sha256": {f: sha_file(os.path.join(a.adapter, f)) for f in sorted(os.listdir(a.adapter)) if os.path.isfile(os.path.join(a.adapter, f))}}
prog("verifying 2023 index"); man = json.load(open(os.path.join(a.w23meta, "index_manifest.json"))); bad = [k for k, v in man.items() if sha_file(os.path.join(a.w23, k)) != v["sha256"]]
env["w23_index_files_verified"] = f"{len(man) - len(bad)}/{len(man)}"
if bad: raise SystemExit(f"2023 index files do not match their manifest: {bad[:5]}")
prog("waiting for 2018 index")
if th: th.join()
if not dl["ok"]: raise SystemExit("2018 index download failed: " + str(dl["err"]))
os.environ["JAVA_HOME"] = dl["java_home"]; os.environ["PATH"] = os.path.join(dl["java_home"], "bin") + os.pathsep + os.environ["PATH"]
import jnius_config; jnius_config.add_options("-Xmx6g"); jnius_config.set_classpath(dl["jar"])
from jnius import autoclass
JS = autoclass("io.anserini.search.SimpleSearcher"); AL = autoclass("java.util.ArrayList"); S = {"w18": JS(dl["index_dir"]), "w23": JS(a.w23)}
def passage(s, docid):
raw = S[s].doc(docid).get("raw") or ""
try: c = json.loads(raw).get("contents", raw)
except Exception: c = raw
first, _, rest = c.partition("\n"); return {"title": first.strip().strip('"'), "text": rest.strip(), "src": s, "docid": docid}
def search(s, texts, k):
out = []
for i in range(0, len(texts), 512):
part = texts[i:i+512]; ql, ids = AL(), AL()
for j, t in enumerate(part): ql.add(t); ids.add(str(j))
res = S[s].batch_search(ql, ids, int(k), 4); out += [[passage(s, h.docid) for h in res.get(str(j))] for j in range(len(part))]
return out
env["index_docs"] = {s: int(S[s].get_total_num_docs()) for s in S}
env["probe_hits"] = {s: sum(any(w in d["title"].lower() for d in row) for (p, w), row in zip(PROBES, search(s, [p for p, _ in PROBES], 3))) for s in S}
texts = [q["query"] for q in qs]; prog("searching")
old = search("w18", texts, 5); c18 = search("w18", texts, K_EACH); c23 = search("w23", texts, K_EACH)
cands = []
for x, y in zip(c18, c23):
seen, row = set(), []
for d in x + y:
key = (d["title"], d["text"][:200])
if key not in seen: seen.add(key); row.append(d)
cands.append(row)
prog("reranking")
import torch
from transformers import AutoTokenizer, AutoModelForSequenceClassification
tok = AutoTokenizer.from_pretrained(RERANKER, revision=RERANKER_REV); rr = AutoModelForSequenceClassification.from_pretrained(RERANKER, revision=RERANKER_REV, torch_dtype=torch.float16 if a.rerank_device == "cuda" else torch.float32).to(a.rerank_device).eval()
new = []; r0 = time.time()
for qi, (q, row) in enumerate(zip(texts, cands)):
pairs = [[q, f"{d['title']}: {d['text']}"] for d in row]; scores = []
for i in range(0, len(pairs), 64):
enc = tok(pairs[i:i+64], padding=True, truncation=True, max_length=384, return_tensors="pt").to(a.rerank_device)
with torch.no_grad(): scores += rr(**enc).logits.view(-1).float().tolist()
new.append([row[i] for i in sorted(range(len(row)), key=lambda i: -scores[i])[:K_FINAL]])
if qi % 200 == 0: prog(f"reranking {qi}/{len(texts)}")
env["rerank_seconds"] = round(time.time() - r0); del rr; torch.cuda.empty_cache() if a.rerank_device == "cuda" else None
prog("answering")
from vllm import LLM, SamplingParams
from vllm.lora.request import LoRARequest
llm = LLM(model=MODEL, revision=REV, dtype="bfloat16", max_model_len=4096, gpu_memory_utilization=0.85, seed=SEED, enable_lora=True, max_lora_rank=16, max_loras=1, enable_prefix_caching=True)
lora = LoRARequest("sentinel", 1, a.adapter); sp = SamplingParams(temperature=0.0, max_tokens=128, seed=SEED)
p_old = None and [OLD_PROMPT + "\n\n".join(f"[{j+1}] {d['title']} — : {d['text'][:600]}" for j, d in enumerate(row)) + f"\n\nQuestion: {q}" for q, row in zip(texts, old)]
p_new = [NEW_PROMPT + "\n\n".join(f"[{j+1}] {d['title']}: {d['text'][:600]}" for j, d in enumerate(row)) + f"\n\nQuestion: {q}" for q, row in zip(texts, new)]
r_new = llm.chat([[{"role": "user", "content": p}] for p in p_new], sp, lora_request=lora, use_tqdm=False)
import re
def cite(ans, row):
na = norm(ans)
if not na: return None, None
for j, d in enumerate(row):
if f" {na} " in f" {norm(d['title'] + ' ' + d['text'][:600])} ": return j + 1, d["title"]
return None, None
recs = []
for q, rn, row_new in zip(qs, r_new, new):
tn = rn.outputs[0].text; ans = unflag(tn.split("Source:")[0].strip())
m = re.search(r"Source:\s*\[?(\d+)\]?", tn); model_n = int(m.group(1)) if m else None
sys_n, sys_title = cite(ans, row_new)
recs.append({"id": q["id"], "set": q["set"], "guess_correct": q["guess_correct"], "new_reply": tn, "answer": ans, "new_correct": bool(fact_ok(ans, q["gold"])),
"system_source_n": sys_n, "system_source_title": sys_title or "no source found", "model_source_n": model_n,
"new_passages": [[d["src"], d["title"], d["text"][:600]] for d in row_new]})
with open(os.path.join(a.out, "search_records.jsonl"), "w") as f:
for x in recs: f.write(json.dumps(x) + "\n")
acc = lambda rows, k: (sum(r[k] for r in rows) / len(rows)) if rows else None
sel = lambda st: [r for r in recs if r["set"] in st]
main_rows = sel(("nq_val", "simpleqa")); conv = sel(("conversation",))
E2_CURRENT = {"all": 0.2512862171676144, "nq_val": 0.2828535669586984, "simpleqa": 0.22720763723150358} # Job E2 current-search results (D57)
summ = {"n_signoff_flagged": len(main_rows), "n_conversation_flagged": len(conv)}
for name, rows in (("all", main_rows), ("nq_val", sel(("nq_val",))), ("simpleqa", sel(("simpleqa",))), ("conversation", conv)):
summ[name] = {"n": len(rows), "guess": acc(rows, "guess_correct"), "stronger_search": acc(rows, "new_correct"), "e2_current_search": E2_CURRENT.get(name)}
P1 = {"e2_current": E2_CURRENT["all"], "stronger": summ["all"]["stronger_search"]}; P1["pass"] = P1["stronger"] >= P1["e2_current"] + 0.05
P2 = {st: {"e2_current": E2_CURRENT[st], "stronger": summ[st]["stronger_search"]} for st in ("nq_val", "simpleqa")}
P2["pass"] = all(P2[st]["stronger"] is not None and P2[st]["stronger"] >= P2[st]["e2_current"] for st in ("nq_val", "simpleqa"))
right = [r for r in main_rows if r["new_correct"]]
P3 = {"right_answers": len(right), "right_answers_with_a_source_containing_the_answer": sum(r["system_source_n"] is not None for r in right)}
P3["rate"] = P3["right_answers_with_a_source_containing_the_answer"] / max(1, len(right)); P3["pass"] = P3["rate"] >= 0.95
summ.update({"P1_five_points_better": P1, "P2_not_worse_on_either_set": P2, "P3_right_answers_cite_a_passage_containing_them": P3, "all_pass": P1["pass"] and P2["pass"] and P3["pass"],
"all_answers_with_a_source": sum(r["system_source_n"] is not None for r in main_rows) / len(main_rows),
"model_gave_a_source_line": sum(r["model_source_n"] is not None for r in main_rows) / len(main_rows),
"share_of_new_passages_from_2023": sum(p[0] == "w23" for r in main_rows for p in r["new_passages"]) / max(1, sum(len(r["new_passages"]) for r in main_rows)),
"seconds": round(time.time() - t0), "env": env, "reranker": {"model": RERANKER, "revision": RERANKER_REV, "candidates_each": K_EACH, "final": K_FINAL}})
json.dump(summ, open(os.path.join(a.out, "summary.json"), "w"), indent=1); prog("done"); print(json.dumps({"all_pass": summ["all_pass"]}))
if __name__ == "__main__": main()
|