Text Generation
PEFT
Safetensors
English
lora
olmo
uncertainty
calibration
abstention
retrieval-augmented-generation
Instructions to use Sentovara/Sentinel-S1-7B with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- PEFT
How to use Sentovara/Sentinel-S1-7B with PEFT:
from peft import PeftModel from transformers import AutoModelForCausalLM base_model = AutoModelForCausalLM.from_pretrained("allenai/Olmo-3-7B-Instruct") model = PeftModel.from_pretrained(base_model, "Sentovara/Sentinel-S1-7B") - Notebooks
- Google Colab
- Kaggle
Download eval/search_e3.py from Sentovara/Sentinel-S1-7B: direct link, hf CLI and curl.
- Browser
- Download file 12.5 kB
-
https://huggingface.co/Sentovara/Sentinel-S1-7B/resolve/main/eval/search_e3.py
- Command line
-
hf download hf://Sentovara/Sentinel-S1-7B/eval/search_e3.py
-
curl -L -o search_e3.py https://huggingface.co/Sentovara/Sentinel-S1-7B/resolve/main/eval/search_e3.py
12.5 kB
| """Sentinel S1 Job E3 (owner GO 14 Nov 2026 09:09 UK): stronger search only, with the system naming the source (D58-D60). Retrieval, | |
| reranking and the answer prompt are byte-identical to Job E2; the current search is not re-run (E2 measured it). Original E2 note: | |
| Job E2: current search vs stronger search, same model (Sentinel S1 rev 4 = Olmo 3 7B | |
| Instruct rev 6e5971d9 + B4 adapter d1a21fd9...), same flagged questions, same marking. Current = Job C2/C4 search exactly (2018 DPR | |
| Wikipedia, BM25 top 5, same prompt). Stronger = BM25 top 50 from the 2018 index + top 50 from the 2023-11-01 Wikipedia index (Job E1), | |
| reranked by BAAI/bge-reranker-v2-m3 (Apache 2.0, pinned revision), best 10 passages, Source given as passage number and mapped to | |
| the article title by the system. Pass marks P1-P3 pre-registered (decision log D51-D55).""" | |
| import argparse, hashlib, json, os, shutil, threading, time | |
| from job_a import MODEL, REV, SEED, fact_ok, norm | |
| WIKI_URL, WIKI_MD5 = "https://rgw.cs.uwaterloo.ca/pyserini/indexes/lucene-index.wikipedia-dpr-100w.20210120.d1b9e6.tar.gz", "7b58c08da992b2ea7e96667f0b176651" | |
| JAR_URL, JAR_SHA256 = "https://repo1.maven.org/maven2/io/anserini/anserini/1.1.1/anserini-1.1.1-fatjar.jar", "69270ba4d160826953347411ce5d7e205ce363766a9cc72ac9da3b945341af83" | |
| RERANKER, RERANKER_REV = "BAAI/bge-reranker-v2-m3", "953dc6f6f85a1b2dbfca4c34a2796e7dde08d41e" | |
| PROBES = [("Eiffel Tower Paris", "eiffel"), ("Albert Einstein theory of relativity", "einstein"), ("William Shakespeare playwright Hamlet", "shakespeare"), | |
| ("Amazon river South America", "amazon"), ("Mount Everest highest mountain", "everest")] | |
| K_EACH, K_FINAL = 50, 10 | |
| OLD_PROMPT = "Answer the question using the Wikipedia passages below. Give a short factual answer only (a few words), then on a new line write 'Source: ' followed by the title of the passage you used.\n\n" | |
| NEW_PROMPT = "Answer the question using the Wikipedia passages below. Give a short factual answer only (a few words), then on a new line write 'Source: ' followed by the number of the passage you used.\n\n" | |
| def unflag(t): | |
| s = t.lstrip(); return s[len("UNSURE:"):].strip() if s.upper().startswith("UNSURE:") else t | |
| def sha_file(p): | |
| h = hashlib.sha256() | |
| with open(p, "rb") as f: | |
| for b in iter(lambda: f.read(1 << 24), b""): h.update(b) | |
| return h.hexdigest() | |
| def main(): | |
| ap = argparse.ArgumentParser(); ap.add_argument("--evalset"); ap.add_argument("--adapter"); ap.add_argument("--w23"); ap.add_argument("--w23meta"); ap.add_argument("--out") | |
| ap.add_argument("--w18_dir", default=""); ap.add_argument("--jar", default=""); ap.add_argument("--java_home", default=""); ap.add_argument("--rerank_device", default="cuda"); ap.add_argument("--limit", type=int, default=0) | |
| a = ap.parse_args(); t0 = time.time(); prog = lambda m: open(os.path.join(a.out, "progress.txt"), "w").write(f"{m} {time.time()-t0:.0f}s\n") | |
| disks = {d: shutil.disk_usage(d).free for d in ("/opt/ml", "/tmp") if os.path.exists(d)}; wdir = os.path.join(max(disks, key=disks.get), "scratch_wiki"); os.makedirs(wdir, exist_ok=True) | |
| dl = {"ok": bool(a.w18_dir), "err": None, "index_dir": a.w18_dir, "jar": a.jar, "java_home": a.java_home} | |
| def fetch(): | |
| s0 = time.time() | |
| try: | |
| import urllib.request, tarfile, jdk | |
| jar = os.path.join(wdir, "anserini-1.1.1-fatjar.jar"); urllib.request.urlretrieve(JAR_URL, jar) | |
| if sha_file(jar) != JAR_SHA256: raise ValueError("Anserini jar hash mismatch") | |
| dl["java_home"] = jdk.install("21", path=os.path.join(wdir, "jdk")); tgz = os.path.join(wdir, "index.tar.gz"); urllib.request.urlretrieve(WIKI_URL, tgz); h = hashlib.md5() | |
| with open(tgz, "rb") as f: | |
| for b in iter(lambda: f.read(1 << 24), b""): h.update(b) | |
| if h.hexdigest() != WIKI_MD5: raise ValueError("index md5 mismatch") | |
| with tarfile.open(tgz) as t: t.extractall(wdir, filter="data") | |
| os.remove(tgz); dl["index_dir"] = [os.path.join(wdir, d) for d in os.listdir(wdir) if d.startswith("lucene-index")][0]; dl["jar"] = jar; dl["ok"] = True | |
| except Exception as e: dl["err"] = repr(e)[:500] | |
| dl["seconds"] = round(time.time() - s0) | |
| th = None | |
| if not a.w18_dir: th = threading.Thread(target=fetch, daemon=True); th.start() | |
| qs = [json.loads(l) for l in open(os.path.join(a.evalset, "flagged_signoff.jsonl"))] + [json.loads(l) for l in open(os.path.join(a.evalset, "flagged_conversations.jsonl"))] | |
| if a.limit: qs = [q for st in ("nq_val", "simpleqa", "conversation") for q in [x for x in qs if x["set"] == st][:a.limit]] | |
| env = {"evalset_sha256": {f: sha_file(os.path.join(a.evalset, f)) for f in sorted(os.listdir(a.evalset))}, "adapter_sha256": {f: sha_file(os.path.join(a.adapter, f)) for f in sorted(os.listdir(a.adapter)) if os.path.isfile(os.path.join(a.adapter, f))}} | |
| prog("verifying 2023 index"); man = json.load(open(os.path.join(a.w23meta, "index_manifest.json"))); bad = [k for k, v in man.items() if sha_file(os.path.join(a.w23, k)) != v["sha256"]] | |
| env["w23_index_files_verified"] = f"{len(man) - len(bad)}/{len(man)}" | |
| if bad: raise SystemExit(f"2023 index files do not match their manifest: {bad[:5]}") | |
| prog("waiting for 2018 index") | |
| if th: th.join() | |
| if not dl["ok"]: raise SystemExit("2018 index download failed: " + str(dl["err"])) | |
| os.environ["JAVA_HOME"] = dl["java_home"]; os.environ["PATH"] = os.path.join(dl["java_home"], "bin") + os.pathsep + os.environ["PATH"] | |
| import jnius_config; jnius_config.add_options("-Xmx6g"); jnius_config.set_classpath(dl["jar"]) | |
| from jnius import autoclass | |
| JS = autoclass("io.anserini.search.SimpleSearcher"); AL = autoclass("java.util.ArrayList"); S = {"w18": JS(dl["index_dir"]), "w23": JS(a.w23)} | |
| def passage(s, docid): | |
| raw = S[s].doc(docid).get("raw") or "" | |
| try: c = json.loads(raw).get("contents", raw) | |
| except Exception: c = raw | |
| first, _, rest = c.partition("\n"); return {"title": first.strip().strip('"'), "text": rest.strip(), "src": s, "docid": docid} | |
| def search(s, texts, k): | |
| out = [] | |
| for i in range(0, len(texts), 512): | |
| part = texts[i:i+512]; ql, ids = AL(), AL() | |
| for j, t in enumerate(part): ql.add(t); ids.add(str(j)) | |
| res = S[s].batch_search(ql, ids, int(k), 4); out += [[passage(s, h.docid) for h in res.get(str(j))] for j in range(len(part))] | |
| return out | |
| env["index_docs"] = {s: int(S[s].get_total_num_docs()) for s in S} | |
| env["probe_hits"] = {s: sum(any(w in d["title"].lower() for d in row) for (p, w), row in zip(PROBES, search(s, [p for p, _ in PROBES], 3))) for s in S} | |
| texts = [q["query"] for q in qs]; prog("searching") | |
| old = search("w18", texts, 5); c18 = search("w18", texts, K_EACH); c23 = search("w23", texts, K_EACH) | |
| cands = [] | |
| for x, y in zip(c18, c23): | |
| seen, row = set(), [] | |
| for d in x + y: | |
| key = (d["title"], d["text"][:200]) | |
| if key not in seen: seen.add(key); row.append(d) | |
| cands.append(row) | |
| prog("reranking") | |
| import torch | |
| from transformers import AutoTokenizer, AutoModelForSequenceClassification | |
| tok = AutoTokenizer.from_pretrained(RERANKER, revision=RERANKER_REV); rr = AutoModelForSequenceClassification.from_pretrained(RERANKER, revision=RERANKER_REV, torch_dtype=torch.float16 if a.rerank_device == "cuda" else torch.float32).to(a.rerank_device).eval() | |
| new = []; r0 = time.time() | |
| for qi, (q, row) in enumerate(zip(texts, cands)): | |
| pairs = [[q, f"{d['title']}: {d['text']}"] for d in row]; scores = [] | |
| for i in range(0, len(pairs), 64): | |
| enc = tok(pairs[i:i+64], padding=True, truncation=True, max_length=384, return_tensors="pt").to(a.rerank_device) | |
| with torch.no_grad(): scores += rr(**enc).logits.view(-1).float().tolist() | |
| new.append([row[i] for i in sorted(range(len(row)), key=lambda i: -scores[i])[:K_FINAL]]) | |
| if qi % 200 == 0: prog(f"reranking {qi}/{len(texts)}") | |
| env["rerank_seconds"] = round(time.time() - r0); del rr; torch.cuda.empty_cache() if a.rerank_device == "cuda" else None | |
| prog("answering") | |
| from vllm import LLM, SamplingParams | |
| from vllm.lora.request import LoRARequest | |
| llm = LLM(model=MODEL, revision=REV, dtype="bfloat16", max_model_len=4096, gpu_memory_utilization=0.85, seed=SEED, enable_lora=True, max_lora_rank=16, max_loras=1, enable_prefix_caching=True) | |
| lora = LoRARequest("sentinel", 1, a.adapter); sp = SamplingParams(temperature=0.0, max_tokens=128, seed=SEED) | |
| p_old = None and [OLD_PROMPT + "\n\n".join(f"[{j+1}] {d['title']} — : {d['text'][:600]}" for j, d in enumerate(row)) + f"\n\nQuestion: {q}" for q, row in zip(texts, old)] | |
| p_new = [NEW_PROMPT + "\n\n".join(f"[{j+1}] {d['title']}: {d['text'][:600]}" for j, d in enumerate(row)) + f"\n\nQuestion: {q}" for q, row in zip(texts, new)] | |
| r_new = llm.chat([[{"role": "user", "content": p}] for p in p_new], sp, lora_request=lora, use_tqdm=False) | |
| import re | |
| def cite(ans, row): | |
| na = norm(ans) | |
| if not na: return None, None | |
| for j, d in enumerate(row): | |
| if f" {na} " in f" {norm(d['title'] + ' ' + d['text'][:600])} ": return j + 1, d["title"] | |
| return None, None | |
| recs = [] | |
| for q, rn, row_new in zip(qs, r_new, new): | |
| tn = rn.outputs[0].text; ans = unflag(tn.split("Source:")[0].strip()) | |
| m = re.search(r"Source:\s*\[?(\d+)\]?", tn); model_n = int(m.group(1)) if m else None | |
| sys_n, sys_title = cite(ans, row_new) | |
| recs.append({"id": q["id"], "set": q["set"], "guess_correct": q["guess_correct"], "new_reply": tn, "answer": ans, "new_correct": bool(fact_ok(ans, q["gold"])), | |
| "system_source_n": sys_n, "system_source_title": sys_title or "no source found", "model_source_n": model_n, | |
| "new_passages": [[d["src"], d["title"], d["text"][:600]] for d in row_new]}) | |
| with open(os.path.join(a.out, "search_records.jsonl"), "w") as f: | |
| for x in recs: f.write(json.dumps(x) + "\n") | |
| acc = lambda rows, k: (sum(r[k] for r in rows) / len(rows)) if rows else None | |
| sel = lambda st: [r for r in recs if r["set"] in st] | |
| main_rows = sel(("nq_val", "simpleqa")); conv = sel(("conversation",)) | |
| E2_CURRENT = {"all": 0.2512862171676144, "nq_val": 0.2828535669586984, "simpleqa": 0.22720763723150358} # Job E2 current-search results (D57) | |
| summ = {"n_signoff_flagged": len(main_rows), "n_conversation_flagged": len(conv)} | |
| for name, rows in (("all", main_rows), ("nq_val", sel(("nq_val",))), ("simpleqa", sel(("simpleqa",))), ("conversation", conv)): | |
| summ[name] = {"n": len(rows), "guess": acc(rows, "guess_correct"), "stronger_search": acc(rows, "new_correct"), "e2_current_search": E2_CURRENT.get(name)} | |
| P1 = {"e2_current": E2_CURRENT["all"], "stronger": summ["all"]["stronger_search"]}; P1["pass"] = P1["stronger"] >= P1["e2_current"] + 0.05 | |
| P2 = {st: {"e2_current": E2_CURRENT[st], "stronger": summ[st]["stronger_search"]} for st in ("nq_val", "simpleqa")} | |
| P2["pass"] = all(P2[st]["stronger"] is not None and P2[st]["stronger"] >= P2[st]["e2_current"] for st in ("nq_val", "simpleqa")) | |
| right = [r for r in main_rows if r["new_correct"]] | |
| P3 = {"right_answers": len(right), "right_answers_with_a_source_containing_the_answer": sum(r["system_source_n"] is not None for r in right)} | |
| P3["rate"] = P3["right_answers_with_a_source_containing_the_answer"] / max(1, len(right)); P3["pass"] = P3["rate"] >= 0.95 | |
| summ.update({"P1_five_points_better": P1, "P2_not_worse_on_either_set": P2, "P3_right_answers_cite_a_passage_containing_them": P3, "all_pass": P1["pass"] and P2["pass"] and P3["pass"], | |
| "all_answers_with_a_source": sum(r["system_source_n"] is not None for r in main_rows) / len(main_rows), | |
| "model_gave_a_source_line": sum(r["model_source_n"] is not None for r in main_rows) / len(main_rows), | |
| "share_of_new_passages_from_2023": sum(p[0] == "w23" for r in main_rows for p in r["new_passages"]) / max(1, sum(len(r["new_passages"]) for r in main_rows)), | |
| "seconds": round(time.time() - t0), "env": env, "reranker": {"model": RERANKER, "revision": RERANKER_REV, "candidates_each": K_EACH, "final": K_FINAL}}) | |
| json.dump(summ, open(os.path.join(a.out, "summary.json"), "w"), indent=1); prog("done"); print(json.dumps({"all_pass": summ["all_pass"]})) | |
| if __name__ == "__main__": main() | |