Text Generation
PEFT
Safetensors
English
lora
olmo
uncertainty
calibration
abstention
retrieval-augmented-generation
Instructions to use Sentovara/Sentinel-S1-7B with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- PEFT
How to use Sentovara/Sentinel-S1-7B with PEFT:
from peft import PeftModel from transformers import AutoModelForCausalLM base_model = AutoModelForCausalLM.from_pretrained("allenai/Olmo-3-7B-Instruct") model = PeftModel.from_pretrained(base_model, "Sentovara/Sentinel-S1-7B") - Notebooks
- Google Colab
- Kaggle
File size: 11,934 Bytes
4e9bf3d | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 | """Sentinel S1 Job C2 (rev 2) — sign-off. Identical to Job C except the search source (owner GO 13 Nov 07:26). Starting model vs Sentinel (same weights + Job B LoRA) on frozen sets never trained on,
identical code, greedy. Then search-when-unsure on the fact questions Sentinel flags. Pass/fail rules as pre-registered (D13, D18, D19)."""
import argparse, hashlib, json, os, shutil, sys, threading, time
from job_a import MODEL, REV, SEED, PROMPT, fact_ok, math_ok
WIKI = "pyserini wikipedia-dpr-100w (Lucene BM25, 21M passages, DPR Wikipedia Dec 2018)"
WIKI_URL, WIKI_MD5 = "https://rgw.cs.uwaterloo.ca/pyserini/indexes/lucene-index.wikipedia-dpr-100w.20210120.d1b9e6.tar.gz", "7b58c08da992b2ea7e96667f0b176651"
JAR_URL, JAR_SHA256 = "https://repo1.maven.org/maven2/io/anserini/anserini/1.1.1/anserini-1.1.1-fatjar.jar", "69270ba4d160826953347411ce5d7e205ce363766a9cc72ac9da3b945341af83"
WIKI_REV = WIKI_MD5
MAXTOK, CH = 1024, 512
PROBES = [("Eiffel Tower Paris", "eiffel"), ("Albert Einstein theory of relativity", "einstein"), ("William Shakespeare playwright Hamlet", "shakespeare"),
("Amazon river South America", "amazon"), ("Mount Everest highest mountain", "everest")]
def flagged(t): return t.lstrip().upper().startswith("UNSURE:")
def unflag(t):
s = t.lstrip(); return s[len("UNSURE:"):].strip() if s.upper().startswith("UNSURE:") else t
def sha(p): return hashlib.sha256(open(p, "rb").read()).hexdigest()
def load(p): return [json.loads(l) for l in open(p) if l.strip()]
def main():
ap = argparse.ArgumentParser(); ap.add_argument("--signoff"); ap.add_argument("--adapter"); ap.add_argument("--out"); a = ap.parse_args()
t0 = time.time(); prog = lambda m: open(os.path.join(a.out, "progress.txt"), "w").write(f"{m} {time.time()-t0:.0f}s\n")
# --- pick the bigger disk for the 40 GB Wikipedia index and start downloading it in the background
disks = {d: shutil.disk_usage(d).free for d in ("/opt/ml", "/tmp") if os.path.exists(d)}
wdir = os.path.join(max(disks, key=disks.get), "scratch_wiki"); os.makedirs(wdir, exist_ok=True)
dl = {"ok": False, "err": None, "seconds": None}
def fetch():
s0 = time.time()
try:
import urllib.request, tarfile, jdk
jar = os.path.join(wdir, "anserini-1.1.1-fatjar.jar"); urllib.request.urlretrieve(JAR_URL, jar)
if hashlib.sha256(open(jar, "rb").read()).hexdigest() != JAR_SHA256: raise ValueError("Anserini jar hash mismatch")
dl["java_home"] = jdk.install("21", path=os.path.join(wdir, "jdk"))
tgz = os.path.join(wdir, "index.tar.gz"); urllib.request.urlretrieve(WIKI_URL, tgz)
h = hashlib.md5()
with open(tgz, "rb") as f:
for b in iter(lambda: f.read(1 << 24), b""): h.update(b)
if h.hexdigest() != WIKI_MD5: raise ValueError("index md5 mismatch")
with tarfile.open(tgz) as t: t.extractall(wdir, filter="data")
os.remove(tgz)
dl["index_dir"] = [os.path.join(wdir, d) for d in os.listdir(wdir) if d.startswith("lucene-index")][0]; dl["jar"] = jar; dl["ok"] = True
except Exception as e: dl["err"] = repr(e)[:500]
dl["seconds"] = round(time.time() - s0)
th = threading.Thread(target=fetch, daemon=True); th.start()
sets = {k: load(os.path.join(a.signoff, f"{k}.jsonl")) for k in ("ifeval", "gsm8k_test", "nq_val", "simpleqa")}
hashes = {k: sha(os.path.join(a.signoff, f"{k}.jsonl")) for k in sets}
adapter_files = {f: sha(os.path.join(a.adapter, f)) for f in sorted(os.listdir(a.adapter)) if os.path.isfile(os.path.join(a.adapter, f))}
import torch, vllm
from vllm import LLM, SamplingParams
from vllm.lora.request import LoRARequest
env = {"model": MODEL, "revision": REV, "vllm": vllm.__version__, "torch": torch.__version__, "gpu": torch.cuda.get_device_name(0),
"signoff_sha256": hashes, "adapter_sha256": adapter_files, "wiki_index": WIKI, "wiki_revision": WIKI_REV, "disk_free_gb": {k: round(v/1e9) for k, v in disks.items()}, "wiki_dir": wdir}
json.dump(env, open(os.path.join(a.out, "env.json"), "w"), indent=1)
llm = LLM(model=MODEL, revision=REV, dtype="bfloat16", max_model_len=3072, gpu_memory_utilization=0.90, seed=SEED,
enable_lora=True, max_lora_rank=16, max_loras=1, enable_prefix_caching=True)
lora = LoRARequest("sentinel", 1, a.adapter)
sp = SamplingParams(temperature=0.0, max_tokens=MAXTOK, seed=SEED)
from ifeval.utils import process_results
recs = {}
for arm in ("base", "sentinel"):
out = []
jobs = [("ifeval", r, r["prompt"]) for r in sets["ifeval"]] + [(k, r, PROMPT[r["kind"]].format(q=r["question"])) for k in ("gsm8k_test", "nq_val", "simpleqa") for r in sets[k]]
for i in range(0, len(jobs), CH):
part = jobs[i:i+CH]
res = llm.chat([[{"role": "user", "content": p}] for _, _, p in part], sp, lora_request=lora if arm == "sentinel" else None, use_tqdm=False)
for (k, r, _), o in zip(part, res):
t = o.outputs[0].text; fin = o.outputs[0].finish_reason
rec = {"set": k, "id": r.get("id", r.get("key")), "reply": t, "finish": fin, "flagged": flagged(t)}
if k == "ifeval": rec["correct"] = bool(process_results(r, [t])["prompt_level_strict_acc"]) # scored exactly as shown (D19)
else: rec["correct"] = bool(fact_ok(unflag(t), r["gold"]) if r["kind"] == "fact" else math_ok(unflag(t), r["gold"]))
out.append(rec)
prog(f"{arm} {min(i+CH, len(jobs))}/{len(jobs)}")
recs[arm] = out
with open(os.path.join(a.out, f"signoff_{arm}.jsonl"), "w") as f:
for x in out: f.write(json.dumps(x) + "\n")
# --- checks 1-4
def acc(rows): return sum(r["correct"] for r in rows) / len(rows) if rows else None
B, S = recs["base"], recs["sentinel"]; sel = lambda rows, ks: [r for r in rows if r["set"] in ks]
QA = ("gsm8k_test", "nq_val", "simpleqa"); FACT = ("nq_val", "simpleqa")
c1 = {"base": acc(sel(B, ("ifeval",))), "sentinel": acc(sel(S, ("ifeval",)))}; c1["pass"] = c1["sentinel"] >= c1["base"] - 0.02
c2 = {"base": acc(sel(B, ("gsm8k_test",))), "sentinel": acc(sel(S, ("gsm8k_test",)))}; c2["pass"] = c2["sentinel"] >= c2["base"] - 0.02
def c3for(ks):
b = sel(B, ks); s = sel(S, ks); un = [r for r in s if not r["flagged"]]; wrong = [r for r in s if not r["correct"]]
return {"base_accuracy": acc(b), "sentinel_unflagged_accuracy": acc(un), "sentinel_unflagged_n": len(un), "sentinel_flag_rate": sum(r["flagged"] for r in s) / len(s),
"sentinel_flag_rate_on_its_wrong_answers": (sum(r["flagged"] for r in wrong) / len(wrong)) if wrong else None, "sentinel_overall_accuracy": acc(s)}
c3 = c3for(QA); c3["pass"] = (c3["sentinel_unflagged_accuracy"] is not None and c3["sentinel_unflagged_accuracy"] >= c3["base_accuracy"] + 0.10
and c3["sentinel_flag_rate_on_its_wrong_answers"] is not None and c3["sentinel_flag_rate_on_its_wrong_answers"] >= 0.50)
c3_breakdown = {"facts_only": c3for(FACT), "maths_only": c3for(("gsm8k_test",)), "nq_val": c3for(("nq_val",)), "simpleqa": c3for(("simpleqa",))} # D18 reporting
c4 = {"sentinel_natural_end_rate": sum(r["finish"] == "stop" for r in S) / len(S), "base_natural_end_rate": sum(r["finish"] == "stop" for r in B) / len(B)}; c4["pass"] = c4["sentinel_natural_end_rate"] >= 0.95
summary = {"check1_instructions_ifeval": c1, "check2_maths_gsm8k": c2, "check3_knows_when_unsure": c3, "check3_breakdown_D18": c3_breakdown, "check4_clean_replies": c4}
json.dump(summary, open(os.path.join(a.out, "signoff_summary.json"), "w"), indent=1)
# --- check 5: search when unsure (fact questions Sentinel flags)
prog("waiting for Wikipedia index"); th.join()
c5 = {"index_download": dl}
flagged_facts = [r for r in sel(S, FACT) if r["flagged"]]; c5["flagged_fact_questions"] = len(flagged_facts)
if dl["ok"] and flagged_facts:
os.environ["JAVA_HOME"] = dl["java_home"]; os.environ["PATH"] = os.path.join(dl["java_home"], "bin") + os.pathsep + os.environ["PATH"]
import jnius_config; jnius_config.add_options("-Xmx4g"); jnius_config.set_classpath(dl["jar"])
from jnius import autoclass
JS = autoclass("io.anserini.search.SimpleSearcher"); AL = autoclass("java.util.ArrayList"); jsr = JS(dl["index_dir"])
c5["index_docs"] = int(jsr.get_total_num_docs())
def passage(docid):
raw = jsr.doc(docid).get("raw") or ""
try: c = json.loads(raw).get("contents", raw)
except Exception: c = raw
first, _, rest = c.partition("\n"); return {"title": first.strip().strip('"'), "section": "", "text": rest.strip()}
def search(qs, k):
q, ids = AL(), AL()
for i, t in enumerate(qs): q.add(t); ids.add(str(i))
res = jsr.batch_search(q, ids, int(k), 4)
return [[passage(h.docid) for h in res.get(str(i))] for i in range(len(qs))]
probe = search([q for q, _ in PROBES], 3); hits = [any(w in str(d.get("title", "")).lower() for d in row) for (q, w), row in zip(PROBES, probe)]
c5["probe_hits"] = f"{sum(hits)}/{len(hits)}"; c5["probe_top_titles"] = [[d.get("title") for d in row] for row in probe]
if sum(hits) >= 4:
qtext = {r["id"]: r for k in FACT for r in sets[k]}; srecs = []
for i in range(0, len(flagged_facts), 256):
part = flagged_facts[i:i+256]; qs = [qtext[r["id"]]["question"] for r in part]; docs = search(qs, 5)
prompts = []
for q, row in zip(qs, docs):
ctx = "\n\n".join(f"[{j+1}] {d.get('title','')} — {d.get('section','')}: {str(d.get('text','')).strip()[:600]}" for j, d in enumerate(row))
prompts.append("Answer the question using the Wikipedia passages below. Give a short factual answer only (a few words), then on a new line write 'Source: ' followed by the title of the passage you used.\n\n" + ctx + f"\n\nQuestion: {q}")
res = llm.chat([[{"role": "user", "content": p}] for p in prompts], SamplingParams(temperature=0.0, max_tokens=128, seed=SEED), lora_request=lora, use_tqdm=False)
for r, row, o in zip(part, docs, res):
t = o.outputs[0].text; ans = unflag(t.split("Source:")[0].strip())
srecs.append({"id": r["id"], "set": r["set"], "guess_correct": r["correct"], "search_reply": t, "search_correct": bool(fact_ok(ans, qtext[r["id"]]["gold"])),
"passage_titles": [d.get("title") for d in row]})
prog(f"search {min(i+256, len(flagged_facts))}/{len(flagged_facts)}")
with open(os.path.join(a.out, "search_records.jsonl"), "w") as f:
for x in srecs: f.write(json.dumps(x) + "\n")
c5["guess_accuracy_on_flagged"] = sum(x["guess_correct"] for x in srecs) / len(srecs)
c5["search_accuracy_on_flagged"] = sum(x["search_correct"] for x in srecs) / len(srecs)
c5["pass"] = c5["search_accuracy_on_flagged"] > c5["guess_accuracy_on_flagged"]
else: c5["pass"] = None; c5["note"] = "index probe gate failed (<4/5): search not evaluated (D19)"
else: c5["pass"] = None; c5["note"] = "index download failed or nothing flagged"
summary["check5_search_when_unsure"] = c5
summary["all_pass"] = all(summary[k].get("pass") is True for k in ("check1_instructions_ifeval", "check2_maths_gsm8k", "check3_knows_when_unsure", "check4_clean_replies", "check5_search_when_unsure"))
summary["seconds"] = round(time.time() - t0); summary["env"] = env
json.dump(summary, open(os.path.join(a.out, "signoff_summary.json"), "w"), indent=1); prog("done"); print(json.dumps({"all_pass": summary["all_pass"]}))
if __name__ == "__main__":
main()
|