Text Generation
PEFT
Safetensors
English
lora
olmo
uncertainty
calibration
abstention
retrieval-augmented-generation
Instructions to use Sentovara/Sentinel-S1-32B with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- PEFT
How to use Sentovara/Sentinel-S1-32B with PEFT:
from peft import PeftModel from transformers import AutoModelForCausalLM base_model = AutoModelForCausalLM.from_pretrained("allenai/Olmo-3.1-32B-Instruct") model = PeftModel.from_pretrained(base_model, "Sentovara/Sentinel-S1-32B") - Notebooks
- Google Colab
- Kaggle
File size: 19,329 Bytes
4d54b76 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 | """Sentinel S1 Job F3 — sign-off of the 32B Sentinel (owner GO 14 Nov 2026 11:47 UK). The 9 checks of rev 4, same methods and pass marks:
checks 1-4 exactly as Job C2/C4 (IFEval, GSM8K test, NQ-val + SimpleQA, clean endings), check 5 with the adopted stronger search (E3 method:
2018 + 2023 Wikipedia, BM25 top 50 each, bge-reranker-v2-m3 top 10, system-named source), and K1-K4 exactly as Job D/D3 on fresh
conversations. Original model = unmodified Olmo 3.1 32B Instruct; Sentinel = same weights + the Job F2 adapter; identical code, greedy."""
import argparse, hashlib, json, os, re, shutil, threading, time
from job_a import SEED, PROMPT, fact_ok, math_ok, norm
from model32 import MODEL32, REV32
WIKI_URL, WIKI_MD5 = "https://rgw.cs.uwaterloo.ca/pyserini/indexes/lucene-index.wikipedia-dpr-100w.20210120.d1b9e6.tar.gz", "7b58c08da992b2ea7e96667f0b176651"
JAR_URL, JAR_SHA256 = "https://repo1.maven.org/maven2/io/anserini/anserini/1.1.1/anserini-1.1.1-fatjar.jar", "69270ba4d160826953347411ce5d7e205ce363766a9cc72ac9da3b945341af83"
RERANKER, RERANKER_REV = "BAAI/bge-reranker-v2-m3", "953dc6f6f85a1b2dbfca4c34a2796e7dde08d41e"
PROBES = [("Eiffel Tower Paris", "eiffel"), ("Albert Einstein theory of relativity", "einstein"), ("William Shakespeare playwright Hamlet", "shakespeare"),
("Amazon river South America", "amazon"), ("Mount Everest highest mountain", "everest")]
K_EACH, K_FINAL, MAXTOK, CH, EARLY_TOK, LAST_TOK = 50, 10, 1024, 512, 512, 1024
NEW_PROMPT = "Answer the question using the Wikipedia passages below. Give a short factual answer only (a few words), then on a new line write 'Source: ' followed by the number of the passage you used.\n\n"
SEVEN_B_REV4 = {"ifeval": [0.8022, 0.8041], "gsm8k": [0.9143, 0.9128], "unflagged": 0.7506, "flags_wrong": 0.9404, "search_after": 0.2505, "conv_unflagged": 0.8228, "conv_flags_wrong": 0.9247, "conv_chat_flagged": 0.06}
def flagged(t): return t.lstrip().upper().startswith("UNSURE:")
def unflag(t):
s = t.lstrip(); return s[len("UNSURE:"):].strip() if s.upper().startswith("UNSURE:") else t
def sha_file(p):
h = hashlib.sha256()
with open(p, "rb") as f:
for b in iter(lambda: f.read(1 << 24), b""): h.update(b)
return h.hexdigest()
def main():
ap = argparse.ArgumentParser()
for k in ("signoff", "convs", "adapter", "w23", "w23meta", "out"): ap.add_argument("--" + k)
ap.add_argument("--tp", type=int, default=4); ap.add_argument("--limit", type=int, default=0); ap.add_argument("--w18_dir", default=""); ap.add_argument("--jar", default=""); ap.add_argument("--java_home", default=""); ap.add_argument("--rerank_device", default="cuda:0")
a = ap.parse_args(); t0 = time.time(); prog = lambda m: open(os.path.join(a.out, "progress.txt"), "w").write(f"{m} {time.time()-t0:.0f}s\n")
disks = {d: shutil.disk_usage(d).free for d in ("/opt/ml", "/tmp") if os.path.exists(d)}; wdir = os.path.join(max(disks, key=disks.get), "scratch_wiki"); os.makedirs(wdir, exist_ok=True)
dl = {"ok": bool(a.w18_dir), "err": None, "index_dir": a.w18_dir, "jar": a.jar, "java_home": a.java_home}
def fetch():
s0 = time.time()
try:
import urllib.request, tarfile, jdk
jar = os.path.join(wdir, "anserini-1.1.1-fatjar.jar"); urllib.request.urlretrieve(JAR_URL, jar)
if sha_file(jar) != JAR_SHA256: raise ValueError("Anserini jar hash mismatch")
dl["java_home"] = jdk.install("21", path=os.path.join(wdir, "jdk")); tgz = os.path.join(wdir, "index.tar.gz"); urllib.request.urlretrieve(WIKI_URL, tgz); h = hashlib.md5()
with open(tgz, "rb") as f:
for b in iter(lambda: f.read(1 << 24), b""): h.update(b)
if h.hexdigest() != WIKI_MD5: raise ValueError("index md5 mismatch")
with tarfile.open(tgz) as t: t.extractall(wdir, filter="data")
os.remove(tgz); dl["index_dir"] = [os.path.join(wdir, d) for d in os.listdir(wdir) if d.startswith("lucene-index")][0]; dl["jar"] = jar; dl["ok"] = True
except Exception as e: dl["err"] = repr(e)[:500]
dl["seconds"] = round(time.time() - s0)
th = None
if not a.w18_dir: th = threading.Thread(target=fetch, daemon=True); th.start()
load = lambda p: [json.loads(l) for l in open(p) if l.strip()]
sets = {k: load(os.path.join(a.signoff, f"{k}.jsonl")) for k in ("ifeval", "gsm8k_test", "nq_val", "simpleqa")}
convs = load(os.path.join(a.convs, "conversations.jsonl"))
if a.limit: sets = {k: v[:a.limit] for k, v in sets.items()}; convs = convs[:a.limit] + [c for c in convs if c["group"].startswith("D_")][:a.limit]
import torch, vllm
from vllm import LLM, SamplingParams
from vllm.lora.request import LoRARequest
env = {"model": MODEL32, "revision": REV32, "vllm": vllm.__version__, "torch": torch.__version__, "gpus": [torch.cuda.get_device_name(i) for i in range(torch.cuda.device_count())],
"signoff_sha256": {k: sha_file(os.path.join(a.signoff, f"{k}.jsonl")) for k in sets}, "conversations_sha256": sha_file(os.path.join(a.convs, "conversations.jsonl")),
"adapter_sha256": {f: sha_file(os.path.join(a.adapter, f)) for f in sorted(os.listdir(a.adapter)) if os.path.isfile(os.path.join(a.adapter, f))}, "tensor_parallel": a.tp}
json.dump(env, open(os.path.join(a.out, "env.json"), "w"), indent=1)
llm = LLM(model=MODEL32, revision=REV32, dtype="bfloat16", tensor_parallel_size=a.tp, max_model_len=8192, gpu_memory_utilization=0.85, seed=SEED, enforce_eager=True, # D72: A10G memory fix (no CUDA graphs)
enable_lora=True, max_lora_rank=16, max_loras=1, enable_prefix_caching=True)
lora = LoRARequest("sentinel", 1, a.adapter)
# ===== checks 1-4: exactly as Job C2/C4
sp = SamplingParams(temperature=0.0, max_tokens=MAXTOK, seed=SEED)
from ifeval.utils import process_results
recs = {}
for arm in ("base", "sentinel"):
out = []
jobs = [("ifeval", r, r["prompt"]) for r in sets["ifeval"]] + [(k, r, PROMPT[r["kind"]].format(q=r["question"])) for k in ("gsm8k_test", "nq_val", "simpleqa") for r in sets[k]]
for i in range(0, len(jobs), CH):
part = jobs[i:i+CH]
res = llm.chat([[{"role": "user", "content": p}] for _, _, p in part], sp, lora_request=lora if arm == "sentinel" else None, use_tqdm=False)
for (k, r, _), o in zip(part, res):
t = o.outputs[0].text; fin = o.outputs[0].finish_reason
rec = {"set": k, "id": r.get("id", r.get("key")), "reply": t, "finish": fin, "flagged": flagged(t)}
if k == "ifeval": rec["correct"] = bool(process_results(r, [t])["prompt_level_strict_acc"])
else: rec["correct"] = bool(fact_ok(unflag(t), r["gold"]) if r["kind"] == "fact" else math_ok(unflag(t), r["gold"]))
out.append(rec)
prog(f"signoff {arm} {min(i+CH, len(jobs))}/{len(jobs)}")
recs[arm] = out
with open(os.path.join(a.out, f"signoff_{arm}.jsonl"), "w") as f:
for x in out: f.write(json.dumps(x) + "\n")
def acc(rows): return sum(r["correct"] for r in rows) / len(rows) if rows else None
B, S = recs["base"], recs["sentinel"]; sel = lambda rows, ks: [r for r in rows if r["set"] in ks]
QA = ("gsm8k_test", "nq_val", "simpleqa"); FACT = ("nq_val", "simpleqa")
c1 = {"base": acc(sel(B, ("ifeval",))), "sentinel": acc(sel(S, ("ifeval",)))}; c1["pass"] = c1["sentinel"] >= c1["base"] - 0.02
c2 = {"base": acc(sel(B, ("gsm8k_test",))), "sentinel": acc(sel(S, ("gsm8k_test",)))}; c2["pass"] = c2["sentinel"] >= c2["base"] - 0.02
def c3for(ks):
b = sel(B, ks); s = sel(S, ks); un = [r for r in s if not r["flagged"]]; wrong = [r for r in s if not r["correct"]]
return {"base_accuracy": acc(b), "sentinel_unflagged_accuracy": acc(un), "sentinel_unflagged_n": len(un), "sentinel_flag_rate": sum(r["flagged"] for r in s) / len(s),
"sentinel_flag_rate_on_its_wrong_answers": (sum(r["flagged"] for r in wrong) / len(wrong)) if wrong else None, "sentinel_overall_accuracy": acc(s)}
c3 = c3for(QA); c3["pass"] = (c3["sentinel_unflagged_accuracy"] is not None and c3["sentinel_unflagged_accuracy"] >= c3["base_accuracy"] + 0.10
and c3["sentinel_flag_rate_on_its_wrong_answers"] is not None and c3["sentinel_flag_rate_on_its_wrong_answers"] >= 0.50)
c3_breakdown = {"facts_only": c3for(FACT), "maths_only": c3for(("gsm8k_test",)), "nq_val": c3for(("nq_val",)), "simpleqa": c3for(("simpleqa",))}
c4 = {"sentinel_natural_end_rate": sum(r["finish"] == "stop" for r in S) / len(S), "base_natural_end_rate": sum(r["finish"] == "stop" for r in B) / len(B)}; c4["pass"] = c4["sentinel_natural_end_rate"] >= 0.95
summary = {"check1_instructions_ifeval": c1, "check2_maths_gsm8k": c2, "check3_knows_when_unsure": c3, "check3_breakdown_D18": c3_breakdown, "check4_clean_replies": c4}
json.dump(summary, open(os.path.join(a.out, "summary.json"), "w"), indent=1)
# ===== K1-K4: exactly as Job D/D3
crec = {}
for arm in ("base", "sentinel"):
msgs = {c["id"]: [] for c in convs}; turns = {c["id"]: [] for c in convs}
for step in range(max(len(c["user_turns"]) for c in convs)):
act = [c for c in convs if len(c["user_turns"]) > step]
for c in act: msgs[c["id"]].append({"role": "user", "content": c["user_turns"][step]})
sps = [SamplingParams(temperature=0.0, seed=SEED, max_tokens=LAST_TOK if step == len(c["user_turns"]) - 1 else EARLY_TOK) for c in act]
res = llm.chat([msgs[c["id"]] for c in act], sps, lora_request=lora if arm == "sentinel" else None, use_tqdm=False)
for c, o in zip(act, res):
t = o.outputs[0].text; msgs[c["id"]].append({"role": "assistant", "content": t}); turns[c["id"]].append({"reply": t, "finish": o.outputs[0].finish_reason})
prog(f"conversations {arm} turn {step+1}")
out = []
for c in convs:
last = turns[c["id"]][-1]; t = last["reply"]
r = {"id": c["id"], "group": c["group"], "kind": c["kind"], "reply": t, "finish": last["finish"], "flagged": flagged(t), "unsure_anywhere": "UNSURE:" in t.upper(),
"earlier_turns_flagged": sum(flagged(x["reply"]) for x in turns[c["id"]][:-1])}
if c["kind"]: r["correct"] = bool(fact_ok(unflag(t), c["gold"]) if c["kind"] == "fact" else math_ok(unflag(t), c["gold"]))
out.append(r)
crec[arm] = out
with open(os.path.join(a.out, f"conv_records_{arm}.jsonl"), "w") as f:
for x in out: f.write(json.dumps(x) + "\n")
with open(os.path.join(a.out, f"conv_transcripts_{arm}.jsonl"), "w") as f:
for c in convs: f.write(json.dumps({"id": c["id"], "messages": msgs[c["id"]]}) + "\n")
CB, CS = crec["base"], crec["sentinel"]; q = lambda rows: [r for r in rows if r["kind"]]
un = [r for r in q(CS) if not r["flagged"]]; wr = [r for r in q(CS) if not r["correct"]]; D = [r for r in CS if r["group"].startswith("D_")]
K1 = {"sentinel_unflagged_accuracy": acc(un), "base_accuracy": acc(q(CB))}; K1["pass"] = K1["sentinel_unflagged_accuracy"] is not None and K1["sentinel_unflagged_accuracy"] >= K1["base_accuracy"] + 0.10
K2 = {"sentinel_flag_rate_on_its_wrong": (sum(r["flagged"] for r in wr) / len(wr)) if wr else None}; K2["pass"] = K2["sentinel_flag_rate_on_its_wrong"] is not None and K2["sentinel_flag_rate_on_its_wrong"] >= 0.50
K3 = {"sentinel_clean_end_rate": sum(r["finish"] == "stop" for r in CS) / len(CS), "base_clean_end_rate": sum(r["finish"] == "stop" for r in CB) / len(CB)}; K3["pass"] = K3["sentinel_clean_end_rate"] >= 0.95
K4 = {"sentinel_flag_rate_no_fact_chats": sum(r["flagged"] for r in D) / len(D), "n": len(D)}; K4["pass"] = K4["sentinel_flag_rate_no_fact_chats"] <= 0.10
summary.update({"K1_unflagged_beats_original_by_10pts": K1, "K2_flags_half_its_wrong": K2, "K3_clean_endings": K3, "K4_no_false_flags_on_chat": K4,
"conv_reported": {"sentinel_overall_accuracy": acc(q(CS)), "sentinel_flag_rate": sum(r["flagged"] for r in q(CS)) / len(q(CS)), "earlier_turns_flagged": sum(r["earlier_turns_flagged"] for r in CS),
"maths_flag_rate_on_wrong": (lambda w: sum(r["flagged"] for r in w) / len(w) if w else None)([r for r in q(CS) if r["kind"] == "math" and not r["correct"]])}})
json.dump(summary, open(os.path.join(a.out, "summary.json"), "w"), indent=1)
# ===== check 5: stronger search (E3 method) on the fact questions Sentinel flags
prog("waiting for 2018 index")
if th: th.join()
c5 = {"index_2018": {k: v for k, v in dl.items() if k != "java_home"}}
man = json.load(open(os.path.join(a.w23meta, "index_manifest.json"))); bad = [k for k, v in man.items() if sha_file(os.path.join(a.w23, k)) != v["sha256"]]
c5["w23_index_files_verified"] = f"{len(man) - len(bad)}/{len(man)}"
ffs = [r for r in sel(S, FACT) if r["flagged"]]; ffc = [r for r in q(CS) if r["kind"] == "fact" and r["flagged"]]; c5["flagged_fact_questions"] = len(ffs); c5["flagged_fact_conversations"] = len(ffc)
if dl["ok"] and not bad and ffs:
os.environ["JAVA_HOME"] = dl["java_home"]; os.environ["PATH"] = os.path.join(dl["java_home"], "bin") + os.pathsep + os.environ["PATH"]
import jnius_config; jnius_config.add_options("-Xmx8g"); jnius_config.set_classpath(dl["jar"])
from jnius import autoclass
JS = autoclass("io.anserini.search.SimpleSearcher"); AL = autoclass("java.util.ArrayList"); SR = {"w18": JS(dl["index_dir"]), "w23": JS(a.w23)}
def passage(s, docid):
raw = SR[s].doc(docid).get("raw") or ""
try: c = json.loads(raw).get("contents", raw)
except Exception: c = raw
first, _, rest = c.partition("\n"); return {"title": first.strip().strip('"'), "text": rest.strip(), "src": s}
def search(s, texts, k):
out = []
for i in range(0, len(texts), 512):
part = texts[i:i+512]; ql, ids = AL(), AL()
for j, t in enumerate(part): ql.add(t); ids.add(str(j))
res = SR[s].batch_search(ql, ids, int(k), 8); out += [[passage(s, h.docid) for h in res.get(str(j))] for j in range(len(part))]
return out
c5["probe_hits"] = {s: sum(any(w in d["title"].lower() for d in row) for (p, w), row in zip(PROBES, search(s, [p for p, _ in PROBES], 3))) for s in SR}
if min(c5["probe_hits"].values()) >= 4:
qtext = {r["id"]: r for k in FACT for r in sets[k]}; cmap = {c["id"]: c for c in convs}
items = [("signoff", r, qtext[r["id"]]["question"], qtext[r["id"]]["gold"]) for r in ffs] + [("conversation", r, cmap[r["id"]]["user_turns"][-1], cmap[r["id"]]["gold"]) for r in ffc]
texts = [it[2] for it in items]; prog("searching")
c18 = search("w18", texts, K_EACH); c23 = search("w23", texts, K_EACH); cands = []
for x, y in zip(c18, c23):
seen, row = set(), []
for d in x + y:
key = (d["title"], d["text"][:200])
if key not in seen: seen.add(key); row.append(d)
cands.append(row)
prog("reranking")
from transformers import AutoTokenizer, AutoModelForSequenceClassification
tok = AutoTokenizer.from_pretrained(RERANKER, revision=RERANKER_REV); rr = AutoModelForSequenceClassification.from_pretrained(RERANKER, revision=RERANKER_REV, dtype=torch.float16 if a.rerank_device.startswith("cuda") else torch.float32).to(a.rerank_device).eval()
top = []
for qi, (qq, row) in enumerate(zip(texts, cands)):
pairs = [[qq, f"{d['title']}: {d['text']}"] for d in row]; scores = []
for i in range(0, len(pairs), 32):
enc = tok(pairs[i:i+32], padding=True, truncation=True, max_length=384, return_tensors="pt").to(a.rerank_device)
with torch.no_grad(): scores += rr(**enc).logits.view(-1).float().tolist()
top.append([row[i] for i in sorted(range(len(row)), key=lambda i: -scores[i])[:K_FINAL]])
if qi % 200 == 0: prog(f"reranking {qi}/{len(texts)}")
del rr
prog("answering with search")
prompts = [NEW_PROMPT + "\n\n".join(f"[{j+1}] {d['title']}: {d['text'][:600]}" for j, d in enumerate(row)) + f"\n\nQuestion: {qq}" for qq, row in zip(texts, top)]
res = llm.chat([[{"role": "user", "content": p}] for p in prompts], SamplingParams(temperature=0.0, max_tokens=128, seed=SEED), lora_request=lora, use_tqdm=False)
def cite(ans, row):
na = norm(ans)
if not na: return None
for j, d in enumerate(row):
if f" {na} " in f" {norm(d['title'] + ' ' + d['text'][:600])} ": return j + 1
return None
srecs = []
for (kind, r, qq, gold), row, o in zip(items, top, res):
t = o.outputs[0].text; ans = unflag(t.split("Source:")[0].strip()); n = cite(ans, row)
srecs.append({"kind": kind, "id": r["id"], "guess_correct": r["correct"], "reply": t, "answer": ans, "search_correct": bool(fact_ok(ans, gold)),
"system_source": row[n - 1]["title"] if n else "no source found", "passages": [[d["src"], d["title"]] for d in row]})
with open(os.path.join(a.out, "search_records.jsonl"), "w") as f:
for x in srecs: f.write(json.dumps(x) + "\n")
ss = [x for x in srecs if x["kind"] == "signoff"]; sc = [x for x in srecs if x["kind"] == "conversation"]
c5["guess_accuracy_on_flagged"] = sum(x["guess_correct"] for x in ss) / len(ss); c5["search_accuracy_on_flagged"] = sum(x["search_correct"] for x in ss) / len(ss)
c5["answers_with_a_source"] = sum(x["system_source"] != "no source found" for x in ss) / len(ss)
c5["conversation_guess"] = (sum(x["guess_correct"] for x in sc) / len(sc)) if sc else None; c5["conversation_search"] = (sum(x["search_correct"] for x in sc) / len(sc)) if sc else None
c5["pass"] = c5["search_accuracy_on_flagged"] > c5["guess_accuracy_on_flagged"]
else: c5["pass"] = None; c5["note"] = "index probe gate failed (<4/5): search not evaluated"
else: c5["pass"] = None; c5["note"] = "2018 index download failed, 2023 index files did not verify, or nothing flagged"
summary["check5_search_when_unsure"] = c5
keys = ("check1_instructions_ifeval", "check2_maths_gsm8k", "check3_knows_when_unsure", "check4_clean_replies", "check5_search_when_unsure",
"K1_unflagged_beats_original_by_10pts", "K2_flags_half_its_wrong", "K3_clean_endings", "K4_no_false_flags_on_chat")
summary["checks_passed"] = f"{sum(summary[k].get('pass') is True for k in keys)}/9"; summary["all_pass"] = all(summary[k].get("pass") is True for k in keys)
summary["for_comparison_7b_rev4"] = SEVEN_B_REV4; summary["seconds"] = round(time.time() - t0); summary["env"] = env
json.dump(summary, open(os.path.join(a.out, "summary.json"), "w"), indent=1); prog("done"); print(json.dumps({"checks_passed": summary["checks_passed"]}))
if __name__ == "__main__": main()
|