Text Generation
PEFT
Safetensors
English
lora
olmo
uncertainty
calibration
abstention
retrieval-augmented-generation
Instructions to use Sentovara/Sentinel-S1-7B with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- PEFT
How to use Sentovara/Sentinel-S1-7B with PEFT:
from peft import PeftModel from transformers import AutoModelForCausalLM base_model = AutoModelForCausalLM.from_pretrained("allenai/Olmo-3-7B-Instruct") model = PeftModel.from_pretrained(base_model, "Sentovara/Sentinel-S1-7B") - Notebooks
- Google Colab
- Kaggle
File size: 11,761 Bytes
4e9bf3d | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 | """Sentinel S1 Job D — conversation test (owner GO 13 Nov 2026 13:42 UK; pre-registration D25-D30, PREREG-JOB-D-2026-11-13.md sha256 465f9a67…).
Original model vs Sentinel (same weights + Job B2 LoRA), identical code, greedy, no system prompt. Each model answers every turn itself;
only the last reply is scored. Pass marks K1-K4 as pre-registered (D28). Search-when-unsure reported only (D29)."""
import argparse, hashlib, json, os, shutil, threading, time
from job_a import MODEL, REV, SEED, fact_ok, math_ok
WIKI_URL, WIKI_MD5 = "https://rgw.cs.uwaterloo.ca/pyserini/indexes/lucene-index.wikipedia-dpr-100w.20210120.d1b9e6.tar.gz", "7b58c08da992b2ea7e96667f0b176651"
JAR_URL, JAR_SHA256 = "https://repo1.maven.org/maven2/io/anserini/anserini/1.1.1/anserini-1.1.1-fatjar.jar", "69270ba4d160826953347411ce5d7e205ce363766a9cc72ac9da3b945341af83"
PROBES = [("Eiffel Tower Paris", "eiffel"), ("Albert Einstein theory of relativity", "einstein"), ("William Shakespeare playwright Hamlet", "shakespeare"),
("Amazon river South America", "amazon"), ("Mount Everest highest mountain", "everest")]
EARLY_TOK, LAST_TOK = 512, 1024
def flagged(t): return t.lstrip().upper().startswith("UNSURE:")
def unflag(t):
s = t.lstrip(); return s[len("UNSURE:"):].strip() if s.upper().startswith("UNSURE:") else t
def sha(p): return hashlib.sha256(open(p, "rb").read()).hexdigest()
def main():
ap = argparse.ArgumentParser(); ap.add_argument("--convs"); ap.add_argument("--adapter"); ap.add_argument("--out"); a = ap.parse_args()
t0 = time.time(); prog = lambda m: open(os.path.join(a.out, "progress.txt"), "w").write(f"{m} {time.time()-t0:.0f}s\n")
disks = {d: shutil.disk_usage(d).free for d in ("/opt/ml", "/tmp") if os.path.exists(d)}
wdir = os.path.join(max(disks, key=disks.get), "scratch_wiki"); os.makedirs(wdir, exist_ok=True); dl = {"ok": False, "err": None, "seconds": None}
def fetch():
s0 = time.time()
try:
import urllib.request, tarfile, jdk
jar = os.path.join(wdir, "anserini-1.1.1-fatjar.jar"); urllib.request.urlretrieve(JAR_URL, jar)
if hashlib.sha256(open(jar, "rb").read()).hexdigest() != JAR_SHA256: raise ValueError("Anserini jar hash mismatch")
dl["java_home"] = jdk.install("21", path=os.path.join(wdir, "jdk"))
tgz = os.path.join(wdir, "index.tar.gz"); urllib.request.urlretrieve(WIKI_URL, tgz); h = hashlib.md5()
with open(tgz, "rb") as f:
for b in iter(lambda: f.read(1 << 24), b""): h.update(b)
if h.hexdigest() != WIKI_MD5: raise ValueError("index md5 mismatch")
with tarfile.open(tgz) as t: t.extractall(wdir, filter="data")
os.remove(tgz); dl["index_dir"] = [os.path.join(wdir, d) for d in os.listdir(wdir) if d.startswith("lucene-index")][0]; dl["jar"] = jar; dl["ok"] = True
except Exception as e: dl["err"] = repr(e)[:500]
dl["seconds"] = round(time.time() - s0)
th = threading.Thread(target=fetch, daemon=True); th.start()
cpath = os.path.join(a.convs, "conversations.jsonl"); convs = [json.loads(l) for l in open(cpath) if l.strip()]
adapter_files = {f: sha(os.path.join(a.adapter, f)) for f in sorted(os.listdir(a.adapter)) if os.path.isfile(os.path.join(a.adapter, f))}
import torch, vllm
from vllm import LLM, SamplingParams
from vllm.lora.request import LoRARequest
env = {"model": MODEL, "revision": REV, "vllm": vllm.__version__, "torch": torch.__version__, "gpu": torch.cuda.get_device_name(0),
"conversations_sha256": sha(cpath), "n_conversations": len(convs), "adapter_sha256": adapter_files, "early_max_tokens": EARLY_TOK, "last_max_tokens": LAST_TOK}
json.dump(env, open(os.path.join(a.out, "env.json"), "w"), indent=1)
llm = LLM(model=MODEL, revision=REV, dtype="bfloat16", max_model_len=8192, gpu_memory_utilization=0.90, seed=SEED,
enable_lora=True, max_lora_rank=16, max_loras=1, enable_prefix_caching=True)
lora = LoRARequest("sentinel", 1, a.adapter)
recs = {}
for arm in ("base", "sentinel"):
msgs = {c["id"]: [] for c in convs}; turns = {c["id"]: [] for c in convs}
for step in range(max(len(c["user_turns"]) for c in convs)):
act = [c for c in convs if len(c["user_turns"]) > step]
for c in act: msgs[c["id"]].append({"role": "user", "content": c["user_turns"][step]})
sps = [SamplingParams(temperature=0.0, seed=SEED, max_tokens=LAST_TOK if step == len(c["user_turns"]) - 1 else EARLY_TOK) for c in act]
res = llm.chat([msgs[c["id"]] for c in act], sps, lora_request=lora if arm == "sentinel" else None, use_tqdm=False)
for c, o in zip(act, res):
t = o.outputs[0].text; msgs[c["id"]].append({"role": "assistant", "content": t}); turns[c["id"]].append({"reply": t, "finish": o.outputs[0].finish_reason})
prog(f"{arm} turn {step+1}")
out = []
for c in convs:
last = turns[c["id"]][-1]; t = last["reply"]
r = {"id": c["id"], "group": c["group"], "kind": c["kind"], "reply": t, "finish": last["finish"], "flagged": flagged(t), "unsure_anywhere": "UNSURE:" in t.upper(),
"earlier_turns_flagged": sum(flagged(x["reply"]) for x in turns[c["id"]][:-1])}
if c["kind"]: r["correct"] = bool(fact_ok(unflag(t), c["gold"]) if c["kind"] == "fact" else math_ok(unflag(t), c["gold"]))
out.append(r)
recs[arm] = out
with open(os.path.join(a.out, f"records_{arm}.jsonl"), "w") as f:
for x in out: f.write(json.dumps(x) + "\n")
with open(os.path.join(a.out, f"transcripts_{arm}.jsonl"), "w") as f:
for c in convs: f.write(json.dumps({"id": c["id"], "messages": msgs[c["id"]]}) + "\n")
B, S = recs["base"], recs["sentinel"]
acc = lambda rows: (sum(r["correct"] for r in rows) / len(rows)) if rows else None
def block(b, s):
un = [r for r in s if not r["flagged"]]; wrong = [r for r in s if not r["correct"]]
return {"n": len(s), "base_accuracy": acc(b), "sentinel_overall_accuracy": acc(s), "sentinel_unflagged_accuracy": acc(un), "sentinel_unflagged_n": len(un),
"sentinel_flag_rate": sum(r["flagged"] for r in s) / len(s), "sentinel_flag_rate_on_its_wrong": (sum(r["flagged"] for r in wrong) / len(wrong)) if wrong else None,
"sentinel_unsure_anywhere_not_leading": sum(r["unsure_anywhere"] and not r["flagged"] for r in s)}
q = lambda rows: [r for r in rows if r["kind"]]
main_b = block(q(B), q(S))
K1 = {"sentinel_unflagged_accuracy": main_b["sentinel_unflagged_accuracy"], "base_accuracy": main_b["base_accuracy"]}
K1["pass"] = K1["sentinel_unflagged_accuracy"] is not None and K1["sentinel_unflagged_accuracy"] >= K1["base_accuracy"] + 0.10
K2 = {"sentinel_flag_rate_on_its_wrong": main_b["sentinel_flag_rate_on_its_wrong"]}; K2["pass"] = K2["sentinel_flag_rate_on_its_wrong"] is not None and K2["sentinel_flag_rate_on_its_wrong"] >= 0.50
K3 = {"sentinel_clean_end_rate": sum(r["finish"] == "stop" for r in S) / len(S), "base_clean_end_rate": sum(r["finish"] == "stop" for r in B) / len(B)}; K3["pass"] = K3["sentinel_clean_end_rate"] >= 0.95
D = [r for r in S if r["group"].startswith("D_")]
K4 = {"sentinel_flag_rate_no_fact_chats": sum(r["flagged"] for r in D) / len(D), "n": len(D)}; K4["pass"] = K4["sentinel_flag_rate_no_fact_chats"] <= 0.10
by = {}
for g in ("A_loose_single", "B_multi_turn", "C_mixed_chat"):
for k in ("fact", "math", None):
sel = lambda rows: [r for r in rows if r["group"] == g and (k is None or r["kind"] == k)]
by[f"{g}/{k or 'all'}"] = block(sel(B), sel(S))
for k in ("fact", "math"): by[f"all/{k}"] = block([r for r in q(B) if r["kind"] == k], [r for r in q(S) if r["kind"] == k])
summary = {"K1_unflagged_beats_original_by_10pts": K1, "K2_flags_half_its_wrong": K2, "K3_clean_endings": K3, "K4_no_false_flags_on_chat": K4,
"question_conversations": main_b, "breakdown": by, "earlier_turns_flagged_sentinel": sum(r["earlier_turns_flagged"] for r in S)}
summary["all_pass"] = all(summary[k]["pass"] is True for k in ("K1_unflagged_beats_original_by_10pts", "K2_flags_half_its_wrong", "K3_clean_endings", "K4_no_false_flags_on_chat"))
json.dump(summary, open(os.path.join(a.out, "summary.json"), "w"), indent=1)
# --- reported only: search-when-unsure on flagged fact conversations, query = user's last message
prog("waiting for Wikipedia index"); th.join(); sr = {"index_download": dl}
cmap = {c["id"]: c for c in convs}; ff = [r for r in S if r["kind"] == "fact" and r["flagged"]]; sr["flagged_fact_conversations"] = len(ff)
if dl["ok"] and ff:
os.environ["JAVA_HOME"] = dl["java_home"]; os.environ["PATH"] = os.path.join(dl["java_home"], "bin") + os.pathsep + os.environ["PATH"]
import jnius_config; jnius_config.add_options("-Xmx4g"); jnius_config.set_classpath(dl["jar"])
from jnius import autoclass
JS = autoclass("io.anserini.search.SimpleSearcher"); AL = autoclass("java.util.ArrayList"); jsr = JS(dl["index_dir"])
def passage(docid):
raw = jsr.doc(docid).get("raw") or ""
try: c = json.loads(raw).get("contents", raw)
except Exception: c = raw
first, _, rest = c.partition("\n"); return {"title": first.strip().strip('"'), "text": rest.strip()}
def search(qs, k):
ql, ids = AL(), AL()
for i, t in enumerate(qs): ql.add(t); ids.add(str(i))
res = jsr.batch_search(ql, ids, int(k), 4); return [[passage(h.docid) for h in res.get(str(i))] for i in range(len(qs))]
probe = search([p for p, _ in PROBES], 3); hits = [any(w in d["title"].lower() for d in row) for (p, w), row in zip(PROBES, probe)]; sr["probe_hits"] = f"{sum(hits)}/{len(hits)}"
if sum(hits) >= 4:
qs = [cmap[r["id"]]["user_turns"][-1] for r in ff]; docs = search(qs, 5); prompts = []
for qq, row in zip(qs, docs):
ctx = "\n\n".join(f"[{j+1}] {d['title']}: {d['text'][:600]}" for j, d in enumerate(row))
prompts.append("Answer the question using the Wikipedia passages below. Give a short factual answer only (a few words), then on a new line write 'Source: ' followed by the title of the passage you used.\n\n" + ctx + f"\n\nQuestion: {qq}")
res = llm.chat([[{"role": "user", "content": p}] for p in prompts], SamplingParams(temperature=0.0, max_tokens=128, seed=SEED), lora_request=lora, use_tqdm=False)
srecs = []
for r, row, o in zip(ff, docs, res):
t = o.outputs[0].text; ans = unflag(t.split("Source:")[0].strip())
srecs.append({"id": r["id"], "guess_correct": r["correct"], "search_reply": t, "search_correct": bool(fact_ok(ans, cmap[r["id"]]["gold"])), "passage_titles": [d["title"] for d in row]})
with open(os.path.join(a.out, "search_records.jsonl"), "w") as f:
for x in srecs: f.write(json.dumps(x) + "\n")
sr["guess_accuracy_on_flagged"] = sum(x["guess_correct"] for x in srecs) / len(srecs); sr["search_accuracy_on_flagged"] = sum(x["search_correct"] for x in srecs) / len(srecs)
else: sr["note"] = "index probe gate failed (<4/5): search not evaluated"
else: sr["note"] = "index download failed or nothing flagged"
summary["search_when_unsure_reported"] = sr; summary["seconds"] = round(time.time() - t0); summary["env"] = env
json.dump(summary, open(os.path.join(a.out, "summary.json"), "w"), indent=1); prog("done"); print(json.dumps({"all_pass": summary["all_pass"]}))
if __name__ == "__main__":
main()
|