Text Generation
PEFT
Safetensors
English
lora
olmo
uncertainty
calibration
abstention
retrieval-augmented-generation
Instructions to use Sentovara/Sentinel-S1-32B with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- PEFT
How to use Sentovara/Sentinel-S1-32B with PEFT:
from peft import PeftModel from transformers import AutoModelForCausalLM base_model = AutoModelForCausalLM.from_pretrained("allenai/Olmo-3.1-32B-Instruct") model = PeftModel.from_pretrained(base_model, "Sentovara/Sentinel-S1-32B") - Notebooks
- Google Colab
- Kaggle
Download eval/job_a.py from Sentovara/Sentinel-S1-32B: direct link, hf CLI and curl.
- Browser
- Download file 4.16 kB
-
https://huggingface.co/Sentovara/Sentinel-S1-32B/resolve/main/eval/job_a.py
- Command line
-
hf download hf://Sentovara/Sentinel-S1-32B/eval/job_a.py
-
curl -L -o job_a.py https://huggingface.co/Sentovara/Sentinel-S1-32B/resolve/main/eval/job_a.py
4.16 kB
| """Sentinel S1 Job A — evidence run. The unmodified starting model answers each frozen question 5 times; | |
| each answer is scored right/wrong by script against the known answer. No training, no model changes.""" | |
| import argparse, hashlib, json, os, re, string, sys, time | |
| MODEL, REV, SEED, N_TRIES = "allenai/Olmo-3-7B-Instruct", "6e5971d9eba42665f5bd5a0fcf047f299ce1dccc", 20261112, 5 | |
| PROMPT = {"fact": "Answer the question with a short factual answer only (a few words, no explanation).\n\nQuestion: {q}", | |
| "math": "Solve the problem. Show your working briefly, then give the final answer on the last line in the form 'Answer: <number>'.\n\nProblem: {q}"} | |
| MAXTOK = {"fact": 48, "math": 640} | |
| def norm(s): | |
| s = s.lower(); s = "".join(c for c in s if c not in set(string.punctuation)) | |
| s = re.sub(r"\b(a|an|the)\b", " ", s); return " ".join(s.split()) | |
| def fact_ok(pred, golds): | |
| p = f" {norm(pred)} " | |
| return any(f" {norm(g)} " in p for g in golds if norm(g)) | |
| NUM = re.compile(r"-?\d[\d,]*\.?\d*") | |
| def math_val(text): | |
| m = re.findall(r"answer\s*[:=]\s*\$?\s*(-?\d[\d,]*\.?\d*)", text, flags=re.I) | |
| cand = m[-1] if m else (NUM.findall(text)[-1] if NUM.findall(text) else None) | |
| if cand is None: return None | |
| try: return float(cand.replace(",", "").rstrip(".")) | |
| except ValueError: return None | |
| def math_ok(pred, golds): | |
| v = math_val(pred) | |
| try: g = float(golds[0]) | |
| except ValueError: return False | |
| return v is not None and abs(v - g) < 1e-6 | |
| def main(): | |
| ap = argparse.ArgumentParser(); ap.add_argument("--questions"); ap.add_argument("--out"); ap.add_argument("--limit", type=int, default=0); a = ap.parse_args() | |
| raw = open(a.questions, "rb").read(); qsha = hashlib.sha256(raw).hexdigest() | |
| qs = [json.loads(l) for l in raw.decode().splitlines() if l.strip()] | |
| if a.limit: qs = qs[:a.limit] | |
| import torch, vllm | |
| from vllm import LLM, SamplingParams | |
| env = {"torch": torch.__version__, "cuda_available": torch.cuda.is_available(), "gpu": torch.cuda.get_device_name(0) if torch.cuda.is_available() else None, | |
| "vllm": vllm.__version__, "model": MODEL, "revision": REV, "questions_sha256": qsha, "n_questions": len(qs), "tries": N_TRIES, "seed": SEED} | |
| json.dump(env, open(os.path.join(a.out, "env.json"), "w"), indent=1) | |
| if not torch.cuda.is_available(): sys.exit("CUDA not available") | |
| llm = LLM(model=MODEL, revision=REV, dtype="bfloat16", max_model_len=2048, gpu_memory_utilization=0.92, seed=SEED, enable_prefix_caching=True) | |
| t0 = time.time(); recs = open(os.path.join(a.out, "records.jsonl"), "w"); summ = {"fact": [0, 0], "math": [0, 0]}; hist = {"fact": [0]*6, "math": [0]*6}; trunc = 0 | |
| CH = 500 | |
| for i in range(0, len(qs), CH): | |
| batch = qs[i:i+CH] | |
| convs = [[{"role": "user", "content": PROMPT[q["kind"]].format(q=q["question"])}] for q in batch] | |
| sps = [SamplingParams(n=N_TRIES, temperature=0.6, top_p=0.95, max_tokens=MAXTOK[q["kind"]], seed=SEED + i + j) for j, q in enumerate(batch)] | |
| outs = llm.chat(convs, sps, use_tqdm=False) | |
| for q, o in zip(batch, outs): | |
| texts = [c.text for c in o.outputs]; fin = [c.finish_reason for c in o.outputs] | |
| ok = [bool(fact_ok(t, q["gold"]) if q["kind"] == "fact" else math_ok(t, q["gold"])) for t in texts] | |
| k = sum(ok); summ[q["kind"]][0] += k; summ[q["kind"]][1] += len(ok); hist[q["kind"]][k] += 1; trunc += sum(f == "length" for f in fin) | |
| recs.write(json.dumps({"id": q["id"], "kind": q["kind"], "question": q["question"], "gold": q["gold"], "outputs": texts, "correct": ok, "finish": fin, "n_correct": k}) + "\n") | |
| recs.flush() | |
| open(os.path.join(a.out, "progress.txt"), "w").write(f"{min(i+CH, len(qs))}/{len(qs)} questions {time.time()-t0:.0f}s\n") | |
| recs.close() | |
| s = {"accuracy_per_try": {k: (v[0]/v[1] if v[1] else None) for k, v in summ.items()}, | |
| "questions_by_n_correct_of_5": hist, "truncated_answers": trunc, "seconds": round(time.time()-t0), **env} | |
| json.dump(s, open(os.path.join(a.out, "summary.json"), "w"), indent=1); print(json.dumps(s)) | |
| if __name__ == "__main__": | |
| main() | |