Sentovara's picture
Load adapter on CPU before moving to GPU; invite questions
787b754 verified
Raw History Blame Contribute Delete
6.25 kB
import re, json, urllib.parse, urllib.request
import gradio as gr
import spaces
import torch
from transformers import AutoModelForCausalLM, AutoTokenizer
from peft import PeftModel
BASE, REV = "allenai/Olmo-3-7B-Instruct", "6e5971d9eba42665f5bd5a0fcf047f299ce1dccc"
ADAPTER = "Sentovara/Sentinel-S1-7B"
SEARCH_PROMPT = ("Answer the question using the Wikipedia passages below. Give a short factual answer only (a few words), "
"then on a new line write 'Source: ' followed by the number of the passage you used.\n\n")
tok = AutoTokenizer.from_pretrained(BASE, revision=REV)
model = AutoModelForCausalLM.from_pretrained(BASE, revision=REV, dtype=torch.bfloat16)
model = PeftModel.from_pretrained(model, ADAPTER, torch_device="cpu").merge_and_unload()
model.to("cuda")
model.eval()
@spaces.GPU(duration=60)
def generate(messages, max_new_tokens=400):
ids = tok.apply_chat_template(messages, add_generation_prompt=True, return_tensors="pt")
if hasattr(ids, "keys"): ids = ids["input_ids"]
ids = ids.to("cuda")
with torch.no_grad():
out = model.generate(ids, max_new_tokens=max_new_tokens, do_sample=False)
return tok.decode(out[0, ids.shape[1]:], skip_special_tokens=True).strip()
def wiki_passages(query, n=5):
q = urllib.parse.urlencode({"action": "query", "list": "search", "srsearch": query, "srlimit": n, "format": "json"})
req = urllib.request.Request("https://en.wikipedia.org/w/api.php?" + q, headers={"User-Agent": "SentinelS1-demo/1.0 (huggingface.co/Sentovara)"})
titles = [h["title"] for h in json.load(urllib.request.urlopen(req, timeout=10))["query"]["search"]]
if not titles: return []
q2 = urllib.parse.urlencode({"action": "query", "prop": "extracts", "exintro": 1, "explaintext": 1, "titles": "|".join(titles), "format": "json"})
req2 = urllib.request.Request("https://en.wikipedia.org/w/api.php?" + q2, headers={"User-Agent": "SentinelS1-demo/1.0 (huggingface.co/Sentovara)"})
pages = json.load(urllib.request.urlopen(req2, timeout=10))["query"]["pages"].values()
by = {p["title"]: p.get("extract", "") for p in pages}
return [(t, by.get(t, "")[:600]) for t in titles if by.get(t)]
def _text(c):
if isinstance(c, str): return c
if isinstance(c, dict): return c.get("text") or ""
if isinstance(c, (list, tuple)): return " ".join(_text(x) for x in c)
return ""
def to_messages(history, user_msg):
msgs = []
for m in history:
if isinstance(m, dict) and m.get("role") in ("user", "assistant"):
t = _text(m.get("content"))
if m["role"] == "assistant":
t = t.split("\n\n🔎")[0].replace("⚠️ **Sentinel is unsure** — treat this as a guess:\n\n", "UNSURE: ")
if t: msgs.append({"role": m["role"], "content": t})
return msgs + [{"role": "user", "content": user_msg}]
def chat(user_msg, history, use_search):
history = history or []
reply = generate(to_messages(history, user_msg))
flagged = reply.startswith("UNSURE:")
shown = reply
if flagged:
shown = "⚠️ **Sentinel is unsure** — treat this as a guess:\n\n" + reply[len("UNSURE:"):].strip()
if use_search:
try:
ps = wiki_passages(user_msg)
except Exception:
ps = []
if ps:
prompt = SEARCH_PROMPT + "\n\n".join(f"[{i+1}] {t}: {x}" for i, (t, x) in enumerate(ps)) + f"\n\nQuestion: {user_msg}"
ans = generate([{"role": "user", "content": prompt}], max_new_tokens=96)
m = re.search(r"Source:\s*\[?(\d+)", ans)
body = ans.split("Source:")[0].strip()
if body.startswith("UNSURE:"):
shown += "\n\n🔎 **Looked it up on Wikipedia:** still not sure — " + body[len("UNSURE:"):].strip()
else:
shown += "\n\n🔎 **Looked it up on Wikipedia:** " + body
if m and 1 <= int(m.group(1)) <= len(ps):
t = ps[int(m.group(1)) - 1][0]
shown += f"\n\nSource: [{t}](https://en.wikipedia.org/wiki/{urllib.parse.quote(t.replace(' ', '_'))})"
else:
shown += "\n\n🔎 Wikipedia search found nothing useful."
history = history + [{"role": "user", "content": user_msg}, {"role": "assistant", "content": shown}]
return "", history
INTRO = """# Sentinel S1 (7B) — an AI that tells you when it doesn't know
Sentinel S1 is [Olmo 3 7B Instruct](https://huggingface.co/allenai/Olmo-3-7B-Instruct) trained by [Sentovara](https://huggingface.co/Sentovara) to flag answers it is likely to get wrong. When it answers plainly it was right 75% of the time on test questions (the original model: 20%), and it flagged 94% of its own wrong answers. Try a hard general-knowledge question and a simple one, or just chat.
*This demo uses a simple live Wikipedia search, not the stronger search used in our tests. It can still be wrong — don't rely on it for medical, legal or financial decisions.* · [Model](https://huggingface.co/Sentovara/Sentinel-S1-7B) · [32B version](https://huggingface.co/Sentovara/Sentinel-S1-32B) · Questions, or want the training recipe? Ask in the Community tab or at [sentovara.com](https://sentovara.com)."""
with gr.Blocks(title="Sentinel S1") as demo:
gr.Markdown(INTRO)
try:
bot = gr.Chatbot(type="messages", height=480)
except TypeError:
bot = gr.Chatbot(height=480)
with gr.Row():
box = gr.Textbox(placeholder="Ask Sentinel something…", scale=5, show_label=False)
send = gr.Button("Send", variant="primary", scale=1)
use_search = gr.Checkbox(value=True, label="When Sentinel is unsure, look it up on Wikipedia")
gr.Examples(["Who composed the music for the 1968 film 2001: A Space Odyssey?", "What is the capital of Australia?",
"Who won the 1987 Tour de France?", "A shop sells pens at 3 for £2. How much do 12 pens cost?", "Write me a haiku about rain."], inputs=box)
clear = gr.Button("Clear chat")
box.submit(chat, [box, bot, use_search], [box, bot]); send.click(chat, [box, bot, use_search], [box, bot])
clear.click(lambda: [], None, bot)
demo.queue(max_size=20).launch()