Spaces:
Running on Zero
Running on Zero
Load adapter on CPU before moving to GPU; invite questions
Browse files
app.py
CHANGED
|
@@ -12,7 +12,9 @@ SEARCH_PROMPT = ("Answer the question using the Wikipedia passages below. Give a
|
|
| 12 |
|
| 13 |
tok = AutoTokenizer.from_pretrained(BASE, revision=REV)
|
| 14 |
model = AutoModelForCausalLM.from_pretrained(BASE, revision=REV, dtype=torch.bfloat16)
|
| 15 |
-
model = PeftModel.from_pretrained(model, ADAPTER
|
|
|
|
|
|
|
| 16 |
|
| 17 |
@spaces.GPU(duration=60)
|
| 18 |
def generate(messages, max_new_tokens=400):
|
|
@@ -82,7 +84,7 @@ def chat(user_msg, history, use_search):
|
|
| 82 |
INTRO = """# Sentinel S1 (7B) — an AI that tells you when it doesn't know
|
| 83 |
Sentinel S1 is [Olmo 3 7B Instruct](https://huggingface.co/allenai/Olmo-3-7B-Instruct) trained by [Sentovara](https://huggingface.co/Sentovara) to flag answers it is likely to get wrong. When it answers plainly it was right 75% of the time on test questions (the original model: 20%), and it flagged 94% of its own wrong answers. Try a hard general-knowledge question and a simple one, or just chat.
|
| 84 |
|
| 85 |
-
*This demo uses a simple live Wikipedia search, not the stronger search used in our tests. It can still be wrong — don't rely on it for medical, legal or financial decisions.* · [Model](https://huggingface.co/Sentovara/Sentinel-S1-7B) · [32B version](https://huggingface.co/Sentovara/Sentinel-S1-32B)"""
|
| 86 |
|
| 87 |
with gr.Blocks(title="Sentinel S1") as demo:
|
| 88 |
gr.Markdown(INTRO)
|
|
|
|
| 12 |
|
| 13 |
tok = AutoTokenizer.from_pretrained(BASE, revision=REV)
|
| 14 |
model = AutoModelForCausalLM.from_pretrained(BASE, revision=REV, dtype=torch.bfloat16)
|
| 15 |
+
model = PeftModel.from_pretrained(model, ADAPTER, torch_device="cpu").merge_and_unload()
|
| 16 |
+
model.to("cuda")
|
| 17 |
+
model.eval()
|
| 18 |
|
| 19 |
@spaces.GPU(duration=60)
|
| 20 |
def generate(messages, max_new_tokens=400):
|
|
|
|
| 84 |
INTRO = """# Sentinel S1 (7B) — an AI that tells you when it doesn't know
|
| 85 |
Sentinel S1 is [Olmo 3 7B Instruct](https://huggingface.co/allenai/Olmo-3-7B-Instruct) trained by [Sentovara](https://huggingface.co/Sentovara) to flag answers it is likely to get wrong. When it answers plainly it was right 75% of the time on test questions (the original model: 20%), and it flagged 94% of its own wrong answers. Try a hard general-knowledge question and a simple one, or just chat.
|
| 86 |
|
| 87 |
+
*This demo uses a simple live Wikipedia search, not the stronger search used in our tests. It can still be wrong — don't rely on it for medical, legal or financial decisions.* · [Model](https://huggingface.co/Sentovara/Sentinel-S1-7B) · [32B version](https://huggingface.co/Sentovara/Sentinel-S1-32B) · Questions, or want the training recipe? Ask in the Community tab or at [sentovara.com](https://sentovara.com)."""
|
| 88 |
|
| 89 |
with gr.Blocks(title="Sentinel S1") as demo:
|
| 90 |
gr.Markdown(INTRO)
|