Sentovara commited on
Commit
787b754
·
verified ·
1 Parent(s): 4b4b289

Load adapter on CPU before moving to GPU; invite questions

Browse files
Files changed (1) hide show
  1. app.py +4 -2
app.py CHANGED
@@ -12,7 +12,9 @@ SEARCH_PROMPT = ("Answer the question using the Wikipedia passages below. Give a
12
 
13
  tok = AutoTokenizer.from_pretrained(BASE, revision=REV)
14
  model = AutoModelForCausalLM.from_pretrained(BASE, revision=REV, dtype=torch.bfloat16)
15
- model = PeftModel.from_pretrained(model, ADAPTER).merge_and_unload().to("cuda").eval()
 
 
16
 
17
  @spaces.GPU(duration=60)
18
  def generate(messages, max_new_tokens=400):
@@ -82,7 +84,7 @@ def chat(user_msg, history, use_search):
82
  INTRO = """# Sentinel S1 (7B) — an AI that tells you when it doesn't know
83
  Sentinel S1 is [Olmo 3 7B Instruct](https://huggingface.co/allenai/Olmo-3-7B-Instruct) trained by [Sentovara](https://huggingface.co/Sentovara) to flag answers it is likely to get wrong. When it answers plainly it was right 75% of the time on test questions (the original model: 20%), and it flagged 94% of its own wrong answers. Try a hard general-knowledge question and a simple one, or just chat.
84
 
85
- *This demo uses a simple live Wikipedia search, not the stronger search used in our tests. It can still be wrong — don't rely on it for medical, legal or financial decisions.* · [Model](https://huggingface.co/Sentovara/Sentinel-S1-7B) · [32B version](https://huggingface.co/Sentovara/Sentinel-S1-32B)"""
86
 
87
  with gr.Blocks(title="Sentinel S1") as demo:
88
  gr.Markdown(INTRO)
 
12
 
13
  tok = AutoTokenizer.from_pretrained(BASE, revision=REV)
14
  model = AutoModelForCausalLM.from_pretrained(BASE, revision=REV, dtype=torch.bfloat16)
15
+ model = PeftModel.from_pretrained(model, ADAPTER, torch_device="cpu").merge_and_unload()
16
+ model.to("cuda")
17
+ model.eval()
18
 
19
  @spaces.GPU(duration=60)
20
  def generate(messages, max_new_tokens=400):
 
84
  INTRO = """# Sentinel S1 (7B) — an AI that tells you when it doesn't know
85
  Sentinel S1 is [Olmo 3 7B Instruct](https://huggingface.co/allenai/Olmo-3-7B-Instruct) trained by [Sentovara](https://huggingface.co/Sentovara) to flag answers it is likely to get wrong. When it answers plainly it was right 75% of the time on test questions (the original model: 20%), and it flagged 94% of its own wrong answers. Try a hard general-knowledge question and a simple one, or just chat.
86
 
87
+ *This demo uses a simple live Wikipedia search, not the stronger search used in our tests. It can still be wrong — don't rely on it for medical, legal or financial decisions.* · [Model](https://huggingface.co/Sentovara/Sentinel-S1-7B) · [32B version](https://huggingface.co/Sentovara/Sentinel-S1-32B) · Questions, or want the training recipe? Ask in the Community tab or at [sentovara.com](https://sentovara.com)."""
88
 
89
  with gr.Blocks(title="Sentinel S1") as demo:
90
  gr.Markdown(INTRO)