sgattup commited on
Commit
455673e
·
verified ·
1 Parent(s): 74513f1

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +41 -25
app.py CHANGED
@@ -1,15 +1,23 @@
1
  """
2
- Estella Explainer — Hugging Face ZeroGPU Space
3
- A math reading coach for grades 3–8.
 
 
4
  """
5
 
6
  import os
 
 
7
 
8
- import spaces
9
  import gradio as gr
 
10
  import torch
11
  from peft import PeftModel
12
- from transformers import AutoModelForCausalLM, AutoTokenizer
 
 
 
 
13
 
14
 
15
  ADAPTER_ID = "sgattup/EstellaExplainerLLM"
@@ -67,7 +75,6 @@ model = PeftModel.from_pretrained(
67
  )
68
 
69
  print("Moving combined model to CUDA.")
70
-
71
  model = model.to("cuda")
72
  model.eval()
73
 
@@ -80,14 +87,15 @@ print("Estella is ready.")
80
 
81
 
82
  @spaces.GPU(duration=60)
83
- def ask_estella(message: str, history: list) -> str:
84
- """Return one reading hint without solving the math problem."""
85
 
86
  if not message or not message.strip():
87
- return (
88
  "Please type a math word problem and I’ll help explain "
89
  "what it is asking!"
90
  )
 
91
 
92
  messages = [
93
  {
@@ -116,30 +124,38 @@ def ask_estella(message: str, history: list) -> str:
116
  )
117
 
118
  input_device = next(model.parameters()).device
119
-
120
  inputs = {
121
  key: value.to(input_device)
122
  for key, value in inputs.items()
123
  }
124
 
125
- input_length = inputs["input_ids"].shape[1]
 
 
 
 
 
126
 
127
- with torch.no_grad():
128
- outputs = model.generate(
129
- **inputs,
130
- max_new_tokens=150,
131
- temperature=0.7,
132
- do_sample=True,
133
- repetition_penalty=1.1,
134
- pad_token_id=tokenizer.eos_token_id,
135
- )
136
 
137
- generated_tokens = outputs[0][input_length:]
 
 
 
 
138
 
139
- return tokenizer.decode(
140
- generated_tokens,
141
- skip_special_tokens=True,
142
- ).strip()
143
 
144
 
145
  demo = gr.ChatInterface(
@@ -161,4 +177,4 @@ demo = gr.ChatInterface(
161
 
162
 
163
  if __name__ == "__main__":
164
- demo.launch()
 
1
  """
2
+ Estella Explainer — streaming Gradio chat interface
3
+
4
+ This version keeps Gradio's default appearance and adds the two important
5
+ interaction patterns used by the DeepSeek demo: token streaming and a queue.
6
  """
7
 
8
  import os
9
+ from threading import Thread
10
+ from typing import Iterator
11
 
 
12
  import gradio as gr
13
+ import spaces
14
  import torch
15
  from peft import PeftModel
16
+ from transformers import (
17
+ AutoModelForCausalLM,
18
+ AutoTokenizer,
19
+ TextIteratorStreamer,
20
+ )
21
 
22
 
23
  ADAPTER_ID = "sgattup/EstellaExplainerLLM"
 
75
  )
76
 
77
  print("Moving combined model to CUDA.")
 
78
  model = model.to("cuda")
79
  model.eval()
80
 
 
87
 
88
 
89
  @spaces.GPU(duration=60)
90
+ def ask_estella(message: str, history: list) -> Iterator[str]:
91
+ """Stream one reading hint without solving the math problem."""
92
 
93
  if not message or not message.strip():
94
+ yield (
95
  "Please type a math word problem and I’ll help explain "
96
  "what it is asking!"
97
  )
98
+ return
99
 
100
  messages = [
101
  {
 
124
  )
125
 
126
  input_device = next(model.parameters()).device
 
127
  inputs = {
128
  key: value.to(input_device)
129
  for key, value in inputs.items()
130
  }
131
 
132
+ streamer = TextIteratorStreamer(
133
+ tokenizer,
134
+ timeout=60.0,
135
+ skip_prompt=True,
136
+ skip_special_tokens=True,
137
+ )
138
 
139
+ generation_kwargs = {
140
+ **inputs,
141
+ "streamer": streamer,
142
+ "max_new_tokens": 150,
143
+ "temperature": 0.7,
144
+ "do_sample": True,
145
+ "repetition_penalty": 1.1,
146
+ "pad_token_id": tokenizer.eos_token_id,
147
+ }
148
 
149
+ generation_thread = Thread(
150
+ target=model.generate,
151
+ kwargs=generation_kwargs,
152
+ )
153
+ generation_thread.start()
154
 
155
+ partial_hint = ""
156
+ for new_text in streamer:
157
+ partial_hint += new_text
158
+ yield partial_hint.strip()
159
 
160
 
161
  demo = gr.ChatInterface(
 
177
 
178
 
179
  if __name__ == "__main__":
180
+ demo.queue(max_size=20).launch()