Dellboy commited on
Commit
44a42c0
·
verified ·
1 Parent(s): a56f58c

Raise the answer budget to 1200 tokens and the GPU window to 120s

Browse files
Files changed (1) hide show
  1. app.py +5 -2
app.py CHANGED
@@ -192,10 +192,13 @@ def build_messages(message: str, history: list | None) -> list[dict]:
192
  return messages
193
 
194
 
195
- @spaces.GPU(duration=60)
 
 
 
196
  def chat(message: str, history: list | None = None, temperature: float = 0.7,
197
  top_p: float = 0.9, repetition_penalty: float = 1.05,
198
- max_new_tokens: int = 512):
199
  """Stream an answer. Yields the answer *so far* on each step, which is what
200
  Gradio's streaming contract expects and what hf_client.py diffs into deltas."""
201
  message = (message or "").strip()
 
192
  return messages
193
 
194
 
195
+ # 120s, not 60. A long structured answer at 1200 tokens takes about 35s of
196
+ # generation on top of a prefill over eleven retrieved passages, and a GPU
197
+ # window that expires mid-answer truncates it with no error.
198
+ @spaces.GPU(duration=120)
199
  def chat(message: str, history: list | None = None, temperature: float = 0.7,
200
  top_p: float = 0.9, repetition_penalty: float = 1.05,
201
+ max_new_tokens: int = 1200):
202
  """Stream an answer. Yields the answer *so far* on each step, which is what
203
  Gradio's streaming contract expects and what hf_client.py diffs into deltas."""
204
  message = (message or "").strip()