Spaces:
Running on Zero
Running on Zero
Raise the answer budget to 1200 tokens and the GPU window to 120s
Browse files
app.py
CHANGED
|
@@ -192,10 +192,13 @@ def build_messages(message: str, history: list | None) -> list[dict]:
|
|
| 192 |
return messages
|
| 193 |
|
| 194 |
|
| 195 |
-
|
|
|
|
|
|
|
|
|
|
| 196 |
def chat(message: str, history: list | None = None, temperature: float = 0.7,
|
| 197 |
top_p: float = 0.9, repetition_penalty: float = 1.05,
|
| 198 |
-
max_new_tokens: int =
|
| 199 |
"""Stream an answer. Yields the answer *so far* on each step, which is what
|
| 200 |
Gradio's streaming contract expects and what hf_client.py diffs into deltas."""
|
| 201 |
message = (message or "").strip()
|
|
|
|
| 192 |
return messages
|
| 193 |
|
| 194 |
|
| 195 |
+
# 120s, not 60. A long structured answer at 1200 tokens takes about 35s of
|
| 196 |
+
# generation on top of a prefill over eleven retrieved passages, and a GPU
|
| 197 |
+
# window that expires mid-answer truncates it with no error.
|
| 198 |
+
@spaces.GPU(duration=120)
|
| 199 |
def chat(message: str, history: list | None = None, temperature: float = 0.7,
|
| 200 |
top_p: float = 0.9, repetition_penalty: float = 1.05,
|
| 201 |
+
max_new_tokens: int = 1200):
|
| 202 |
"""Stream an answer. Yields the answer *so far* on each step, which is what
|
| 203 |
Gradio's streaming contract expects and what hf_client.py diffs into deltas."""
|
| 204 |
message = (message or "").strip()
|