Spaces:
Running on Zero
Running on Zero
Calibrate ZeroGPU duration from measured ~8s/turn throughput
Browse files
app.py
CHANGED
|
@@ -377,13 +377,17 @@ def stream_turn(messages, max_new_tokens: int, temperature: float, top_p: float,
|
|
| 377 |
|
| 378 |
|
| 379 |
def _estimate_duration(*args, **kwargs) -> int:
|
| 380 |
-
|
| 381 |
-
|
| 382 |
-
|
|
|
|
| 383 |
max_turns = int(args[4])
|
| 384 |
-
|
| 385 |
-
|
| 386 |
-
|
|
|
|
|
|
|
|
|
|
| 387 |
|
| 388 |
|
| 389 |
# --------------------------------------------------------------------------------------
|
|
|
|
| 377 |
|
| 378 |
|
| 379 |
def _estimate_duration(*args, **kwargs) -> int:
|
| 380 |
+
"""GPU budget, sized from measured throughput (~8 s/turn at 1536 max_new_tokens)."""
|
| 381 |
+
max_turns, max_new = 8, 1536
|
| 382 |
+
try:
|
| 383 |
+
if len(args) >= 5 and args[4] is not None:
|
| 384 |
max_turns = int(args[4])
|
| 385 |
+
if len(args) >= 6 and args[5] is not None:
|
| 386 |
+
max_new = int(args[5])
|
| 387 |
+
except Exception:
|
| 388 |
+
pass
|
| 389 |
+
per_turn = 6.0 + 6.0 * (max_new / 1536.0)
|
| 390 |
+
return int(min(280, 15 + max_turns * per_turn))
|
| 391 |
|
| 392 |
|
| 393 |
# --------------------------------------------------------------------------------------
|