multimodalart HF Staff commited on
Commit
ed584bc
·
verified ·
1 Parent(s): 26aa770

Calibrate ZeroGPU duration from measured ~8s/turn throughput

Browse files
Files changed (1) hide show
  1. app.py +10 -6
app.py CHANGED
@@ -377,13 +377,17 @@ def stream_turn(messages, max_new_tokens: int, temperature: float, top_p: float,
377
 
378
 
379
  def _estimate_duration(*args, **kwargs) -> int:
380
- max_turns = 8
381
- if len(args) >= 5:
382
- try:
 
383
  max_turns = int(args[4])
384
- except Exception:
385
- pass
386
- return int(min(280, 40 + max_turns * 22))
 
 
 
387
 
388
 
389
  # --------------------------------------------------------------------------------------
 
377
 
378
 
379
  def _estimate_duration(*args, **kwargs) -> int:
380
+ """GPU budget, sized from measured throughput (~8 s/turn at 1536 max_new_tokens)."""
381
+ max_turns, max_new = 8, 1536
382
+ try:
383
+ if len(args) >= 5 and args[4] is not None:
384
  max_turns = int(args[4])
385
+ if len(args) >= 6 and args[5] is not None:
386
+ max_new = int(args[5])
387
+ except Exception:
388
+ pass
389
+ per_turn = 6.0 + 6.0 * (max_new / 1536.0)
390
+ return int(min(280, 15 + max_turns * per_turn))
391
 
392
 
393
  # --------------------------------------------------------------------------------------