AndrianBalanescu commited on
Commit
fc7efb2
·
1 Parent(s): f725c6a

fix(dispatcher): re-raise ZeroGPUQuotaError caught by generic except handlers

Browse files

Quota stops raised inside the Gradio SSE loop and the async-jobs poll loop
were swallowed by their own 'except Exception' handlers, downgraded to
None and finally to a generic RuntimeError. The cron therefore marked all
6 tracks 'failed' on 09-12/09-13 instead of saving resume state and
deferring to the next day, burning zero of the remaining quota.

Re-raise ZeroGPUQuotaError explicitly in both loops so the nightly cron
persists resume state and recovers the album automatically.

moldovan-qwen/data/studio_settings.json DELETED
@@ -1,125 +0,0 @@
1
- {
2
- "hf_token": "",
3
- "openrouter_key": "",
4
- "minimax_key": "",
5
- "elevenlabs_key": "",
6
- "fal_key": "",
7
- "active_models": {
8
- "chat": "Qwen/Qwen2.5-7B-Instruct",
9
- "music": "MiniMaxAI/MiniMax-Music3",
10
- "tts": "edge-tts-ro-RO",
11
- "video": "Wan-AI/Wan2.1-T2V-14B",
12
- "image": "black-forest-labs/FLUX.1-schnell"
13
- },
14
- "providers_enabled": {
15
- "huggingface": true,
16
- "openrouter": true,
17
- "minimax": false,
18
- "elevenlabs": false,
19
- "local_fallback": true
20
- },
21
- "custom_endpoints": {
22
- "whisper": "http://homelab:8086/v1/transcribe",
23
- "flow_space": "https://abalanescu-flow2.hf.space/v1"
24
- },
25
- "custom_providers": [
26
- {
27
- "name": "Local vLLM",
28
- "base_url": "http://100.70.158.21:8000/v1"
29
- }
30
- ],
31
- "model_catalog": {
32
- "audio": [
33
- {
34
- "id": "MiniMaxAI/MiniMax-Music3",
35
- "label": "MiniMax Music 3 (ZeroGPU Neural Song)",
36
- "provider": "huggingface",
37
- "capabilities": [
38
- "music"
39
- ]
40
- },
41
- {
42
- "id": "edge-tts-ro-RO",
43
- "label": "Edge-TTS ro-RO (Emil)",
44
- "provider": "edge",
45
- "capabilities": [
46
- "tts"
47
- ]
48
- }
49
- ],
50
- "video": [
51
- {
52
- "id": "Wan-AI/Wan2.1-T2V-14B",
53
- "label": "Wan 2.1 Text to Video",
54
- "provider": "huggingface",
55
- "capabilities": [
56
- "video"
57
- ]
58
- },
59
- {
60
- "id": "Wan-AI/Wan2.1-T2V-1.3B",
61
- "label": "Wan 2.1 Fast Video",
62
- "provider": "huggingface",
63
- "capabilities": [
64
- "video"
65
- ]
66
- },
67
- {
68
- "id": "THUDM/CogVideoX-5b",
69
- "label": "CogVideoX 5B",
70
- "provider": "huggingface",
71
- "capabilities": [
72
- "video"
73
- ]
74
- },
75
- {
76
- "id": "Lightricks/LTX-Video",
77
- "label": "LTX Video",
78
- "provider": "huggingface",
79
- "capabilities": [
80
- "video"
81
- ]
82
- },
83
- {
84
- "id": "black-forest-labs/FLUX.1-dev",
85
- "label": "FLUX.1 Dev",
86
- "provider": "huggingface",
87
- "capabilities": [
88
- "image",
89
- "keyframes"
90
- ]
91
- },
92
- {
93
- "id": "black-forest-labs/FLUX.1-schnell",
94
- "label": "FLUX.1 Schnell",
95
- "provider": "huggingface",
96
- "capabilities": [
97
- "image",
98
- "keyframes"
99
- ]
100
- }
101
- ],
102
- "text": [
103
- {
104
- "id": "Qwen/Qwen2.5-72B-Instruct",
105
- "label": "Qwen 2.5 72B",
106
- "provider": "huggingface",
107
- "capabilities": [
108
- "chat",
109
- "lyrics",
110
- "scripts"
111
- ]
112
- },
113
- {
114
- "id": "Qwen/Qwen3-32B",
115
- "label": "Qwen3 32B",
116
- "provider": "huggingface",
117
- "capabilities": [
118
- "chat",
119
- "lyrics",
120
- "scripts"
121
- ]
122
- }
123
- ]
124
- }
125
- }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
moldovan-qwen/engine/model_dispatcher.py CHANGED
@@ -606,6 +606,8 @@ class ModelDispatcher:
606
  if dl_resp.status_code == 200 and len(dl_resp.content) > 1000:
607
  return dl_resp.content
608
  return None
 
 
609
  except Exception as e:
610
  print(f"[ModelDispatcher] Gradio queue streaming error: {e}")
611
  return None
@@ -685,6 +687,8 @@ class ModelDispatcher:
685
  if 'exceeded your Pro ZeroGPU quota' in _fail_text or 'reached its GPU limit' in _fail_text:
686
  raise ZeroGPUQuotaError(_fail_text[:300])
687
  return None
 
 
688
  except Exception as e:
689
  print(f"[ModelDispatcher] Async job poll warning: {e}")
690
  return None
 
606
  if dl_resp.status_code == 200 and len(dl_resp.content) > 1000:
607
  return dl_resp.content
608
  return None
609
+ except ZeroGPUQuotaError:
610
+ raise # account-level quota stop: must reach the cron, never swallow
611
  except Exception as e:
612
  print(f"[ModelDispatcher] Gradio queue streaming error: {e}")
613
  return None
 
687
  if 'exceeded your Pro ZeroGPU quota' in _fail_text or 'reached its GPU limit' in _fail_text:
688
  raise ZeroGPUQuotaError(_fail_text[:300])
689
  return None
690
+ except ZeroGPUQuotaError:
691
+ raise # account-level quota stop: must reach the cron
692
  except Exception as e:
693
  print(f"[ModelDispatcher] Async job poll warning: {e}")
694
  return None