Spaces:
Running on Zero
Running on Zero
AndrianBalanescu commited on
Commit ·
fc7efb2
1
Parent(s): f725c6a
fix(dispatcher): re-raise ZeroGPUQuotaError caught by generic except handlers
Browse filesQuota stops raised inside the Gradio SSE loop and the async-jobs poll loop
were swallowed by their own 'except Exception' handlers, downgraded to
None and finally to a generic RuntimeError. The cron therefore marked all
6 tracks 'failed' on 09-12/09-13 instead of saving resume state and
deferring to the next day, burning zero of the remaining quota.
Re-raise ZeroGPUQuotaError explicitly in both loops so the nightly cron
persists resume state and recovers the album automatically.
moldovan-qwen/data/studio_settings.json
DELETED
|
@@ -1,125 +0,0 @@
|
|
| 1 |
-
{
|
| 2 |
-
"hf_token": "",
|
| 3 |
-
"openrouter_key": "",
|
| 4 |
-
"minimax_key": "",
|
| 5 |
-
"elevenlabs_key": "",
|
| 6 |
-
"fal_key": "",
|
| 7 |
-
"active_models": {
|
| 8 |
-
"chat": "Qwen/Qwen2.5-7B-Instruct",
|
| 9 |
-
"music": "MiniMaxAI/MiniMax-Music3",
|
| 10 |
-
"tts": "edge-tts-ro-RO",
|
| 11 |
-
"video": "Wan-AI/Wan2.1-T2V-14B",
|
| 12 |
-
"image": "black-forest-labs/FLUX.1-schnell"
|
| 13 |
-
},
|
| 14 |
-
"providers_enabled": {
|
| 15 |
-
"huggingface": true,
|
| 16 |
-
"openrouter": true,
|
| 17 |
-
"minimax": false,
|
| 18 |
-
"elevenlabs": false,
|
| 19 |
-
"local_fallback": true
|
| 20 |
-
},
|
| 21 |
-
"custom_endpoints": {
|
| 22 |
-
"whisper": "http://homelab:8086/v1/transcribe",
|
| 23 |
-
"flow_space": "https://abalanescu-flow2.hf.space/v1"
|
| 24 |
-
},
|
| 25 |
-
"custom_providers": [
|
| 26 |
-
{
|
| 27 |
-
"name": "Local vLLM",
|
| 28 |
-
"base_url": "http://100.70.158.21:8000/v1"
|
| 29 |
-
}
|
| 30 |
-
],
|
| 31 |
-
"model_catalog": {
|
| 32 |
-
"audio": [
|
| 33 |
-
{
|
| 34 |
-
"id": "MiniMaxAI/MiniMax-Music3",
|
| 35 |
-
"label": "MiniMax Music 3 (ZeroGPU Neural Song)",
|
| 36 |
-
"provider": "huggingface",
|
| 37 |
-
"capabilities": [
|
| 38 |
-
"music"
|
| 39 |
-
]
|
| 40 |
-
},
|
| 41 |
-
{
|
| 42 |
-
"id": "edge-tts-ro-RO",
|
| 43 |
-
"label": "Edge-TTS ro-RO (Emil)",
|
| 44 |
-
"provider": "edge",
|
| 45 |
-
"capabilities": [
|
| 46 |
-
"tts"
|
| 47 |
-
]
|
| 48 |
-
}
|
| 49 |
-
],
|
| 50 |
-
"video": [
|
| 51 |
-
{
|
| 52 |
-
"id": "Wan-AI/Wan2.1-T2V-14B",
|
| 53 |
-
"label": "Wan 2.1 Text to Video",
|
| 54 |
-
"provider": "huggingface",
|
| 55 |
-
"capabilities": [
|
| 56 |
-
"video"
|
| 57 |
-
]
|
| 58 |
-
},
|
| 59 |
-
{
|
| 60 |
-
"id": "Wan-AI/Wan2.1-T2V-1.3B",
|
| 61 |
-
"label": "Wan 2.1 Fast Video",
|
| 62 |
-
"provider": "huggingface",
|
| 63 |
-
"capabilities": [
|
| 64 |
-
"video"
|
| 65 |
-
]
|
| 66 |
-
},
|
| 67 |
-
{
|
| 68 |
-
"id": "THUDM/CogVideoX-5b",
|
| 69 |
-
"label": "CogVideoX 5B",
|
| 70 |
-
"provider": "huggingface",
|
| 71 |
-
"capabilities": [
|
| 72 |
-
"video"
|
| 73 |
-
]
|
| 74 |
-
},
|
| 75 |
-
{
|
| 76 |
-
"id": "Lightricks/LTX-Video",
|
| 77 |
-
"label": "LTX Video",
|
| 78 |
-
"provider": "huggingface",
|
| 79 |
-
"capabilities": [
|
| 80 |
-
"video"
|
| 81 |
-
]
|
| 82 |
-
},
|
| 83 |
-
{
|
| 84 |
-
"id": "black-forest-labs/FLUX.1-dev",
|
| 85 |
-
"label": "FLUX.1 Dev",
|
| 86 |
-
"provider": "huggingface",
|
| 87 |
-
"capabilities": [
|
| 88 |
-
"image",
|
| 89 |
-
"keyframes"
|
| 90 |
-
]
|
| 91 |
-
},
|
| 92 |
-
{
|
| 93 |
-
"id": "black-forest-labs/FLUX.1-schnell",
|
| 94 |
-
"label": "FLUX.1 Schnell",
|
| 95 |
-
"provider": "huggingface",
|
| 96 |
-
"capabilities": [
|
| 97 |
-
"image",
|
| 98 |
-
"keyframes"
|
| 99 |
-
]
|
| 100 |
-
}
|
| 101 |
-
],
|
| 102 |
-
"text": [
|
| 103 |
-
{
|
| 104 |
-
"id": "Qwen/Qwen2.5-72B-Instruct",
|
| 105 |
-
"label": "Qwen 2.5 72B",
|
| 106 |
-
"provider": "huggingface",
|
| 107 |
-
"capabilities": [
|
| 108 |
-
"chat",
|
| 109 |
-
"lyrics",
|
| 110 |
-
"scripts"
|
| 111 |
-
]
|
| 112 |
-
},
|
| 113 |
-
{
|
| 114 |
-
"id": "Qwen/Qwen3-32B",
|
| 115 |
-
"label": "Qwen3 32B",
|
| 116 |
-
"provider": "huggingface",
|
| 117 |
-
"capabilities": [
|
| 118 |
-
"chat",
|
| 119 |
-
"lyrics",
|
| 120 |
-
"scripts"
|
| 121 |
-
]
|
| 122 |
-
}
|
| 123 |
-
]
|
| 124 |
-
}
|
| 125 |
-
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
moldovan-qwen/engine/model_dispatcher.py
CHANGED
|
@@ -606,6 +606,8 @@ class ModelDispatcher:
|
|
| 606 |
if dl_resp.status_code == 200 and len(dl_resp.content) > 1000:
|
| 607 |
return dl_resp.content
|
| 608 |
return None
|
|
|
|
|
|
|
| 609 |
except Exception as e:
|
| 610 |
print(f"[ModelDispatcher] Gradio queue streaming error: {e}")
|
| 611 |
return None
|
|
@@ -685,6 +687,8 @@ class ModelDispatcher:
|
|
| 685 |
if 'exceeded your Pro ZeroGPU quota' in _fail_text or 'reached its GPU limit' in _fail_text:
|
| 686 |
raise ZeroGPUQuotaError(_fail_text[:300])
|
| 687 |
return None
|
|
|
|
|
|
|
| 688 |
except Exception as e:
|
| 689 |
print(f"[ModelDispatcher] Async job poll warning: {e}")
|
| 690 |
return None
|
|
|
|
| 606 |
if dl_resp.status_code == 200 and len(dl_resp.content) > 1000:
|
| 607 |
return dl_resp.content
|
| 608 |
return None
|
| 609 |
+
except ZeroGPUQuotaError:
|
| 610 |
+
raise # account-level quota stop: must reach the cron, never swallow
|
| 611 |
except Exception as e:
|
| 612 |
print(f"[ModelDispatcher] Gradio queue streaming error: {e}")
|
| 613 |
return None
|
|
|
|
| 687 |
if 'exceeded your Pro ZeroGPU quota' in _fail_text or 'reached its GPU limit' in _fail_text:
|
| 688 |
raise ZeroGPUQuotaError(_fail_text[:300])
|
| 689 |
return None
|
| 690 |
+
except ZeroGPUQuotaError:
|
| 691 |
+
raise # account-level quota stop: must reach the cron
|
| 692 |
except Exception as e:
|
| 693 |
print(f"[ModelDispatcher] Async job poll warning: {e}")
|
| 694 |
return None
|