Commit ยท
e9f689e
1
Parent(s): 19a11de
fix: wait out the 60s cooldown instead of silently failing extraction when keys are rate-limited
Browse files- api/projects.py +13 -1
- utils/key_manager.py +26 -12
- workflow/step3_extract.py +12 -6
api/projects.py
CHANGED
|
@@ -29,11 +29,23 @@ async def get_active_project(project_id: Optional[int] = None, current_user: dic
|
|
| 29 |
if df.empty:
|
| 30 |
return {"has_active": False}
|
| 31 |
row = df.iloc[0]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 32 |
res = {
|
| 33 |
"has_active": True,
|
| 34 |
"current_step": int(row["current_step"]),
|
| 35 |
"updated_at": row["updated_at"],
|
| 36 |
-
"state_data":
|
| 37 |
}
|
| 38 |
if "project_id" in row:
|
| 39 |
res["project_id"] = int(row["project_id"])
|
|
|
|
| 29 |
if df.empty:
|
| 30 |
return {"has_active": False}
|
| 31 |
row = df.iloc[0]
|
| 32 |
+
|
| 33 |
+
import json as _json
|
| 34 |
+
from api.routers.data_review import reconstruct_project_inputs
|
| 35 |
+
raw = row["state_data"]
|
| 36 |
+
state_data = _json.loads(raw) if isinstance(raw, str) else (raw or {})
|
| 37 |
+
if state_data.get("current_step", 0) >= 3:
|
| 38 |
+
reconstruct_project_inputs(state_data)
|
| 39 |
+
safe_execute(
|
| 40 |
+
"UPDATE qto_active_projects SET state_data=%s WHERE user_id=%s AND project_id=%s",
|
| 41 |
+
(_json.dumps(state_data, ensure_ascii=False, default=str), current_user["id"], row.get("project_id") or project_id)
|
| 42 |
+
)
|
| 43 |
+
|
| 44 |
res = {
|
| 45 |
"has_active": True,
|
| 46 |
"current_step": int(row["current_step"]),
|
| 47 |
"updated_at": row["updated_at"],
|
| 48 |
+
"state_data": state_data
|
| 49 |
}
|
| 50 |
if "project_id" in row:
|
| 51 |
res["project_id"] = int(row["project_id"])
|
utils/key_manager.py
CHANGED
|
@@ -34,17 +34,17 @@ class KeyManager:
|
|
| 34 |
def __init__(self, keys: List[str]):
|
| 35 |
self._keys = [k.strip() for k in keys if k and k.strip()]
|
| 36 |
self._lock = threading.Lock()
|
| 37 |
-
|
| 38 |
-
#
|
| 39 |
self._combinations = []
|
| 40 |
-
for
|
| 41 |
-
for
|
| 42 |
self._combinations.append((k, m))
|
| 43 |
-
|
| 44 |
self._index = 0
|
| 45 |
self._usage: dict = {}
|
| 46 |
for k, m in self._combinations:
|
| 47 |
-
self._usage[(k, m)] = {"count": 0, "date": date.today(), "exhausted": False}
|
| 48 |
|
| 49 |
# โโ Public API โโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโ
|
| 50 |
|
|
@@ -60,11 +60,12 @@ class KeyManager:
|
|
| 60 |
return None, "gemini-3.1-flash-lite" # fallback if all exhausted
|
| 61 |
|
| 62 |
def mark_rate_limited(self, key: str, model: str):
|
| 63 |
-
"""ุงุณุชุฏุนู ูุฐุง ุนูุฏ ุงุณุชูุจุงู 429 โ
|
| 64 |
with self._lock:
|
| 65 |
combo = (key, model)
|
| 66 |
if combo in self._usage:
|
| 67 |
self._usage[combo]["exhausted"] = True
|
|
|
|
| 68 |
|
| 69 |
def mark_exhausted(self, key: str, model: str):
|
| 70 |
"""ุนููู
ุชุฑููุจุฉ ุจุงุนุชุจุงุฑูุง ู
ูุชููุฉ ูุฏููุงู."""
|
|
@@ -109,12 +110,17 @@ class KeyManager:
|
|
| 109 |
# โโ Internal โโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโ
|
| 110 |
|
| 111 |
def _active_combo(self) -> Optional[Tuple[str, str]]:
|
| 112 |
-
"""ูุจุญุซ ุนู ุฃูู ุชุฑููุจุฉ ุบูุฑ ู
ูุชููุฉ."""
|
| 113 |
if not self._combinations:
|
| 114 |
return None
|
|
|
|
| 115 |
for _ in range(len(self._combinations)):
|
| 116 |
combo = self._combinations[self._index % len(self._combinations)]
|
| 117 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 118 |
return combo
|
| 119 |
self._index += 1
|
| 120 |
return None
|
|
@@ -124,11 +130,19 @@ class KeyManager:
|
|
| 124 |
self._index = (self._index + 1) % len(self._combinations)
|
| 125 |
|
| 126 |
def _reset_daily_counts(self):
|
| 127 |
-
"""ูุตููุฑ ุงูุนุฏุงุฏ ูู ููู
ุฌุฏูุฏ."""
|
| 128 |
today = date.today()
|
| 129 |
for combo in self._combinations:
|
| 130 |
-
|
| 131 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 132 |
|
| 133 |
|
| 134 |
# โโ ุชุญู
ูู ุงูู
ูุงุชูุญ ู
ู .env ุฃู ู
ุชุบูุฑุงุช ุงูุจูุฆุฉ โโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโ
|
|
|
|
| 34 |
def __init__(self, keys: List[str]):
|
| 35 |
self._keys = [k.strip() for k in keys if k and k.strip()]
|
| 36 |
self._lock = threading.Lock()
|
| 37 |
+
|
| 38 |
+
# Rotate keys first, then models โ spreads RPM load across keys before hitting same model twice
|
| 39 |
self._combinations = []
|
| 40 |
+
for k in self._keys:
|
| 41 |
+
for m in ROTATION_MODELS:
|
| 42 |
self._combinations.append((k, m))
|
| 43 |
+
|
| 44 |
self._index = 0
|
| 45 |
self._usage: dict = {}
|
| 46 |
for k, m in self._combinations:
|
| 47 |
+
self._usage[(k, m)] = {"count": 0, "date": date.today(), "exhausted": False, "exhausted_at": 0}
|
| 48 |
|
| 49 |
# โโ Public API โโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโ
|
| 50 |
|
|
|
|
| 60 |
return None, "gemini-3.1-flash-lite" # fallback if all exhausted
|
| 61 |
|
| 62 |
def mark_rate_limited(self, key: str, model: str):
|
| 63 |
+
"""ุงุณุชุฏุนู ูุฐุง ุนูุฏ ุงุณุชูุจุงู 429 โ ูุชุฎุทู ูุฐู ุงูุชุฑููุจุฉ ูู
ุฏุฉ 60 ุซุงููุฉ ููุท."""
|
| 64 |
with self._lock:
|
| 65 |
combo = (key, model)
|
| 66 |
if combo in self._usage:
|
| 67 |
self._usage[combo]["exhausted"] = True
|
| 68 |
+
self._usage[combo]["exhausted_at"] = time.time()
|
| 69 |
|
| 70 |
def mark_exhausted(self, key: str, model: str):
|
| 71 |
"""ุนููู
ุชุฑููุจุฉ ุจุงุนุชุจุงุฑูุง ู
ูุชููุฉ ูุฏููุงู."""
|
|
|
|
| 110 |
# โโ Internal โโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโ
|
| 111 |
|
| 112 |
def _active_combo(self) -> Optional[Tuple[str, str]]:
|
| 113 |
+
"""ูุจุญุซ ุนู ุฃูู ุชุฑููุจุฉ ุบูุฑ ู
ูุชููุฉ. ุงูู
ูุงุชูุญ ุงูู
ุนูููุฉ ุชูุชูู ุจุนุฏ 60 ุซุงููุฉ ุชููุงุฆูุงู."""
|
| 114 |
if not self._combinations:
|
| 115 |
return None
|
| 116 |
+
now = time.time()
|
| 117 |
for _ in range(len(self._combinations)):
|
| 118 |
combo = self._combinations[self._index % len(self._combinations)]
|
| 119 |
+
u = self._usage[combo]
|
| 120 |
+
# Auto-recover after 60 seconds
|
| 121 |
+
if u["exhausted"] and (now - u.get("exhausted_at", 0)) > 60:
|
| 122 |
+
u["exhausted"] = False
|
| 123 |
+
if not u["exhausted"]:
|
| 124 |
return combo
|
| 125 |
self._index += 1
|
| 126 |
return None
|
|
|
|
| 130 |
self._index = (self._index + 1) % len(self._combinations)
|
| 131 |
|
| 132 |
def _reset_daily_counts(self):
|
| 133 |
+
"""ูุตููุฑ ุงูุนุฏุงุฏ ูู ููู
ุฌุฏูุฏ ูููุดุบูู auto-exhaustion ุนูุฏ ุจููุบ ุงูุญุฏ."""
|
| 134 |
today = date.today()
|
| 135 |
for combo in self._combinations:
|
| 136 |
+
k, m = combo
|
| 137 |
+
u = self._usage[combo]
|
| 138 |
+
if u["date"] != today:
|
| 139 |
+
self._usage[combo] = {"count": 0, "date": today, "exhausted": False, "exhausted_at": 0}
|
| 140 |
+
else:
|
| 141 |
+
# Auto-mark exhausted when daily limit reached
|
| 142 |
+
limit = MODEL_DAILY_LIMITS.get(m, MODEL_DAILY_LIMITS["default"])
|
| 143 |
+
if u["count"] >= limit and not u["exhausted"]:
|
| 144 |
+
u["exhausted"] = True
|
| 145 |
+
u["exhausted_at"] = time.time()
|
| 146 |
|
| 147 |
|
| 148 |
# โโ ุชุญู
ูู ุงูู
ูุงุชูุญ ู
ู .env ุฃู ู
ุชุบูุฑุงุช ุงูุจูุฆุฉ โโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโ
|
workflow/step3_extract.py
CHANGED
|
@@ -394,11 +394,14 @@ async def _ask_ai_with_retry(img_bytes: bytes, full_prompt: str, mgr, force_reex
|
|
| 394 |
print(f"Cache read error: {cache_ex}")
|
| 395 |
|
| 396 |
last_err = ""
|
| 397 |
-
for attempt in range(
|
| 398 |
current_key, current_model = mgr.get_key_and_model()
|
| 399 |
if not current_key or current_key == "NO_API_KEY_FOUND":
|
| 400 |
-
last_err = "No valid API key found."
|
| 401 |
-
|
|
|
|
|
|
|
|
|
|
| 402 |
|
| 403 |
try:
|
| 404 |
client = genai.Client(api_key=current_key, http_options={'timeout': 45})
|
|
@@ -772,12 +775,15 @@ Return ONLY this JSON structure (null for not found):
|
|
| 772 |
|
| 773 |
last_err = ""
|
| 774 |
|
| 775 |
-
for attempt in range(
|
| 776 |
current_key, current_model = mgr.get_key_and_model()
|
| 777 |
|
| 778 |
if not current_key or current_key == "NO_API_KEY_FOUND":
|
| 779 |
-
last_err = "No valid API key found."
|
| 780 |
-
|
|
|
|
|
|
|
|
|
|
| 781 |
|
| 782 |
try:
|
| 783 |
# Added a shorter timeout to fail fast on stuck networks
|
|
|
|
| 394 |
print(f"Cache read error: {cache_ex}")
|
| 395 |
|
| 396 |
last_err = ""
|
| 397 |
+
for attempt in range(6):
|
| 398 |
current_key, current_model = mgr.get_key_and_model()
|
| 399 |
if not current_key or current_key == "NO_API_KEY_FOUND":
|
| 400 |
+
last_err = "No valid API key found. All keys are in cooldown."
|
| 401 |
+
print(f"All keys in 60s cooldown. Waiting 15 seconds... (Attempt {attempt+1}/6)")
|
| 402 |
+
import time
|
| 403 |
+
time.sleep(15)
|
| 404 |
+
continue
|
| 405 |
|
| 406 |
try:
|
| 407 |
client = genai.Client(api_key=current_key, http_options={'timeout': 45})
|
|
|
|
| 775 |
|
| 776 |
last_err = ""
|
| 777 |
|
| 778 |
+
for attempt in range(6): # retry up to 6 times
|
| 779 |
current_key, current_model = mgr.get_key_and_model()
|
| 780 |
|
| 781 |
if not current_key or current_key == "NO_API_KEY_FOUND":
|
| 782 |
+
last_err = "No valid API key found. All keys in cooldown."
|
| 783 |
+
print(f"All keys in 60s cooldown. Waiting 15 seconds... (Attempt {attempt+1}/6)")
|
| 784 |
+
import time
|
| 785 |
+
time.sleep(15)
|
| 786 |
+
continue
|
| 787 |
|
| 788 |
try:
|
| 789 |
# Added a shorter timeout to fail fast on stuck networks
|