basel00omar commited on
Commit
e9f689e
ยท
1 Parent(s): 19a11de

fix: wait out the 60s cooldown instead of silently failing extraction when keys are rate-limited

Browse files
Files changed (3) hide show
  1. api/projects.py +13 -1
  2. utils/key_manager.py +26 -12
  3. workflow/step3_extract.py +12 -6
api/projects.py CHANGED
@@ -29,11 +29,23 @@ async def get_active_project(project_id: Optional[int] = None, current_user: dic
29
  if df.empty:
30
  return {"has_active": False}
31
  row = df.iloc[0]
 
 
 
 
 
 
 
 
 
 
 
 
32
  res = {
33
  "has_active": True,
34
  "current_step": int(row["current_step"]),
35
  "updated_at": row["updated_at"],
36
- "state_data": row["state_data"]
37
  }
38
  if "project_id" in row:
39
  res["project_id"] = int(row["project_id"])
 
29
  if df.empty:
30
  return {"has_active": False}
31
  row = df.iloc[0]
32
+
33
+ import json as _json
34
+ from api.routers.data_review import reconstruct_project_inputs
35
+ raw = row["state_data"]
36
+ state_data = _json.loads(raw) if isinstance(raw, str) else (raw or {})
37
+ if state_data.get("current_step", 0) >= 3:
38
+ reconstruct_project_inputs(state_data)
39
+ safe_execute(
40
+ "UPDATE qto_active_projects SET state_data=%s WHERE user_id=%s AND project_id=%s",
41
+ (_json.dumps(state_data, ensure_ascii=False, default=str), current_user["id"], row.get("project_id") or project_id)
42
+ )
43
+
44
  res = {
45
  "has_active": True,
46
  "current_step": int(row["current_step"]),
47
  "updated_at": row["updated_at"],
48
+ "state_data": state_data
49
  }
50
  if "project_id" in row:
51
  res["project_id"] = int(row["project_id"])
utils/key_manager.py CHANGED
@@ -34,17 +34,17 @@ class KeyManager:
34
  def __init__(self, keys: List[str]):
35
  self._keys = [k.strip() for k in keys if k and k.strip()]
36
  self._lock = threading.Lock()
37
-
38
- # ุฅู†ุดุงุก ู…ุตููˆูุฉ ุงู„ุชุฏูˆูŠุฑ: ู…ูุชุงุญ1-ู†ู…ูˆุฐุฌ1ุŒ ู…ูุชุงุญ2-ู†ู…ูˆุฐุฌ1ุŒ ... ู„ุชูˆุฒูŠุน ุญู…ู„ ุงู„ู€ RPM ุนู„ู‰ ุงู„ู…ูุงุชูŠุญ ุฃูˆู„ุงู‹
39
  self._combinations = []
40
- for m in ROTATION_MODELS:
41
- for k in self._keys:
42
  self._combinations.append((k, m))
43
-
44
  self._index = 0
45
  self._usage: dict = {}
46
  for k, m in self._combinations:
47
- self._usage[(k, m)] = {"count": 0, "date": date.today(), "exhausted": False}
48
 
49
  # โ”€โ”€ Public API โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€
50
 
@@ -60,11 +60,12 @@ class KeyManager:
60
  return None, "gemini-3.1-flash-lite" # fallback if all exhausted
61
 
62
  def mark_rate_limited(self, key: str, model: str):
63
- """ุงุณุชุฏุนู ู‡ุฐุง ุนู†ุฏ ุงุณุชู‚ุจุงู„ 429 โ€” ุณูŠุชุฎุทู‰ ู‡ุฐู‡ ุงู„ุชุฑูƒูŠุจุฉ."""
64
  with self._lock:
65
  combo = (key, model)
66
  if combo in self._usage:
67
  self._usage[combo]["exhausted"] = True
 
68
 
69
  def mark_exhausted(self, key: str, model: str):
70
  """ุนู„ู‘ู… ุชุฑูƒูŠุจุฉ ุจุงุนุชุจุงุฑู‡ุง ู…ู†ุชู‡ูŠุฉ ูŠุฏูˆูŠุงู‹."""
@@ -109,12 +110,17 @@ class KeyManager:
109
  # โ”€โ”€ Internal โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€
110
 
111
  def _active_combo(self) -> Optional[Tuple[str, str]]:
112
- """ูŠุจุญุซ ุนู† ุฃูˆู„ ุชุฑูƒูŠุจุฉ ุบูŠุฑ ู…ู†ุชู‡ูŠุฉ."""
113
  if not self._combinations:
114
  return None
 
115
  for _ in range(len(self._combinations)):
116
  combo = self._combinations[self._index % len(self._combinations)]
117
- if not self._usage[combo]["exhausted"]:
 
 
 
 
118
  return combo
119
  self._index += 1
120
  return None
@@ -124,11 +130,19 @@ class KeyManager:
124
  self._index = (self._index + 1) % len(self._combinations)
125
 
126
  def _reset_daily_counts(self):
127
- """ูŠุตูู‘ุฑ ุงู„ุนุฏุงุฏ ูƒู„ ูŠูˆู… ุฌุฏูŠุฏ."""
128
  today = date.today()
129
  for combo in self._combinations:
130
- if self._usage[combo]["date"] != today:
131
- self._usage[combo] = {"count": 0, "date": today, "exhausted": False}
 
 
 
 
 
 
 
 
132
 
133
 
134
  # โ”€โ”€ ุชุญู…ูŠู„ ุงู„ู…ูุงุชูŠุญ ู…ู† .env ุฃูˆ ู…ุชุบูŠุฑุงุช ุงู„ุจูŠุฆุฉ โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€
 
34
  def __init__(self, keys: List[str]):
35
  self._keys = [k.strip() for k in keys if k and k.strip()]
36
  self._lock = threading.Lock()
37
+
38
+ # Rotate keys first, then models โ€” spreads RPM load across keys before hitting same model twice
39
  self._combinations = []
40
+ for k in self._keys:
41
+ for m in ROTATION_MODELS:
42
  self._combinations.append((k, m))
43
+
44
  self._index = 0
45
  self._usage: dict = {}
46
  for k, m in self._combinations:
47
+ self._usage[(k, m)] = {"count": 0, "date": date.today(), "exhausted": False, "exhausted_at": 0}
48
 
49
  # โ”€โ”€ Public API โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€
50
 
 
60
  return None, "gemini-3.1-flash-lite" # fallback if all exhausted
61
 
62
  def mark_rate_limited(self, key: str, model: str):
63
+ """ุงุณุชุฏุนู ู‡ุฐุง ุนู†ุฏ ุงุณุชู‚ุจุงู„ 429 โ€” ูŠุชุฎุทู‰ ู‡ุฐู‡ ุงู„ุชุฑูƒูŠุจุฉ ู„ู…ุฏุฉ 60 ุซุงู†ูŠุฉ ูู‚ุท."""
64
  with self._lock:
65
  combo = (key, model)
66
  if combo in self._usage:
67
  self._usage[combo]["exhausted"] = True
68
+ self._usage[combo]["exhausted_at"] = time.time()
69
 
70
  def mark_exhausted(self, key: str, model: str):
71
  """ุนู„ู‘ู… ุชุฑูƒูŠุจุฉ ุจุงุนุชุจุงุฑู‡ุง ู…ู†ุชู‡ูŠุฉ ูŠุฏูˆูŠุงู‹."""
 
110
  # โ”€โ”€ Internal โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€
111
 
112
  def _active_combo(self) -> Optional[Tuple[str, str]]:
113
+ """ูŠุจุญุซ ุนู† ุฃูˆู„ ุชุฑูƒูŠุจุฉ ุบูŠุฑ ู…ู†ุชู‡ูŠุฉ. ุงู„ู…ูุงุชูŠุญ ุงู„ู…ุนู„ู‘ู‚ุฉ ุชู†ุชู‡ูŠ ุจุนุฏ 60 ุซุงู†ูŠุฉ ุชู„ู‚ุงุฆูŠุงู‹."""
114
  if not self._combinations:
115
  return None
116
+ now = time.time()
117
  for _ in range(len(self._combinations)):
118
  combo = self._combinations[self._index % len(self._combinations)]
119
+ u = self._usage[combo]
120
+ # Auto-recover after 60 seconds
121
+ if u["exhausted"] and (now - u.get("exhausted_at", 0)) > 60:
122
+ u["exhausted"] = False
123
+ if not u["exhausted"]:
124
  return combo
125
  self._index += 1
126
  return None
 
130
  self._index = (self._index + 1) % len(self._combinations)
131
 
132
  def _reset_daily_counts(self):
133
+ """ูŠุตูู‘ุฑ ุงู„ุนุฏุงุฏ ูƒู„ ูŠูˆู… ุฌุฏูŠุฏ ูˆูŠูุดุบู‘ู„ auto-exhaustion ุนู†ุฏ ุจู„ูˆุบ ุงู„ุญุฏ."""
134
  today = date.today()
135
  for combo in self._combinations:
136
+ k, m = combo
137
+ u = self._usage[combo]
138
+ if u["date"] != today:
139
+ self._usage[combo] = {"count": 0, "date": today, "exhausted": False, "exhausted_at": 0}
140
+ else:
141
+ # Auto-mark exhausted when daily limit reached
142
+ limit = MODEL_DAILY_LIMITS.get(m, MODEL_DAILY_LIMITS["default"])
143
+ if u["count"] >= limit and not u["exhausted"]:
144
+ u["exhausted"] = True
145
+ u["exhausted_at"] = time.time()
146
 
147
 
148
  # โ”€โ”€ ุชุญู…ูŠู„ ุงู„ู…ูุงุชูŠุญ ู…ู† .env ุฃูˆ ู…ุชุบูŠุฑุงุช ุงู„ุจูŠุฆุฉ โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€
workflow/step3_extract.py CHANGED
@@ -394,11 +394,14 @@ async def _ask_ai_with_retry(img_bytes: bytes, full_prompt: str, mgr, force_reex
394
  print(f"Cache read error: {cache_ex}")
395
 
396
  last_err = ""
397
- for attempt in range(3):
398
  current_key, current_model = mgr.get_key_and_model()
399
  if not current_key or current_key == "NO_API_KEY_FOUND":
400
- last_err = "No valid API key found."
401
- break
 
 
 
402
 
403
  try:
404
  client = genai.Client(api_key=current_key, http_options={'timeout': 45})
@@ -772,12 +775,15 @@ Return ONLY this JSON structure (null for not found):
772
 
773
  last_err = ""
774
 
775
- for attempt in range(3): # retry up to 3 times
776
  current_key, current_model = mgr.get_key_and_model()
777
 
778
  if not current_key or current_key == "NO_API_KEY_FOUND":
779
- last_err = "No valid API key found."
780
- break
 
 
 
781
 
782
  try:
783
  # Added a shorter timeout to fail fast on stuck networks
 
394
  print(f"Cache read error: {cache_ex}")
395
 
396
  last_err = ""
397
+ for attempt in range(6):
398
  current_key, current_model = mgr.get_key_and_model()
399
  if not current_key or current_key == "NO_API_KEY_FOUND":
400
+ last_err = "No valid API key found. All keys are in cooldown."
401
+ print(f"All keys in 60s cooldown. Waiting 15 seconds... (Attempt {attempt+1}/6)")
402
+ import time
403
+ time.sleep(15)
404
+ continue
405
 
406
  try:
407
  client = genai.Client(api_key=current_key, http_options={'timeout': 45})
 
775
 
776
  last_err = ""
777
 
778
+ for attempt in range(6): # retry up to 6 times
779
  current_key, current_model = mgr.get_key_and_model()
780
 
781
  if not current_key or current_key == "NO_API_KEY_FOUND":
782
+ last_err = "No valid API key found. All keys in cooldown."
783
+ print(f"All keys in 60s cooldown. Waiting 15 seconds... (Attempt {attempt+1}/6)")
784
+ import time
785
+ time.sleep(15)
786
+ continue
787
 
788
  try:
789
  # Added a shorter timeout to fail fast on stuck networks