basel00omar commited on
Commit
f7abdcf
·
1 Parent(s): d0d7322

Fix extraction hanging indefinitely by adding fast fail logic for API rate limits and errors

Browse files
Files changed (1) hide show
  1. workflow/step3_extract.py +32 -5
workflow/step3_extract.py CHANGED
@@ -428,10 +428,17 @@ async def _ask_ai_with_retry(img_bytes: bytes, full_prompt: str, mgr, force_reex
428
 
429
  last_err = ""
430
  max_attempts = max(6, mgr.key_count() * 2)
 
 
 
431
  for attempt in range(max_attempts):
432
  current_key, current_model = mgr.get_key_and_model()
433
  if not current_key or current_key == "NO_API_KEY_FOUND":
434
  last_err = "No valid API key found. All keys are in cooldown."
 
 
 
 
435
  print(f"All keys in cooldown. Waiting 15 seconds... (Attempt {attempt+1}/{max_attempts})")
436
  import time
437
  time.sleep(15)
@@ -471,6 +478,10 @@ async def _ask_ai_with_retry(img_bytes: bytes, full_prompt: str, mgr, force_reex
471
  except Exception as e:
472
  last_err = str(e)
473
  mgr.mark_rate_limited(current_key, current_model)
 
 
 
 
474
  await asyncio.sleep(2)
475
 
476
  return json.dumps({"_error": last_err})
@@ -920,13 +931,19 @@ Return ONLY this JSON structure (null for not found):
920
  print(f"Cache read error: {cache_ex}")
921
 
922
  last_err = ""
923
-
924
  max_attempts = max(6, mgr.key_count() * 2)
 
 
 
925
  for attempt in range(max_attempts):
926
  current_key, current_model = mgr.get_key_and_model()
927
 
928
  if not current_key or current_key == "NO_API_KEY_FOUND":
929
  last_err = "No valid API key found. All keys in cooldown."
 
 
 
 
930
  print(f"All keys in cooldown. Waiting 15 seconds... (Attempt {attempt+1}/{max_attempts})")
931
  import time
932
  time.sleep(15)
@@ -1016,14 +1033,25 @@ Return ONLY this JSON structure (null for not found):
1016
  print(f"Cache write error: {cache_ex}")
1017
 
1018
  return data
 
1019
  except Exception as e:
1020
  last_err = str(e)
1021
  if any(code in last_err for code in ("429", "RESOURCE_EXHAUSTED", "quota")):
1022
  mgr.mark_rate_limited(current_key, current_model)
1023
- continue
 
 
 
 
 
 
 
 
1024
  time.sleep(2)
1025
- time.sleep(2 * (attempt + 1))
1026
-
 
 
1027
  # API Fallback Mechanism: Return standard fallback object instead of failing completely
1028
  fallback_data = {
1029
  "_ok": True,
@@ -1061,7 +1089,6 @@ Return ONLY this JSON structure (null for not found):
1061
 
1062
  return fallback_data
1063
 
1064
-
1065
  def merge_schedule_json_to_results(results: dict):
1066
  """
1067
  Loads _schedule_data.json, _arch_data.json, and _openings_data.json
 
428
 
429
  last_err = ""
430
  max_attempts = max(6, mgr.key_count() * 2)
431
+ cooldown_waits = 0
432
+ consecutive_failures = 0
433
+
434
  for attempt in range(max_attempts):
435
  current_key, current_model = mgr.get_key_and_model()
436
  if not current_key or current_key == "NO_API_KEY_FOUND":
437
  last_err = "No valid API key found. All keys are in cooldown."
438
+ cooldown_waits += 1
439
+ if cooldown_waits > 3: # Max 45 seconds total waiting for cooldown
440
+ print("Aborting _ask_ai_with_retry after 45s of total cooldown.")
441
+ break
442
  print(f"All keys in cooldown. Waiting 15 seconds... (Attempt {attempt+1}/{max_attempts})")
443
  import time
444
  time.sleep(15)
 
478
  except Exception as e:
479
  last_err = str(e)
480
  mgr.mark_rate_limited(current_key, current_model)
481
+ consecutive_failures += 1
482
+ if consecutive_failures > mgr.key_count() + 1:
483
+ print(f"Aborting _ask_ai_with_retry after {consecutive_failures} consecutive failures. Last error: {last_err}")
484
+ break
485
  await asyncio.sleep(2)
486
 
487
  return json.dumps({"_error": last_err})
 
931
  print(f"Cache read error: {cache_ex}")
932
 
933
  last_err = ""
 
934
  max_attempts = max(6, mgr.key_count() * 2)
935
+ cooldown_waits = 0
936
+ consecutive_failures = 0
937
+
938
  for attempt in range(max_attempts):
939
  current_key, current_model = mgr.get_key_and_model()
940
 
941
  if not current_key or current_key == "NO_API_KEY_FOUND":
942
  last_err = "No valid API key found. All keys in cooldown."
943
+ cooldown_waits += 1
944
+ if cooldown_waits > 3: # Max 45s wait
945
+ print("Aborting synchronous extraction after 45s of total cooldown.")
946
+ break
947
  print(f"All keys in cooldown. Waiting 15 seconds... (Attempt {attempt+1}/{max_attempts})")
948
  import time
949
  time.sleep(15)
 
1033
  print(f"Cache write error: {cache_ex}")
1034
 
1035
  return data
1036
+
1037
  except Exception as e:
1038
  last_err = str(e)
1039
  if any(code in last_err for code in ("429", "RESOURCE_EXHAUSTED", "quota")):
1040
  mgr.mark_rate_limited(current_key, current_model)
1041
+ else:
1042
+ # If it's a 403 or other error, still mark it to avoid spamming the same broken key
1043
+ mgr.mark_rate_limited(current_key, current_model)
1044
+
1045
+ consecutive_failures += 1
1046
+ if consecutive_failures > mgr.key_count() + 1:
1047
+ print(f"Aborting synchronous extraction after {consecutive_failures} consecutive failures. Last error: {last_err}")
1048
+ break
1049
+
1050
  time.sleep(2)
1051
+ # Cap exponential backoff to avoid hanging
1052
+ sleep_time = min(10, 2 * (attempt + 1))
1053
+ time.sleep(sleep_time)
1054
+
1055
  # API Fallback Mechanism: Return standard fallback object instead of failing completely
1056
  fallback_data = {
1057
  "_ok": True,
 
1089
 
1090
  return fallback_data
1091
 
 
1092
  def merge_schedule_json_to_results(results: dict):
1093
  """
1094
  Loads _schedule_data.json, _arch_data.json, and _openings_data.json