Commit ·
f7abdcf
1
Parent(s): d0d7322
Fix extraction hanging indefinitely by adding fast fail logic for API rate limits and errors
Browse files- workflow/step3_extract.py +32 -5
workflow/step3_extract.py
CHANGED
|
@@ -428,10 +428,17 @@ async def _ask_ai_with_retry(img_bytes: bytes, full_prompt: str, mgr, force_reex
|
|
| 428 |
|
| 429 |
last_err = ""
|
| 430 |
max_attempts = max(6, mgr.key_count() * 2)
|
|
|
|
|
|
|
|
|
|
| 431 |
for attempt in range(max_attempts):
|
| 432 |
current_key, current_model = mgr.get_key_and_model()
|
| 433 |
if not current_key or current_key == "NO_API_KEY_FOUND":
|
| 434 |
last_err = "No valid API key found. All keys are in cooldown."
|
|
|
|
|
|
|
|
|
|
|
|
|
| 435 |
print(f"All keys in cooldown. Waiting 15 seconds... (Attempt {attempt+1}/{max_attempts})")
|
| 436 |
import time
|
| 437 |
time.sleep(15)
|
|
@@ -471,6 +478,10 @@ async def _ask_ai_with_retry(img_bytes: bytes, full_prompt: str, mgr, force_reex
|
|
| 471 |
except Exception as e:
|
| 472 |
last_err = str(e)
|
| 473 |
mgr.mark_rate_limited(current_key, current_model)
|
|
|
|
|
|
|
|
|
|
|
|
|
| 474 |
await asyncio.sleep(2)
|
| 475 |
|
| 476 |
return json.dumps({"_error": last_err})
|
|
@@ -920,13 +931,19 @@ Return ONLY this JSON structure (null for not found):
|
|
| 920 |
print(f"Cache read error: {cache_ex}")
|
| 921 |
|
| 922 |
last_err = ""
|
| 923 |
-
|
| 924 |
max_attempts = max(6, mgr.key_count() * 2)
|
|
|
|
|
|
|
|
|
|
| 925 |
for attempt in range(max_attempts):
|
| 926 |
current_key, current_model = mgr.get_key_and_model()
|
| 927 |
|
| 928 |
if not current_key or current_key == "NO_API_KEY_FOUND":
|
| 929 |
last_err = "No valid API key found. All keys in cooldown."
|
|
|
|
|
|
|
|
|
|
|
|
|
| 930 |
print(f"All keys in cooldown. Waiting 15 seconds... (Attempt {attempt+1}/{max_attempts})")
|
| 931 |
import time
|
| 932 |
time.sleep(15)
|
|
@@ -1016,14 +1033,25 @@ Return ONLY this JSON structure (null for not found):
|
|
| 1016 |
print(f"Cache write error: {cache_ex}")
|
| 1017 |
|
| 1018 |
return data
|
|
|
|
| 1019 |
except Exception as e:
|
| 1020 |
last_err = str(e)
|
| 1021 |
if any(code in last_err for code in ("429", "RESOURCE_EXHAUSTED", "quota")):
|
| 1022 |
mgr.mark_rate_limited(current_key, current_model)
|
| 1023 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1024 |
time.sleep(2)
|
| 1025 |
-
|
| 1026 |
-
|
|
|
|
|
|
|
| 1027 |
# API Fallback Mechanism: Return standard fallback object instead of failing completely
|
| 1028 |
fallback_data = {
|
| 1029 |
"_ok": True,
|
|
@@ -1061,7 +1089,6 @@ Return ONLY this JSON structure (null for not found):
|
|
| 1061 |
|
| 1062 |
return fallback_data
|
| 1063 |
|
| 1064 |
-
|
| 1065 |
def merge_schedule_json_to_results(results: dict):
|
| 1066 |
"""
|
| 1067 |
Loads _schedule_data.json, _arch_data.json, and _openings_data.json
|
|
|
|
| 428 |
|
| 429 |
last_err = ""
|
| 430 |
max_attempts = max(6, mgr.key_count() * 2)
|
| 431 |
+
cooldown_waits = 0
|
| 432 |
+
consecutive_failures = 0
|
| 433 |
+
|
| 434 |
for attempt in range(max_attempts):
|
| 435 |
current_key, current_model = mgr.get_key_and_model()
|
| 436 |
if not current_key or current_key == "NO_API_KEY_FOUND":
|
| 437 |
last_err = "No valid API key found. All keys are in cooldown."
|
| 438 |
+
cooldown_waits += 1
|
| 439 |
+
if cooldown_waits > 3: # Max 45 seconds total waiting for cooldown
|
| 440 |
+
print("Aborting _ask_ai_with_retry after 45s of total cooldown.")
|
| 441 |
+
break
|
| 442 |
print(f"All keys in cooldown. Waiting 15 seconds... (Attempt {attempt+1}/{max_attempts})")
|
| 443 |
import time
|
| 444 |
time.sleep(15)
|
|
|
|
| 478 |
except Exception as e:
|
| 479 |
last_err = str(e)
|
| 480 |
mgr.mark_rate_limited(current_key, current_model)
|
| 481 |
+
consecutive_failures += 1
|
| 482 |
+
if consecutive_failures > mgr.key_count() + 1:
|
| 483 |
+
print(f"Aborting _ask_ai_with_retry after {consecutive_failures} consecutive failures. Last error: {last_err}")
|
| 484 |
+
break
|
| 485 |
await asyncio.sleep(2)
|
| 486 |
|
| 487 |
return json.dumps({"_error": last_err})
|
|
|
|
| 931 |
print(f"Cache read error: {cache_ex}")
|
| 932 |
|
| 933 |
last_err = ""
|
|
|
|
| 934 |
max_attempts = max(6, mgr.key_count() * 2)
|
| 935 |
+
cooldown_waits = 0
|
| 936 |
+
consecutive_failures = 0
|
| 937 |
+
|
| 938 |
for attempt in range(max_attempts):
|
| 939 |
current_key, current_model = mgr.get_key_and_model()
|
| 940 |
|
| 941 |
if not current_key or current_key == "NO_API_KEY_FOUND":
|
| 942 |
last_err = "No valid API key found. All keys in cooldown."
|
| 943 |
+
cooldown_waits += 1
|
| 944 |
+
if cooldown_waits > 3: # Max 45s wait
|
| 945 |
+
print("Aborting synchronous extraction after 45s of total cooldown.")
|
| 946 |
+
break
|
| 947 |
print(f"All keys in cooldown. Waiting 15 seconds... (Attempt {attempt+1}/{max_attempts})")
|
| 948 |
import time
|
| 949 |
time.sleep(15)
|
|
|
|
| 1033 |
print(f"Cache write error: {cache_ex}")
|
| 1034 |
|
| 1035 |
return data
|
| 1036 |
+
|
| 1037 |
except Exception as e:
|
| 1038 |
last_err = str(e)
|
| 1039 |
if any(code in last_err for code in ("429", "RESOURCE_EXHAUSTED", "quota")):
|
| 1040 |
mgr.mark_rate_limited(current_key, current_model)
|
| 1041 |
+
else:
|
| 1042 |
+
# If it's a 403 or other error, still mark it to avoid spamming the same broken key
|
| 1043 |
+
mgr.mark_rate_limited(current_key, current_model)
|
| 1044 |
+
|
| 1045 |
+
consecutive_failures += 1
|
| 1046 |
+
if consecutive_failures > mgr.key_count() + 1:
|
| 1047 |
+
print(f"Aborting synchronous extraction after {consecutive_failures} consecutive failures. Last error: {last_err}")
|
| 1048 |
+
break
|
| 1049 |
+
|
| 1050 |
time.sleep(2)
|
| 1051 |
+
# Cap exponential backoff to avoid hanging
|
| 1052 |
+
sleep_time = min(10, 2 * (attempt + 1))
|
| 1053 |
+
time.sleep(sleep_time)
|
| 1054 |
+
|
| 1055 |
# API Fallback Mechanism: Return standard fallback object instead of failing completely
|
| 1056 |
fallback_data = {
|
| 1057 |
"_ok": True,
|
|
|
|
| 1089 |
|
| 1090 |
return fallback_data
|
| 1091 |
|
|
|
|
| 1092 |
def merge_schedule_json_to_results(results: dict):
|
| 1093 |
"""
|
| 1094 |
Loads _schedule_data.json, _arch_data.json, and _openings_data.json
|