Spaces:
Sleeping
Sleeping
Update core/pipeline_agent.py
Browse files- core/pipeline_agent.py +43 -11
core/pipeline_agent.py
CHANGED
|
@@ -133,7 +133,11 @@ class SSciPipelineAgent:
|
|
| 133 |
sys_prompt = f"Translate the following text into {target_lang}. Preserve all formatting, line breaks, and [CITE-XX] tags exactly. Output ONLY the translation."
|
| 134 |
return self.call_deepseek(sys_prompt, text, max_tokens=max_tokens, temp=0.2, model_name="deepseek-chat")
|
| 135 |
|
| 136 |
-
@retry(
|
|
|
|
|
|
|
|
|
|
|
|
|
| 137 |
def _requests_get_with_retry(self, url, headers=None, params=None, timeout=15):
|
| 138 |
resp = requests.get(url, headers=headers, params=params, timeout=timeout)
|
| 139 |
if resp.status_code == 429:
|
|
@@ -307,7 +311,14 @@ class SSciPipelineAgent:
|
|
| 307 |
yield content
|
| 308 |
except Exception: pass
|
| 309 |
return
|
| 310 |
-
except GeneratorExit:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 311 |
except Exception as e:
|
| 312 |
logging.error(f"❌ [流式中断] 尝试 {attempt+1}/3 异常: {e}")
|
| 313 |
if attempt == 2: raise
|
|
@@ -488,7 +499,12 @@ class SSciPipelineAgent:
|
|
| 488 |
return resp.json().get("results", [])
|
| 489 |
|
| 490 |
def fetch_references_parallel(self, topic, keywords, min_required=50, progress_callback=None):
|
| 491 |
-
with self.lock:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 492 |
llm_rag_context = []
|
| 493 |
seen_identifiers = set()
|
| 494 |
filters_to_try = ["language:en,type:article,has_doi:true,is_paratext:false", "language:en,type:article,is_paratext:false", "type:article,is_paratext:false", "language:en,is_paratext:false"]
|
|
@@ -496,6 +512,10 @@ class SSciPipelineAgent:
|
|
| 496 |
|
| 497 |
for expansion_phase in range(3):
|
| 498 |
if len(self.citation_map) >= min_required: break
|
|
|
|
|
|
|
|
|
|
|
|
|
| 499 |
|
| 500 |
if expansion_phase == 0: search_terms = self.generate_search_queries(topic, keywords)
|
| 501 |
else:
|
|
@@ -514,10 +534,10 @@ class SSciPipelineAgent:
|
|
| 514 |
if not current_batch_terms: continue
|
| 515 |
for t in current_batch_terms: all_used_terms.add(t)
|
| 516 |
|
| 517 |
-
if not
|
| 518 |
for term in current_batch_terms:
|
| 519 |
if len(self.citation_map) >= self.lit_target_papers: break
|
| 520 |
-
if
|
| 521 |
|
| 522 |
offset = 0
|
| 523 |
while len(self.citation_map) < self.lit_target_papers and offset < 500:
|
|
@@ -578,20 +598,32 @@ class SSciPipelineAgent:
|
|
| 578 |
offset += 50
|
| 579 |
if progress_callback: progress_callback()
|
| 580 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 581 |
except Exception as e:
|
| 582 |
-
logging.error(f"❌
|
| 583 |
self.s2_fused = True
|
| 584 |
-
logging.warning("🚨 [全局熔断] S2 引擎物理熔断!将跳过 S2,移交 OpenAlex 备用引擎。")
|
| 585 |
break
|
| 586 |
|
| 587 |
-
if len(self.citation_map) < self.lit_target_papers and not
|
| 588 |
for filter_str in filters_to_try:
|
| 589 |
if len(self.citation_map) >= self.lit_target_papers: break
|
| 590 |
-
if
|
| 591 |
|
| 592 |
for term in current_batch_terms:
|
| 593 |
if len(self.citation_map) >= self.lit_target_papers: break
|
| 594 |
-
if
|
| 595 |
|
| 596 |
page_batch = 1
|
| 597 |
while len(self.citation_map) < self.lit_target_papers and page_batch <= self.lit_max_page_batches:
|
|
@@ -609,7 +641,7 @@ class SSciPipelineAgent:
|
|
| 609 |
logging.warning("🚨 [全局熔断] OpenAlex 引擎也发生物理熔断!")
|
| 610 |
break
|
| 611 |
|
| 612 |
-
if
|
| 613 |
|
| 614 |
added_in_this_batch = 0
|
| 615 |
for item in raw_results:
|
|
|
|
| 133 |
sys_prompt = f"Translate the following text into {target_lang}. Preserve all formatting, line breaks, and [CITE-XX] tags exactly. Output ONLY the translation."
|
| 134 |
return self.call_deepseek(sys_prompt, text, max_tokens=max_tokens, temp=0.2, model_name="deepseek-chat")
|
| 135 |
|
| 136 |
+
@retry(
|
| 137 |
+
stop=stop_after_attempt(3),
|
| 138 |
+
wait=wait_exponential(multiplier=1, min=2, max=10),
|
| 139 |
+
retry=retry_if_exception_type((requests.exceptions.Timeout, requests.exceptions.ConnectionError))
|
| 140 |
+
)
|
| 141 |
def _requests_get_with_retry(self, url, headers=None, params=None, timeout=15):
|
| 142 |
resp = requests.get(url, headers=headers, params=params, timeout=timeout)
|
| 143 |
if resp.status_code == 429:
|
|
|
|
| 311 |
yield content
|
| 312 |
except Exception: pass
|
| 313 |
return
|
| 314 |
+
except GeneratorExit:
|
| 315 |
+
raise
|
| 316 |
+
except (requests.exceptions.ConnectionError, ConnectionResetError) as e:
|
| 317 |
+
logging.error(f"❌ [流式连接重置] 尝试 {attempt+1}/3 异常: {e}")
|
| 318 |
+
if attempt == 2:
|
| 319 |
+
raise
|
| 320 |
+
time.sleep(2 ** attempt)
|
| 321 |
+
continue
|
| 322 |
except Exception as e:
|
| 323 |
logging.error(f"❌ [流式中断] 尝试 {attempt+1}/3 异常: {e}")
|
| 324 |
if attempt == 2: raise
|
|
|
|
| 499 |
return resp.json().get("results", [])
|
| 500 |
|
| 501 |
def fetch_references_parallel(self, topic, keywords, min_required=50, progress_callback=None):
|
| 502 |
+
with self.lock:
|
| 503 |
+
self.citation_map.clear()
|
| 504 |
+
# 每轮检索前重置熔断标志,允许恢复尝试
|
| 505 |
+
self.s2_fused = False
|
| 506 |
+
self.oa_fused = False
|
| 507 |
+
|
| 508 |
llm_rag_context = []
|
| 509 |
seen_identifiers = set()
|
| 510 |
filters_to_try = ["language:en,type:article,has_doi:true,is_paratext:false", "language:en,type:article,is_paratext:false", "type:article,is_paratext:false", "language:en,is_paratext:false"]
|
|
|
|
| 512 |
|
| 513 |
for expansion_phase in range(3):
|
| 514 |
if len(self.citation_map) >= min_required: break
|
| 515 |
+
|
| 516 |
+
# 重置熔断标志,让之前失败的源有机会重新尝试
|
| 517 |
+
self.s2_fused = False
|
| 518 |
+
self.oa_fused = False
|
| 519 |
|
| 520 |
if expansion_phase == 0: search_terms = self.generate_search_queries(topic, keywords)
|
| 521 |
else:
|
|
|
|
| 534 |
if not current_batch_terms: continue
|
| 535 |
for t in current_batch_terms: all_used_terms.add(t)
|
| 536 |
|
| 537 |
+
if not self.s2_fused:
|
| 538 |
for term in current_batch_terms:
|
| 539 |
if len(self.citation_map) >= self.lit_target_papers: break
|
| 540 |
+
if self.s2_fused: break
|
| 541 |
|
| 542 |
offset = 0
|
| 543 |
while len(self.citation_map) < self.lit_target_papers and offset < 500:
|
|
|
|
| 598 |
offset += 50
|
| 599 |
if progress_callback: progress_callback()
|
| 600 |
|
| 601 |
+
except requests.exceptions.HTTPError as e:
|
| 602 |
+
if e.response.status_code == 400:
|
| 603 |
+
# 400 表示 offset 超出范围,已无更多数据,正常终止翻页
|
| 604 |
+
logging.info(f"ℹ️ [S2] 关键词 '{term}' 已无更多结果 (offset={offset}),正常结束。")
|
| 605 |
+
break
|
| 606 |
+
elif e.response.status_code == 429:
|
| 607 |
+
logging.warning(f"⚠️ [S2 限流] 等待 5 秒后重试...")
|
| 608 |
+
time.sleep(5)
|
| 609 |
+
continue
|
| 610 |
+
else:
|
| 611 |
+
logging.error(f"❌ [S2 致命错误] 状态码 {e.response.status_code},熔断 S2。")
|
| 612 |
+
self.s2_fused = True
|
| 613 |
+
break
|
| 614 |
except Exception as e:
|
| 615 |
+
logging.error(f"❌ [S2 网络异常] {e},熔断 S2。")
|
| 616 |
self.s2_fused = True
|
|
|
|
| 617 |
break
|
| 618 |
|
| 619 |
+
if len(self.citation_map) < self.lit_target_papers and not self.oa_fused:
|
| 620 |
for filter_str in filters_to_try:
|
| 621 |
if len(self.citation_map) >= self.lit_target_papers: break
|
| 622 |
+
if self.oa_fused: break
|
| 623 |
|
| 624 |
for term in current_batch_terms:
|
| 625 |
if len(self.citation_map) >= self.lit_target_papers: break
|
| 626 |
+
if self.oa_fused: break
|
| 627 |
|
| 628 |
page_batch = 1
|
| 629 |
while len(self.citation_map) < self.lit_target_papers and page_batch <= self.lit_max_page_batches:
|
|
|
|
| 641 |
logging.warning("🚨 [全局熔断] OpenAlex 引擎也发生物理熔断!")
|
| 642 |
break
|
| 643 |
|
| 644 |
+
if self.oa_fused: break
|
| 645 |
|
| 646 |
added_in_this_batch = 0
|
| 647 |
for item in raw_results:
|