GuXiaopeng commited on
Commit
e85d27c
·
verified ·
1 Parent(s): dd23cbc

Update core/pipeline_agent.py

Browse files
Files changed (1) hide show
  1. core/pipeline_agent.py +43 -11
core/pipeline_agent.py CHANGED
@@ -133,7 +133,11 @@ class SSciPipelineAgent:
133
  sys_prompt = f"Translate the following text into {target_lang}. Preserve all formatting, line breaks, and [CITE-XX] tags exactly. Output ONLY the translation."
134
  return self.call_deepseek(sys_prompt, text, max_tokens=max_tokens, temp=0.2, model_name="deepseek-chat")
135
 
136
- @retry(stop=stop_after_attempt(3), wait=wait_random_exponential(multiplier=1, max=10), retry=retry_if_exception_type((requests.exceptions.Timeout, requests.exceptions.ConnectionError, requests.exceptions.HTTPError)))
 
 
 
 
137
  def _requests_get_with_retry(self, url, headers=None, params=None, timeout=15):
138
  resp = requests.get(url, headers=headers, params=params, timeout=timeout)
139
  if resp.status_code == 429:
@@ -307,7 +311,14 @@ class SSciPipelineAgent:
307
  yield content
308
  except Exception: pass
309
  return
310
- except GeneratorExit: raise
 
 
 
 
 
 
 
311
  except Exception as e:
312
  logging.error(f"❌ [流式中断] 尝试 {attempt+1}/3 异常: {e}")
313
  if attempt == 2: raise
@@ -488,7 +499,12 @@ class SSciPipelineAgent:
488
  return resp.json().get("results", [])
489
 
490
  def fetch_references_parallel(self, topic, keywords, min_required=50, progress_callback=None):
491
- with self.lock: self.citation_map.clear()
 
 
 
 
 
492
  llm_rag_context = []
493
  seen_identifiers = set()
494
  filters_to_try = ["language:en,type:article,has_doi:true,is_paratext:false", "language:en,type:article,is_paratext:false", "type:article,is_paratext:false", "language:en,is_paratext:false"]
@@ -496,6 +512,10 @@ class SSciPipelineAgent:
496
 
497
  for expansion_phase in range(3):
498
  if len(self.citation_map) >= min_required: break
 
 
 
 
499
 
500
  if expansion_phase == 0: search_terms = self.generate_search_queries(topic, keywords)
501
  else:
@@ -514,10 +534,10 @@ class SSciPipelineAgent:
514
  if not current_batch_terms: continue
515
  for t in current_batch_terms: all_used_terms.add(t)
516
 
517
- if not getattr(self, 's2_fused', False):
518
  for term in current_batch_terms:
519
  if len(self.citation_map) >= self.lit_target_papers: break
520
- if getattr(self, 's2_fused', False): break
521
 
522
  offset = 0
523
  while len(self.citation_map) < self.lit_target_papers and offset < 500:
@@ -578,20 +598,32 @@ class SSciPipelineAgent:
578
  offset += 50
579
  if progress_callback: progress_callback()
580
 
 
 
 
 
 
 
 
 
 
 
 
 
 
581
  except Exception as e:
582
- logging.error(f"❌ Semantic Scholar API 拒绝响应或异常: {e}")
583
  self.s2_fused = True
584
- logging.warning("🚨 [全局熔断] S2 引擎物理熔断!将跳过 S2,移交 OpenAlex 备用引擎。")
585
  break
586
 
587
- if len(self.citation_map) < self.lit_target_papers and not getattr(self, 'oa_fused', False):
588
  for filter_str in filters_to_try:
589
  if len(self.citation_map) >= self.lit_target_papers: break
590
- if getattr(self, 'oa_fused', False): break
591
 
592
  for term in current_batch_terms:
593
  if len(self.citation_map) >= self.lit_target_papers: break
594
- if getattr(self, 'oa_fused', False): break
595
 
596
  page_batch = 1
597
  while len(self.citation_map) < self.lit_target_papers and page_batch <= self.lit_max_page_batches:
@@ -609,7 +641,7 @@ class SSciPipelineAgent:
609
  logging.warning("🚨 [全局熔断] OpenAlex 引擎也发生物理熔断!")
610
  break
611
 
612
- if getattr(self, 'oa_fused', False): break
613
 
614
  added_in_this_batch = 0
615
  for item in raw_results:
 
133
  sys_prompt = f"Translate the following text into {target_lang}. Preserve all formatting, line breaks, and [CITE-XX] tags exactly. Output ONLY the translation."
134
  return self.call_deepseek(sys_prompt, text, max_tokens=max_tokens, temp=0.2, model_name="deepseek-chat")
135
 
136
+ @retry(
137
+ stop=stop_after_attempt(3),
138
+ wait=wait_exponential(multiplier=1, min=2, max=10),
139
+ retry=retry_if_exception_type((requests.exceptions.Timeout, requests.exceptions.ConnectionError))
140
+ )
141
  def _requests_get_with_retry(self, url, headers=None, params=None, timeout=15):
142
  resp = requests.get(url, headers=headers, params=params, timeout=timeout)
143
  if resp.status_code == 429:
 
311
  yield content
312
  except Exception: pass
313
  return
314
+ except GeneratorExit:
315
+ raise
316
+ except (requests.exceptions.ConnectionError, ConnectionResetError) as e:
317
+ logging.error(f"❌ [流式连接重置] 尝试 {attempt+1}/3 异常: {e}")
318
+ if attempt == 2:
319
+ raise
320
+ time.sleep(2 ** attempt)
321
+ continue
322
  except Exception as e:
323
  logging.error(f"❌ [流式中断] 尝试 {attempt+1}/3 异常: {e}")
324
  if attempt == 2: raise
 
499
  return resp.json().get("results", [])
500
 
501
  def fetch_references_parallel(self, topic, keywords, min_required=50, progress_callback=None):
502
+ with self.lock:
503
+ self.citation_map.clear()
504
+ # 每轮检索前重置熔断标志,允许恢复尝试
505
+ self.s2_fused = False
506
+ self.oa_fused = False
507
+
508
  llm_rag_context = []
509
  seen_identifiers = set()
510
  filters_to_try = ["language:en,type:article,has_doi:true,is_paratext:false", "language:en,type:article,is_paratext:false", "type:article,is_paratext:false", "language:en,is_paratext:false"]
 
512
 
513
  for expansion_phase in range(3):
514
  if len(self.citation_map) >= min_required: break
515
+
516
+ # 重置熔断标志,让之前失败的源有机会重新尝试
517
+ self.s2_fused = False
518
+ self.oa_fused = False
519
 
520
  if expansion_phase == 0: search_terms = self.generate_search_queries(topic, keywords)
521
  else:
 
534
  if not current_batch_terms: continue
535
  for t in current_batch_terms: all_used_terms.add(t)
536
 
537
+ if not self.s2_fused:
538
  for term in current_batch_terms:
539
  if len(self.citation_map) >= self.lit_target_papers: break
540
+ if self.s2_fused: break
541
 
542
  offset = 0
543
  while len(self.citation_map) < self.lit_target_papers and offset < 500:
 
598
  offset += 50
599
  if progress_callback: progress_callback()
600
 
601
+ except requests.exceptions.HTTPError as e:
602
+ if e.response.status_code == 400:
603
+ # 400 表示 offset 超出范围,已无更多数据,正常终止翻页
604
+ logging.info(f"ℹ️ [S2] 关键词 '{term}' 已无更多结果 (offset={offset}),正常结束。")
605
+ break
606
+ elif e.response.status_code == 429:
607
+ logging.warning(f"⚠️ [S2 限流] 等待 5 秒后重试...")
608
+ time.sleep(5)
609
+ continue
610
+ else:
611
+ logging.error(f"❌ [S2 致命错误] 状态码 {e.response.status_code},熔断 S2。")
612
+ self.s2_fused = True
613
+ break
614
  except Exception as e:
615
+ logging.error(f"❌ [S2 网络异常] {e},熔断 S2。")
616
  self.s2_fused = True
 
617
  break
618
 
619
+ if len(self.citation_map) < self.lit_target_papers and not self.oa_fused:
620
  for filter_str in filters_to_try:
621
  if len(self.citation_map) >= self.lit_target_papers: break
622
+ if self.oa_fused: break
623
 
624
  for term in current_batch_terms:
625
  if len(self.citation_map) >= self.lit_target_papers: break
626
+ if self.oa_fused: break
627
 
628
  page_batch = 1
629
  while len(self.citation_map) < self.lit_target_papers and page_batch <= self.lit_max_page_batches:
 
641
  logging.warning("🚨 [全局熔断] OpenAlex 引擎也发生物理熔断!")
642
  break
643
 
644
+ if self.oa_fused: break
645
 
646
  added_in_this_batch = 0
647
  for item in raw_results: