GuXiaopeng commited on
Commit
13de46a
·
verified ·
1 Parent(s): bb6127c

Update core/pipeline_agent.py

Browse files
Files changed (1) hide show
  1. core/pipeline_agent.py +23 -15
core/pipeline_agent.py CHANGED
@@ -10,9 +10,17 @@ import threading
10
  import math
11
  import random
12
  import tiktoken
 
13
  import logging
14
  from tenacity import retry, stop_after_attempt, wait_exponential, retry_if_exception_type
15
 
 
 
 
 
 
 
 
16
  try:
17
  from langdetect import detect
18
  LANGDETECT_AVAILABLE = True
@@ -119,13 +127,17 @@ class SSciPipelineAgent:
119
  except Exception:
120
  return False
121
 
122
- @retry(stop=stop_after_attempt(3), wait=wait_exponential(multiplier=1, min=2, max=10), retry=retry_if_exception_type((requests.exceptions.Timeout, requests.exceptions.ConnectionError, requests.exceptions.HTTPError)))
123
  def _requests_get_with_retry(self, url, headers=None, params=None, timeout=15):
124
  resp = requests.get(url, headers=headers, params=params, timeout=timeout)
125
  if resp.status_code == 429:
126
- time.sleep(5)
 
 
 
 
127
  resp.raise_for_status()
128
- resp.raise_for_status()
129
  return resp
130
 
131
  def _is_english_dominant(self, text):
@@ -174,6 +186,7 @@ class SSciPipelineAgent:
174
 
175
  @retry(stop=stop_after_attempt(3), wait=wait_exponential(multiplier=1, min=2, max=10))
176
  def call_deepseek(self, system_prompt, user_prompt, max_tokens=8000, temp=0.3, model_name="deepseek-reasoner"):
 
177
  print(f"🚀 [API-流式保活] 正在调用 {model_name}...")
178
  with self.lock: self.ds_calls += 1
179
 
@@ -188,10 +201,9 @@ class SSciPipelineAgent:
188
  full_content = ""
189
  try:
190
  with requests.post(self.deepseek_api_url, headers={"Authorization": f"Bearer {self.ds_key}"}, json=payload, stream=True, timeout=120) as response:
191
- # 🚨 如果 API 报错(如 402欠费, 401未授权, 502宕机),立刻抓取错误信息并抛到前端网页!
192
  if response.status_code != 200:
193
  error_msg = f"DeepSeek API 拒绝访问 (状态码: {response.status_code})。详情: {response.text}"
194
- st.toast(f"❌ {error_msg}", icon="🚨") # 在网页右下角弹窗报警
195
  raise requests.exceptions.HTTPError(error_msg)
196
 
197
  for line in response.iter_lines():
@@ -238,11 +250,8 @@ class SSciPipelineAgent:
238
  yield content
239
 
240
  except Exception:
241
- # 🚨 核心修复:把裸的 except: 改为 except Exception:
242
- # GeneratorExit 属于 BaseException,这样就不会被拦截,允许 Streamlit 正常杀死流式输出
243
  pass
244
  except GeneratorExit:
245
- # 🚨 明确允许生成器退出,彻底消灭僵尸线程
246
  response.close()
247
  raise
248
 
@@ -395,25 +404,22 @@ class SSciPipelineAgent:
395
  except: time.sleep(2)
396
  return []
397
 
398
- @retry(stop=stop_after_attempt(2), wait=wait_exponential(multiplier=1, min=1, max=3))
399
- @retry(stop=stop_after_attempt(2), wait=wait_exponential(multiplier=1, min=1, max=3))
400
  def _fetch_openalex_page(self, search_term, page, filter_str):
401
  headers = {"User-Agent": "mailto:ssci_agent@example.com"}
402
  if self.openalex_key: headers["Authorization"] = f"Bearer {self.openalex_key}"
403
 
404
- # 🚨 核心修复:
405
- # 1. 改为 title_and_abstract.search 提高命中精度
406
- # 2. 移除 sort 参数,利用 OpenAlex 默认的 Relevance 相关性排序,打破同质化去重陷阱
407
  params = {
408
- "title_and_abstract.search": search_term,
409
  "per-page": self.lit_per_page,
410
  "page": page,
411
  "filter": filter_str
412
  }
 
 
413
  resp = self._requests_get_with_retry(self.openalex_api_url, headers=headers, params=params, timeout=15)
414
  return resp.json().get("results", [])
415
 
416
- def fetch_references_parallel(self, topic, keywords, min_required=50):
417
  with self.lock: self.citation_map.clear()
418
  llm_rag_context = []
419
  seen_identifiers = set()
@@ -522,6 +528,7 @@ class SSciPipelineAgent:
522
 
523
  if added_in_this_batch == 0: break
524
  page_batch += 1
 
525
 
526
  if len(self.citation_map) < self.lit_target_papers:
527
  for term in current_batch_terms:
@@ -583,6 +590,7 @@ class SSciPipelineAgent:
583
  if added_in_this_batch == 0: break
584
  offset += 50
585
  time.sleep(0.5)
 
586
 
587
  return "\n".join(llm_rag_context)
588
 
 
10
  import math
11
  import random
12
  import tiktoken
13
+ import sys
14
  import logging
15
  from tenacity import retry, stop_after_attempt, wait_exponential, retry_if_exception_type
16
 
17
+ # 配置全局日志,直接输出到 HF 容器标准输出
18
+ logging.basicConfig(
19
+ level=logging.INFO,
20
+ format="%(asctime)s [%(levelname)s] %(message)s",
21
+ handlers=[logging.StreamHandler(sys.stdout)]
22
+ )
23
+
24
  try:
25
  from langdetect import detect
26
  LANGDETECT_AVAILABLE = True
 
127
  except Exception:
128
  return False
129
 
130
+ @retry(stop=stop_after_attempt(4), wait=wait_exponential(multiplier=1, min=2, max=10), retry=retry_if_exception_type((requests.exceptions.Timeout, requests.exceptions.ConnectionError, requests.exceptions.HTTPError)))
131
  def _requests_get_with_retry(self, url, headers=None, params=None, timeout=15):
132
  resp = requests.get(url, headers=headers, params=params, timeout=timeout)
133
  if resp.status_code == 429:
134
+ logging.warning(f"⚠️ [API 限流] 触发 429 Too Many Requests,Tenacity 将自动等待并重试。URL: {url}")
135
+ resp.raise_for_status()
136
+
137
+ if resp.status_code != 200:
138
+ logging.error(f"❌ [API 错误] 状态码: {resp.status_code}, 响应: {resp.text}")
139
  resp.raise_for_status()
140
+
141
  return resp
142
 
143
  def _is_english_dominant(self, text):
 
186
 
187
  @retry(stop=stop_after_attempt(3), wait=wait_exponential(multiplier=1, min=2, max=10))
188
  def call_deepseek(self, system_prompt, user_prompt, max_tokens=8000, temp=0.3, model_name="deepseek-reasoner"):
189
+ logging.info(f"🚀 [DeepSeek API] 调用模型: {model_name}, 提示词预计 Tokens: {self._count_tokens(user_prompt)}")
190
  print(f"🚀 [API-流式保活] 正在调用 {model_name}...")
191
  with self.lock: self.ds_calls += 1
192
 
 
201
  full_content = ""
202
  try:
203
  with requests.post(self.deepseek_api_url, headers={"Authorization": f"Bearer {self.ds_key}"}, json=payload, stream=True, timeout=120) as response:
 
204
  if response.status_code != 200:
205
  error_msg = f"DeepSeek API 拒绝访问 (状态码: {response.status_code})。详情: {response.text}"
206
+ st.toast(f"❌ {error_msg}", icon="🚨")
207
  raise requests.exceptions.HTTPError(error_msg)
208
 
209
  for line in response.iter_lines():
 
250
  yield content
251
 
252
  except Exception:
 
 
253
  pass
254
  except GeneratorExit:
 
255
  response.close()
256
  raise
257
 
 
404
  except: time.sleep(2)
405
  return []
406
 
 
 
407
  def _fetch_openalex_page(self, search_term, page, filter_str):
408
  headers = {"User-Agent": "mailto:ssci_agent@example.com"}
409
  if self.openalex_key: headers["Authorization"] = f"Bearer {self.openalex_key}"
410
 
 
 
 
411
  params = {
412
+ "search": search_term,
413
  "per-page": self.lit_per_page,
414
  "page": page,
415
  "filter": filter_str
416
  }
417
+
418
+ logging.info(f"🔍 [OpenAlex 检索] 关键词: '{search_term}', 页码: {page}")
419
  resp = self._requests_get_with_retry(self.openalex_api_url, headers=headers, params=params, timeout=15)
420
  return resp.json().get("results", [])
421
 
422
+ def fetch_references_parallel(self, topic, keywords, min_required=50, progress_callback=None):
423
  with self.lock: self.citation_map.clear()
424
  llm_rag_context = []
425
  seen_identifiers = set()
 
528
 
529
  if added_in_this_batch == 0: break
530
  page_batch += 1
531
+ if progress_callback: progress_callback()
532
 
533
  if len(self.citation_map) < self.lit_target_papers:
534
  for term in current_batch_terms:
 
590
  if added_in_this_batch == 0: break
591
  offset += 50
592
  time.sleep(0.5)
593
+ if progress_callback: progress_callback()
594
 
595
  return "\n".join(llm_rag_context)
596