Spaces:
Sleeping
Sleeping
Update core/pipeline_agent.py
Browse files- core/pipeline_agent.py +23 -15
core/pipeline_agent.py
CHANGED
|
@@ -10,9 +10,17 @@ import threading
|
|
| 10 |
import math
|
| 11 |
import random
|
| 12 |
import tiktoken
|
|
|
|
| 13 |
import logging
|
| 14 |
from tenacity import retry, stop_after_attempt, wait_exponential, retry_if_exception_type
|
| 15 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 16 |
try:
|
| 17 |
from langdetect import detect
|
| 18 |
LANGDETECT_AVAILABLE = True
|
|
@@ -119,13 +127,17 @@ class SSciPipelineAgent:
|
|
| 119 |
except Exception:
|
| 120 |
return False
|
| 121 |
|
| 122 |
-
@retry(stop=stop_after_attempt(
|
| 123 |
def _requests_get_with_retry(self, url, headers=None, params=None, timeout=15):
|
| 124 |
resp = requests.get(url, headers=headers, params=params, timeout=timeout)
|
| 125 |
if resp.status_code == 429:
|
| 126 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 127 |
resp.raise_for_status()
|
| 128 |
-
|
| 129 |
return resp
|
| 130 |
|
| 131 |
def _is_english_dominant(self, text):
|
|
@@ -174,6 +186,7 @@ class SSciPipelineAgent:
|
|
| 174 |
|
| 175 |
@retry(stop=stop_after_attempt(3), wait=wait_exponential(multiplier=1, min=2, max=10))
|
| 176 |
def call_deepseek(self, system_prompt, user_prompt, max_tokens=8000, temp=0.3, model_name="deepseek-reasoner"):
|
|
|
|
| 177 |
print(f"🚀 [API-流式保活] 正在调用 {model_name}...")
|
| 178 |
with self.lock: self.ds_calls += 1
|
| 179 |
|
|
@@ -188,10 +201,9 @@ class SSciPipelineAgent:
|
|
| 188 |
full_content = ""
|
| 189 |
try:
|
| 190 |
with requests.post(self.deepseek_api_url, headers={"Authorization": f"Bearer {self.ds_key}"}, json=payload, stream=True, timeout=120) as response:
|
| 191 |
-
# 🚨 如果 API 报错(如 402欠费, 401未授权, 502宕机),立刻抓取错误信息并抛到前端网页!
|
| 192 |
if response.status_code != 200:
|
| 193 |
error_msg = f"DeepSeek API 拒绝访问 (状态码: {response.status_code})。详情: {response.text}"
|
| 194 |
-
st.toast(f"❌ {error_msg}", icon="🚨")
|
| 195 |
raise requests.exceptions.HTTPError(error_msg)
|
| 196 |
|
| 197 |
for line in response.iter_lines():
|
|
@@ -238,11 +250,8 @@ class SSciPipelineAgent:
|
|
| 238 |
yield content
|
| 239 |
|
| 240 |
except Exception:
|
| 241 |
-
# 🚨 核心修复:把裸的 except: 改为 except Exception:
|
| 242 |
-
# GeneratorExit 属于 BaseException,这样就不会被拦截,允许 Streamlit 正常杀死流式输出
|
| 243 |
pass
|
| 244 |
except GeneratorExit:
|
| 245 |
-
# 🚨 明确允许生成器退出,彻底消灭僵尸线程
|
| 246 |
response.close()
|
| 247 |
raise
|
| 248 |
|
|
@@ -395,25 +404,22 @@ class SSciPipelineAgent:
|
|
| 395 |
except: time.sleep(2)
|
| 396 |
return []
|
| 397 |
|
| 398 |
-
@retry(stop=stop_after_attempt(2), wait=wait_exponential(multiplier=1, min=1, max=3))
|
| 399 |
-
@retry(stop=stop_after_attempt(2), wait=wait_exponential(multiplier=1, min=1, max=3))
|
| 400 |
def _fetch_openalex_page(self, search_term, page, filter_str):
|
| 401 |
headers = {"User-Agent": "mailto:ssci_agent@example.com"}
|
| 402 |
if self.openalex_key: headers["Authorization"] = f"Bearer {self.openalex_key}"
|
| 403 |
|
| 404 |
-
# 🚨 核心修复:
|
| 405 |
-
# 1. 改为 title_and_abstract.search 提高命中精度
|
| 406 |
-
# 2. 移除 sort 参数,利用 OpenAlex 默认的 Relevance 相关性排序,打破同质化去重陷阱
|
| 407 |
params = {
|
| 408 |
-
"
|
| 409 |
"per-page": self.lit_per_page,
|
| 410 |
"page": page,
|
| 411 |
"filter": filter_str
|
| 412 |
}
|
|
|
|
|
|
|
| 413 |
resp = self._requests_get_with_retry(self.openalex_api_url, headers=headers, params=params, timeout=15)
|
| 414 |
return resp.json().get("results", [])
|
| 415 |
|
| 416 |
-
def fetch_references_parallel(self, topic, keywords, min_required=50):
|
| 417 |
with self.lock: self.citation_map.clear()
|
| 418 |
llm_rag_context = []
|
| 419 |
seen_identifiers = set()
|
|
@@ -522,6 +528,7 @@ class SSciPipelineAgent:
|
|
| 522 |
|
| 523 |
if added_in_this_batch == 0: break
|
| 524 |
page_batch += 1
|
|
|
|
| 525 |
|
| 526 |
if len(self.citation_map) < self.lit_target_papers:
|
| 527 |
for term in current_batch_terms:
|
|
@@ -583,6 +590,7 @@ class SSciPipelineAgent:
|
|
| 583 |
if added_in_this_batch == 0: break
|
| 584 |
offset += 50
|
| 585 |
time.sleep(0.5)
|
|
|
|
| 586 |
|
| 587 |
return "\n".join(llm_rag_context)
|
| 588 |
|
|
|
|
| 10 |
import math
|
| 11 |
import random
|
| 12 |
import tiktoken
|
| 13 |
+
import sys
|
| 14 |
import logging
|
| 15 |
from tenacity import retry, stop_after_attempt, wait_exponential, retry_if_exception_type
|
| 16 |
|
| 17 |
+
# 配置全局日志,直接输出到 HF 容器标准输出
|
| 18 |
+
logging.basicConfig(
|
| 19 |
+
level=logging.INFO,
|
| 20 |
+
format="%(asctime)s [%(levelname)s] %(message)s",
|
| 21 |
+
handlers=[logging.StreamHandler(sys.stdout)]
|
| 22 |
+
)
|
| 23 |
+
|
| 24 |
try:
|
| 25 |
from langdetect import detect
|
| 26 |
LANGDETECT_AVAILABLE = True
|
|
|
|
| 127 |
except Exception:
|
| 128 |
return False
|
| 129 |
|
| 130 |
+
@retry(stop=stop_after_attempt(4), wait=wait_exponential(multiplier=1, min=2, max=10), retry=retry_if_exception_type((requests.exceptions.Timeout, requests.exceptions.ConnectionError, requests.exceptions.HTTPError)))
|
| 131 |
def _requests_get_with_retry(self, url, headers=None, params=None, timeout=15):
|
| 132 |
resp = requests.get(url, headers=headers, params=params, timeout=timeout)
|
| 133 |
if resp.status_code == 429:
|
| 134 |
+
logging.warning(f"⚠️ [API 限流] 触发 429 Too Many Requests,Tenacity 将自动等待并重试。URL: {url}")
|
| 135 |
+
resp.raise_for_status()
|
| 136 |
+
|
| 137 |
+
if resp.status_code != 200:
|
| 138 |
+
logging.error(f"❌ [API 错误] 状态码: {resp.status_code}, 响应: {resp.text}")
|
| 139 |
resp.raise_for_status()
|
| 140 |
+
|
| 141 |
return resp
|
| 142 |
|
| 143 |
def _is_english_dominant(self, text):
|
|
|
|
| 186 |
|
| 187 |
@retry(stop=stop_after_attempt(3), wait=wait_exponential(multiplier=1, min=2, max=10))
|
| 188 |
def call_deepseek(self, system_prompt, user_prompt, max_tokens=8000, temp=0.3, model_name="deepseek-reasoner"):
|
| 189 |
+
logging.info(f"🚀 [DeepSeek API] 调用模型: {model_name}, 提示词预计 Tokens: {self._count_tokens(user_prompt)}")
|
| 190 |
print(f"🚀 [API-流式保活] 正在调用 {model_name}...")
|
| 191 |
with self.lock: self.ds_calls += 1
|
| 192 |
|
|
|
|
| 201 |
full_content = ""
|
| 202 |
try:
|
| 203 |
with requests.post(self.deepseek_api_url, headers={"Authorization": f"Bearer {self.ds_key}"}, json=payload, stream=True, timeout=120) as response:
|
|
|
|
| 204 |
if response.status_code != 200:
|
| 205 |
error_msg = f"DeepSeek API 拒绝访问 (状态码: {response.status_code})。详情: {response.text}"
|
| 206 |
+
st.toast(f"❌ {error_msg}", icon="🚨")
|
| 207 |
raise requests.exceptions.HTTPError(error_msg)
|
| 208 |
|
| 209 |
for line in response.iter_lines():
|
|
|
|
| 250 |
yield content
|
| 251 |
|
| 252 |
except Exception:
|
|
|
|
|
|
|
| 253 |
pass
|
| 254 |
except GeneratorExit:
|
|
|
|
| 255 |
response.close()
|
| 256 |
raise
|
| 257 |
|
|
|
|
| 404 |
except: time.sleep(2)
|
| 405 |
return []
|
| 406 |
|
|
|
|
|
|
|
| 407 |
def _fetch_openalex_page(self, search_term, page, filter_str):
|
| 408 |
headers = {"User-Agent": "mailto:ssci_agent@example.com"}
|
| 409 |
if self.openalex_key: headers["Authorization"] = f"Bearer {self.openalex_key}"
|
| 410 |
|
|
|
|
|
|
|
|
|
|
| 411 |
params = {
|
| 412 |
+
"search": search_term,
|
| 413 |
"per-page": self.lit_per_page,
|
| 414 |
"page": page,
|
| 415 |
"filter": filter_str
|
| 416 |
}
|
| 417 |
+
|
| 418 |
+
logging.info(f"🔍 [OpenAlex 检索] 关键词: '{search_term}', 页码: {page}")
|
| 419 |
resp = self._requests_get_with_retry(self.openalex_api_url, headers=headers, params=params, timeout=15)
|
| 420 |
return resp.json().get("results", [])
|
| 421 |
|
| 422 |
+
def fetch_references_parallel(self, topic, keywords, min_required=50, progress_callback=None):
|
| 423 |
with self.lock: self.citation_map.clear()
|
| 424 |
llm_rag_context = []
|
| 425 |
seen_identifiers = set()
|
|
|
|
| 528 |
|
| 529 |
if added_in_this_batch == 0: break
|
| 530 |
page_batch += 1
|
| 531 |
+
if progress_callback: progress_callback()
|
| 532 |
|
| 533 |
if len(self.citation_map) < self.lit_target_papers:
|
| 534 |
for term in current_batch_terms:
|
|
|
|
| 590 |
if added_in_this_batch == 0: break
|
| 591 |
offset += 50
|
| 592 |
time.sleep(0.5)
|
| 593 |
+
if progress_callback: progress_callback()
|
| 594 |
|
| 595 |
return "\n".join(llm_rag_context)
|
| 596 |
|