Blame for core/meta_extractor.py

# -*- coding: utf-8 -*-
import json
import concurrent.futures
from typing import List, Dict
import re
import logging
from config import APP_CONFIG

class MetaExtractor:
    def __init__(self, agent):
        self.agent = agent
        self.max_workers = APP_CONFIG["lit_max_workers"]

    def extract_single_paper_meta(self, text: str) -> Dict:
        snippet = self.agent._truncate_context(text, max_tokens=3500)
        prompt = APP_CONFIG["prompt_templates"]["meta_extract"]
        
        try:
            # 修改点：显式将模型降级为 deepseek-chat (V3)，以获得更快的并行提取速度
            response = self.agent.call_deepseek(
                prompt, 
                snippet, 
                max_tokens=1000, 
                model_name="deepseek-chat"
            )
            
            # 强制提取 JSON 部分，过滤模型可能的解释性前缀或后缀
            json_match = re.search(r'\{.*\}', response, re.DOTALL)
            if json_match:
                return json.loads(json_match.group())
            return {}
        except Exception as e:
            logging.error(f"Meta Extraction error: {e}")
            return {}

    def run_batch_extraction(self, paper_texts: List[str]) -> List[Dict]:
        golden_db = []
        # 使用配置中的 lit_max_workers 进行多线程并行提取
        with concurrent.futures.ThreadPoolExecutor(max_workers=self.max_workers) as executor:
            future_to_paper = {executor.submit(self.extract_single_paper_meta, text): text for text in paper_texts}
            for future in concurrent.futures.as_completed(future_to_paper):
                try:
                    data = future.result()
                    if data: 
                        golden_db.append(data)
                except Exception as e:
                    logging.error(f"Batch Future error: {e}")
        return golden_db

def extract_golden_metadata(agent, paper_texts: List[str]):
    extractor = MetaExtractor(agent)
    return extractor.run_batch_extraction(paper_texts)