Update evaluate.py

evaluate.py

@@ -81,9 +81,6 @@ SUB_TYPE_TO_ABBR = {
     "Answer Refusal": "AR"
 }
 
-# Sub type 简称到全称的映射（用于调试）
-ABBR_TO_SUB_TYPE = {v: k for k, v in SUB_TYPE_TO_ABBR.items()}
-
 # 指标列表
 METRICS = ["LLM", "P", "R", "F1", "BLEU-1"]
 
@@ -175,7 +172,6 @@ class LexicalMetricsCalculator:
         return 2 * prec * rec / (prec + rec) if (prec + rec) > 0 else 0.0
 
     def _bleu1(self, prediction: str, reference: str, **kwargs) -> float:
-        # Tokenize
         pred_tokens = self.text_processor.tokenize(prediction, remove_stopwords=False)
         ref_tokens = self.text_processor.tokenize(reference, remove_stopwords=False)
 
@@ -184,27 +180,25 @@ class LexicalMetricsCalculator:
         if pred_tokens == ref_tokens:
             return 1.0
         
-        # Use sentence_bleu to calculate BLEU-1 (weights=(1, 0, 0, 0) means use only 1-gram)
         try:
-            # Add smoothing function to avoid zero score issues
             smoothie = SmoothingFunction().method4
             bleu_score = sentence_bleu(
-                [ref_tokens],  # Reference needs to be a list of lists
+                [ref_tokens],
                 pred_tokens,
-                weights=(1.0, 0, 0, 0),  # BLEU-1
+                weights=(1.0, 0, 0, 0),
                 smoothing_function=smoothie
             )
             return float(bleu_score)
-        except Exception as e:
+        except Exception:
             return 0.0
 
 
 # =========================================================
-# 第二部分：LLM-as-judge 评估器
+# 第二部分：LLM-as-judge 评估器（支持批量评估）
 # =========================================================
 
 class LLMJudgeEvaluator:
-    """LLM-as-judge 评估器"""
+    """LLM-as-judge 评估器 - 支持批量评估"""
     
     SCORE_RUBRIC = """
 **Score 0 (Incorrect / Miss):**
@@ -232,6 +226,7 @@ class LLMJudgeEvaluator:
         self.base_url = self.config.get('base_url') or os.getenv('OPENAI_BASE_URL', 'https://api.openai.com/v1')
         self.model_name = self.config.get('model_name', 'gpt-4o-mini')
         self.temperature = self.config.get('temperature', 0)
+        self.batch_size = self.config.get('batch_size', 5)
         self._client = None
         
     def _get_client(self):
@@ -241,60 +236,120 @@ class LLMJudgeEvaluator:
             self._client = OpenAI(api_key=self.api_key, base_url=self.base_url)
         return self._client
     
-    def _build_prompt(self, question: str, ground_truth: str, assistant_answer: str) -> str:
-        return f"""You are an impartial judge evaluating an AI assistant's answer.
+    def _build_batch_prompt(self, qa_pairs: List[Dict[str, str]]) -> str:
+        """构建批量评估的提示词，包含临时 ID"""
+        prompt = f"""You are an impartial judge evaluating an AI assistant's answers.
+You will evaluate {len(qa_pairs)} questions and provide scores for each.
 
 ### Scoring Rubric
 {self.SCORE_RUBRIC}
 
-### Question
-{question}
-
-### Ground Truth
-{ground_truth}
+### Questions and Answers
+"""
+        for item in qa_pairs:
+            prompt += f"""
+--- Question ID: {item['temp_id']} ---
+Question: {item['question']}
+Ground Truth: {item['ground_truth']}
+Assistant Answer: {item['assistant_answer'] if item['assistant_answer'] else "(No answer provided)"}
 
-### Assistant Answer
-{assistant_answer if assistant_answer else "(No answer provided)"}
+"""
 
+        prompt += """
 ### Output Format
-Output strictly in JSON format:
-{{"score": 0.75, "reasoning": "<short explanation>"}}
-
-Score must be one of: 0, 0.25, 0.5, 0.75, or 1.
+Output strictly in JSON format as a list of objects, each containing the question_id and score:
+[
+    {"question_id": 0, "score": 0.75, "reasoning": "<explanation>"},
+    {"question_id": 1, "score": 0.5, "reasoning": "<explanation>"},
+    ...
+]
+
+Each score must be one of: 0, 0.25, 0.5, 0.75, or 1.
+The number of objects must equal the number of questions.
+The question_id must match the ID provided above.
 """
+        return prompt
+    
+    def _parse_batch_response(self, response_text: str, qa_pairs: List[Dict[str, str]]) -> List[Dict[str, Any]]:
+        """解析批量评估的响应，按临时 ID 匹配"""
+        expected_count = len(qa_pairs)
+        
+        # 创建 ID 到结果的映射
+        result_map = {item['temp_id']: None for item in qa_pairs}
+        
+        try:
+            # 尝试解析 JSON 数组
+            json_match = re.search(r'\[.*\]', response_text, re.DOTALL)
+            if json_match:
+                parsed_results = json.loads(json_match.group())
+                
+                if isinstance(parsed_results, list):
+                    for r in parsed_results:
+                        if isinstance(r, dict) and 'question_id' in r:
+                            qid = r['question_id']
+                            score = float(r.get('score', 0))
+                            valid_scores = [0, 0.25, 0.5, 0.75, 1]
+                            if score not in valid_scores:
+                                score = min(valid_scores, key=lambda x: abs(x - score))
+                            result_map[qid] = {
+                                'score': score,
+                                'reasoning': r.get('reasoning', ''),
+                                'success': True
+                            }
+            
+            # 为每个问题返回结果（未匹配的使用默认值）
+            results = []
+            for item in qa_pairs:
+                qid = item['temp_id']
+                if result_map[qid] is not None:
+                    results.append(result_map[qid])
+                else:
+                    logger.warning(f"问题 ID {qid} 未在响应中找到，使用默认值")
+                    results.append({'score': 0, 'reasoning': 'Not found in response', 'success': False})
+            
+            return results
+            
+        except Exception as e:
+            logger.error(f"解析批量响应失败: {e}")
+            return [{'score': 0, 'reasoning': str(e), 'success': False} for _ in range(expected_count)]
     
     @tenacity.retry(
         stop=tenacity.stop_after_attempt(3),
         wait=tenacity.wait_exponential(multiplier=1, min=1, max=10),
         retry=tenacity.retry_if_exception_type(Exception),
     )
-    def evaluate_single(self, question: str, ground_truth: str, assistant_answer: str) -> Dict[str, Any]:
+    def evaluate_batch(self, qa_pairs: List[Dict[str, str]]) -> List[Dict[str, Any]]:
+        """批量评估多个问答对"""
+        if not qa_pairs:
+            return []
+        
         try:
             client = self._get_client()
-            prompt = self._build_prompt(question, ground_truth, assistant_answer)
+            prompt = self._build_batch_prompt(qa_pairs)
             
             response = client.chat.completions.create(
                 model=self.model_name,
                 messages=[{"role": "user", "content": prompt}],
                 temperature=self.temperature,
-                timeout=30
+                timeout=120
             )
             
             response_text = response.choices[0].message.content.strip()
-            json_match = re.search(r'\{.*\}', response_text, re.DOTALL)
+            return self._parse_batch_response(response_text, qa_pairs)
             
-            if json_match:
-                result = json.loads(json_match.group())
-                score = float(result.get('score', 0))
-                valid_scores = [0, 0.25, 0.5, 0.75, 1]
-                if score not in valid_scores:
-                    score = min(valid_scores, key=lambda x: abs(x - score))
-                return {'score': score, 'reasoning': result.get('reasoning', ''), 'success': True}
-            else:
-                return {'score': 0, 'reasoning': 'Parse failed', 'success': False}
         except Exception as e:
-            logger.error(f"LLM评估失败: {e}")
-            return {'score': 0, 'reasoning': str(e), 'success': False}
+            logger.error(f"批量LLM评估失败: {e}")
+            return [{'score': 0, 'reasoning': str(e), 'success': False} for _ in range(len(qa_pairs))]
+    
+    def evaluate_single(self, question: str, ground_truth: str, assistant_answer: str) -> Dict[str, Any]:
+        """单个评估（向后兼容）"""
+        result = self.evaluate_batch([{
+            'temp_id': 0,
+            'question': question,
+            'ground_truth': ground_truth,
+            'assistant_answer': assistant_answer
+        }])
+        return result[0] if result else {'score': 0, 'reasoning': 'Failed', 'success': False}
 
 
 # =========================================================
@@ -305,21 +360,18 @@ class ScoreCollector:
     """分数收集器，用于按类别统计加权平均"""
     
     def __init__(self):
-        self.scores = defaultdict(list)  # {category: [scores]}
-        self.weights = defaultdict(int)  # {category: count}
+        self.scores = defaultdict(list)
+        self.weights = defaultdict(int)
     
     def add(self, category: str, score: float):
-        """添加一个分数"""
         self.scores[category].append(score)
         self.weights[category] += 1
     
     def get_weighted_average(self, category: str = None) -> float:
-        """获取加权平均分数"""
         if category:
             scores = self.scores.get(category, [])
             return np.mean(scores) if scores else 0.0
         
-        # 计算所有分数的加权平均
         total_score = 0
         total_weight = 0
         for cat, scores in self.scores.items():
@@ -328,7 +380,6 @@ class ScoreCollector:
         return total_score / total_weight if total_weight > 0 else 0.0
     
     def get_all_averages(self) -> Dict[str, float]:
-        """获取所有类别的平均值"""
         return {cat: np.mean(scores) if scores else 0.0 
                 for cat, scores in self.scores.items()}
 
@@ -348,7 +399,7 @@ class SubmissionEvaluator:
         if self.enable_llm_judge and llm_config:
             try:
                 self.llm_judge = LLMJudgeEvaluator(llm_config)
-                logger.info("LLM-as-judge 已启用")
+                logger.info(f"LLM-as-judge 已启用，批量大小: {self.llm_judge.batch_size}")
             except Exception as e:
                 logger.warning(f"LLM-as-judge 初始化失败: {e}")
                 self.enable_llm_judge = False
@@ -363,129 +414,206 @@ class SubmissionEvaluator:
             raise ValueError(f"文件格式错误: 缺少 'predictions' 字段")
         return data['predictions']
     
-    def evaluate_single_prediction(self, pred: Dict, enable_llm: bool = True) -> Dict[str, Any]:
-        """评估单个预测，返回各指标的分数"""
+    def evaluate_single_prediction(self, pred: Dict) -> Dict[str, float]:
+        """评估单个预测的词法指标（不包含 LLM）"""
         system_answer = pred.get('system_answer', '').strip()
         original_answer = pred.get('original_answer', '').strip()
         
-        # 计算词法指标
         lexical = self.lexical_calculator.calculate(system_answer, original_answer)
         
-        result = {
+        return {
             'P': lexical['precision'],
             'R': lexical['recall'],
             'F1': lexical['f1'],
-            'BLEU-1': lexical['bleu1'],
-            'LLM': 0.0
+            'BLEU-1': lexical['bleu1']
         }
-        
-        # LLM 评估
-        if self.enable_llm_judge and enable_llm and self.llm_judge:
-            llm_result = self.llm_judge.evaluate_single(
-                question=pred.get('question_text', ''),
-                ground_truth=original_answer,
-                assistant_answer=system_answer
-            )
-            if llm_result['success']:
-                result['LLM'] = llm_result['score']
-        
-        return result
     
-    def evaluate_dataset(self, predictions: List[Dict], dataset_name: str) -> Dict[str, Any]:
+    def evaluate_dataset(self, predictions: List[Dict], dataset_name: str, enable_llm: bool = True) -> Dict[str, Any]:
         """
         评估单个数据集，按 main_type 和 sub_type 分类统计
-        
-        Returns:
-            包含按指标、按 main_type、按 sub_type 分类的分数
         """
-        # 初始化收集器
-        # 按指标和 main_type 收集
-        metric_main_collectors = {metric: {main_type: ScoreCollector() for main_type in MAIN_TYPE_MAPPING.values()} 
-                                   for metric in METRICS}
+        # 存储每个问题的词法指标
+        all_lexical_metrics = []
+        category_lexical_metrics = {cat: [] for cat in SUB_TYPE_TO_ABBR.values()}
         
-        # 按指标和 sub_type 收集
-        metric_sub_collectors = {metric: defaultdict(ScoreCollector) for metric in METRICS}
+        # 按 main_type 和 sub_type 分类的收集器（词法指标）
+        main_type_lexical = {mt: [] for mt in ['recall', 'reasoning', 'application']}
+        sub_type_lexical = {st: [] for st in SUB_TYPE_TO_ABBR.values()}
         
-        # 总问题数统计
-        total_count = len(predictions)
-        main_type_counts = defaultdict(int)
-        sub_type_counts = defaultdict(int)
+        # 存储每个问题的完整信息（用于后续分类）
+        questions_info = []
         
-        for pred in predictions:
+        # 第一遍：计算词法指标，收集问题信息
+        for idx, pred in enumerate(predictions):
+            system_answer = pred.get('system_answer', '').strip()
+            original_answer = pred.get('original_answer', '').strip()
+            question_text = pred.get('question_text', '')
+            
             # 获取类型信息
             main_type_full = pred.get('question_type', {}).get('main_type', '')
             sub_type_full = pred.get('question_type', {}).get('sub_type', '')
             
-            # 映射到内部标识
             main_type = MAIN_TYPE_MAPPING.get(main_type_full, '')
-            sub_type_abbr = SUB_TYPE_TO_ABBR.get(sub_type_full, sub_type_full)
-            
-            # 统计数量
-            if main_type:
-                main_type_counts[main_type] += 1
-            if sub_type_abbr:
-                sub_type_counts[sub_type_abbr] += 1
-            
-            # 评估当前预测
-            scores = self.evaluate_single_prediction(pred)
-            
-            # 按 main_type 收集分数
-            if main_type:
-                for metric, score in scores.items():
-                    metric_main_collectors[metric][main_type].add(main_type, score)
-            
-            # 按 sub_type 收集分数
-            if sub_type_abbr:
-                for metric, score in scores.items():
-                    metric_sub_collectors[metric][sub_type_abbr].add(sub_type_abbr, score)
+            sub_type_abbr = SUB_TYPE_TO_ABBR.get(sub_type_full, '')
+            
+            # 计算词法指标
+            lexical_scores = self.evaluate_single_prediction(pred)
+            
+            # 存储信息
+            questions_info.append({
+                'index': idx,
+                'main_type': main_type,
+                'sub_type': sub_type_abbr,
+                'lexical_scores': lexical_scores,
+                'question_text': question_text,
+                'original_answer': original_answer,
+                'system_answer': system_answer
+            })
+            
+            # 收集词法指标
+            all_lexical_metrics.append(lexical_scores)
+            
+            # 按类别收集词法指标
+            if sub_type_abbr and sub_type_abbr in category_lexical_metrics:
+                category_lexical_metrics[sub_type_abbr].append(lexical_scores)
+            
+            # 按 main_type 收集
+            if main_type and main_type in main_type_lexical:
+                main_type_lexical[main_type].append(lexical_scores)
+            
+            # 按 sub_type 收集
+            if sub_type_abbr and sub_type_abbr in sub_type_lexical:
+                sub_type_lexical[sub_type_abbr].append(lexical_scores)
+        
+        # =====================================================
+        # LLM-as-judge 批量评估
+        # =====================================================
+        llm_scores_by_index = {}  # 索引 -> LLM分数
+        
+        if self.enable_llm_judge and enable_llm and self.llm_judge:
+            # 准备批量评估数据
+            batch_size = self.llm_judge.batch_size
+            total_questions = len(questions_info)
+            total_batches = (total_questions + batch_size - 1) // batch_size
+            
+            logger.info(f"开始批量LLM评估，共 {total_questions} 个问题，分成 {total_batches} 批（每批最多 {batch_size} 个）")
+            
+            for batch_start in range(0, total_questions, batch_size):
+                batch_end = min(batch_start + batch_size, total_questions)
+                batch_questions = questions_info[batch_start:batch_end]
+                current_batch_num = batch_start // batch_size + 1
+                
+                logger.info(f"评估第 {current_batch_num}/{total_batches} 批，包含 {len(batch_questions)} 个问题...")
+                
+                # 准备批量数据，添加临时 ID
+                batch_data = []
+                for i, q in enumerate(batch_questions):
+                    batch_data.append({
+                        'temp_id': batch_start + i,  # 使用原始索引作为临时 ID
+                        'question': q['question_text'],
+                        'ground_truth': q['original_answer'],
+                        'assistant_answer': q['system_answer']
+                    })
+                
+                # 批量调用 LLM
+                batch_results = self.llm_judge.evaluate_batch(batch_data)
+                
+                # 存储结果
+                for i, result in enumerate(batch_results):
+                    original_idx = batch_start + i
+                    if result['success']:
+                        llm_scores_by_index[original_idx] = result['score']
+                    else:
+                        llm_scores_by_index[original_idx] = 0.0
+                
+                success_count = len([r for r in batch_results if r['success']])
+                logger.info(f"第 {current_batch_num} 批完成，成功 {success_count}/{len(batch_questions)} 个")
+            
+            logger.info(f"LLM批量评估完成，总成功 {len(llm_scores_by_index)}/{total_questions} 个")
+        
+        # =====================================================
+        # 收集完整的分数（词法 + LLM）
+        # =====================================================
+        
+        # 用于收集按类别的 LLM 分数
+        main_type_llm = {mt: [] for mt in ['recall', 'reasoning', 'application']}
+        sub_type_llm = {st: [] for st in SUB_TYPE_TO_ABBR.values()}
+        
+        for q in questions_info:
+            idx = q['index']
+            llm_score = llm_scores_by_index.get(idx, 0.0)
+            
+            # 按 main_type 收集 LLM 分数
+            if q['main_type'] and q['main_type'] in main_type_llm:
+                main_type_llm[q['main_type']].append(llm_score)
+            
+            # 按 sub_type 收集 LLM 分数
+            if q['sub_type'] and q['sub_type'] in sub_type_llm:
+                sub_type_llm[q['sub_type']].append(llm_score)
+        
+        # =====================================================
+        # 计算各类别的平均分
+        # =====================================================
+        
+        def calc_avg(scores_list):
+            return np.mean(scores_list) if scores_list else 0.0
+        
+        # 计算整体词法平均
+        overall_lexical = {
+            'P': calc_avg([m['P'] for m in all_lexical_metrics]),
+            'R': calc_avg([m['R'] for m in all_lexical_metrics]),
+            'F1': calc_avg([m['F1'] for m in all_lexical_metrics]),
+            'BLEU-1': calc_avg([m['BLEU-1'] for m in all_lexical_metrics])
+        }
+        
+        # 计算整体 LLM 平均
+        overall_llm = calc_avg(list(llm_scores_by_index.values()))
         
         # 构建结果
         result = {
-            'total': total_count,
-            'main_type_counts': dict(main_type_counts),
-            'sub_type_counts': dict(sub_type_counts)
+            'total': len(predictions),
+            'main_type_counts': {mt: len(main_type_lexical[mt]) for mt in main_type_lexical},
+            'sub_type_counts': {st: len(sub_type_lexical[st]) for st in sub_type_lexical}
         }
         
-        # 按指标组织结果
+        # 为每个指标构建结果
         for metric in METRICS:
-            result[metric] = {}
-            
-            # 计算 overall（所有问题的平均）
-            all_scores = []
-            for pred in predictions:
-                scores = self.evaluate_single_prediction(pred)
-                all_scores.append(scores[metric])
-            result[metric]['overall'] = np.mean(all_scores) if all_scores else 0.0
-            
-            # 按 main_type 计算
-            result[metric]['recall'] = {
-                'score': metric_main_collectors[metric].get('recall', ScoreCollector()).get_weighted_average(),
-                'sub_types': {}
-            }
-            result[metric]['reasoning'] = {
-                'score': metric_main_collectors[metric].get('reasoning', ScoreCollector()).get_weighted_average(),
-                'sub_types': {}
-            }
-            result[metric]['application'] = {
-                'score': metric_main_collectors[metric].get('application', ScoreCollector()).get_weighted_average(),
-                'sub_types': {}
+            result[metric] = {
+                'overall': 0.0,
+                'recall': {'score': 0.0, 'sub_types': {}},
+                'reasoning': {'score': 0.0, 'sub_types': {}},
+                'application': {'score': 0.0, 'sub_types': {}}
             }
             
-            # 按 sub_type 填充子类型分数
-            # Recall 子类型
-            for sub in ['UPR', 'CRR', 'KR']:
-                collector = metric_sub_collectors[metric].get(sub, ScoreCollector())
-                result[metric]['recall']['sub_types'][sub] = collector.get_weighted_average()
-            
-            # Reasoning 子类型
-            for sub in ['TR', 'MCR', 'RET']:
-                collector = metric_sub_collectors[metric].get(sub, ScoreCollector())
-                result[metric]['reasoning']['sub_types'][sub] = collector.get_weighted_average()
-            
-            # Application 子类型
-            for sub in ['TTL', 'CD', 'AR']:
-                collector = metric_sub_collectors[metric].get(sub, ScoreCollector())
-                result[metric]['application']['sub_types'][sub] = collector.get_weighted_average()
+            if metric == 'LLM':
+                # LLM 指标
+                result[metric]['overall'] = overall_llm
+                for mt in ['recall', 'reasoning', 'application']:
+                    result[metric][mt]['score'] = calc_avg(main_type_llm.get(mt, []))
+                for st, abbr in SUB_TYPE_TO_ABBR.items():
+                    if abbr in ['UPR', 'CRR', 'KR']:
+                        result[metric]['recall']['sub_types'][abbr] = calc_avg(sub_type_llm.get(abbr, []))
+                    elif abbr in ['TR', 'MCR', 'RET']:
+                        result[metric]['reasoning']['sub_types'][abbr] = calc_avg(sub_type_llm.get(abbr, []))
+                    elif abbr in ['TTL', 'CD', 'AR']:
+                        result[metric]['application']['sub_types'][abbr] = calc_avg(sub_type_llm.get(abbr, []))
+            else:
+                # 词法指标
+                metric_key = metric if metric != 'BLEU-1' else 'BLEU-1'
+                result[metric]['overall'] = overall_lexical.get(metric_key, 0.0)
+                
+                for mt in ['recall', 'reasoning', 'application']:
+                    scores = [m[metric_key] for m in main_type_lexical.get(mt, [])]
+                    result[metric][mt]['score'] = calc_avg(scores)
+                
+                for st, abbr in SUB_TYPE_TO_ABBR.items():
+                    scores = [m[metric_key] for m in sub_type_lexical.get(abbr, [])]
+                    if abbr in ['UPR', 'CRR', 'KR']:
+                        result[metric]['recall']['sub_types'][abbr] = calc_avg(scores)
+                    elif abbr in ['TR', 'MCR', 'RET']:
+                        result[metric]['reasoning']['sub_types'][abbr] = calc_avg(scores)
+                    elif abbr in ['TTL', 'CD', 'AR']:
+                        result[metric]['application']['sub_types'][abbr] = calc_avg(scores)
         
         return result
     
@@ -676,7 +804,8 @@ def evaluate_submission(dyadic_path: str, multiparty_path: str,
             'api_key': os.getenv('OPENAI_API_KEY'),
             'base_url': os.getenv('OPENAI_BASE_URL', 'https://api.openai.com/v1'),
             'model_name': os.getenv('LLM_JUDGE_MODEL', 'gpt-4o-mini'),
-            'temperature': 0
+            'temperature': 0,
+            'batch_size': 5  # 每批最多5个问题
         }
         _evaluator = SubmissionEvaluator(llm_config=llm_config, enable_llm_judge=True)