Buckets:
| """ | |
| Geometry Evaluator | |
| ارزیاب هندسه | |
| This module provides evaluation capabilities for geometric problem solutions, | |
| including accuracy, completeness, and quality metrics. | |
| """ | |
| import os | |
| import logging | |
| import json | |
| import numpy as np | |
| from typing import Dict, List, Any, Optional, Tuple | |
| from datetime import datetime | |
| import re | |
| logger = logging.getLogger(__name__) | |
| class GeometryEvaluator: | |
| """ | |
| Evaluator for geometric problem solutions | |
| ارزیاب راهحلهای مسائل هندسی | |
| """ | |
| def __init__(self, config: Dict[str, Any]): | |
| """ | |
| Initialize the geometry evaluator | |
| Args: | |
| config: Evaluation configuration | |
| """ | |
| self.config = config | |
| # Evaluation metrics | |
| self.metrics = config.get( | |
| "metrics", | |
| [ | |
| "accuracy", | |
| "proof_quality", | |
| "time_efficiency", | |
| "explanation_quality", | |
| "confidence_calibration", | |
| ], | |
| ) | |
| # Benchmarks | |
| self.benchmarks = config.get("benchmarks", []) | |
| logger.info("Geometry Evaluator initialized") | |
| def evaluate_solution( | |
| self, | |
| problem: str, | |
| solution: str, | |
| expected_answer: str = None, | |
| method: str = "unknown", | |
| ) -> Dict[str, Any]: | |
| """ | |
| Evaluate a geometric solution | |
| Args: | |
| problem: Original problem statement | |
| solution: Generated solution | |
| expected_answer: Expected answer (if available) | |
| method: Solving method used | |
| Returns: | |
| Dictionary containing evaluation results | |
| """ | |
| try: | |
| evaluation = { | |
| "method": method, | |
| "timestamp": datetime.now().isoformat(), | |
| "metrics": {}, | |
| } | |
| # Calculate individual metrics | |
| for metric in self.metrics: | |
| if metric == "accuracy": | |
| evaluation["metrics"]["accuracy"] = self._calculate_accuracy( | |
| solution, expected_answer | |
| ) | |
| elif metric == "proof_quality": | |
| evaluation["metrics"]["proof_quality"] = ( | |
| self._calculate_proof_quality(solution) | |
| ) | |
| elif metric == "time_efficiency": | |
| evaluation["metrics"]["time_efficiency"] = ( | |
| self._calculate_time_efficiency(solution) | |
| ) | |
| elif metric == "explanation_quality": | |
| evaluation["metrics"]["explanation_quality"] = ( | |
| self._calculate_explanation_quality(solution) | |
| ) | |
| elif metric == "confidence_calibration": | |
| evaluation["metrics"]["confidence_calibration"] = ( | |
| self._calculate_confidence_calibration(solution) | |
| ) | |
| # Calculate overall score | |
| evaluation["overall_score"] = self._calculate_overall_score( | |
| evaluation["metrics"] | |
| ) | |
| # Add qualitative analysis | |
| evaluation["qualitative_analysis"] = self._qualitative_analysis(solution) | |
| logger.info(f"Solution evaluation completed for method: {method}") | |
| return evaluation | |
| except Exception as e: | |
| logger.error(f"Error evaluating solution: {e}") | |
| return { | |
| "method": method, | |
| "error": str(e), | |
| "timestamp": datetime.now().isoformat(), | |
| } | |
| def _calculate_accuracy(self, solution: str, expected_answer: str = None) -> float: | |
| """ | |
| Calculate accuracy of the solution | |
| Args: | |
| solution: Generated solution | |
| expected_answer: Expected answer | |
| Returns: | |
| Accuracy score between 0 and 1 | |
| """ | |
| try: | |
| if not expected_answer: | |
| # If no expected answer, use heuristics | |
| return self._heuristic_accuracy(solution) | |
| # Extract numerical answers from both solutions | |
| solution_numbers = self._extract_numbers(solution) | |
| expected_numbers = self._extract_numbers(expected_answer) | |
| if not solution_numbers or not expected_numbers: | |
| return 0.5 # Default score if no numbers found | |
| # Calculate accuracy based on numerical matches | |
| matches = 0 | |
| total = max(len(solution_numbers), len(expected_numbers)) | |
| for sol_num in solution_numbers: | |
| for exp_num in expected_numbers: | |
| if abs(sol_num - exp_num) < 0.01: # Tolerance for floating point | |
| matches += 1 | |
| break | |
| accuracy = matches / total if total > 0 else 0 | |
| return min(accuracy, 1.0) | |
| except Exception as e: | |
| logger.error(f"Error calculating accuracy: {e}") | |
| return 0.5 | |
| def _heuristic_accuracy(self, solution: str) -> float: | |
| """ | |
| Calculate accuracy using heuristics when no expected answer is available | |
| Args: | |
| solution: Generated solution | |
| Returns: | |
| Heuristic accuracy score | |
| """ | |
| try: | |
| score = 0.5 # Base score | |
| # Check for mathematical content | |
| if any( | |
| keyword in solution.lower() for keyword in ["محاسبه", "نتیجه", "پاسخ"] | |
| ): | |
| score += 0.1 | |
| # Check for step-by-step reasoning | |
| if any( | |
| keyword in solution.lower() | |
| for keyword in ["مرحله", "گام", "ابتدا", "سپس"] | |
| ): | |
| score += 0.1 | |
| # Check for mathematical symbols | |
| if re.search(r"[=+\-*/^√π]", solution): | |
| score += 0.1 | |
| # Check for geometric terminology | |
| if any( | |
| keyword in solution.lower() | |
| for keyword in ["مثلث", "دایره", "زاویه", "ضلع"] | |
| ): | |
| score += 0.1 | |
| # Check for proof elements | |
| if any( | |
| keyword in solution.lower() for keyword in ["قضیه", "اثبات", "بنابراین"] | |
| ): | |
| score += 0.1 | |
| return min(score, 1.0) | |
| except Exception as e: | |
| logger.error(f"Error in heuristic accuracy: {e}") | |
| return 0.5 | |
| def _extract_numbers(self, text: str) -> List[float]: | |
| """ | |
| Extract numerical values from text | |
| Args: | |
| text: Text to extract numbers from | |
| Returns: | |
| List of extracted numbers | |
| """ | |
| try: | |
| # Find all numbers in the text | |
| numbers = re.findall(r"-?\d+\.?\d*", text) | |
| return [float(num) for num in numbers if num] | |
| except Exception as e: | |
| logger.error(f"Error extracting numbers: {e}") | |
| return [] | |
| def _calculate_proof_quality(self, solution: str) -> float: | |
| """ | |
| Calculate quality of mathematical proof | |
| Args: | |
| solution: Generated solution | |
| Returns: | |
| Proof quality score between 0 and 1 | |
| """ | |
| try: | |
| score = 0.0 | |
| # Check for logical structure | |
| if any(keyword in solution.lower() for keyword in ["فرض", "مفروض"]): | |
| score += 0.2 | |
| if any( | |
| keyword in solution.lower() for keyword in ["بنابراین", "پس", "نتیجه"] | |
| ): | |
| score += 0.2 | |
| # Check for theorem references | |
| if any(keyword in solution.lower() for keyword in ["قضیه", "لم", "نتیجه"]): | |
| score += 0.2 | |
| # Check for step-by-step reasoning | |
| step_indicators = ["مرحله", "گام", "ابتدا", "سپس", "در نهایت"] | |
| step_count = sum( | |
| 1 for indicator in step_indicators if indicator in solution.lower() | |
| ) | |
| score += min(step_count * 0.1, 0.3) | |
| # Check for mathematical rigor | |
| if re.search(r"[=+\-*/^√π]", solution): | |
| score += 0.1 | |
| return min(score, 1.0) | |
| except Exception as e: | |
| logger.error(f"Error calculating proof quality: {e}") | |
| return 0.5 | |
| def _calculate_time_efficiency(self, solution: str) -> float: | |
| """ | |
| Calculate time efficiency based on solution length and complexity | |
| Args: | |
| solution: Generated solution | |
| Returns: | |
| Time efficiency score between 0 and 1 | |
| """ | |
| try: | |
| # Shorter solutions are generally more efficient | |
| length_score = max(0, 1 - len(solution) / 2000) # Normalize by 2000 chars | |
| # Check for concise mathematical expressions | |
| math_density = len(re.findall(r"[=+\-*/^√π]", solution)) / max( | |
| len(solution), 1 | |
| ) | |
| math_score = min(math_density * 10, 1.0) | |
| # Combine scores | |
| efficiency = (length_score + math_score) / 2 | |
| return min(efficiency, 1.0) | |
| except Exception as e: | |
| logger.error(f"Error calculating time efficiency: {e}") | |
| return 0.5 | |
| def _calculate_explanation_quality(self, solution: str) -> float: | |
| """ | |
| Calculate quality of explanation | |
| Args: | |
| solution: Generated solution | |
| Returns: | |
| Explanation quality score between 0 and 1 | |
| """ | |
| try: | |
| score = 0.0 | |
| # Check for clarity indicators | |
| clarity_indicators = ["به وضوح", "به روشنی", "ساده", "آسان"] | |
| score += min( | |
| len([ind for ind in clarity_indicators if ind in solution.lower()]) | |
| * 0.1, | |
| 0.3, | |
| ) | |
| # Check for examples | |
| if any( | |
| keyword in solution.lower() for keyword in ["مثال", "مثلاً", "برای مثال"] | |
| ): | |
| score += 0.2 | |
| # Check for Persian language quality | |
| persian_indicators = ["است", "میباشد", "خواهد بود", "دارد"] | |
| score += min( | |
| len([ind for ind in persian_indicators if ind in solution.lower()]) | |
| * 0.05, | |
| 0.2, | |
| ) | |
| # Check for completeness | |
| if len(solution) > 100: # Minimum length for good explanation | |
| score += 0.2 | |
| # Check for structure | |
| if any( | |
| keyword in solution.lower() | |
| for keyword in ["اول", "دوم", "سوم", "در نهایت"] | |
| ): | |
| score += 0.1 | |
| return min(score, 1.0) | |
| except Exception as e: | |
| logger.error(f"Error calculating explanation quality: {e}") | |
| return 0.5 | |
| def _calculate_confidence_calibration(self, solution: str) -> float: | |
| """ | |
| Calculate confidence calibration based on solution certainty | |
| Args: | |
| solution: Generated solution | |
| Returns: | |
| Confidence calibration score between 0 and 1 | |
| """ | |
| try: | |
| # Check for confidence indicators | |
| high_confidence = ["قطعاً", "حتماً", "بدون شک", "یقیناً"] | |
| medium_confidence = ["احتمالاً", "به احتمال زیاد", "معمولاً"] | |
| low_confidence = ["شاید", "ممکن است", "احتمالاً نه"] | |
| high_count = sum( | |
| 1 for phrase in high_confidence if phrase in solution.lower() | |
| ) | |
| medium_count = sum( | |
| 1 for phrase in medium_confidence if phrase in solution.lower() | |
| ) | |
| low_count = sum( | |
| 1 for phrase in low_confidence if phrase in solution.lower() | |
| ) | |
| # Calculate calibration score | |
| if high_count > 0 and low_count == 0: | |
| calibration = 0.8 # High confidence, no uncertainty | |
| elif medium_count > 0 and high_count == 0: | |
| calibration = 0.6 # Moderate confidence | |
| elif low_count > 0: | |
| calibration = 0.4 # Low confidence | |
| else: | |
| calibration = 0.7 # Default moderate confidence | |
| return calibration | |
| except Exception as e: | |
| logger.error(f"Error calculating confidence calibration: {e}") | |
| return 0.5 | |
| def _calculate_overall_score(self, metrics: Dict[str, float]) -> float: | |
| """ | |
| Calculate overall evaluation score | |
| Args: | |
| metrics: Individual metric scores | |
| Returns: | |
| Overall score between 0 and 1 | |
| """ | |
| try: | |
| if not metrics: | |
| return 0.5 | |
| # Weighted average of metrics | |
| weights = { | |
| "accuracy": 0.3, | |
| "proof_quality": 0.25, | |
| "explanation_quality": 0.2, | |
| "time_efficiency": 0.15, | |
| "confidence_calibration": 0.1, | |
| } | |
| weighted_sum = 0 | |
| total_weight = 0 | |
| for metric, score in metrics.items(): | |
| weight = weights.get(metric, 0.1) | |
| weighted_sum += score * weight | |
| total_weight += weight | |
| return weighted_sum / total_weight if total_weight > 0 else 0.5 | |
| except Exception as e: | |
| logger.error(f"Error calculating overall score: {e}") | |
| return 0.5 | |
| def _qualitative_analysis(self, solution: str) -> Dict[str, Any]: | |
| """ | |
| Perform qualitative analysis of the solution | |
| Args: | |
| solution: Generated solution | |
| Returns: | |
| Dictionary containing qualitative analysis | |
| """ | |
| try: | |
| analysis = {"strengths": [], "weaknesses": [], "suggestions": []} | |
| # Identify strengths | |
| if any( | |
| keyword in solution.lower() for keyword in ["قضیه", "اثبات", "بنابراین"] | |
| ): | |
| analysis["strengths"].append("استفاده از قضایای ریاضی") | |
| if any( | |
| keyword in solution.lower() for keyword in ["مرحله", "گام", "ابتدا"] | |
| ): | |
| analysis["strengths"].append("ساختار گام به گام") | |
| if len(solution) > 200: | |
| analysis["strengths"].append("توضیح جامع") | |
| # Identify weaknesses | |
| if len(solution) < 50: | |
| analysis["weaknesses"].append("توضیح کوتاه") | |
| if not re.search(r"[=+\-*/^√π]", solution): | |
| analysis["weaknesses"].append("عدم استفاده از نمادهای ریاضی") | |
| if not any( | |
| keyword in solution.lower() for keyword in ["نتیجه", "پاسخ", "محاسبه"] | |
| ): | |
| analysis["weaknesses"].append("عدم ارائه نتیجه نهایی") | |
| # Provide suggestions | |
| if len(solution) < 100: | |
| analysis["suggestions"].append("توضیح بیشتری ارائه دهید") | |
| if not any(keyword in solution.lower() for keyword in ["مثال", "مثلاً"]): | |
| analysis["suggestions"].append("مثالهای بیشتری اضافه کنید") | |
| return analysis | |
| except Exception as e: | |
| logger.error(f"Error in qualitative analysis: {e}") | |
| return {"strengths": [], "weaknesses": [], "suggestions": []} | |
| def compare_solutions(self, solutions: Dict[str, Dict[str, Any]]) -> Dict[str, Any]: | |
| """ | |
| Compare multiple solutions | |
| Args: | |
| solutions: Dictionary of solutions with their evaluations | |
| Returns: | |
| Comparison results | |
| """ | |
| try: | |
| comparison = { | |
| "methods": list(solutions.keys()), | |
| "scores": {}, | |
| "rankings": {}, | |
| "best_method": "", | |
| "analysis": {}, | |
| } | |
| # Extract scores | |
| for method, solution_data in solutions.items(): | |
| if "overall_score" in solution_data: | |
| comparison["scores"][method] = solution_data["overall_score"] | |
| # Rank methods | |
| if comparison["scores"]: | |
| sorted_methods = sorted( | |
| comparison["scores"].items(), key=lambda x: x[1], reverse=True | |
| ) | |
| comparison["rankings"] = { | |
| method: rank + 1 for rank, (method, _) in enumerate(sorted_methods) | |
| } | |
| comparison["best_method"] = sorted_methods[0][0] | |
| # Generate analysis | |
| if len(comparison["scores"]) > 1: | |
| best_score = max(comparison["scores"].values()) | |
| worst_score = min(comparison["scores"].values()) | |
| score_range = best_score - worst_score | |
| comparison["analysis"] = { | |
| "score_range": score_range, | |
| "performance_gap": f"{score_range:.2f}", | |
| "consistency": "High" if score_range < 0.2 else "Low", | |
| } | |
| logger.info("Solution comparison completed") | |
| return comparison | |
| except Exception as e: | |
| logger.error(f"Error comparing solutions: {e}") | |
| return {"error": str(e)} | |
| def generate_evaluation_report(self, evaluations: Dict[str, Any]) -> str: | |
| """ | |
| Generate a comprehensive evaluation report | |
| Args: | |
| evaluations: Dictionary of evaluations | |
| Returns: | |
| Formatted evaluation report | |
| """ | |
| try: | |
| report = "گزارش ارزیابی راهحلهای هندسی\n" | |
| report += "=" * 50 + "\n\n" | |
| for method, evaluation in evaluations.items(): | |
| if "error" in evaluation: | |
| report += f"❌ {method}: خطا - {evaluation['error']}\n\n" | |
| continue | |
| report += f"📊 روش: {method}\n" | |
| report += f"امتیاز کلی: {evaluation.get('overall_score', 0):.2f}\n\n" | |
| # Metrics | |
| metrics = evaluation.get("metrics", {}) | |
| for metric, score in metrics.items(): | |
| report += f" {metric}: {score:.2f}\n" | |
| # Qualitative analysis | |
| qualitative = evaluation.get("qualitative_analysis", {}) | |
| if qualitative: | |
| report += "\nنقاط قوت:\n" | |
| for strength in qualitative.get("strengths", []): | |
| report += f" ✅ {strength}\n" | |
| report += "\nنقاط ضعف:\n" | |
| for weakness in qualitative.get("weaknesses", []): | |
| report += f" ❌ {weakness}\n" | |
| report += "\nپیشنهادات:\n" | |
| for suggestion in qualitative.get("suggestions", []): | |
| report += f" 💡 {suggestion}\n" | |
| report += "\n" + "-" * 30 + "\n\n" | |
| return report | |
| except Exception as e: | |
| logger.error(f"Error generating evaluation report: {e}") | |
| return f"خطا در تولید گزارش: {str(e)}" | |
| def save_evaluation(self, evaluation: Dict[str, Any], filename: str = None): | |
| """ | |
| Save evaluation results to file | |
| Args: | |
| evaluation: Evaluation results | |
| filename: Output filename | |
| """ | |
| try: | |
| if filename is None: | |
| timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") | |
| filename = f"results/evaluation_{timestamp}.json" | |
| os.makedirs(os.path.dirname(filename), exist_ok=True) | |
| with open(filename, "w", encoding="utf-8") as f: | |
| json.dump(evaluation, f, ensure_ascii=False, indent=2) | |
| logger.info(f"Evaluation saved to {filename}") | |
| except Exception as e: | |
| logger.error(f"Error saving evaluation: {e}") | |
| def get_evaluator_info(self) -> Dict[str, Any]: | |
| """ | |
| Get information about the evaluator | |
| Returns: | |
| Dictionary containing evaluator information | |
| """ | |
| return { | |
| "metrics": self.metrics, | |
| "benchmarks": len(self.benchmarks), | |
| "available_evaluations": [ | |
| "accuracy", | |
| "proof_quality", | |
| "time_efficiency", | |
| "explanation_quality", | |
| "confidence_calibration", | |
| ], | |
| } | |
Xet Storage Details
- Size:
- 20.9 kB
- Xet hash:
- b80df75973430260fd10027c381f26e6f02a2c263a45339ff2c306d7314eada5
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.