Buckets:

tahamajs
/

Sysmem2_in_AI

Files

xet

tahamajs/Sysmem2_in_AI / ComputerAssignments /CA5_geometric_reasoning /src /evaluation /benchmark.py

tahamajs

5 months ago

download

raw

10.1 kB

import json
from typing import List, Dict, Tuple, Any, Optional
from dataclasses import dataclass
from .metrics import ProofQuality, EvaluationMetrics
from ..core.geometry import OlympiadProblem

@dataclass
class GeometryBenchmark:

    id: str
    name: str
    difficulty: str
    problem_statement: str
    expected_solution_steps: List[str]
    key_insights: List[str]
    time_limit: int
    topics: List[str]

class GeometrySolverEvaluator:

    def __init__(self):
        self.benchmarks = self._load_benchmarks()

    def evaluate_solver(
        self, solver, benchmark_subset: str = "all"
    ) -> EvaluationMetrics:

        if benchmark_subset == "all":
            problems = self.benchmarks
        else:
            problems = [b for b in self.benchmarks if benchmark_subset in b.topics]

        metrics = EvaluationMetrics(total_problems=len(problems))

        total_time = 0.0
        total_confidence = 0.0
        solved_count = 0

        for benchmark in problems:
            print(f"Evaluating on: {benchmark.name}")


            problem = self._benchmark_to_problem(benchmark)


            solution = solver.solve_problem(problem, timeout=benchmark.time_limit)


            is_solved = (
                solution["best_proof"] is not None and solution["confidence"] > 0.5
            )

            if is_solved:
                solved_count += 1
                metrics.difficulty_breakdown[benchmark.difficulty] = (
                    metrics.difficulty_breakdown.get(benchmark.difficulty, 0) + 1
                )


                strategy = solution["best_proof"]["strategy"]
                if strategy not in metrics.strategy_success_rates:
                    metrics.strategy_success_rates[strategy] = {
                        "attempts": 0,
                        "successes": 0,
                    }
                metrics.strategy_success_rates[strategy]["successes"] += 1


            for strategy in solution["strategies_attempted"]:
                if strategy not in metrics.strategy_success_rates:
                    metrics.strategy_success_rates[strategy] = {
                        "attempts": 0,
                        "successes": 0,
                    }
                metrics.strategy_success_rates[strategy]["attempts"] += 1

            total_time += solution["time_taken"]
            total_confidence += solution["confidence"]


            if is_solved:
                quality_score = self._evaluate_proof_quality(
                    solution["best_proof"], benchmark
                )
                metrics.proof_quality_scores.append(quality_score.value)


        metrics.solved_problems = solved_count
        metrics.total_time = total_time
        metrics.average_time = total_time / len(problems) if problems else 0
        metrics.average_confidence = total_confidence / len(problems) if problems else 0

        metrics.calculate_derived_metrics()
        return metrics

    def _benchmark_to_problem(self, benchmark: GeometryBenchmark) -> OlympiadProblem:
        from ..core.geometry import GeometricPoint, GeometricTriangle

        problem = OlympiadProblem(benchmark.name, benchmark.problem_statement)


        statement_lower = benchmark.problem_statement.lower()


        import re

        points = re.findall(r"\b[A-Z]\b", benchmark.problem_statement)
        points = list(set(points))[:6]

        for i, point_name in enumerate(points):

            x = (i % 3) * 2
            y = (i // 3) * 2
            problem.add_point(GeometricPoint(point_name, x, y))


        triangle_matches = re.findall(r"triangle\s+([A-Z]{3})", statement_lower)
        for match in triangle_matches:
            if len(match) == 3 and all(p in points for p in match):
                vertices = [problem.points[p] for p in match]
                triangle = GeometricTriangle(match, vertices)
                problem.add_triangle(triangle)


        if benchmark.expected_solution_steps:
            problem.set_goal(benchmark.expected_solution_steps[-1])

        return problem

    def _evaluate_proof_quality(
        self, proof: Dict[str, Any], benchmark: GeometryBenchmark
    ) -> ProofQuality:

        steps = proof.get("steps", [])
        confidence = proof.get("confidence", 0)


        completeness = len(steps) >= 3
        clarity = confidence > 0.7
        correctness = proof.get("success", False)


        insight_coverage = 0
        if benchmark.key_insights:
            for insight in benchmark.key_insights:
                if any(insight.lower() in step.lower() for step in steps):
                    insight_coverage += 1
            insight_coverage /= len(benchmark.key_insights)
        else:
            insight_coverage = 1.0


        if not correctness:
            return ProofQuality.INVALID
        elif completeness and clarity and insight_coverage > 0.7:
            return ProofQuality.EXCELLENT
        elif completeness and (clarity or insight_coverage > 0.5):
            return ProofQuality.GOOD
        elif completeness or clarity:
            return ProofQuality.ACCEPTABLE
        else:
            return ProofQuality.POOR

    def _load_benchmarks(self) -> List[GeometryBenchmark]:
        benchmarks = []


        benchmarks.append(
            GeometryBenchmark(
                "triangle_sum",
                "Triangle Angle Sum",
                "Elementary",
                "In triangle ABC, prove that the sum of angles is 180 degrees.",
                [
                    "Consider triangle ABC",
                    "Apply angle sum theorem",
                    "Therefore angles sum to 180°",
                ],
                ["angle sum theorem"],
                30,
                ["triangles", "angles"],
            )
        )

        benchmarks.append(
            GeometryBenchmark(
                "pythagorean",
                "Pythagorean Theorem",
                "High School",
                "In right triangle ABC with right angle at C, prove that AB² = AC² + BC².",
                [
                    "Identify right angle",
                    "Apply Pythagorean theorem",
                    "Therefore AB² = AC² + BC²",
                ],
                ["right angle", "pythagorean"],
                45,
                ["triangles", "right triangles"],
            )
        )


        benchmarks.append(
            GeometryBenchmark(
                "circumcenter",
                "Triangle Circumcenter",
                "High School",
                "Prove that the perpendicular bisectors of a triangle meet at a point equidistant from all vertices.",
                [
                    "Consider perpendicular bisectors",
                    "Show they intersect",
                    "Prove equidistant property",
                ],
                ["perpendicular bisector", "equidistant"],
                60,
                ["triangles", "circles", "construction"],
            )
        )

        benchmarks.append(
            GeometryBenchmark(
                "inscribed_angle",
                "Inscribed Angle Theorem",
                "High School",
                "In circle O, prove that an inscribed angle is half the central angle subtending the same arc.",
                [
                    "Consider inscribed and central angles",
                    "Use arc relationship",
                    "Prove angle ratio is 1:2",
                ],
                ["inscribed angle", "central angle", "arc"],
                60,
                ["circles", "angles"],
            )
        )


        benchmarks.append(
            GeometryBenchmark(
                "apollonius",
                "Apollonius Circle",
                "Olympiad",
                "Given triangle ABC and point P, construct a circle through P such that the tangent from A to the circle has specified length.",
                [
                    "Analyze locus conditions",
                    "Apply power of point",
                    "Construct Apollonius circle",
                ],
                ["power of point", "locus", "apollonius"],
                120,
                ["circles", "construction", "locus"],
            )
        )

        return benchmarks

    def generate_report(self, metrics: EvaluationMetrics) -> str:

        report = []
        report.append("=== Geometry Solver Evaluation Report ===")
        report.append(f"\nOverall Performance:")
        report.append(
            f"  Problems Solved: {metrics.solved_problems}/{metrics.total_problems}"
        )
        report.append(f"  Success Rate: {metrics.success_rate:.1%}")
        report.append(f"  Average Time: {metrics.average_time:.2f} seconds")
        report.append(f"  Average Confidence: {metrics.average_confidence:.1%}")
        report.append(
            f"  Average Proof Quality: {metrics.average_proof_quality:.1f}/5.0"
        )

        report.append(f"\nDifficulty Breakdown:")
        for difficulty, count in metrics.difficulty_breakdown.items():
            report.append(f"  {difficulty}: {count} problems solved")

        report.append(f"\nStrategy Performance:")
        for strategy, data in metrics.strategy_success_rates.items():
            report.append(f"  {strategy.capitalize()}:")
            report.append(f"    Attempts: {data['attempts']}")
            report.append(f"    Successes: {data['successes']}")
            report.append(f"    Success Rate: {data.get('success_rate', 0):.1%}")

        if metrics.proof_quality_scores:
            report.append(f"\nProof Quality Distribution:")
            quality_counts = {
                i: metrics.proof_quality_scores.count(i) for i in range(1, 6)
            }
            for quality, count in quality_counts.items():
                quality_name = ProofQuality(quality).name.title()
                percentage = count / len(metrics.proof_quality_scores) * 100
                report.append(f"  {quality_name}: {count} ({percentage:.1f}%)")

        return "\n".join(report)

Xet Storage Details

Size:: 10.1 kB
Xet hash:: 9f22c11736d1fe189c399fa9e72eaf2de1cc15915cb2e52aa0de7ab6f86fd4b0

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.

	import json
	from typing import List, Dict, Tuple, Any, Optional
	from dataclasses import dataclass
	from .metrics import ProofQuality, EvaluationMetrics
	from ..core.geometry import OlympiadProblem

	@dataclass
	class GeometryBenchmark:

	id: str
	name: str
	difficulty: str
	problem_statement: str
	expected_solution_steps: List[str]
	key_insights: List[str]
	time_limit: int
	topics: List[str]

	class GeometrySolverEvaluator:

	def __init__(self):
	self.benchmarks = self._load_benchmarks()

	def evaluate_solver(
	self, solver, benchmark_subset: str = "all"
	) -> EvaluationMetrics:

	if benchmark_subset == "all":
	problems = self.benchmarks
	else:
	problems = [b for b in self.benchmarks if benchmark_subset in b.topics]

	metrics = EvaluationMetrics(total_problems=len(problems))

	total_time = 0.0
	total_confidence = 0.0
	solved_count = 0

	for benchmark in problems:
	print(f"Evaluating on: {benchmark.name}")


	problem = self._benchmark_to_problem(benchmark)


	solution = solver.solve_problem(problem, timeout=benchmark.time_limit)


	is_solved = (
	solution["best_proof"] is not None and solution["confidence"] > 0.5
	)

	if is_solved:
	solved_count += 1
	metrics.difficulty_breakdown[benchmark.difficulty] = (
	metrics.difficulty_breakdown.get(benchmark.difficulty, 0) + 1
	)


	strategy = solution["best_proof"]["strategy"]
	if strategy not in metrics.strategy_success_rates:
	metrics.strategy_success_rates[strategy] = {
	"attempts": 0,
	"successes": 0,
	}
	metrics.strategy_success_rates[strategy]["successes"] += 1


	for strategy in solution["strategies_attempted"]:
	if strategy not in metrics.strategy_success_rates:
	metrics.strategy_success_rates[strategy] = {
	"attempts": 0,
	"successes": 0,
	}
	metrics.strategy_success_rates[strategy]["attempts"] += 1

	total_time += solution["time_taken"]
	total_confidence += solution["confidence"]


	if is_solved:
	quality_score = self._evaluate_proof_quality(
	solution["best_proof"], benchmark
	)
	metrics.proof_quality_scores.append(quality_score.value)


	metrics.solved_problems = solved_count
	metrics.total_time = total_time
	metrics.average_time = total_time / len(problems) if problems else 0
	metrics.average_confidence = total_confidence / len(problems) if problems else 0

	metrics.calculate_derived_metrics()
	return metrics

	def _benchmark_to_problem(self, benchmark: GeometryBenchmark) -> OlympiadProblem:
	from ..core.geometry import GeometricPoint, GeometricTriangle

	problem = OlympiadProblem(benchmark.name, benchmark.problem_statement)


	statement_lower = benchmark.problem_statement.lower()


	import re

	points = re.findall(r"\b[A-Z]\b", benchmark.problem_statement)
	points = list(set(points))[:6]

	for i, point_name in enumerate(points):

	x = (i % 3) * 2
	y = (i // 3) * 2
	problem.add_point(GeometricPoint(point_name, x, y))


	triangle_matches = re.findall(r"triangle\s+([A-Z]{3})", statement_lower)
	for match in triangle_matches:
	if len(match) == 3 and all(p in points for p in match):
	vertices = [problem.points[p] for p in match]
	triangle = GeometricTriangle(match, vertices)
	problem.add_triangle(triangle)


	if benchmark.expected_solution_steps:
	problem.set_goal(benchmark.expected_solution_steps[-1])

	return problem

	def _evaluate_proof_quality(
	self, proof: Dict[str, Any], benchmark: GeometryBenchmark
	) -> ProofQuality:

	steps = proof.get("steps", [])
	confidence = proof.get("confidence", 0)


	completeness = len(steps) >= 3
	clarity = confidence > 0.7
	correctness = proof.get("success", False)


	insight_coverage = 0
	if benchmark.key_insights:
	for insight in benchmark.key_insights:
	if any(insight.lower() in step.lower() for step in steps):
	insight_coverage += 1
	insight_coverage /= len(benchmark.key_insights)
	else:
	insight_coverage = 1.0


	if not correctness:
	return ProofQuality.INVALID
	elif completeness and clarity and insight_coverage > 0.7:
	return ProofQuality.EXCELLENT
	elif completeness and (clarity or insight_coverage > 0.5):
	return ProofQuality.GOOD
	elif completeness or clarity:
	return ProofQuality.ACCEPTABLE
	else:
	return ProofQuality.POOR

	def _load_benchmarks(self) -> List[GeometryBenchmark]:
	benchmarks = []


	benchmarks.append(
	GeometryBenchmark(
	"triangle_sum",
	"Triangle Angle Sum",
	"Elementary",
	"In triangle ABC, prove that the sum of angles is 180 degrees.",
	[
	"Consider triangle ABC",
	"Apply angle sum theorem",
	"Therefore angles sum to 180°",
	],
	["angle sum theorem"],
	30,
	["triangles", "angles"],
	)
	)

	benchmarks.append(
	GeometryBenchmark(
	"pythagorean",
	"Pythagorean Theorem",
	"High School",
	"In right triangle ABC with right angle at C, prove that AB² = AC² + BC².",
	[
	"Identify right angle",
	"Apply Pythagorean theorem",
	"Therefore AB² = AC² + BC²",
	],
	["right angle", "pythagorean"],
	45,
	["triangles", "right triangles"],
	)
	)


	benchmarks.append(
	GeometryBenchmark(
	"circumcenter",
	"Triangle Circumcenter",
	"High School",
	"Prove that the perpendicular bisectors of a triangle meet at a point equidistant from all vertices.",
	[
	"Consider perpendicular bisectors",
	"Show they intersect",
	"Prove equidistant property",
	],
	["perpendicular bisector", "equidistant"],
	60,
	["triangles", "circles", "construction"],
	)
	)

	benchmarks.append(
	GeometryBenchmark(
	"inscribed_angle",
	"Inscribed Angle Theorem",
	"High School",
	"In circle O, prove that an inscribed angle is half the central angle subtending the same arc.",
	[
	"Consider inscribed and central angles",
	"Use arc relationship",
	"Prove angle ratio is 1:2",
	],
	["inscribed angle", "central angle", "arc"],
	60,
	["circles", "angles"],
	)
	)


	benchmarks.append(
	GeometryBenchmark(
	"apollonius",
	"Apollonius Circle",
	"Olympiad",
	"Given triangle ABC and point P, construct a circle through P such that the tangent from A to the circle has specified length.",
	[
	"Analyze locus conditions",
	"Apply power of point",
	"Construct Apollonius circle",
	],
	["power of point", "locus", "apollonius"],
	120,
	["circles", "construction", "locus"],
	)
	)

	return benchmarks

	def generate_report(self, metrics: EvaluationMetrics) -> str:

	report = []
	report.append("=== Geometry Solver Evaluation Report ===")
	report.append(f"\nOverall Performance:")
	report.append(
	f" Problems Solved: {metrics.solved_problems}/{metrics.total_problems}"
	)
	report.append(f" Success Rate: {metrics.success_rate:.1%}")
	report.append(f" Average Time: {metrics.average_time:.2f} seconds")
	report.append(f" Average Confidence: {metrics.average_confidence:.1%}")
	report.append(
	f" Average Proof Quality: {metrics.average_proof_quality:.1f}/5.0"
	)

	report.append(f"\nDifficulty Breakdown:")
	for difficulty, count in metrics.difficulty_breakdown.items():
	report.append(f" {difficulty}: {count} problems solved")

	report.append(f"\nStrategy Performance:")
	for strategy, data in metrics.strategy_success_rates.items():
	report.append(f" {strategy.capitalize()}:")
	report.append(f" Attempts: {data['attempts']}")
	report.append(f" Successes: {data['successes']}")
	report.append(f" Success Rate: {data.get('success_rate', 0):.1%}")

	if metrics.proof_quality_scores:
	report.append(f"\nProof Quality Distribution:")
	quality_counts = {
	i: metrics.proof_quality_scores.count(i) for i in range(1, 6)
	}
	for quality, count in quality_counts.items():
	quality_name = ProofQuality(quality).name.title()
	percentage = count / len(metrics.proof_quality_scores) * 100
	report.append(f" {quality_name}: {count} ({percentage:.1f}%)")

	return "\n".join(report)