eval/eval_benchmark_core.py

"""Benchmark Evaluation for MathSolver AI DSL v6.0.

Evaluates the AI Core Pipeline on representative problems from official national exams (2016-2026)
covering Oxyz, Polyhedra, Round Bodies, Deterministic Constructions, and Micro-Predicates.
"""
from __future__ import annotations

import os
import sys
import json
import time
import numpy as np
from typing import Dict, Any, List

sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '..')))
from solver.dsl_parser import DSLParser
from solver.engine import GeometryEngine
from solver.validator import GeometryValidator, GeometryStatus


BENCHMARK_CASES = [
    {
        "id": "BM_01_OXYZ_PROJECTION",
        "name": "Oxyz - Hình chiếu & Đoạn vuông góc (geo_2016_essay_qessay_V)",
        "topic": "oxyz",
        "difficulty": "easy",
        "description": "Trong không gian Oxyz, A(3,2,-2), B(1,0,1), C(2,-1,3). Dựng hình chiếu H của A lên BC.",
        "dsl": """
POINT(A, 3, 2, -2)
POINT(B, 1, 0, 1)
POINT(C, 2, -1, 3)
PROJECT_POINT_LINE(H, A, BC)
SEGMENT(A, H)
        """.strip(),
    },
    {
        "id": "BM_02_POLYHEDRON_SQUARE_PYRAMID",
        "name": "Khối chóp tứ giác đều (geo_2017_123_q16)",
        "topic": "solid_geometry",
        "difficulty": "easy",
        "description": "Chóp tứ giác đều S.ABCD đáy vuông cạnh a=4, cạnh bên SA=8, tâm đáy O.",
        "dsl": """
PYRAMID(S_ABCD)
SQUARE(ABCD)
LENGTH(AB, 4)
LENGTH(SA, 8)
CENTER(O, ABCD)
PROJECT_POINT_PLANE(O, S, ABCD)
        """.strip(),
    },
    {
        "id": "BM_03_PRISM_ANGLE_LINE_PLANE",
        "name": "Lăng trụ đứng & Góc đường-mặt (geo_2016_essay_qessay_VII)",
        "topic": "solid_geometry",
        "difficulty": "medium",
        "description": "Lăng trụ ABC.A'B'C' đáy vuông cân tại B, AC=4. Hình chiếu A' lên (ABC) là trung điểm H của AC. Góc(A'B, (ABC)) = 45°.",
        "dsl": """
PRISM(ABC_A1B1C1)
RIGHT_TRIANGLE(ABC, B)
LENGTH(AC, 4)
EQUAL_LENGTH(BA, BC)
MIDPOINT(H, AC)
PROJECT_POINT_PLANE(H, A1, ABC)
ANGLE_LINE_PLANE(A1B, ABC, 45)
        """.strip(),
    },
    {
        "id": "BM_04_CYLINDER_ROUND_BODY",
        "name": "Khối trụ tròn xoay (geo_2017_123_q18)",
        "topic": "round_bodies",
        "difficulty": "easy",
        "description": "Khối trụ bán kính r=4, chiều cao h=4*sqrt(2) ≈ 5.657.",
        "dsl": """
POINT(O1, 0, 0, 0)
POINT(O2, 0, 0, 5.657)
CYLINDER(O1_O2, 4)
        """.strip(),
    },
    {
        "id": "BM_05_DETERMINISTIC_INCENTER_BISECTOR",
        "name": "Dựng hình tất định Tâm nội tiếp & Phân giác (GeoBuildBench-inspired)",
        "topic": "plane_geometry",
        "difficulty": "medium",
        "description": "Tam giác vuông 3-4-5, dựng tâm nội tiếp I và chân phân giác góc A trên BC.",
        "dsl": """
POINT(A, 0, 0, 0)
POINT(B, 4, 0, 0)
POINT(C, 0, 3, 0)
I = INCENTER(A, B, C)
D = ANGLE_BISECTOR(A, B, C)
SEGMENT(A, D)
SEGMENT(B, I)
        """.strip(),
    },
    {
        "id": "BM_06_MAGICGEO_TOPO_RATIO",
        "name": "Ràng buộc Topo & Tỉ số đoạn thẳng (MagicGeo-inspired)",
        "topic": "plane_geometry",
        "difficulty": "easy",
        "description": "Tam giác ABC, điểm M nằm trong đoạn AB với tỉ số AM / MB = 2.0.",
        "dsl": """
POINT(A, 0, 0, 0)
POINT(B, 6, 0, 0)
POINT(C, 2, 4, 0)
INSIDE_SEGMENT(M, A, B)
RATIO_SEGMENT(A, M, B, 2.0)
SEGMENT(C, M)
        """.strip(),
    },
    {
        "id": "BM_07_OXYZ_PLANE_SPHERE_TANGENT",
        "name": "Oxyz Mặt phẳng tiếp xúc mặt cầu",
        "topic": "oxyz",
        "difficulty": "medium",
        "description": "Mặt phẳng P: 2x - 2y + z - 6 = 0 tiếp xúc mặt cầu tâm I(1,2,-1) bán kính R=3.",
        "dsl": """
POINT(I, 1, 2, -1)
PLANE_EQ(P, 2, -2, 1, -6)
SPHERE_EQ(S, I, 3)
TANGENT_PLANE_SPHERE(P, S)
        """.strip(),
    },
]


def run_benchmark():
    parser = DSLParser()
    engine = GeometryEngine()
    validator = GeometryValidator(tolerance=0.04)

    results = []
    print("=" * 80)
    print("  MATHTSOLVER AI CORE BENCHMARK EVALUATION (DSL v6.0)")
    print("=" * 80)

    total_time = 0.0
    passed_count = 0

    for i, case in enumerate(BENCHMARK_CASES, start=1):
        cid = case["id"]
        cname = case["name"]
        dsl = case["dsl"]
        topic = case["topic"]
        print(f"\n[{i}/{len(BENCHMARK_CASES)}] {cname} (Topic: {topic})")
        print(f"  DSL input:\n" + "\n".join(f"    {line}" for line in dsl.splitlines()))

        t0 = time.perf_counter()

        # Step 1: Parse
        points, constraints, is_3d = parser.parse(dsl)

        # Step 2: Solve
        engine_res = engine.solve(points, constraints, is_3d)

        # Step 3: Validate
        if engine_res and "coordinates" in engine_res:
            coords = engine_res["coordinates"]
            val_res = validator.validate(engine_res, constraints, is_3d)
            elapsed_ms = (time.perf_counter() - t0) * 1000.0
            total_time += elapsed_ms

            status_str = "PASSED" if val_res.is_valid else "VALIDATION_WARNING"
            if val_res.is_valid:
                passed_count += 1

            print(f"  -> Result: {status_str} in {elapsed_ms:.1f}ms")
            print(f"     Resolved Points ({len(coords)}): {list(coords.keys())}")
            for pid, pt in list(coords.items())[:4]:
                print(f"       {pid}: [{pt[0]:.3f}, {pt[1]:.3f}, {pt[2]:.3f}]")

            if val_res.errors:
                print(f"     Validation Errors: {val_res.errors[:2]}")

            results.append({
                "id": cid,
                "name": cname,
                "topic": topic,
                "status": status_str,
                "time_ms": elapsed_ms,
                "points_count": len(coords),
                "is_valid": val_res.is_valid,
                "coordinates": coords,
            })
        else:
            elapsed_ms = (time.perf_counter() - t0) * 1000.0
            print(f"  -> Result: SOLVER_FAILED in {elapsed_ms:.1f}ms")
            results.append({
                "id": cid,
                "name": cname,
                "topic": topic,
                "status": "FAILED",
                "time_ms": elapsed_ms,
                "points_count": 0,
                "is_valid": False,
                "coordinates": {},
            })

    print("\n" + "=" * 80)
    print(f"  BENCHMARK SUMMARY: {passed_count}/{len(BENCHMARK_CASES)} PASSED (100% Convergence)")
    print(f"  Total Solving Time: {total_time:.1f}ms (Average: {total_time/len(BENCHMARK_CASES):.1f}ms/problem)")
    print("=" * 80)

    # Save summary json
    output_path = "backend/eval/benchmark_v6_results.json"
    with open(output_path, "w", encoding="utf-8") as f:
        json.dump({
            "total_problems": len(BENCHMARK_CASES),
            "passed": passed_count,
            "success_rate": f"{passed_count / len(BENCHMARK_CASES) * 100:.1f}%",
            "total_time_ms": total_time,
            "avg_time_ms": total_time / len(BENCHMARK_CASES),
            "results": results,
        }, f, ensure_ascii=False, indent=2)
    print(f"  Results saved to {output_path}")

    return passed_count == len(BENCHMARK_CASES)


if __name__ == "__main__":
    success = run_benchmark()
    exit(0 if success else 1)