scratch/test_all_cases.py

import sys
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))

from src.engine import ReguAIEngine
from src.core.case_catalog import CaseStudyCatalog

sys.stdout.reconfigure(encoding="utf-8")

engine = ReguAIEngine()
cat = CaseStudyCatalog()

domains = cat.list_domains()
print(f"Total domains: {len(domains)}")

all_passed = True
for d in domains:
    print(f"\n--- Domain: {d['domain_name']} ---")
    for c in d.get("case_studies", []):
        case_id = c["case_id"]
        expected = c.get("expected_conformity")
        doc = cat.get_case_document_text(case_id)
        report = engine.evaluate_system(doc)
        
        is_pass = report.overall_conforms
        has_violations = len(report.violations) > 0
        
        # Check expected alignment
        if "CONFORMANT (PASSED)" in expected:
            matches = is_pass and not has_violations
        elif "NON-CONFORMANT" in expected or "FAILED" in expected or "PROHIBITED" in expected:
            matches = not is_pass and has_violations
        else: # BORDERLINE
            matches = True
            
        status_str = "OK" if matches else "MISMATCH"
        if not matches:
            all_passed = False
        print(f"  [{status_str}] {case_id}: Expected '{expected}' | Got conforms={is_pass} (Violations: {len(report.violations)})")
        if not matches:
            for v in report.violations:
                print(f"       -> Violation: {v.regulatory_article} | {v.message[:80]}...")

print(f"\nOverall Alignment: {'ALL MATCHED' if all_passed else 'SOME MISMATCHES'}")