engine/evaluation/deterministic.py

"""Experimental deterministic A/S/E/C classifier.

This is NOT the official evaluator.  It exists so that the LLM judge can be
audited against a rule-based reimplementation of the same rubric (see
DETERMINISTIC_EVALUATION_ANALYSIS.md).  Switching official scoring to it is an
organizer decision.
"""

from __future__ import annotations

import json
from typing import Any, Dict, Optional

from .parser import normalize_number, numbers_equal

__all__ = ["deterministic_label", "parse_answer_object"]


def parse_answer_object(text: str) -> Optional[Dict[str, Any]]:
    """Recover an ``{"extracted_value": .., "calculated_value": ..}`` object.

    Accepts the object embedded in surrounding prose (the outermost ``{...}``
    span), matching the prompt's "minor formatting differences are acceptable".
    Returns ``None`` when no such object with both keys can be found.
    """
    if text is None:
        return None
    candidate = str(text).strip()
    if not candidate:
        return None
    start = candidate.find("{")
    end = candidate.rfind("}")
    if start == -1 or end <= start:
        return None
    try:
        parsed = json.loads(candidate[start : end + 1])
    except json.JSONDecodeError:
        return None
    if not isinstance(parsed, dict):
        return None
    if "extracted_value" not in parsed or "calculated_value" not in parsed:
        return None
    return parsed


def deterministic_label(gold_text: str, pred_text: str) -> str:
    """Apply the FinMR rubric hierarchy: structure -> extracted -> calculated."""
    gold = parse_answer_object(gold_text)
    if gold is None:
        # A malformed gold record is an organizer-side data error, not a
        # participant error; surface it loudly instead of scoring the case.
        raise ValueError(f"Gold answer is not a valid FinMR answer object: {gold_text!r}")

    pred = parse_answer_object(pred_text)
    if pred is None:
        return "S"
    # The prompt says "exactly two keys"; extra keys are treated as structural
    # noise only when the two required keys are absent, which parse_answer_object
    # already enforces.  Non-scalar values are structurally invalid.
    for key in ("extracted_value", "calculated_value"):
        value = pred[key]
        if isinstance(value, (dict, list)):
            return "S"
        if value is None or str(value).strip() == "":
            return "S"

    if not numbers_equal(gold["extracted_value"], pred["extracted_value"]):
        return "E"
    if not numbers_equal(gold["calculated_value"], pred["calculated_value"]):
        return "C"
    return "A"


def is_mismatch_case(extracted: Any, calculated: Any) -> Optional[bool]:
    """Diagnostic only: does the reported value disagree with the calculated one?

    Returns ``None`` when either side is not numeric.  This is a *diagnostic*
    derived from the two official outputs - Task 3 does not score match/mismatch
    (see TASK_SPEC.md section "Match / mismatch is a diagnostic, not a task").
    """
    left = normalize_number(extracted)
    right = normalize_number(calculated)
    if left is None or right is None:
        return None
    return left != right