File size: 3,170 Bytes
f4cc109 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 | """Experimental deterministic A/S/E/C classifier.
This is NOT the official evaluator. It exists so that the LLM judge can be
audited against a rule-based reimplementation of the same rubric (see
DETERMINISTIC_EVALUATION_ANALYSIS.md). Switching official scoring to it is an
organizer decision.
"""
from __future__ import annotations
import json
from typing import Any, Dict, Optional
from .parser import normalize_number, numbers_equal
__all__ = ["deterministic_label", "parse_answer_object"]
def parse_answer_object(text: str) -> Optional[Dict[str, Any]]:
"""Recover an ``{"extracted_value": .., "calculated_value": ..}`` object.
Accepts the object embedded in surrounding prose (the outermost ``{...}``
span), matching the prompt's "minor formatting differences are acceptable".
Returns ``None`` when no such object with both keys can be found.
"""
if text is None:
return None
candidate = str(text).strip()
if not candidate:
return None
start = candidate.find("{")
end = candidate.rfind("}")
if start == -1 or end <= start:
return None
try:
parsed = json.loads(candidate[start : end + 1])
except json.JSONDecodeError:
return None
if not isinstance(parsed, dict):
return None
if "extracted_value" not in parsed or "calculated_value" not in parsed:
return None
return parsed
def deterministic_label(gold_text: str, pred_text: str) -> str:
"""Apply the FinMR rubric hierarchy: structure -> extracted -> calculated."""
gold = parse_answer_object(gold_text)
if gold is None:
# A malformed gold record is an organizer-side data error, not a
# participant error; surface it loudly instead of scoring the case.
raise ValueError(f"Gold answer is not a valid FinMR answer object: {gold_text!r}")
pred = parse_answer_object(pred_text)
if pred is None:
return "S"
# The prompt says "exactly two keys"; extra keys are treated as structural
# noise only when the two required keys are absent, which parse_answer_object
# already enforces. Non-scalar values are structurally invalid.
for key in ("extracted_value", "calculated_value"):
value = pred[key]
if isinstance(value, (dict, list)):
return "S"
if value is None or str(value).strip() == "":
return "S"
if not numbers_equal(gold["extracted_value"], pred["extracted_value"]):
return "E"
if not numbers_equal(gold["calculated_value"], pred["calculated_value"]):
return "C"
return "A"
def is_mismatch_case(extracted: Any, calculated: Any) -> Optional[bool]:
"""Diagnostic only: does the reported value disagree with the calculated one?
Returns ``None`` when either side is not numeric. This is a *diagnostic*
derived from the two official outputs - Task 3 does not score match/mismatch
(see TASK_SPEC.md section "Match / mismatch is a diagnostic, not a task").
"""
left = normalize_number(extracted)
right = normalize_number(calculated)
if left is None or right is None:
return None
return left != right
|