""" TalkingHeadBench evaluation harness. Usage ----- :: # Validate all cases in a single file against their schemas (no node calls): python src/evaluate.py --test-set tests/test_set/subenv1_cases.json --subenv 1 --dry-run # Run Sub-env 1 graders on every case in a file: python src/evaluate.py --test-set tests/test_set/subenv1_cases.json --subenv 1 # Run all sub-environments from a directory of JSON files: python src/evaluate.py --test-set tests/test_set/ --subenv all # Dry-run the entire directory: python src/evaluate.py --test-set tests/test_set/ --subenv all --dry-run Exit codes ---------- - 0: success (all cases validated / scored) - 1: schema validation failure or any other fatal error Test-case JSON format --------------------- Each file must be a JSON object with a ``"cases"`` array:: { "cases": [ { "id": "001", "observation": { ... }, "ground_truth": { ... } } ] } The schema for ``observation`` and ``ground_truth`` depends on ``--subenv``: --subenv 1 observation keys: image_obs: ImageDiagnosticsObservation (all fields) proposed_config: dict (e.g. {"cfg": 7.5, "eta": 0.08}) ground_truth keys: image: GroundTruthImageAnnotation param: GroundTruthParamAnnotation --subenv 2 observation: ClipSignalObservation (all fields) ground_truth: GroundTruthClipAnnotation --subenv 3 observation: PhonemeRiskObservation (all fields) ground_truth: GroundTruthBehavioralAnnotation --subenv all The file is expected to carry one of the three formats above; the harness auto-detects by checking which keys are present in ``observation``. Alternatively, supply a directory — each .json file is loaded independently and auto-detected. Per-case output format (example for Sub-env 1) ----------------------------------------------- :: Case 001 score=0.745 [regime=1.00 risk=0.80 prompt=0.43] Case 002 score=0.512 [regime=0.70 risk=0.50 prompt=0.21] ───────────────────────────────────────────────────────────── Mean: 0.629 Std: 0.117 Min: 0.512 Max: 0.745 """ from __future__ import annotations import argparse import json import math import statistics import sys from pathlib import Path from typing import Any, Callable from pydantic import ValidationError PROJECT_ROOT = Path(__file__).resolve().parents[1] if str(PROJECT_ROOT) not in sys.path: sys.path.insert(0, str(PROJECT_ROOT)) # --------------------------------------------------------------------------- # Schema imports # --------------------------------------------------------------------------- from src.schemas.ground_truth import ( GroundTruthBehavioralAnnotation, GroundTruthClipAnnotation, GroundTruthImageAnnotation, GroundTruthParamAnnotation, ) from src.schemas.subenv1 import ImageDiagnosticsObservation, ParamAnomalyObservation from src.schemas.subenv2 import ( ClipDispositionObservation, ClipEvidenceDossier, ClipSignalObservation, ) from src.schemas.subenv3 import PhonemeRiskObservation # --------------------------------------------------------------------------- # Agent / grader imports # --------------------------------------------------------------------------- from src.envs.subenv1.node1_image_diagnostician import diagnose_image from src.envs.subenv1.node2_param_anomaly import detect_param_anomalies from src.envs.subenv1.node3_grader import grade_anomaly_detection from src.envs.subenv2.node5_disposition import recommend_clip_disposition from src.envs.subenv2.node6_grader import grade_clip_disposition from src.envs.subenv3.node8_phoneme_risk import assess_phoneme_risk from src.envs.subenv3.node9_grader import grade_behavioral_audit from src.pipeline import ( _grade_image_diagnostics, _build_param_anomaly_obs, ) from src.utils.grader_utils import jaccard_similarity, set_f1 # ─── Display constants ─────────────────────────────────────────────────────── _SEPARATOR = "─" * 61 _SAFETY_LEVELS = ["safe", "minor_concerns", "moderate_risk", "high_risk", "unsafe"] # --------------------------------------------------------------------------- # Auto-detection helper # --------------------------------------------------------------------------- def detect_subenv(observation: dict) -> int: """Detect sub-environment from observation field names.""" # Sub-env 1: ImageDiagnosticsObservation if "face_occupancy_ratio" in observation: return 1 # Sub-env 1 (wrapped): {"image_obs": {...}, "proposed_config": {...}} if "image_obs" in observation: return 1 # Sub-env 2: ClipSignalObservation or ClipDispositionObservation if "face_embedding_variance" in observation or "evidence_dossier" in observation: return 2 # Sub-env 3: WeightSignalObservation or PhonemeRiskObservation if "lora_rank" in observation or "weight_evidence" in observation: return 3 raise ValueError( f"Cannot auto-detect sub-environment from observation keys. " f"Keys present: {list(observation.keys())}. " f"Expected 'face_occupancy_ratio' or 'image_obs' (sub-env 1), " f"'face_embedding_variance' (sub-env 2), or " f"'lora_rank' (sub-env 3)." ) def _coerce_subenv1_obs(obs_dict: dict) -> tuple[dict[str, Any], dict[str, Any]]: """Support both legacy wrapped and flat extractor Sub-env 1 observations.""" if "image_obs" in obs_dict and isinstance(obs_dict["image_obs"], dict): image_obs = obs_dict["image_obs"] proposed_config = obs_dict.get("proposed_config", {}) else: image_obs = obs_dict proposed_config = obs_dict.get("proposed_config", {}) if not isinstance(proposed_config, dict): raise ValueError( f"'proposed_config' must be a dict, got {type(proposed_config).__name__}" ) return image_obs, proposed_config def _coerce_subenv1_gt(gt_dict: dict) -> tuple[dict[str, Any], dict[str, Any]]: """Support both legacy nested and flat annotation-ready Sub-env 1 ground truth.""" if "image" in gt_dict and isinstance(gt_dict["image"], dict): image_gt = gt_dict["image"] else: image_gt = gt_dict if "param" in gt_dict and isinstance(gt_dict["param"], dict): param_gt = gt_dict["param"] else: # Flat extraction cases are image-first and do not include param GT yet. param_gt = { "config_risk_level": "safe", "anomalies": [], "predicted_failure_modes": [], "valid_fix_directions": [], } return image_gt, param_gt # --------------------------------------------------------------------------- # Schema validation helpers # --------------------------------------------------------------------------- def _validate_subenv1(obs_dict: dict, gt_dict: dict, case_id: str) -> None: """Validate a Sub-env 1 case against its Pydantic schemas.""" try: image_obs_dict, proposed_config = _coerce_subenv1_obs(obs_dict) ImageDiagnosticsObservation(**image_obs_dict) except ValidationError as exc: raise ValueError( f"Case {case_id}: ImageDiagnosticsObservation validation failed\n{exc}" ) from exc except ValueError as exc: raise ValueError(f"Case {case_id}: {exc}") from exc if not isinstance(proposed_config, dict): raise ValueError(f"Case {case_id}: 'proposed_config' must be a dict") try: image_gt_dict, param_gt_dict = _coerce_subenv1_gt(gt_dict) GroundTruthImageAnnotation(**image_gt_dict) except ValidationError as exc: raise ValueError( f"Case {case_id}: GroundTruthImageAnnotation validation failed\n{exc}" ) from exc except ValueError as exc: raise ValueError(f"Case {case_id}: {exc}") from exc try: GroundTruthParamAnnotation(**param_gt_dict) except ValidationError as exc: raise ValueError( f"Case {case_id}: GroundTruthParamAnnotation validation failed\n{exc}" ) from exc def _validate_subenv2(obs_dict: dict, gt_dict: dict, case_id: str) -> None: """Validate a Sub-env 2 case against its Pydantic schemas.""" try: ClipSignalObservation(**obs_dict) except ValidationError as exc: raise ValueError( f"Case {case_id}: ClipSignalObservation validation failed\n{exc}" ) from exc try: gt_dict = _coerce_subenv2_gt(gt_dict) except ValueError as exc: raise ValueError(f"Case {case_id}: {exc}") from exc try: GroundTruthClipAnnotation(**gt_dict) except ValidationError as exc: raise ValueError( f"Case {case_id}: GroundTruthClipAnnotation validation failed\n{exc}" ) from exc def _validate_subenv3(obs_dict: dict, gt_dict: dict, case_id: str) -> None: """Validate a Sub-env 3 case against its Pydantic schemas.""" try: PhonemeRiskObservation(**obs_dict) except ValidationError as exc: raise ValueError( f"Case {case_id}: PhonemeRiskObservation validation failed\n{exc}" ) from exc try: GroundTruthBehavioralAnnotation(**gt_dict) except ValidationError as exc: raise ValueError( f"Case {case_id}: GroundTruthBehavioralAnnotation validation failed\n{exc}" ) from exc # --------------------------------------------------------------------------- # Grader runners — return (score, breakdown_str, dim_dict) # --------------------------------------------------------------------------- def _run_subenv1(obs_dict: dict, gt_dict: dict, case_id: str): """Run Sub-env 1 graders and return (score, breakdown, dims). Executes Node 1 → Node 2 → Node 3 graders. Breakdown dimensions: regime — regime classification accuracy (0.35 weight in node1) risk — risk factor recall (0.35 weight in node1) prompt — prompt modification validity (0.30 weight in node1) node2 — full composite Node 2 score Final sub-env 1 score = 0.50 * node1_score + 0.50 * node2_score. """ image_obs_dict, proposed_config = _coerce_subenv1_obs(obs_dict) image_gt_dict, param_gt_dict = _coerce_subenv1_gt(gt_dict) image_obs = ImageDiagnosticsObservation(**image_obs_dict) gt_image = GroundTruthImageAnnotation(**image_gt_dict) gt_param = GroundTruthParamAnnotation(**param_gt_dict) # Node 1 node1_action = diagnose_image(image_obs) # Node 2 node2_obs = _build_param_anomaly_obs(node1_action, image_obs, proposed_config) node2_action = detect_param_anomalies(node2_obs) # Node 3 — grade node1 (sub-dimensions) # Regime accuracy if node1_action.regime_classification == gt_image.regime_classification: regime_score = 1.0 elif node1_action.regime_classification in gt_image.acceptable_regimes: regime_score = 0.7 else: regime_score = 0.0 # Risk factor recall pred_risks = set(node1_action.identified_risk_factors) true_risks = set(gt_image.identified_risk_factors) risk_score = ( len(pred_risks & true_risks) / len(true_risks) if true_risks else 1.0 ) # Prompt modification validity valid_mods = set(gt_image.valid_prompt_modifications) agent_mods = set(node1_action.recommended_prompt_modifications) if agent_mods: prompt_score = len(agent_mods & valid_mods) / len(agent_mods) else: prompt_score = 0.0 if valid_mods else 1.0 node1_score = ( 0.35 * regime_score + 0.35 * risk_score + 0.30 * prompt_score ) # Node 3 — grade node2 node2_score = grade_anomaly_detection(node2_action, gt_param) final_score = 0.50 * node1_score + 0.50 * node2_score dims = { "regime": regime_score, "risk": risk_score, "prompt": prompt_score, "node2": node2_score, } breakdown = ( f"regime={regime_score:.2f} risk={risk_score:.2f} " f"prompt={prompt_score:.2f} node2={node2_score:.2f}" ) return final_score, breakdown, dims def _coerce_subenv2_gt(gt_dict: dict) -> dict[str, Any]: """Support annotation-ready Sub-env 2 ground truth placeholders. If ``disposition`` is ``"ANNOTATE"``, this maps to a concrete ``GroundTruthClipAnnotation`` payload using ``_annotation_notes`` hints. """ if not isinstance(gt_dict, dict): raise ValueError( f"Sub-env 2 ground_truth must be a dict, got {type(gt_dict).__name__}" ) disposition = gt_dict.get("disposition") if not (isinstance(disposition, str) and disposition.upper() == "ANNOTATE"): return gt_dict notes = gt_dict.get("_annotation_notes", {}) if not isinstance(notes, dict): notes = {} suggested = str(notes.get("suggested_disposition", "defer")).lower() if suggested not in {"accept", "reject", "fix", "defer"}: suggested = "defer" try: confidence = float(notes.get("suggested_confidence", gt_dict.get("confidence", 0.5))) except (TypeError, ValueError): confidence = 0.5 confidence = max(0.0, min(1.0, confidence)) default_ambiguity = 0.5 if suggested == "defer" else 0.0 try: ambiguity = float(gt_dict.get("disposition_ambiguity", default_ambiguity)) except (TypeError, ValueError): ambiguity = default_ambiguity ambiguity = max(0.0, min(1.0, ambiguity)) valid_fix_steps = gt_dict.get("valid_fix_steps", []) if not isinstance(valid_fix_steps, list): valid_fix_steps = [] valid_override_justifications = gt_dict.get("valid_override_justifications", []) if not isinstance(valid_override_justifications, list): valid_override_justifications = [] expected_reasoning_elements = gt_dict.get("expected_reasoning_elements", []) if not isinstance(expected_reasoning_elements, list): expected_reasoning_elements = [] if not expected_reasoning_elements or all( str(v).upper() == "ANNOTATE" for v in expected_reasoning_elements ): expected_reasoning_elements = [ "dataset phoneme gaps", "pose gaps", "critical gaps", ] return { "disposition": suggested, "confidence": confidence, "disposition_ambiguity": ambiguity, "valid_fix_steps": valid_fix_steps, "valid_override_justifications": valid_override_justifications, "expected_reasoning_elements": expected_reasoning_elements, } def _resolve_subenv2_dossier_builder() -> Callable[[ClipSignalObservation], ClipEvidenceDossier]: """Resolve the Sub-env 2 evidence-dossier mapper from pipeline helpers.""" from src import pipeline as pipeline_module fn = getattr(pipeline_module, "build_evidence_dossier", None) if callable(fn): return fn fn = getattr(pipeline_module, "_heuristic_clip_evidence_dossier", None) if callable(fn): return fn raise RuntimeError( "No Sub-env 2 dossier builder found in pipeline. " "Expected build_evidence_dossier or _heuristic_clip_evidence_dossier." ) def _run_subenv2(obs_dict: dict, gt_dict: dict, case_id: str): """Run Sub-env 2 graders and return (score, breakdown, dims). Executes Node 5 → Node 6 grader. Breakdown dimensions (mirroring node6 internal scoring): base — base disposition score (0.40 max) fix_quality — fix instruction quality (0.20 max) reasoning — dataset impact reasoning (0.20 max) override — override penalty (subtractive, 0.10 max) """ clip_signal_obs = ClipSignalObservation(**obs_dict) gt_clip = GroundTruthClipAnnotation(**_coerce_subenv2_gt(gt_dict)) dossier_builder = _resolve_subenv2_dossier_builder() dossier = dossier_builder(clip_signal_obs) if isinstance(dossier, ClipEvidenceDossier): evidence_dossier = dossier elif isinstance(dossier, dict): evidence_dossier = ClipEvidenceDossier(**dossier) else: raise ValueError( f"Sub-env 2 dossier builder returned unsupported type: {type(dossier).__name__}" ) clip_obs = ClipDispositionObservation( evidence_dossier=evidence_dossier, minimum_clips_needed=20, phoneme_gap_severity={}, pose_gap_severity={}, budget_remaining=10, reference_risk_profile="medium", estimated_drift_risk=0.3, marginal_training_damage=0.2, marginal_coverage_gain=0.5, ) # Node 5 action = recommend_clip_disposition(clip_obs) # Node 6 — compute sub-dimensions manually for display, then get final score = grade_clip_disposition(action, gt_clip) # Reconstruct sub-dimension contributions for display base_score = 0.0 if action.disposition == gt_clip.disposition: calibrated = abs(action.confidence - gt_clip.confidence) < 0.15 base_score = 0.40 if calibrated else 0.28 elif action.disposition == "fix" and gt_clip.disposition == "reject": base_score = 0.20 elif action.disposition == "defer": if gt_clip.disposition_ambiguity >= 0.5: base_score = 0.15 if action.defer_reason else 0.10 fix_score = 0.0 if action.disposition == "fix" and action.fix_instructions: valid_steps = sum( 1 for s in action.fix_instructions if s in gt_clip.valid_fix_steps ) fp = valid_steps / len(action.fix_instructions) fix_score = 0.20 if fp >= 0.8 else (0.10 if fp >= 0.5 else 0.0) reasoning_score = 0.0 kw_elements = gt_clip.expected_reasoning_elements agent_reasoning = action.dataset_impact_reasoning.lower() matched = sum(1 for kw in kw_elements if kw in agent_reasoning) reasoning_score = ( 0.20 if matched >= len(kw_elements) * 0.8 else 0.10 if matched >= 1 else 0.00 ) override_penalty = -0.0 has_override_labels = ( isinstance(gt_clip.valid_override_justifications, list) and len(gt_clip.valid_override_justifications) > 0 ) if has_override_labels and action.override_decision == "applied": if not action.override_justification: override_penalty = -0.10 elif action.override_justification not in gt_clip.valid_override_justifications: override_penalty = -0.05 dims = { "base": base_score, "fix": fix_score, "reasoning": reasoning_score, "override_penalty": override_penalty, "disposition": action.disposition, } breakdown = ( f"base={base_score:.2f} fix={fix_score:.2f} " f"reasoning={reasoning_score:.2f} override={override_penalty:+.2f} " f"→{action.disposition}" ) return score, breakdown, dims def _run_subenv3(obs_dict: dict, gt_dict: dict, case_id: str): """Run Sub-env 3 graders and return (score, breakdown, dims). Executes Node 8 → Node 9 grader. Breakdown dimensions (node9 weights): ranking — top-5 phoneme ranking overlap (0.15) triggers — behavior trigger set F1 (0.30) clusters — phoneme cluster Jaccard (0.20) safety — safety level ordinal distance (0.15) mitigation — mitigation precision (0.20) """ phoneme_obs = PhonemeRiskObservation(**obs_dict) gt_behavioral = GroundTruthBehavioralAnnotation(**gt_dict) # Node 8 action = assess_phoneme_risk(phoneme_obs) # Node 9 score = grade_behavioral_audit(action, gt_behavioral) # Sub-dimension breakdown (mirrors node9 logic verbatim) agent_top5 = {e.phoneme for e in action.phoneme_risk_ranking[:5]} true_top5 = {e.phoneme for e in gt_behavioral.phoneme_risk_ranking[:5]} ranking_score = len(agent_top5 & true_top5) / 5 agent_triggers = { (t.trigger_phoneme, t.triggered_behavior) for t in action.predicted_behavior_triggers } true_triggers = { (t.trigger_phoneme, t.triggered_behavior) for t in gt_behavioral.predicted_behavior_triggers } trigger_score = set_f1(agent_triggers, true_triggers) agent_clusters = {frozenset(c.phonemes) for c in action.risky_phoneme_clusters} true_clusters = {frozenset(c.phonemes) for c in gt_behavioral.risky_phoneme_clusters} cluster_score = jaccard_similarity(agent_clusters, true_clusters) try: ai = _SAFETY_LEVELS.index(action.model_behavioral_safety) ti = _SAFETY_LEVELS.index(gt_behavioral.model_behavioral_safety) safety_score = 1.0 - abs(ai - ti) / (len(_SAFETY_LEVELS) - 1) except ValueError: safety_score = 0.0 agent_mits = {(m.target, m.action) for m in action.mitigation_recommendations} valid_mits = gt_behavioral.valid_mitigation_set mit_score = ( len(agent_mits & valid_mits) / len(agent_mits) if agent_mits else (0.0 if valid_mits else 1.0) ) dims = { "ranking": ranking_score, "triggers": trigger_score, "clusters": cluster_score, "safety": safety_score, "mitigation": mit_score, } breakdown = ( f"ranking={ranking_score:.2f} triggers={trigger_score:.2f} " f"clusters={cluster_score:.2f} safety={safety_score:.2f} " f"mitigation={mit_score:.2f}" ) return score, breakdown, dims # --------------------------------------------------------------------------- # File loading # --------------------------------------------------------------------------- def _load_cases(path: Path) -> list[dict]: """Load and return the ``cases`` list from a JSON test-set file.""" try: raw = path.read_text(encoding="utf-8") except OSError as exc: raise SystemExit(f"ERROR: Cannot read file '{path}': {exc}") from exc try: data = json.loads(raw) except json.JSONDecodeError as exc: raise SystemExit(f"ERROR: Invalid JSON in '{path}': {exc}") from exc if not isinstance(data, dict) or "cases" not in data: raise SystemExit( f"ERROR: '{path}' must be a JSON object with a top-level 'cases' array. " f"Got keys: {list(data.keys()) if isinstance(data, dict) else type(data).__name__}" ) if not isinstance(data["cases"], list): raise SystemExit( f"ERROR: '{path}': 'cases' must be a list, " f"got {type(data['cases']).__name__}" ) return data["cases"] def _collect_json_files(test_set_path: Path) -> list[Path]: """Return a sorted list of .json files from a path (file or directory).""" if test_set_path.is_file(): return [test_set_path] if test_set_path.is_dir(): files = sorted(test_set_path.glob("*.json")) if not files: raise SystemExit( f"ERROR: No .json files found in directory '{test_set_path}'" ) return files raise SystemExit( f"ERROR: '{test_set_path}' is neither a file nor a directory" ) # --------------------------------------------------------------------------- # Summary printer # --------------------------------------------------------------------------- def _print_summary(scores: list[float]) -> None: """Print the mean/std/min/max summary line.""" if not scores: print(" (no cases scored)") return mean = statistics.mean(scores) std = statistics.stdev(scores) if len(scores) > 1 else 0.0 lo = min(scores) hi = max(scores) print(_SEPARATOR) print(f"Mean: {mean:.3f} Std: {std:.3f} Min: {lo:.3f} Max: {hi:.3f}") # --------------------------------------------------------------------------- # Per-file processor # --------------------------------------------------------------------------- def _process_file( json_path: Path, subenv_arg: str, dry_run: bool, ) -> list[float]: """Process all cases in a single JSON file. Returns the list of per-case scores (empty in --dry-run mode). """ cases = _load_cases(json_path) print(f"\nFile: {json_path} ({len(cases)} case(s))") scores: list[float] = [] total_validated = 0 for raw_case in cases: case_id: str = str(raw_case.get("id", "???")) obs_dict: dict = raw_case.get("observation", {}) gt_dict: dict = raw_case.get("ground_truth", {}) # ── Determine sub-environment ──────────────────────────────────────── if subenv_arg == "all": try: subenv = detect_subenv(obs_dict) except ValueError as exc: print(f"ERROR: Case {case_id}: {exc}", file=sys.stderr) sys.exit(1) else: subenv = int(subenv_arg) # ── Schema validation ──────────────────────────────────────────────── try: if subenv == 1: _validate_subenv1(obs_dict, gt_dict, case_id) elif subenv == 2: _validate_subenv2(obs_dict, gt_dict, case_id) elif subenv == 3: _validate_subenv3(obs_dict, gt_dict, case_id) except ValueError as exc: print(f"ERROR: {exc}", file=sys.stderr) sys.exit(1) total_validated += 1 if dry_run: continue # ── Run graders ────────────────────────────────────────────────────── try: if subenv == 1: score, breakdown, _ = _run_subenv1(obs_dict, gt_dict, case_id) elif subenv == 2: score, breakdown, _ = _run_subenv2(obs_dict, gt_dict, case_id) else: score, breakdown, _ = _run_subenv3(obs_dict, gt_dict, case_id) except Exception as exc: # noqa: BLE001 print( f"ERROR: Case {case_id}: node/grader raised an exception: {exc}", file=sys.stderr, ) sys.exit(1) scores.append(score) print(f"Case {case_id:<6} score={score:.3f} [{breakdown}]") if dry_run: print(f"Schema OK: {total_validated} case(s)") else: _print_summary(scores) return scores # --------------------------------------------------------------------------- # CLI entry-point # --------------------------------------------------------------------------- def _build_parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser( prog="evaluate", description=( "TalkingHeadBench evaluation harness. " "Runs graders for one or all sub-environments against a JSON test set." ), formatter_class=argparse.RawDescriptionHelpFormatter, epilog=__doc__, ) parser.add_argument( "--test-set", required=True, metavar="PATH", help=( "Path to a JSON test-set file, or a directory of JSON files. " "Each file must be a JSON object with a top-level 'cases' array." ), ) parser.add_argument( "--subenv", required=True, choices=["1", "2", "3", "all"], metavar="{1,2,3,all}", help=( "Which sub-environment to evaluate. " "Use 'all' to auto-detect from observation keys." ), ) parser.add_argument( "--dry-run", action="store_true", default=False, help=( "Validate all observation and ground-truth dicts against their " "Pydantic schemas, then exit 0. No node calls are made." ), ) return parser def main(argv: list[str] | None = None) -> None: """Parse arguments and run the evaluation harness.""" parser = _build_parser() args = parser.parse_args(argv) test_set_path = Path(args.test_set) json_files = _collect_json_files(test_set_path) mode_label = "DRY-RUN (schema validation only)" if args.dry_run else f"sub-env {args.subenv}" print(f"TalkingHeadBench evaluate — mode: {mode_label}") print(f"Test set: {test_set_path} ({len(json_files)} file(s))") all_scores: list[float] = [] for json_path in json_files: file_scores = _process_file(json_path, args.subenv, args.dry_run) all_scores.extend(file_scores) # If multiple files were scored, print a cross-file summary if not args.dry_run and len(json_files) > 1 and all_scores: print(f"\nOverall summary across {len(json_files)} file(s):") _print_summary(all_scores) sys.exit(0) if __name__ == "__main__": main()