"""Verify and summarize aligned benchmark reports without pooling physical units. Python API: build_comparison(data_path, runs, output), where runs maps each name to {"report": benchmark-directory, "checkpoints": [checkpoint-path, ...]}. CLI: python -m tfm.evaluation.comparison --data DATA --config RUNS.json --output comparison.json Missing reports are explicitly unrun. Re-running refreshes report fingerprints after isolated latency measurements have been added to existing reports. """ from __future__ import annotations import argparse import json from pathlib import Path import numpy as np from tfm.data import FIELD_INDEX, Normalization, WindowDataset from tfm.training.train import load_checkpoint, load_manifest_episodes from .benchmark import sampling_provenance, sha256 from .calibration import validate_calibration def _summarize_model(model, normalization): if model["status"] != "completed": return {"status": model["status"], "reason": model.get("reason")} forecasts = [] for family in sorted({row["family"] for row in model["metrics"]}): rows = [row for row in model["metrics"] if row["family"] == family] calibrated = [row for row in rows if row["interval_kind"] == "heldout_calibrated"] calibrated_count = sum(row["count"] for row in calibrated) count = sum(row["count"] for row in rows) forecasts.append( { "family": family, "field_horizon_balanced_standardized_mae": float( np.mean([row["standardized_mae"] for row in rows]) ), "coverage_weighted_by_count": sum(row["coverage"] * row["count"] for row in calibrated) / calibrated_count if calibrated_count else None, "calibrated_query_count": calibrated_count, "total_observed_query_count": count, "calibrated_query_fraction": calibrated_count / count if count else 0, "field_horizon_balanced_standardized_interval_width": float( np.mean( [ row["mean_interval_width"] / normalization.scale[row["field_id"]] for row in calibrated ] ) ) if calibrated else None, "uncalibrated_fields": sorted( {row["field"] for row in rows if row["interval_kind"] != "heldout_calibrated"} ), } ) interventions = [] for row in model.get("counterfactual", {}).get("summary_by_family", []): interventions.append( { "family": row["family"], "sample_count": row["sample_count"], "field_horizon_balanced_standardized_action_effect_mae": float( np.mean( [ effect["action_effect_mae"] / normalization.scale[FIELD_INDEX[effect["field"]]] for effect in row["effects"] ] ) ), "ranking_accuracy": row["ranking_accuracy"], "mean_regret_family_objective_units": row["mean_regret"], } ) return { "forecast": forecasts, "intervention": interventions, "calibration_status": model.get("calibration_status"), } def build_comparison(data_path, runs, output): data_path = Path(data_path) manifest_path = data_path / "manifest.json" if data_path.is_dir() else data_path manifest, splits = load_manifest_episodes(data_path) manifest_ids = {key: {str(ep.system_id) for ep in episodes} for key, episodes in splits.items()} result = { "version": "tfm-comparison-v1", "status": "smoke_only", "generator_version": manifest["generator_version"], "manifest_sha256": sha256(manifest_path), "partition_episode_counts": {key: len(value) for key, value in splits.items()}, "interpretation": "Short synthetic runs, no research or scaling claim. Prediction and action-effect MAE equally average field/horizon rows within family. Coverage is weighted by calibrated observed query counts and omits unsupported fields, which are listed. Widths divide by each field's training standard deviation; physical widths remain in source reports. Regret has family-specific objective units and must not be pooled. Latency retains the source measurement scope and contention limitations.", "runs": {}, } reference_sampling, baseline_reference = None, {} for name, specification in runs.items(): report_path = Path(specification["report"]) if report_path.is_dir() or not report_path.suffix: report_path = report_path / "benchmark.json" if not report_path.exists(): result["runs"][name] = {"status": "unrun", "reason": "Benchmark report not yet available"} continue report = json.loads(report_path.read_text()) paths = [Path(path) for path in specification["checkpoints"]] checkpoints = [load_checkpoint(path)[1] for path in paths] checkpoint = checkpoints[0] norm_path = paths[0].parent / "normalization.json" normalization = Normalization.load(norm_path) if report["checkpoint_sha256"] != [sha256(path) for path in paths]: raise ValueError("Comparison report checkpoint fingerprint mismatch") if ( report["normalization_sha256"] != sha256(norm_path) or report["manifest_sha256"] != result["manifest_sha256"] ): raise ValueError("Comparison normalization or dataset fingerprint mismatch") keys = ( "training_system_ids", "validation_selection_system_ids", "calibration_system_ids", "test_system_ids", ) groups = {key: set(checkpoint[key]) for key in keys} for i, key in enumerate(keys): if any(groups[key] & groups[other] for other in keys[:i]): raise ValueError("Comparison checkpoint partition leakage") if any(set(member[key]) != groups[key] for member in checkpoints): raise ValueError("Comparison ensemble member partitions differ") if ( groups["training_system_ids"] != manifest_ids["train"] or groups["test_system_ids"] != manifest_ids["test"] ): raise ValueError("Comparison checkpoints differ from manifest training or test partitions") if ( groups["validation_selection_system_ids"] | groups["calibration_system_ids"] != manifest_ids["validation"] ): raise ValueError("Comparison validation partition provenance differs") if set(normalization.training_system_ids) != groups["training_system_ids"]: raise ValueError("Comparison normalization training provenance differs") if set(report["test_system_ids"]) != groups["test_system_ids"]: raise ValueError("Comparison report test provenance differs") cal_path = report_path.parent / "calibration.json" calibration = json.loads(cal_path.read_text()) validate_calibration( calibration, checkpoint_sha256=report["checkpoint_sha256"], normalization_version=1, normalization_sha256=report["normalization_sha256"], forbidden_system_ids=groups["training_system_ids"] | groups["validation_selection_system_ids"] | groups["test_system_ids"], ) if set(calibration["calibration_system_ids"]) != groups["calibration_system_ids"]: raise ValueError("Comparison calibration systems differ from reserved partition") dataset = WindowDataset( splits["test"], normalization, history_steps=checkpoint["model_config"]["history_steps"], future_steps=8, samples=report["samples"], seed=report["seed"], ) expected = sampling_provenance(dataset, manifest_path)["window_sampling"] if ( report["window_sampling"] != expected or reference_sampling is not None and expected != reference_sampling ): raise ValueError("Comparison sampled windows differ") reference_sampling = expected models = { model_name: _summarize_model(model, normalization) for model_name, model in report["models"].items() } for baseline in ("persistence", "history_linear_system_id"): if baseline in baseline_reference and models[baseline] != baseline_reference[baseline]: raise ValueError("Comparison aligned baseline results differ") baseline_reference[baseline] = models[baseline] generalization = json.loads((report_path.parent / "generalization.json").read_text()) result["runs"][name] = { "status": "completed", "provenance_checks": "passed", "source_report": str(report_path), "source_report_sha256": sha256(report_path), "calibration_sha256": sha256(cal_path), "checkpoint_sha256": report["checkpoint_sha256"], "normalization_sha256": report["normalization_sha256"], "split_system_counts": {key: len(ids) for key, ids in groups.items()}, "parameter_count": report["parameter_count"], "inference": report["inference"], "window_sampling": expected, "models": models, "generalization": [ { "name": exp["config"]["name"], "axis": exp["config"]["axis"], "status": exp["status"], "reason": exp.get("reason"), "evaluated_target_fraction": exp.get("result", {}).get("evaluated_target_fraction"), } for exp in generalization["experiments"] ], } Path(output).write_text(json.dumps(result, indent=2, allow_nan=False) + "\n") return result def main(argv=None): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--data", required=True) parser.add_argument( "--config", required=True, help="JSON mapping of names to report directories and checkpoint path lists", ) parser.add_argument("--output", required=True) args = parser.parse_args(argv) build_comparison(args.data, json.loads(Path(args.config).read_text()), args.output) if __name__ == "__main__": main()