Release experimental Transition Foundation Model v0.1 with reproducible source and validation
1baf215 verified Download tfm/evaluation/benchmark.py from maxffarrell/transition-foundation-model: direct link, hf CLI and curl.
- Browser
- Download file 30 kB
-
https://huggingface.co/maxffarrell/transition-foundation-model/resolve/main/tfm/evaluation/benchmark.py
- Command line
-
hf download hf://maxffarrell/transition-foundation-model/tfm/evaluation/benchmark.py
-
curl -L -o benchmark.py https://huggingface.co/maxffarrell/transition-foundation-model/resolve/main/tfm/evaluation/benchmark.py
30 kB
| """Reproducible physical-unit benchmark on independently sampled held-out windows. | |
| Usage: python -m tfm.evaluation.benchmark --data DATA --checkpoint RUN/checkpoint.pt | |
| --output REPORT --device cpu --samples 512 --counterfactual-samples 12 | |
| Repeated --checkpoint arguments combine ensemble predictive distributions. | |
| """ | |
| from __future__ import annotations | |
| import argparse | |
| import copy | |
| import hashlib | |
| import json | |
| import resource | |
| import sys | |
| import time | |
| from pathlib import Path | |
| import numpy as np | |
| import torch | |
| from torch.utils.data import DataLoader | |
| from tfm.data import FIELD_INDEX, FIELD_SPECS, ROLE_IDS, Normalization, WindowDataset | |
| from tfm.data.schema import GENERATOR_VERSION, SCHEMAS | |
| from tfm.training.train import ( | |
| SCHEMA_VERSION, | |
| load_checkpoint, | |
| load_manifest_episodes, | |
| resolve_device, | |
| to_device, | |
| ) | |
| from .baselines import CatBoostBaseline, HistoryLinearBaseline, PersistenceBaseline | |
| from .calibration import fit_calibration, save_calibration | |
| from .counterfactual import COST_DEFINITIONS, action_effect_metrics, branch_rollouts, family_cost | |
| from .experiments import default_experiments, run_experiments, stress_observations | |
| from .metrics import metrics_by_group | |
| def sha256(path): | |
| return hashlib.sha256(Path(path).read_bytes()).hexdigest() | |
| def _ids(episodes): | |
| return sorted({str(e.system_id) for e in episodes}) | |
| class Ensemble: | |
| def __init__(self, models): | |
| self.models = models | |
| def __call__(self, batch): | |
| predictions = [model(batch) for model in self.models] | |
| means = torch.stack([x[0] for x in predictions]) | |
| stds = torch.stack([x[1].exp() for x in predictions]) | |
| mean = means.mean(0) | |
| variance = (stds.square() + means.square()).mean(0) - mean.square() | |
| return mean, variance.clamp_min(1e-12).sqrt().log() | |
| def to(self, device): | |
| for model in self.models: | |
| model.to(device) | |
| return self | |
| def eval(self): | |
| for model in self.models: | |
| model.eval() | |
| return self | |
| def parameters(self): | |
| for model in self.models: | |
| yield from model.parameters() | |
| def _concat(batches): | |
| return {key: np.concatenate([batch[key] for batch in batches], axis=0) for key in batches[0]} | |
| def collect_predictions(model, dataset, device, batch_size=64, stress=None, seed=0): | |
| means, log_stds, arrays = [], [], [] | |
| with torch.inference_mode(): | |
| for i, batch in enumerate(DataLoader(dataset, batch_size=batch_size, shuffle=False)): | |
| device_batch = to_device(batch, device) | |
| inputs = stress_observations(device_batch, stress, seed + i) if stress else device_batch | |
| # Targets remain outside the callable input contract. | |
| mean, log_std = model({k: v for k, v in inputs.items() if k not in ("target", "target_mask")}) | |
| if stress: | |
| # Decode from corrupted observed current, as the deployed API does. | |
| anchor_difference = (inputs["current"] - device_batch["current"]).gather( | |
| 1, inputs["query_fields"] | |
| ) | |
| mean = mean + anchor_difference | |
| queried_available = inputs["current_mask"].gather(1, inputs["query_fields"]).bool() | |
| batch["target_mask"] = batch["target_mask"].bool() & queried_available.cpu() | |
| means.append(mean.cpu().numpy()) | |
| log_stds.append(log_std.cpu().numpy()) | |
| arrays.append({k: v.numpy() for k, v in batch.items()}) | |
| if not means: | |
| raise ValueError("No benchmark windows") | |
| return np.concatenate(means), np.concatenate(log_stds), _concat(arrays) | |
| def _families(dataset): | |
| return [str(dataset.episodes[ep].family_id) for ep, _ in dataset.indices] | |
| def sampling_provenance(dataset, manifest_path): | |
| anchors = [(str(dataset.episodes[ep].system_id), int(step)) for ep, step in dataset.indices] | |
| families = _families(dataset) | |
| return { | |
| "manifest_sha256": sha256(manifest_path), | |
| "window_sampling": { | |
| "seed": dataset.seed, | |
| "history_steps": dataset.history_steps, | |
| "future_steps": dataset.future_steps, | |
| "sampling": "uniform eligible anchors with replacement", | |
| "anchor_sha256": hashlib.sha256(json.dumps(anchors, separators=(",", ":")).encode()).hexdigest(), | |
| "family_window_counts": {family: families.count(family) for family in sorted(set(families))}, | |
| }, | |
| } | |
| def summarize(mean, log_std, arrays, families, normalization, calibration=None): | |
| return metrics_by_group( | |
| mean, | |
| log_std, | |
| arrays["target"], | |
| arrays["query_fields"], | |
| arrays["query_horizons"], | |
| arrays["target_mask"], | |
| families, | |
| normalization, | |
| calibration=calibration, | |
| field_names=[spec.name for spec in FIELD_SPECS], | |
| ) | |
| def profile_inference(model, example, devices=("cpu",), repetitions=30, warmup=5): | |
| """Synchronized model-only latency; excludes validation and tensor transfers.""" | |
| profiles = [] | |
| one = {k: v[:1] for k, v in example.items() if k not in ("target", "target_mask")} | |
| for name in devices: | |
| if name == "mps" and not torch.backends.mps.is_available(): | |
| profiles.append({"device": name, "status": "unavailable"}) | |
| continue | |
| if hasattr(model, "to"): | |
| model.to(name) | |
| if hasattr(model, "eval"): | |
| model.eval() | |
| batch = to_device(one, name) | |
| memory_samples, durations = [], [] | |
| with torch.inference_mode(): | |
| for index in range(warmup + repetitions): | |
| if name == "mps": | |
| torch.mps.synchronize() | |
| start = time.perf_counter_ns() | |
| model(batch) | |
| if name == "mps": | |
| torch.mps.synchronize() | |
| memory_samples.append(torch.mps.driver_allocated_memory()) | |
| duration = (time.perf_counter_ns() - start) / 1e6 | |
| if index >= warmup: | |
| durations.append(duration) | |
| rss = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss | |
| profiles.append( | |
| { | |
| "device": name, | |
| "status": "completed", | |
| "batch_size": 1, | |
| "precision": "float32", | |
| "repetitions": repetitions, | |
| "warmup": warmup, | |
| "p50_ms": float(np.percentile(durations, 50)), | |
| "p95_ms": float(np.percentile(durations, 95)), | |
| "process_peak_rss_bytes": int(rss if sys.platform == "darwin" else rss * 1024), | |
| "sampled_mps_driver_allocation_peak_bytes": max(memory_samples) if memory_samples else None, | |
| "memory_limitation": "RSS is lifetime process high water, includes data and prior models; MPS driver allocation sampled after calls is a lower bound, not resettable model peak.", | |
| "timing_scope": "forward only; inputs resident; MPS synchronized before and after every call", | |
| } | |
| ) | |
| return profiles | |
| def _candidate_actions(episode, t, steps=8): | |
| base = episode.actions[t : t + steps].copy() | |
| candidates = [base] | |
| if episode.family_id == "thermal": | |
| for power in (-5.0, 0.0, 5.0): | |
| action = base.copy() | |
| action[:, FIELD_INDEX["hvac_power"]] = power | |
| candidates.append(action) | |
| elif episode.family_id == "tank": | |
| for pump in (0.0, 0.5, 1.0): | |
| action = base.copy() | |
| action[:, FIELD_INDEX["pump_output"]] = pump | |
| action[:, FIELD_INDEX["valve_position"]] = 0.7 | |
| candidates.append(action) | |
| elif episode.family_id == "battery": | |
| for power in (-5.0, 0.0, 5.0): | |
| action = base.copy() | |
| action[:, FIELD_INDEX["battery_power"]] = power | |
| candidates.append(action) | |
| else: | |
| raise ValueError("Counterfactual family has no declared action candidates") | |
| return np.asarray(candidates) | |
| def evaluate_counterfactual(model, dataset, normalization, device, samples=12): | |
| from tfm.simulators import make_simulator | |
| records = [] | |
| for index in range(min(samples, len(dataset))): | |
| epi, t = dataset.indices[index] | |
| episode = dataset.episodes[epi] | |
| sim = make_simulator(episode.family_id, episode.seed) | |
| for j in range(t): | |
| sim.step(episode.actions[j], episode.exogenous[j]) | |
| snapshot = sim.snapshot() | |
| actions = _candidate_actions(episode, t) | |
| exogenous = episode.exogenous[t : t + 8] | |
| truth_compact = branch_rollouts(sim, snapshot, actions, exogenous) | |
| names = [name for name in sim.schema if FIELD_SPECS[FIELD_INDEX[name]].role == "state"] | |
| fields = [FIELD_INDEX[name] for name in names] | |
| truth = np.zeros((len(actions), 9, len(FIELD_SPECS))) | |
| truth[:, :, fields] = truth_compact | |
| sample = dataset[index] | |
| batch = { | |
| key: value.unsqueeze(0).repeat((len(actions),) + (1,) * value.ndim) | |
| for key, value in sample.items() | |
| if key not in ("target", "target_mask") | |
| } | |
| normalized_actions = (actions - normalization.mean) / normalization.scale | |
| normalized_actions[:, :, ROLE_IDS != 1] = 0 | |
| batch["future_actions"] = torch.tensor(normalized_actions, dtype=torch.float32) | |
| with torch.inference_mode(): | |
| mean, _ = model(to_device(batch, device)) | |
| mean = mean.cpu().numpy() | |
| horizons = sorted({float(h) for h in sample["query_horizons"].numpy() if h > 0}) | |
| observed_current = normalization.inverse(sample["current"].numpy()) | |
| for field in fields: | |
| if not sample["current_mask"][field]: | |
| seen = np.flatnonzero(sample["history_mask"][:, field].numpy()) | |
| observed_current[field] = ( | |
| ( | |
| sample["history"][seen[-1], field].item() * normalization.scale[field] | |
| + normalization.mean[field] | |
| ) | |
| if len(seen) | |
| else normalization.mean[field] | |
| ) | |
| predicted = np.repeat(observed_current[None, None, :], len(actions), axis=0) | |
| predicted = np.repeat(predicted, len(horizons), axis=1) | |
| lookup = {h: j for j, h in enumerate(horizons)} | |
| for q, (field, horizon) in enumerate(zip(sample["query_fields"], sample["query_horizons"])): | |
| field, horizon = int(field), float(horizon) | |
| if field in fields and horizon in lookup: | |
| predicted[:, lookup[horizon], field] += mean[:, q] * normalization.scale[field] | |
| selected_steps = np.array([round(h / 15) for h in horizons]) | |
| true_at_queries = truth[:, selected_steps] | |
| # Sum costs only at declared queried horizons, equally weighted. | |
| query_actions = actions[:, selected_steps - 1] | |
| predicted_cost = family_cost(episode.family_id, predicted, query_actions, FIELD_INDEX).sum(axis=1) | |
| true_cost = family_cost(episode.family_id, true_at_queries, query_actions, FIELD_INDEX).sum(axis=1) | |
| result = action_effect_metrics( | |
| predicted[:, :, fields], | |
| true_at_queries[:, :, fields], | |
| predicted_cost, | |
| true_cost, | |
| field_names=names, | |
| horizons=horizons, | |
| ) | |
| records.append( | |
| { | |
| "system_id": str(episode.system_id), | |
| "family": str(episode.family_id), | |
| "anchor_step": int(t), | |
| **result, | |
| } | |
| ) | |
| summaries = [] | |
| for family in sorted({r["family"] for r in records}): | |
| selected = [r for r in records if r["family"] == family] | |
| effects = {} | |
| for row in selected: | |
| for effect in row["effects"]: | |
| key = (effect["field"], effect["horizon_minutes"]) | |
| effects.setdefault(key, []).append(effect) | |
| pairs = sum(row["ranking_comparable_pairs"] for row in selected) | |
| summaries.append( | |
| { | |
| "family": family, | |
| "sample_count": len(selected), | |
| "mean_regret": float(np.mean([r["regret"] for r in selected])), | |
| "ranking_accuracy": sum( | |
| (r["ranking_accuracy"] or 0) * r["ranking_comparable_pairs"] for r in selected | |
| ) | |
| / pairs | |
| if pairs | |
| else None, | |
| "ranking_comparable_pairs": pairs, | |
| "effects": [ | |
| { | |
| "field": field, | |
| "horizon_minutes": horizon, | |
| "sample_count": len(rows), | |
| "action_effect_mae": float(np.mean([r["action_effect_mae"] for r in rows])), | |
| "action_effect_rmse": float( | |
| np.sqrt(np.mean([r["action_effect_rmse"] ** 2 for r in rows])) | |
| ), | |
| } | |
| for (field, horizon), rows in sorted(effects.items()) | |
| ], | |
| } | |
| ) | |
| return { | |
| "status": "completed" if records else "unrun", | |
| "samples": len(records), | |
| "method": "Replay exact simulator to sampled held-out anchor, restore one snapshot for each candidate; identical exogenous trajectory and RNG noise.", | |
| "cost_aggregation": "Equal-weight sum at 15,30,60,120 minute queried horizons", | |
| "current_reconstruction": "Observed current state; missing fields use last history observation, then training mean. No hidden current state enters predictions.", | |
| "oracle_reference": "Simulation-only hindsight cost minimum among supplied candidates using exact shared-noise branches; future noise is available to the oracle, not to learned methods.", | |
| "cost_definitions": COST_DEFINITIONS, | |
| "candidate_definition": "Observed plan plus constant -5/0/5 kW thermal/battery or 0/0.5/1 pump tank plans; tank valve 0.7.", | |
| "records": records, | |
| "summary_by_family": summaries, | |
| } | |
| def _remap_batch(batch, source, destination): | |
| result = dict(batch) | |
| device = batch["history"].device | |
| source_scale = torch.tensor(source.scale, dtype=torch.float32, device=device) | |
| source_mean = torch.tensor(source.mean, dtype=torch.float32, device=device) | |
| destination_scale = torch.tensor(destination.scale, dtype=torch.float32, device=device) | |
| destination_mean = torch.tensor(destination.mean, dtype=torch.float32, device=device) | |
| for key, mask_key in (("history", "history_mask"), ("current", "current_mask")): | |
| result[key] = ( | |
| (batch[key] * source_scale + source_mean - destination_mean) / destination_scale | |
| ) * batch[mask_key] | |
| for key, role in (("future_actions", 1), ("future_exogenous", 2)): | |
| active = batch["field_mask"][:, None, :] * (batch["role_ids"][:, None, :] == role) | |
| result[key] = ( | |
| (batch[key] * source_scale + source_mean - destination_mean) / destination_scale | |
| ) * active | |
| return result | |
| class SpecialistAdapter: | |
| def __init__(self, model, source, destination): | |
| self.model, self.source, self.destination = model, source, destination | |
| def __call__(self, batch): | |
| mean, log_std = self.model(_remap_batch(batch, self.source, self.destination)) | |
| ratio = torch.tensor( | |
| self.destination.scale / self.source.scale, dtype=mean.dtype, device=mean.device | |
| )[batch["query_fields"]] | |
| return mean * ratio, log_std + ratio.log() | |
| def to(self, device): | |
| self.model.to(device) | |
| return self | |
| def eval(self): | |
| self.model.eval() | |
| return self | |
| def parameters(self): | |
| return self.model.parameters() | |
| def benchmark(args): | |
| manifest, splits = load_manifest_episodes(args.data) | |
| if args.counterfactual_samples > 0 and manifest.get("generator_version") != GENERATOR_VERSION: | |
| raise ValueError( | |
| "Exact counterfactual replay requires data from the installed simulator generator version" | |
| ) | |
| device = resolve_device(args.device) | |
| paths = [Path(p) for p in args.checkpoint] | |
| model_members, checkpoints = zip(*(load_checkpoint(path, device) for path in paths)) | |
| norm_path = paths[0].parent / "normalization.json" | |
| normalization = Normalization.load(norm_path) | |
| norm_json = json.loads(norm_path.read_text()) | |
| norm_version = norm_json.get("version", "tfm-normalization-v1") | |
| for path, checkpoint in zip(paths, checkpoints): | |
| if checkpoint.get("normalization_sha256") != sha256(path.parent / "normalization.json"): | |
| raise ValueError("Checkpoint normalization fingerprint is missing or does not match its sidecar") | |
| if "schema_version" in checkpoint and checkpoint["schema_version"] != SCHEMA_VERSION: | |
| raise ValueError("Checkpoint schema/generator version does not match the installed vocabulary") | |
| for path in paths[1:]: | |
| other = Normalization.load(path.parent / "normalization.json") | |
| if not np.array_equal(other.mean, normalization.mean) or not np.array_equal( | |
| other.scale, normalization.scale | |
| ): | |
| raise ValueError("Ensemble members require identical training normalization") | |
| model = Ensemble(model_members) if len(model_members) > 1 else model_members[0] | |
| reference = checkpoints[0] | |
| for checkpoint in checkpoints: | |
| for key in ( | |
| "training_system_ids", | |
| "validation_selection_system_ids", | |
| "calibration_system_ids", | |
| "test_system_ids", | |
| ): | |
| if checkpoint.get(key, []) != reference.get(key, []): | |
| raise ValueError("Ensemble members need identical recorded partitions") | |
| train_ids, test_ids = _ids(splits["train"]), _ids(splits["test"]) | |
| if set(test_ids) & ( | |
| set(reference.get("training_system_ids", [])) | |
| | set(reference.get("validation_selection_system_ids", [])) | |
| ): | |
| raise ValueError("Benchmark test systems occurred in checkpoint training or model selection") | |
| if set(normalization.training_system_ids) & set(test_ids): | |
| raise ValueError("Normalization contains test systems") | |
| if set(normalization.training_system_ids) != set(reference.get("training_system_ids", [])): | |
| raise ValueError("Normalization training systems differ from checkpoint training provenance") | |
| history_steps = reference["model_config"].get("history_steps", 24) | |
| dataset = WindowDataset( | |
| splits["test"], | |
| normalization, | |
| history_steps=history_steps, | |
| future_steps=8, | |
| samples=args.samples, | |
| seed=args.seed, | |
| ) | |
| experiment = getattr(args, "experiment", None) | |
| if experiment and experiment.axis == "E": | |
| selected = [ | |
| pair | |
| for pair in dataset.indices | |
| if dataset.episodes[pair[0]].family_id == experiment.excluded_family | |
| ] | |
| if not selected: | |
| raise ValueError("Leave-family-out test dataset contains no windows of the excluded family") | |
| dataset.indices = selected | |
| families = _families(dataset) | |
| output = Path(args.output) | |
| output.mkdir(parents=True, exist_ok=True) | |
| calibration_ids = reference.get("calibration_system_ids", []) | |
| calibration_episodes = [e for e in splits["validation"] if str(e.system_id) in calibration_ids] | |
| calibration_data = ( | |
| WindowDataset( | |
| calibration_episodes, | |
| normalization, | |
| history_steps=history_steps, | |
| future_steps=8, | |
| samples=args.calibration_samples, | |
| seed=args.seed + 1, | |
| ) | |
| if calibration_episodes | |
| else None | |
| ) | |
| reports = {} | |
| models = { | |
| "foundation_model": model, | |
| "persistence": PersistenceBaseline(), | |
| "history_linear_system_id": HistoryLinearBaseline(ROLE_IDS), | |
| } | |
| if args.catboost: | |
| training_data = WindowDataset( | |
| splits["train"], | |
| normalization, | |
| history_steps=history_steps, | |
| future_steps=8, | |
| samples=args.catboost_samples, | |
| seed=args.seed + 2, | |
| ) | |
| training_batch = next(iter(DataLoader(training_data, batch_size=len(training_data)))) | |
| numpy_batch = {k: v.numpy() for k, v in training_batch.items()} | |
| try: | |
| models["catboost"] = CatBoostBaseline(seed=args.seed).fit( | |
| numpy_batch, training_system_ids=train_ids, fitted_system_ids=train_ids | |
| ) | |
| except ImportError: | |
| reports["catboost"] = {"status": "unrun", "reason": "Optional catboost package is not installed"} | |
| else: | |
| reports["catboost"] = { | |
| "status": "unrun", | |
| "reason": "Enable --catboost to fit the optional training-only baseline", | |
| } | |
| fingerprints = [sha256(path) for path in paths] | |
| for name, candidate in models.items(): | |
| calibration = None | |
| if calibration_data: | |
| cal_mean, cal_log_std, cal_arrays = collect_predictions( | |
| candidate, calibration_data, device, args.batch_size | |
| ) | |
| calibration = fit_calibration( | |
| cal_mean, | |
| np.exp(cal_log_std), | |
| cal_arrays["target"], | |
| cal_arrays["query_fields"], | |
| cal_arrays["query_horizons"], | |
| cal_arrays["target_mask"], | |
| calibration_system_ids=_ids(calibration_episodes), | |
| training_system_ids=reference.get("training_system_ids", []), | |
| validation_selection_system_ids=reference.get("validation_selection_system_ids", []), | |
| test_system_ids=test_ids, | |
| checkpoint_sha256=fingerprints if name == "foundation_model" else [], | |
| normalization_version=norm_version, | |
| normalization_sha256=sha256(norm_path), | |
| ) | |
| save_calibration( | |
| calibration, | |
| output / ("calibration.json" if name == "foundation_model" else f"calibration_{name}.json"), | |
| ) | |
| stress = experiment if experiment and experiment.axis == "C" else None | |
| mean, log_std, arrays = collect_predictions( | |
| candidate, dataset, device, args.batch_size, stress, args.seed | |
| ) | |
| applied_calibration = None if stress else calibration | |
| metric_rows = summarize(mean, log_std, arrays, families, normalization, applied_calibration) | |
| calibrated_count = sum( | |
| row["count"] for row in metric_rows if row["interval_kind"] == "heldout_calibrated" | |
| ) | |
| total_count = sum(row["count"] for row in metric_rows) | |
| reports[name] = { | |
| "status": "completed", | |
| "metrics": metric_rows, | |
| "calibrated_query_fraction": calibrated_count / total_count if total_count else 0, | |
| "calibration_status": "uncalibrated_under_distribution_shift" | |
| if stress | |
| else "heldout_partial" | |
| if calibration and calibrated_count < total_count | |
| else "heldout" | |
| if calibration | |
| else "unavailable_no_disjoint_calibration_systems", | |
| } | |
| if stress: | |
| reports[name]["evaluated_target_fraction"] = float( | |
| arrays["target_mask"].sum() | |
| / max(1, sum(int(dataset[i]["target_mask"].sum()) for i in range(len(dataset)))) | |
| ) | |
| reports[name]["missing_current_policy"] = ( | |
| "Queries without observed current sensors are excluded, matching API query rejection" | |
| ) | |
| if not stress: | |
| reports[name]["counterfactual"] = evaluate_counterfactual( | |
| candidate, dataset, normalization, device, args.counterfactual_samples | |
| ) | |
| for specification in args.specialist_checkpoint: | |
| family, path = specification.split("=", 1) | |
| if family not in SCHEMAS or family not in families: | |
| reports[f"specialist_{family}"] = { | |
| "status": "unrun", | |
| "reason": "Family is unknown or absent from sampled benchmark windows", | |
| } | |
| continue | |
| specialist, metadata = load_checkpoint(path, device) | |
| if metadata.get("training_families") != [family]: | |
| reports[f"specialist_{family}"] = { | |
| "status": "unrun", | |
| "reason": "Checkpoint provenance does not prove training on this family alone", | |
| } | |
| continue | |
| specialist_norm_path = Path(path).parent / "normalization.json" | |
| if metadata.get("normalization_sha256") != sha256(specialist_norm_path): | |
| raise ValueError("Specialist normalization fingerprint is missing or does not match") | |
| if set(test_ids) & ( | |
| set(metadata.get("training_system_ids", [])) | |
| | set(metadata.get("validation_selection_system_ids", [])) | |
| ): | |
| raise ValueError("Specialist checkpoint has test-system leakage") | |
| specialist_norm = Normalization.load(specialist_norm_path) | |
| adapter = SpecialistAdapter(specialist, normalization, specialist_norm) | |
| family_dataset = copy.copy(dataset) | |
| family_dataset.indices = [ | |
| pair for pair in dataset.indices if dataset.episodes[pair[0]].family_id == family | |
| ] | |
| mean, log_std, arrays = collect_predictions(adapter, family_dataset, device, args.batch_size) | |
| rows = summarize(mean, log_std, arrays, _families(family_dataset), normalization) | |
| reports[f"specialist_{family}"] = { | |
| "status": "completed", | |
| "checkpoint": str(path), | |
| "metrics": [r for r in rows if r["family"] == family], | |
| "calibration_status": "uncalibrated", | |
| "window_alignment": "Identical heldout windows, remapped training normalization", | |
| "counterfactual": evaluate_counterfactual( | |
| adapter, family_dataset, normalization, device, args.counterfactual_samples | |
| ), | |
| } | |
| if not args.specialist_checkpoint: | |
| reports["specialists"] = {"status": "unrun", "reason": "No specialist checkpoint artifacts supplied"} | |
| sample_batch = next(iter(DataLoader(dataset, batch_size=1))) | |
| profile = profile_inference( | |
| model, sample_batch, devices=("cpu", "mps"), repetitions=args.latency_repetitions | |
| ) | |
| model.to(device) | |
| result = { | |
| "version": "tfm-benchmark-v1", | |
| "seed": args.seed, | |
| "samples": len(dataset), | |
| "device": str(device), | |
| "checkpoint_sha256": fingerprints, | |
| "normalization_sha256": sha256(norm_path), | |
| "generator_version": manifest.get("generator_version"), | |
| "test_system_ids": test_ids, | |
| "calibration_system_ids": _ids(calibration_episodes), | |
| "parameter_count": sum(p.numel() for p in model.parameters()), | |
| "inference": profile, | |
| "models": reports, | |
| "external_datasets": { | |
| "status": "unrun", | |
| "reason": "No external physical dataset adapter or checkpoint experiment was supplied; synthetic results only.", | |
| }, | |
| } | |
| manifest_path = Path(args.data) | |
| if manifest_path.is_dir(): | |
| manifest_path = manifest_path / "manifest.json" | |
| result.update(sampling_provenance(dataset, manifest_path)) | |
| (output / "benchmark.json").write_text(json.dumps(result, indent=2, allow_nan=False) + "\n") | |
| configs = default_experiments() | |
| configs[0].data, configs[0].checkpoint = str(args.data), str(paths[0]) | |
| if args.stress_tests: | |
| for config in configs: | |
| if config.axis == "C": | |
| config.data, config.checkpoint = str(args.data), str(paths[0]) | |
| def evaluate_config(config): | |
| if config.axis == "B": | |
| return {"benchmark": "benchmark.json", "model": "foundation_model"} | |
| mean, log_std, arrays = collect_predictions( | |
| model, dataset, device, args.batch_size, config, args.seed | |
| ) | |
| return { | |
| "metrics": summarize(mean, log_std, arrays, families, normalization), | |
| "evaluated_target_fraction": float( | |
| arrays["target_mask"].sum() | |
| / max(1, sum(int(dataset[i]["target_mask"].sum()) for i in range(len(dataset)))) | |
| ), | |
| "missing_current_policy": "Queries without observed current sensors excluded, matching API query rejection", | |
| "interval_status": "uncalibrated_under_distribution_shift", | |
| } | |
| run_experiments(configs, evaluate_config, output / "generalization.json") | |
| return result | |
| def parser(): | |
| result = argparse.ArgumentParser(description=__doc__) | |
| result.add_argument("--data", required=True) | |
| result.add_argument("--checkpoint", required=True, action="append") | |
| result.add_argument("--output", required=True) | |
| result.add_argument("--samples", type=int, default=512) | |
| result.add_argument("--calibration-samples", type=int, default=512) | |
| result.add_argument("--counterfactual-samples", type=int, default=12) | |
| result.add_argument("--latency-repetitions", type=int, default=30) | |
| result.add_argument("--batch-size", type=int, default=64) | |
| result.add_argument("--seed", type=int, default=2026) | |
| result.add_argument("--device", choices=("auto", "cpu", "mps"), default="auto") | |
| result.add_argument("--catboost", action="store_true") | |
| result.add_argument("--catboost-samples", type=int, default=1024) | |
| result.add_argument("--specialist-checkpoint", action="append", default=[], help="family=path") | |
| result.add_argument("--stress-tests", action="store_true") | |
| return result | |
| def main(argv=None): | |
| args = parser().parse_args(argv) | |
| if min(args.samples, args.calibration_samples, args.batch_size, args.latency_repetitions) < 1: | |
| raise ValueError("Sample counts, latency repetitions and batch size must be positive") | |
| if args.counterfactual_samples < 0: | |
| raise ValueError("Counterfactual sample count must be nonnegative") | |
| benchmark(args) | |
| if __name__ == "__main__": | |
| main() | |