maxffarrell's picture
Release experimental Transition Foundation Model v0.1 with reproducible source and validation
1baf215 verified
Raw History Blame Contribute Delete
50 kB
{ "version": "tfm-comparison-v1", "status": "smoke_only", "generator_version": "0.1.2", "manifest_sha256": "3541d17c0a76b9e3e9d462c462b30fe5c17eb76968b8ed0d6196ebb6d0b7558c", "partition_episode_counts": { "train": 63, "validation": 14, "test": 13 }, "interpretation": "Short synthetic runs, no research or scaling claim. Prediction and action-effect MAE equally average field/horizon rows within family. Coverage is weighted by calibrated observed query counts and omits unsupported fields, which are listed. Widths divide by each field's training standard deviation; physical widths remain in source reports. Regret has family-specific objective units and must not be pooled. Latency retains the source measurement scope and contention limitations.", "runs": { "transformer": { "status": "completed", "provenance_checks": "passed", "source_report": "artifacts/smoke/report-transformer-v2/benchmark.json", "source_report_sha256": "a0b6e74770b87f1300cfe97c40bc05e48a689b40e4453c35cc9e90323cb4690b", "calibration_sha256": "ef5ef17c35d6a4f4f8614b21c011a591594b97d87cb1532008e28b08b501b198", "checkpoint_sha256": [ "6b0ac9c50d7d581c0495f85dd01b42c48004afdbede88a86e445347bdce9d739" ], "normalization_sha256": "ddc73b06488090bd4f79571416bf8244638bbd6f28f899fb3ef92411e5f0e41e", "split_system_counts": { "training_system_ids": 63, "validation_selection_system_ids": 8, "calibration_system_ids": 6, "test_system_ids": 13 }, "parameter_count": 1570434, "inference": [ { "device": "cpu", "status": "completed", "batch_size": 1, "precision": "float32", "repetitions": 30, "warmup": 5, "p50_ms": 5.8506665, "p95_ms": 6.984247949999999, "process_peak_rss_bytes": 2904473600, "sampled_mps_driver_allocation_peak_bytes": null, "memory_limitation": "RSS is lifetime process high water, includes data and prior models; MPS driver allocation sampled after calls is a lower bound, not resettable model peak.", "timing_scope": "forward only; inputs resident; MPS synchronized before and after every call" }, { "device": "mps", "status": "unavailable" } ], "window_sampling": { "seed": 2026, "history_steps": 24, "future_steps": 8, "sampling": "uniform eligible anchors with replacement", "anchor_sha256": "8b6e0caa2a477ef4cdce719cfc04caa8e80561e124d803dac61ffaefe8dff8f2", "family_window_counts": { "battery": 184, "tank": 165, "thermal": 163 } }, "models": { "foundation_model": { "forecast": [ { "family": "battery", "field_horizon_balanced_standardized_mae": 0.2464418007681767, "coverage_weighted_by_count": 0.9348859315589354, "calibrated_query_count": 2104, "total_observed_query_count": 2104, "calibrated_query_fraction": 1.0, "field_horizon_balanced_standardized_interval_width": 1.5368033744363387, "uncalibrated_fields": [] }, { "family": "tank", "field_horizon_balanced_standardized_mae": 0.20518572337459773, "coverage_weighted_by_count": 0.841858648965248, "calibrated_query_count": 2561, "total_observed_query_count": 2561, "calibrated_query_fraction": 1.0, "field_horizon_balanced_standardized_interval_width": 1.0525175486113494, "uncalibrated_fields": [] }, { "family": "thermal", "field_horizon_balanced_standardized_mae": 0.08072376647032797, "coverage_weighted_by_count": 0.9260763606823721, "calibrated_query_count": 1231, "total_observed_query_count": 1686, "calibrated_query_fraction": 0.7301304863582444, "field_horizon_balanced_standardized_interval_width": 0.47078061521257536, "uncalibrated_fields": [ "mass_temp" ] } ], "intervention": [ { "family": "battery", "sample_count": 12, "field_horizon_balanced_standardized_action_effect_mae": 0.3353849789275787, "ranking_accuracy": 0.8888888888888888, "mean_regret_family_objective_units": 0.04787804231797377 }, { "family": "tank", "sample_count": 9, "field_horizon_balanced_standardized_action_effect_mae": 0.4100909468108305, "ranking_accuracy": 0.7777777777777778, "mean_regret_family_objective_units": 5.476421404145535 }, { "family": "thermal", "sample_count": 9, "field_horizon_balanced_standardized_action_effect_mae": 0.061770662692730134, "ranking_accuracy": 0.9074074074074074, "mean_regret_family_objective_units": 3.2748828767368185 } ], "calibration_status": "heldout_partial" }, "persistence": { "forecast": [ { "family": "battery", "field_horizon_balanced_standardized_mae": 0.2741769899924596, "coverage_weighted_by_count": 0.9120722433460076, "calibrated_query_count": 2104, "total_observed_query_count": 2104, "calibrated_query_fraction": 1.0, "field_horizon_balanced_standardized_interval_width": 1.9144873287577546, "uncalibrated_fields": [] }, { "family": "tank", "field_horizon_balanced_standardized_mae": 0.20508833997882903, "coverage_weighted_by_count": 0.91097227645451, "calibrated_query_count": 2561, "total_observed_query_count": 2561, "calibrated_query_fraction": 1.0, "field_horizon_balanced_standardized_interval_width": 1.3987040263066008, "uncalibrated_fields": [] }, { "family": "thermal", "field_horizon_balanced_standardized_mae": 0.1350649856030941, "coverage_weighted_by_count": 0.9870024370430545, "calibrated_query_count": 1231, "total_observed_query_count": 1686, "calibrated_query_fraction": 0.7301304863582444, "field_horizon_balanced_standardized_interval_width": 1.2741399954043633, "uncalibrated_fields": [ "mass_temp" ] } ], "intervention": [ { "family": "battery", "sample_count": 12, "field_horizon_balanced_standardized_action_effect_mae": 0.44577424080535133, "ranking_accuracy": 0.625, "mean_regret_family_objective_units": 0.1149686559871352 }, { "family": "tank", "sample_count": 9, "field_horizon_balanced_standardized_action_effect_mae": 0.4309230445413622, "ranking_accuracy": 0.9629629629629629, "mean_regret_family_objective_units": 0.19212913518986896 }, { "family": "thermal", "sample_count": 9, "field_horizon_balanced_standardized_action_effect_mae": 0.13642123593907265, "ranking_accuracy": 0.6388888888888888, "mean_regret_family_objective_units": 24.588706704133514 } ], "calibration_status": "heldout_partial" }, "history_linear_system_id": { "forecast": [ { "family": "battery", "field_horizon_balanced_standardized_mae": 0.1964516549681624, "coverage_weighted_by_count": 0.9068441064638784, "calibrated_query_count": 2104, "total_observed_query_count": 2104, "calibrated_query_fraction": 1.0, "field_horizon_balanced_standardized_interval_width": 1.0612048471296183, "uncalibrated_fields": [] }, { "family": "tank", "field_horizon_balanced_standardized_mae": 0.19398614921374246, "coverage_weighted_by_count": 0.8867629832096837, "calibrated_query_count": 2561, "total_observed_query_count": 2561, "calibrated_query_fraction": 1.0, "field_horizon_balanced_standardized_interval_width": 1.0373429280371522, "uncalibrated_fields": [] }, { "family": "thermal", "field_horizon_balanced_standardized_mae": 0.053480925814559065, "coverage_weighted_by_count": 0.9512591389114541, "calibrated_query_count": 1231, "total_observed_query_count": 1686, "calibrated_query_fraction": 0.7301304863582444, "field_horizon_balanced_standardized_interval_width": 0.37496149991779415, "uncalibrated_fields": [ "mass_temp" ] } ], "intervention": [ { "family": "battery", "sample_count": 12, "field_horizon_balanced_standardized_action_effect_mae": 0.21353556679131813, "ranking_accuracy": 0.8194444444444444, "mean_regret_family_objective_units": 0.07438788574709695 }, { "family": "tank", "sample_count": 9, "field_horizon_balanced_standardized_action_effect_mae": 0.375519408741499, "ranking_accuracy": 0.6666666666666666, "mean_regret_family_objective_units": 14.565733903777224 }, { "family": "thermal", "sample_count": 9, "field_horizon_balanced_standardized_action_effect_mae": 0.05728579907697506, "ranking_accuracy": 0.9074074074074074, "mean_regret_family_objective_units": 4.096646376501237 } ], "calibration_status": "heldout_partial" }, "catboost": { "forecast": [ { "family": "battery", "field_horizon_balanced_standardized_mae": 0.17793322975436845, "coverage_weighted_by_count": 0.9325095057034221, "calibrated_query_count": 2104, "total_observed_query_count": 2104, "calibrated_query_fraction": 1.0, "field_horizon_balanced_standardized_interval_width": 1.2700347987341436, "uncalibrated_fields": [] }, { "family": "tank", "field_horizon_balanced_standardized_mae": 0.16986428969539702, "coverage_weighted_by_count": 0.8891058180398282, "calibrated_query_count": 2561, "total_observed_query_count": 2561, "calibrated_query_fraction": 1.0, "field_horizon_balanced_standardized_interval_width": 0.9215257671904333, "uncalibrated_fields": [] }, { "family": "thermal", "field_horizon_balanced_standardized_mae": 0.08332891451815765, "coverage_weighted_by_count": 0.9358245329000813, "calibrated_query_count": 1231, "total_observed_query_count": 1686, "calibrated_query_fraction": 0.7301304863582444, "field_horizon_balanced_standardized_interval_width": 0.7638302007604652, "uncalibrated_fields": [ "mass_temp" ] } ], "intervention": [ { "family": "battery", "sample_count": 12, "field_horizon_balanced_standardized_action_effect_mae": 0.23051647289866473, "ranking_accuracy": 0.8888888888888888, "mean_regret_family_objective_units": 0.05557148176567784 }, { "family": "tank", "sample_count": 9, "field_horizon_balanced_standardized_action_effect_mae": 0.26401079379275233, "ranking_accuracy": 0.12962962962962962, "mean_regret_family_objective_units": 51.16753917152375 }, { "family": "thermal", "sample_count": 9, "field_horizon_balanced_standardized_action_effect_mae": 0.08128041531309356, "ranking_accuracy": 0.8518518518518519, "mean_regret_family_objective_units": 3.19755500037222 } ], "calibration_status": "heldout_partial" }, "specialists": { "status": "unrun", "reason": "No specialist checkpoint artifacts supplied" } }, "generalization": [ { "name": "heldout_systems", "axis": "B", "status": "completed", "reason": null, "evaluated_target_fraction": null }, { "name": "sensor_failures", "axis": "C", "status": "completed", "reason": null, "evaluated_target_fraction": 0.7616123445126751 }, { "name": "sensor_bias", "axis": "C", "status": "completed", "reason": null, "evaluated_target_fraction": 1.0 }, { "name": "configuration_shift", "axis": "D", "status": "unrun", "reason": "Dataset and compatible trained checkpoint were not supplied", "evaluated_target_fraction": null }, { "name": "leave_thermal_out", "axis": "E", "status": "unrun", "reason": "Dataset and compatible trained checkpoint were not supplied", "evaluated_target_fraction": null }, { "name": "leave_tank_out", "axis": "E", "status": "unrun", "reason": "Dataset and compatible trained checkpoint were not supplied", "evaluated_target_fraction": null }, { "name": "leave_battery_out", "axis": "E", "status": "unrun", "reason": "Dataset and compatible trained checkpoint were not supplied", "evaluated_target_fraction": null } ] }, "mlp": { "status": "completed", "provenance_checks": "passed", "source_report": "artifacts/smoke/report-mlp-v2/benchmark.json", "source_report_sha256": "349d0ff2d2f4ef7076de4b22a95fff87e6904b6d85bd0362a74d33093c7b0377", "calibration_sha256": "d37e7f30f43186e917b336b03904ad609d2b0d296e44bc7e0097dcafe2a27797", "checkpoint_sha256": [ "3f7e5adfaff54b553f0e019fd9d328dc2176aa777a6176adea2321969573199c" ], "normalization_sha256": "ddc73b06488090bd4f79571416bf8244638bbd6f28f899fb3ef92411e5f0e41e", "split_system_counts": { "training_system_ids": 63, "validation_selection_system_ids": 8, "calibration_system_ids": 6, "test_system_ids": 13 }, "parameter_count": 876034, "inference": [ { "device": "cpu", "status": "completed", "batch_size": 1, "precision": "float32", "repetitions": 30, "warmup": 5, "p50_ms": 1.711187, "p95_ms": 4.087550149999997, "process_peak_rss_bytes": 380567552, "sampled_mps_driver_allocation_peak_bytes": null, "memory_limitation": "RSS is lifetime process high water, includes data and prior models; MPS driver allocation sampled after calls is a lower bound, not resettable model peak.", "timing_scope": "forward only; inputs resident; MPS synchronized before and after every call" }, { "device": "mps", "status": "unavailable" } ], "window_sampling": { "seed": 2026, "history_steps": 24, "future_steps": 8, "sampling": "uniform eligible anchors with replacement", "anchor_sha256": "8b6e0caa2a477ef4cdce719cfc04caa8e80561e124d803dac61ffaefe8dff8f2", "family_window_counts": { "battery": 184, "tank": 165, "thermal": 163 } }, "models": { "catboost": { "status": "unrun", "reason": "Enable --catboost to fit the optional training-only baseline" }, "foundation_model": { "forecast": [ { "family": "battery", "field_horizon_balanced_standardized_mae": 0.2655715017269055, "coverage_weighted_by_count": 0.9462927756653993, "calibrated_query_count": 2104, "total_observed_query_count": 2104, "calibrated_query_fraction": 1.0, "field_horizon_balanced_standardized_interval_width": 1.790856030421164, "uncalibrated_fields": [] }, { "family": "tank", "field_horizon_balanced_standardized_mae": 0.21289619465824217, "coverage_weighted_by_count": 0.88481062085123, "calibrated_query_count": 2561, "total_observed_query_count": 2561, "calibrated_query_fraction": 1.0, "field_horizon_balanced_standardized_interval_width": 1.1492993770553106, "uncalibrated_fields": [] }, { "family": "thermal", "field_horizon_balanced_standardized_mae": 0.09717030023845534, "coverage_weighted_by_count": 0.9610073111291633, "calibrated_query_count": 1231, "total_observed_query_count": 1686, "calibrated_query_fraction": 0.7301304863582444, "field_horizon_balanced_standardized_interval_width": 0.6933850772112384, "uncalibrated_fields": [ "mass_temp" ] } ], "intervention": [ { "family": "battery", "sample_count": 12, "field_horizon_balanced_standardized_action_effect_mae": 0.40193630005660164, "ranking_accuracy": 0.8333333333333334, "mean_regret_family_objective_units": 0.058006828239173934 }, { "family": "tank", "sample_count": 9, "field_horizon_balanced_standardized_action_effect_mae": 0.4218413598223273, "ranking_accuracy": 0.8518518518518519, "mean_regret_family_objective_units": 17.03598703076836 }, { "family": "thermal", "sample_count": 9, "field_horizon_balanced_standardized_action_effect_mae": 0.10134565391581384, "ranking_accuracy": 0.8703703703703703, "mean_regret_family_objective_units": 2.646552774845556 } ], "calibration_status": "heldout_partial" }, "persistence": { "forecast": [ { "family": "battery", "field_horizon_balanced_standardized_mae": 0.2741769899924596, "coverage_weighted_by_count": 0.9120722433460076, "calibrated_query_count": 2104, "total_observed_query_count": 2104, "calibrated_query_fraction": 1.0, "field_horizon_balanced_standardized_interval_width": 1.9144873287577546, "uncalibrated_fields": [] }, { "family": "tank", "field_horizon_balanced_standardized_mae": 0.20508833997882903, "coverage_weighted_by_count": 0.91097227645451, "calibrated_query_count": 2561, "total_observed_query_count": 2561, "calibrated_query_fraction": 1.0, "field_horizon_balanced_standardized_interval_width": 1.3987040263066008, "uncalibrated_fields": [] }, { "family": "thermal", "field_horizon_balanced_standardized_mae": 0.1350649856030941, "coverage_weighted_by_count": 0.9870024370430545, "calibrated_query_count": 1231, "total_observed_query_count": 1686, "calibrated_query_fraction": 0.7301304863582444, "field_horizon_balanced_standardized_interval_width": 1.2741399954043633, "uncalibrated_fields": [ "mass_temp" ] } ], "intervention": [ { "family": "battery", "sample_count": 12, "field_horizon_balanced_standardized_action_effect_mae": 0.44577424080535133, "ranking_accuracy": 0.625, "mean_regret_family_objective_units": 0.1149686559871352 }, { "family": "tank", "sample_count": 9, "field_horizon_balanced_standardized_action_effect_mae": 0.4309230445413622, "ranking_accuracy": 0.9629629629629629, "mean_regret_family_objective_units": 0.19212913518986896 }, { "family": "thermal", "sample_count": 9, "field_horizon_balanced_standardized_action_effect_mae": 0.13642123593907265, "ranking_accuracy": 0.6388888888888888, "mean_regret_family_objective_units": 24.588706704133514 } ], "calibration_status": "heldout_partial" }, "history_linear_system_id": { "forecast": [ { "family": "battery", "field_horizon_balanced_standardized_mae": 0.1964516549681624, "coverage_weighted_by_count": 0.9068441064638784, "calibrated_query_count": 2104, "total_observed_query_count": 2104, "calibrated_query_fraction": 1.0, "field_horizon_balanced_standardized_interval_width": 1.0612048471296183, "uncalibrated_fields": [] }, { "family": "tank", "field_horizon_balanced_standardized_mae": 0.19398614921374246, "coverage_weighted_by_count": 0.8867629832096837, "calibrated_query_count": 2561, "total_observed_query_count": 2561, "calibrated_query_fraction": 1.0, "field_horizon_balanced_standardized_interval_width": 1.0373429280371522, "uncalibrated_fields": [] }, { "family": "thermal", "field_horizon_balanced_standardized_mae": 0.053480925814559065, "coverage_weighted_by_count": 0.9512591389114541, "calibrated_query_count": 1231, "total_observed_query_count": 1686, "calibrated_query_fraction": 0.7301304863582444, "field_horizon_balanced_standardized_interval_width": 0.37496149991779415, "uncalibrated_fields": [ "mass_temp" ] } ], "intervention": [ { "family": "battery", "sample_count": 12, "field_horizon_balanced_standardized_action_effect_mae": 0.21353556679131813, "ranking_accuracy": 0.8194444444444444, "mean_regret_family_objective_units": 0.07438788574709695 }, { "family": "tank", "sample_count": 9, "field_horizon_balanced_standardized_action_effect_mae": 0.375519408741499, "ranking_accuracy": 0.6666666666666666, "mean_regret_family_objective_units": 14.565733903777224 }, { "family": "thermal", "sample_count": 9, "field_horizon_balanced_standardized_action_effect_mae": 0.05728579907697506, "ranking_accuracy": 0.9074074074074074, "mean_regret_family_objective_units": 4.096646376501237 } ], "calibration_status": "heldout_partial" }, "specialists": { "status": "unrun", "reason": "No specialist checkpoint artifacts supplied" } }, "generalization": [ { "name": "heldout_systems", "axis": "B", "status": "completed", "reason": null, "evaluated_target_fraction": null }, { "name": "sensor_failures", "axis": "C", "status": "unrun", "reason": "Dataset and compatible trained checkpoint were not supplied", "evaluated_target_fraction": null }, { "name": "sensor_bias", "axis": "C", "status": "unrun", "reason": "Dataset and compatible trained checkpoint were not supplied", "evaluated_target_fraction": null }, { "name": "configuration_shift", "axis": "D", "status": "unrun", "reason": "Dataset and compatible trained checkpoint were not supplied", "evaluated_target_fraction": null }, { "name": "leave_thermal_out", "axis": "E", "status": "unrun", "reason": "Dataset and compatible trained checkpoint were not supplied", "evaluated_target_fraction": null }, { "name": "leave_tank_out", "axis": "E", "status": "unrun", "reason": "Dataset and compatible trained checkpoint were not supplied", "evaluated_target_fraction": null }, { "name": "leave_battery_out", "axis": "E", "status": "unrun", "reason": "Dataset and compatible trained checkpoint were not supplied", "evaluated_target_fraction": null } ] }, "ensemble": { "status": "completed", "provenance_checks": "passed", "source_report": "artifacts/smoke/report-ensemble-v2/benchmark.json", "source_report_sha256": "f97f91deb10661ed6c3ddeae8b9d25813ad6d4fb4cd9cc3be5564220b69e67cb", "calibration_sha256": "f96d9a18b64711dca5bb76fd21a9671c298195ecd3b83a9ec1daf9762a2c7e40", "checkpoint_sha256": [ "6b0ac9c50d7d581c0495f85dd01b42c48004afdbede88a86e445347bdce9d739", "80ffcf05367f581ed48d34e649ffde7f3738a7c31fb7e0539cb2245e88eae09e", "623420955dc271068182f2cd6fb460d0ae64792c25dbd79c0aae91b3ba777137" ], "normalization_sha256": "ddc73b06488090bd4f79571416bf8244638bbd6f28f899fb3ef92411e5f0e41e", "split_system_counts": { "training_system_ids": 63, "validation_selection_system_ids": 8, "calibration_system_ids": 6, "test_system_ids": 13 }, "parameter_count": 4711302, "inference": [ { "device": "cpu", "status": "completed", "batch_size": 1, "precision": "float32", "repetitions": 30, "warmup": 5, "p50_ms": 21.5922915, "p95_ms": 26.497902299999996, "process_peak_rss_bytes": 589479936, "sampled_mps_driver_allocation_peak_bytes": null, "memory_limitation": "RSS is lifetime process high water, includes data and prior models; MPS driver allocation sampled after calls is a lower bound, not resettable model peak.", "timing_scope": "forward only; inputs resident; MPS synchronized before and after every call" }, { "device": "mps", "status": "unavailable" } ], "window_sampling": { "seed": 2026, "history_steps": 24, "future_steps": 8, "sampling": "uniform eligible anchors with replacement", "anchor_sha256": "8b6e0caa2a477ef4cdce719cfc04caa8e80561e124d803dac61ffaefe8dff8f2", "family_window_counts": { "battery": 184, "tank": 165, "thermal": 163 } }, "models": { "catboost": { "status": "unrun", "reason": "Enable --catboost to fit the optional training-only baseline" }, "foundation_model": { "forecast": [ { "family": "battery", "field_horizon_balanced_standardized_mae": 0.24396890867501497, "coverage_weighted_by_count": 0.938212927756654, "calibrated_query_count": 2104, "total_observed_query_count": 2104, "calibrated_query_fraction": 1.0, "field_horizon_balanced_standardized_interval_width": 1.5705312779267908, "uncalibrated_fields": [] }, { "family": "tank", "field_horizon_balanced_standardized_mae": 0.20474898838438094, "coverage_weighted_by_count": 0.8617727450214759, "calibrated_query_count": 2561, "total_observed_query_count": 2561, "calibrated_query_fraction": 1.0, "field_horizon_balanced_standardized_interval_width": 1.0582992655848964, "uncalibrated_fields": [] }, { "family": "thermal", "field_horizon_balanced_standardized_mae": 0.08036906473959486, "coverage_weighted_by_count": 0.925264012997563, "calibrated_query_count": 1231, "total_observed_query_count": 1686, "calibrated_query_fraction": 0.7301304863582444, "field_horizon_balanced_standardized_interval_width": 0.46081873453475863, "uncalibrated_fields": [ "mass_temp" ] } ], "intervention": [ { "family": "battery", "sample_count": 12, "field_horizon_balanced_standardized_action_effect_mae": 0.33930385699822924, "ranking_accuracy": 0.8888888888888888, "mean_regret_family_objective_units": 0.04787804231797377 }, { "family": "tank", "sample_count": 9, "field_horizon_balanced_standardized_action_effect_mae": 0.40268880790635786, "ranking_accuracy": 0.8888888888888888, "mean_regret_family_objective_units": 5.476421404145535 }, { "family": "thermal", "sample_count": 9, "field_horizon_balanced_standardized_action_effect_mae": 0.06546775249797411, "ranking_accuracy": 0.8888888888888888, "mean_regret_family_objective_units": 3.2748828767368185 } ], "calibration_status": "heldout_partial" }, "persistence": { "forecast": [ { "family": "battery", "field_horizon_balanced_standardized_mae": 0.2741769899924596, "coverage_weighted_by_count": 0.9120722433460076, "calibrated_query_count": 2104, "total_observed_query_count": 2104, "calibrated_query_fraction": 1.0, "field_horizon_balanced_standardized_interval_width": 1.9144873287577546, "uncalibrated_fields": [] }, { "family": "tank", "field_horizon_balanced_standardized_mae": 0.20508833997882903, "coverage_weighted_by_count": 0.91097227645451, "calibrated_query_count": 2561, "total_observed_query_count": 2561, "calibrated_query_fraction": 1.0, "field_horizon_balanced_standardized_interval_width": 1.3987040263066008, "uncalibrated_fields": [] }, { "family": "thermal", "field_horizon_balanced_standardized_mae": 0.1350649856030941, "coverage_weighted_by_count": 0.9870024370430545, "calibrated_query_count": 1231, "total_observed_query_count": 1686, "calibrated_query_fraction": 0.7301304863582444, "field_horizon_balanced_standardized_interval_width": 1.2741399954043633, "uncalibrated_fields": [ "mass_temp" ] } ], "intervention": [ { "family": "battery", "sample_count": 12, "field_horizon_balanced_standardized_action_effect_mae": 0.44577424080535133, "ranking_accuracy": 0.625, "mean_regret_family_objective_units": 0.1149686559871352 }, { "family": "tank", "sample_count": 9, "field_horizon_balanced_standardized_action_effect_mae": 0.4309230445413622, "ranking_accuracy": 0.9629629629629629, "mean_regret_family_objective_units": 0.19212913518986896 }, { "family": "thermal", "sample_count": 9, "field_horizon_balanced_standardized_action_effect_mae": 0.13642123593907265, "ranking_accuracy": 0.6388888888888888, "mean_regret_family_objective_units": 24.588706704133514 } ], "calibration_status": "heldout_partial" }, "history_linear_system_id": { "forecast": [ { "family": "battery", "field_horizon_balanced_standardized_mae": 0.1964516549681624, "coverage_weighted_by_count": 0.9068441064638784, "calibrated_query_count": 2104, "total_observed_query_count": 2104, "calibrated_query_fraction": 1.0, "field_horizon_balanced_standardized_interval_width": 1.0612048471296183, "uncalibrated_fields": [] }, { "family": "tank", "field_horizon_balanced_standardized_mae": 0.19398614921374246, "coverage_weighted_by_count": 0.8867629832096837, "calibrated_query_count": 2561, "total_observed_query_count": 2561, "calibrated_query_fraction": 1.0, "field_horizon_balanced_standardized_interval_width": 1.0373429280371522, "uncalibrated_fields": [] }, { "family": "thermal", "field_horizon_balanced_standardized_mae": 0.053480925814559065, "coverage_weighted_by_count": 0.9512591389114541, "calibrated_query_count": 1231, "total_observed_query_count": 1686, "calibrated_query_fraction": 0.7301304863582444, "field_horizon_balanced_standardized_interval_width": 0.37496149991779415, "uncalibrated_fields": [ "mass_temp" ] } ], "intervention": [ { "family": "battery", "sample_count": 12, "field_horizon_balanced_standardized_action_effect_mae": 0.21353556679131813, "ranking_accuracy": 0.8194444444444444, "mean_regret_family_objective_units": 0.07438788574709695 }, { "family": "tank", "sample_count": 9, "field_horizon_balanced_standardized_action_effect_mae": 0.375519408741499, "ranking_accuracy": 0.6666666666666666, "mean_regret_family_objective_units": 14.565733903777224 }, { "family": "thermal", "sample_count": 9, "field_horizon_balanced_standardized_action_effect_mae": 0.05728579907697506, "ranking_accuracy": 0.9074074074074074, "mean_regret_family_objective_units": 4.096646376501237 } ], "calibration_status": "heldout_partial" }, "specialists": { "status": "unrun", "reason": "No specialist checkpoint artifacts supplied" } }, "generalization": [ { "name": "heldout_systems", "axis": "B", "status": "completed", "reason": null, "evaluated_target_fraction": null }, { "name": "sensor_failures", "axis": "C", "status": "unrun", "reason": "Dataset and compatible trained checkpoint were not supplied", "evaluated_target_fraction": null }, { "name": "sensor_bias", "axis": "C", "status": "unrun", "reason": "Dataset and compatible trained checkpoint were not supplied", "evaluated_target_fraction": null }, { "name": "configuration_shift", "axis": "D", "status": "unrun", "reason": "Dataset and compatible trained checkpoint were not supplied", "evaluated_target_fraction": null }, { "name": "leave_thermal_out", "axis": "E", "status": "unrun", "reason": "Dataset and compatible trained checkpoint were not supplied", "evaluated_target_fraction": null }, { "name": "leave_tank_out", "axis": "E", "status": "unrun", "reason": "Dataset and compatible trained checkpoint were not supplied", "evaluated_target_fraction": null }, { "name": "leave_battery_out", "axis": "E", "status": "unrun", "reason": "Dataset and compatible trained checkpoint were not supplied", "evaluated_target_fraction": null } ] }, "gru": { "status": "completed", "provenance_checks": "passed", "source_report": "artifacts/smoke/report-gru-v2/benchmark.json", "source_report_sha256": "4f8ba6827462589f5f80df0d8e34cb0b50da2f5d2d7d6d506337bd17a852271a", "calibration_sha256": "67d780e83bde04238192bdb5ea2caa7353d3be63fbf762e28e500da0a8a78ad0", "checkpoint_sha256": [ "ae952a6d9b3f9dccafe6dcc92195b25a656f65c5edd022ed75841c2d2acc0663" ], "normalization_sha256": "ddc73b06488090bd4f79571416bf8244638bbd6f28f899fb3ef92411e5f0e41e", "split_system_counts": { "training_system_ids": 63, "validation_selection_system_ids": 8, "calibration_system_ids": 6, "test_system_ids": 13 }, "parameter_count": 664322, "inference": [ { "device": "cpu", "status": "completed", "batch_size": 1, "precision": "float32", "repetitions": 30, "warmup": 5, "p50_ms": 3.8872915, "p95_ms": 4.4816310999999995, "process_peak_rss_bytes": 456704000, "sampled_mps_driver_allocation_peak_bytes": null, "memory_limitation": "RSS is lifetime process high water, includes data and prior models; MPS driver allocation sampled after calls is a lower bound, not resettable model peak.", "timing_scope": "forward only; inputs resident; MPS synchronized before and after every call" }, { "device": "mps", "status": "unavailable" } ], "window_sampling": { "seed": 2026, "history_steps": 24, "future_steps": 8, "sampling": "uniform eligible anchors with replacement", "anchor_sha256": "8b6e0caa2a477ef4cdce719cfc04caa8e80561e124d803dac61ffaefe8dff8f2", "family_window_counts": { "battery": 184, "tank": 165, "thermal": 163 } }, "models": { "catboost": { "status": "unrun", "reason": "Enable --catboost to fit the optional training-only baseline" }, "foundation_model": { "forecast": [ { "family": "battery", "field_horizon_balanced_standardized_mae": 0.2691992074251175, "coverage_weighted_by_count": 0.94106463878327, "calibrated_query_count": 2104, "total_observed_query_count": 2104, "calibrated_query_fraction": 1.0, "field_horizon_balanced_standardized_interval_width": 1.769772475077991, "uncalibrated_fields": [] }, { "family": "tank", "field_horizon_balanced_standardized_mae": 0.22744196222629398, "coverage_weighted_by_count": 0.8875439281530652, "calibrated_query_count": 2561, "total_observed_query_count": 2561, "calibrated_query_fraction": 1.0, "field_horizon_balanced_standardized_interval_width": 1.1845489464009766, "uncalibrated_fields": [] }, { "family": "thermal", "field_horizon_balanced_standardized_mae": 0.10626375791616738, "coverage_weighted_by_count": 0.9553208773354996, "calibrated_query_count": 1231, "total_observed_query_count": 1686, "calibrated_query_fraction": 0.7301304863582444, "field_horizon_balanced_standardized_interval_width": 0.6831428043987473, "uncalibrated_fields": [ "mass_temp" ] } ], "intervention": [ { "family": "battery", "sample_count": 12, "field_horizon_balanced_standardized_action_effect_mae": 0.3903253484541181, "ranking_accuracy": 0.8472222222222222, "mean_regret_family_objective_units": 0.050546485460566226 }, { "family": "tank", "sample_count": 9, "field_horizon_balanced_standardized_action_effect_mae": 0.4149932857908747, "ranking_accuracy": 0.8703703703703703, "mean_regret_family_objective_units": 9.831722235589073 }, { "family": "thermal", "sample_count": 9, "field_horizon_balanced_standardized_action_effect_mae": 0.10282845522232471, "ranking_accuracy": 0.8703703703703703, "mean_regret_family_objective_units": 2.646552774845556 } ], "calibration_status": "heldout_partial" }, "persistence": { "forecast": [ { "family": "battery", "field_horizon_balanced_standardized_mae": 0.2741769899924596, "coverage_weighted_by_count": 0.9120722433460076, "calibrated_query_count": 2104, "total_observed_query_count": 2104, "calibrated_query_fraction": 1.0, "field_horizon_balanced_standardized_interval_width": 1.9144873287577546, "uncalibrated_fields": [] }, { "family": "tank", "field_horizon_balanced_standardized_mae": 0.20508833997882903, "coverage_weighted_by_count": 0.91097227645451, "calibrated_query_count": 2561, "total_observed_query_count": 2561, "calibrated_query_fraction": 1.0, "field_horizon_balanced_standardized_interval_width": 1.3987040263066008, "uncalibrated_fields": [] }, { "family": "thermal", "field_horizon_balanced_standardized_mae": 0.1350649856030941, "coverage_weighted_by_count": 0.9870024370430545, "calibrated_query_count": 1231, "total_observed_query_count": 1686, "calibrated_query_fraction": 0.7301304863582444, "field_horizon_balanced_standardized_interval_width": 1.2741399954043633, "uncalibrated_fields": [ "mass_temp" ] } ], "intervention": [ { "family": "battery", "sample_count": 12, "field_horizon_balanced_standardized_action_effect_mae": 0.44577424080535133, "ranking_accuracy": 0.625, "mean_regret_family_objective_units": 0.1149686559871352 }, { "family": "tank", "sample_count": 9, "field_horizon_balanced_standardized_action_effect_mae": 0.4309230445413622, "ranking_accuracy": 0.9629629629629629, "mean_regret_family_objective_units": 0.19212913518986896 }, { "family": "thermal", "sample_count": 9, "field_horizon_balanced_standardized_action_effect_mae": 0.13642123593907265, "ranking_accuracy": 0.6388888888888888, "mean_regret_family_objective_units": 24.588706704133514 } ], "calibration_status": "heldout_partial" }, "history_linear_system_id": { "forecast": [ { "family": "battery", "field_horizon_balanced_standardized_mae": 0.1964516549681624, "coverage_weighted_by_count": 0.9068441064638784, "calibrated_query_count": 2104, "total_observed_query_count": 2104, "calibrated_query_fraction": 1.0, "field_horizon_balanced_standardized_interval_width": 1.0612048471296183, "uncalibrated_fields": [] }, { "family": "tank", "field_horizon_balanced_standardized_mae": 0.19398614921374246, "coverage_weighted_by_count": 0.8867629832096837, "calibrated_query_count": 2561, "total_observed_query_count": 2561, "calibrated_query_fraction": 1.0, "field_horizon_balanced_standardized_interval_width": 1.0373429280371522, "uncalibrated_fields": [] }, { "family": "thermal", "field_horizon_balanced_standardized_mae": 0.053480925814559065, "coverage_weighted_by_count": 0.9512591389114541, "calibrated_query_count": 1231, "total_observed_query_count": 1686, "calibrated_query_fraction": 0.7301304863582444, "field_horizon_balanced_standardized_interval_width": 0.37496149991779415, "uncalibrated_fields": [ "mass_temp" ] } ], "intervention": [ { "family": "battery", "sample_count": 12, "field_horizon_balanced_standardized_action_effect_mae": 0.21353556679131813, "ranking_accuracy": 0.8194444444444444, "mean_regret_family_objective_units": 0.07438788574709695 }, { "family": "tank", "sample_count": 9, "field_horizon_balanced_standardized_action_effect_mae": 0.375519408741499, "ranking_accuracy": 0.6666666666666666, "mean_regret_family_objective_units": 14.565733903777224 }, { "family": "thermal", "sample_count": 9, "field_horizon_balanced_standardized_action_effect_mae": 0.05728579907697506, "ranking_accuracy": 0.9074074074074074, "mean_regret_family_objective_units": 4.096646376501237 } ], "calibration_status": "heldout_partial" }, "specialists": { "status": "unrun", "reason": "No specialist checkpoint artifacts supplied" } }, "generalization": [ { "name": "heldout_systems", "axis": "B", "status": "completed", "reason": null, "evaluated_target_fraction": null }, { "name": "sensor_failures", "axis": "C", "status": "unrun", "reason": "Dataset and compatible trained checkpoint were not supplied", "evaluated_target_fraction": null }, { "name": "sensor_bias", "axis": "C", "status": "unrun", "reason": "Dataset and compatible trained checkpoint were not supplied", "evaluated_target_fraction": null }, { "name": "configuration_shift", "axis": "D", "status": "unrun", "reason": "Dataset and compatible trained checkpoint were not supplied", "evaluated_target_fraction": null }, { "name": "leave_thermal_out", "axis": "E", "status": "unrun", "reason": "Dataset and compatible trained checkpoint were not supplied", "evaluated_target_fraction": null }, { "name": "leave_tank_out", "axis": "E", "status": "unrun", "reason": "Dataset and compatible trained checkpoint were not supplied", "evaluated_target_fraction": null }, { "name": "leave_battery_out", "axis": "E", "status": "unrun", "reason": "Dataset and compatible trained checkpoint were not supplied", "evaluated_target_fraction": null } ] } } }
{
"version": "tfm-comparison-v1",
"status": "smoke_only",
"generator_version": "0.1.2",
"manifest_sha256": "3541d17c0a76b9e3e9d462c462b30fe5c17eb76968b8ed0d6196ebb6d0b7558c",
"partition_episode_counts": {
"train": 63,
"validation": 14,
"test": 13
},
"interpretation": "Short synthetic runs, no research or scaling claim. Prediction and action-effect MAE equally average field/horizon rows within family. Coverage is weighted by calibrated observed query counts and omits unsupported fields, which are listed. Widths divide by each field's training standard deviation; physical widths remain in source reports. Regret has family-specific objective units and must not be pooled. Latency retains the source measurement scope and contention limitations.",
"runs": {
"transformer": {
"status": "completed",
"provenance_checks": "passed",
"source_report": "artifacts/smoke/report-transformer-v2/benchmark.json",
"source_report_sha256": "a0b6e74770b87f1300cfe97c40bc05e48a689b40e4453c35cc9e90323cb4690b",
"calibration_sha256": "ef5ef17c35d6a4f4f8614b21c011a591594b97d87cb1532008e28b08b501b198",
"checkpoint_sha256": [
"6b0ac9c50d7d581c0495f85dd01b42c48004afdbede88a86e445347bdce9d739"
],
"normalization_sha256": "ddc73b06488090bd4f79571416bf8244638bbd6f28f899fb3ef92411e5f0e41e",
"split_system_counts": {
"training_system_ids": 63,
"validation_selection_system_ids": 8,
"calibration_system_ids": 6,
"test_system_ids": 13
},
"parameter_count": 1570434,
"inference": [
{
"device": "cpu",
"status": "completed",
"batch_size": 1,
"precision": "float32",
"repetitions": 30,
"warmup": 5,
"p50_ms": 5.8506665,
"p95_ms": 6.984247949999999,
"process_peak_rss_bytes": 2904473600,
"sampled_mps_driver_allocation_peak_bytes": null,
"memory_limitation": "RSS is lifetime process high water, includes data and prior models; MPS driver allocation sampled after calls is a lower bound, not resettable model peak.",
"timing_scope": "forward only; inputs resident; MPS synchronized before and after every call"
},
{
"device": "mps",
"status": "unavailable"
}
],
"window_sampling": {
"seed": 2026,
"history_steps": 24,
"future_steps": 8,
"sampling": "uniform eligible anchors with replacement",
"anchor_sha256": "8b6e0caa2a477ef4cdce719cfc04caa8e80561e124d803dac61ffaefe8dff8f2",
"family_window_counts": {
"battery": 184,
"tank": 165,
"thermal": 163
}
},
"models": {
"foundation_model": {
"forecast": [
{
"family": "battery",
"field_horizon_balanced_standardized_mae": 0.2464418007681767,
"coverage_weighted_by_count": 0.9348859315589354,
"calibrated_query_count": 2104,
"total_observed_query_count": 2104,
"calibrated_query_fraction": 1.0,
"field_horizon_balanced_standardized_interval_width": 1.5368033744363387,
"uncalibrated_fields": []
},
{
"family": "tank",
"field_horizon_balanced_standardized_mae": 0.20518572337459773,
"coverage_weighted_by_count": 0.841858648965248,
"calibrated_query_count": 2561,
"total_observed_query_count": 2561,
"calibrated_query_fraction": 1.0,
"field_horizon_balanced_standardized_interval_width": 1.0525175486113494,
"uncalibrated_fields": []
},
{
"family": "thermal",
"field_horizon_balanced_standardized_mae": 0.08072376647032797,
"coverage_weighted_by_count": 0.9260763606823721,
"calibrated_query_count": 1231,
"total_observed_query_count": 1686,
"calibrated_query_fraction": 0.7301304863582444,
"field_horizon_balanced_standardized_interval_width": 0.47078061521257536,
"uncalibrated_fields": [
"mass_temp"
]
}
],
"intervention": [
{
"family": "battery",