Upload live inference API deployment files

.dockerignore

@@ -0,0 +1,16 @@
+.git
+.DS_Store
+*__pycache__*
+*.pyc
+xx_private
+
+data/synthetic_v0
+data/model_runs/synthetic_v0_baseline
+
+data/synthetic_v1/raw_normalized
+data/synthetic_v1/examples
+data/synthetic_v1/schemas
+data/synthetic_v1/validation
+data/synthetic_v1/workbook_rules
+data/synthetic_v1/features/*
+!data/synthetic_v1/features/window_features_all.csv

.gitattributes

@@ -34,3 +34,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
 data/synthetic_v1/features/window_features_all.csv filter=lfs diff=lfs merge=lfs -text
+data/model_runs/synthetic_v1_baseline/predictions_all.csv filter=lfs diff=lfs merge=lfs -text

Dockerfile

@@ -0,0 +1,16 @@
+FROM python:3.11-slim
+
+ENV PYTHONUNBUFFERED=1
+ENV DCV_MODEL_RUN_DIR=data/model_runs/synthetic_v1_baseline
+ENV DCV_FEATURE_TABLE=data/synthetic_v1/features/window_features_all.csv
+
+WORKDIR /app
+
+COPY src/datacenter_verification_api/requirements.txt /tmp/requirements-api.txt
+RUN pip install --no-cache-dir -r /tmp/requirements-api.txt
+
+COPY src ./src
+COPY data/model_runs/synthetic_v1_baseline ./data/model_runs/synthetic_v1_baseline
+COPY data/synthetic_v1/features/window_features_all.csv ./data/synthetic_v1/features/window_features_all.csv
+
+CMD ["sh", "-c", "uvicorn src.datacenter_verification_api.app:app --host 0.0.0.0 --port ${PORT:-7860}"]

data/model_runs/synthetic_v1_baseline/README.md

@@ -0,0 +1,87 @@
+# synthetic_v1_baseline model run
+
+This directory contains a public runnable baseline for the `synthetic_v1` datacenter training-run verification dataset.
+
+## Dataset
+
+- Feature table: `data/synthetic_v1/features/window_features_all.csv`
+- Rows: 14712
+- Episode split: grouped by `episode_id`, scenario-stratified, seed `20260510`
+- Split rows: train 8870, validation 2937, test 2905
+- Split episodes: train 458, validation 152, test 150
+
+## Model
+
+- Supervised model: calibrated scikit-learn histogram gradient boosting classifier
+- Calibration: validation split only, held-out test evaluated once
+- Rule baseline: deterministic evidence rules in `src/datacenter_verification_modeling/rule_baseline.py`
+- Leakage exclusions: identifiers, labels, site id, episode id, raw manifest hash, scenario metadata, counterfactual metadata, and synthetic-only audit columns
+
+## Headline Test Metrics
+
+- Accuracy: 0.9725
+- Macro F1: 0.9781
+- Weighted F1: 0.9723
+- Log loss: 0.1057
+- Label 3/4 precision by predicted label: 0.9935
+- Label 3/4 recall by predicted label: 0.9967
+- `p_large_training >= 0.5` precision: 0.9935
+- `p_large_training >= 0.5` recall: 0.9967
+- Rule baseline macro F1: 0.3636
+
+## Error Scenarios
+
+- Largest false-positive scenarios at `p_large_training >= 0.5`: underclocked_energy_capped_training: 4
+- Largest false-negative scenarios at `p_large_training >= 0.5`: fragmented_training_linked: 2
+
+## Calibration
+
+- Brier score for `p_large_training`: 0.0018
+- Expected calibration error for `p_large_training`: 0.0105
+
+## Stress Splits
+
+Stress outputs are in:
+
+- `stress_metrics.json`
+- `stress_summary.md`
+
+Key stress checks from the lightweight retraining script:
+
+- Episode-grouped random split macro F1: 0.9820; large-training recall: 0.9967
+- Time holdout macro F1: 0.9863; large-training recall: 1.0000
+- Underclocked holdout large-training recall: 1.0000; false negatives: 0
+- Fragmented linked training holdout remains hard: 91 large-training false negatives
+- Source ablation dropping runtime and ML logs changed macro F1 by -0.089 versus the random stress baseline
+- Middle `p_large_training` bins are populated in stress checks: {'0.1_to_0.3': 18, '0.3_to_0.7': 2, '0.7_to_0.9': 0}
+
+## Reproduce
+
+```bash
+python src/datacenter_verification_modeling/train_model.py \
+  --features data/synthetic_v1/features/window_features_all.csv \
+  --output data/model_runs/synthetic_v1_baseline \
+  --seed 20260510
+```
+
+```bash
+python src/datacenter_verification_modeling/evaluate_model.py \
+  --model-run data/model_runs/synthetic_v1_baseline \
+  --features data/synthetic_v1/features/window_features_all.csv
+```
+
+```bash
+python src/datacenter_verification_modeling/predict.py \
+  --model-run data/model_runs/synthetic_v1_baseline \
+  --features data/synthetic_v1/features/window_features_all.csv \
+  --output data/model_runs/synthetic_v1_baseline/predictions_all.csv
+```
+
+## Limitations
+
+- This model is trained on synthetic data only.
+- Performance numbers are not real-world deployment claims.
+- Adjacent windows are correlated, so group splitting by `episode_id` is mandatory.
+- Synthetic labels are generated from rules and latent scenarios, so the model may learn generator assumptions.
+- Real datacenter deployment would require calibration on real telemetry and controlled drills.
+- The model should assist audit triage; it should not be treated as sole proof of a violation.

data/model_runs/synthetic_v1_baseline/calibration_metrics.json

@@ -0,0 +1,119 @@
+{
+  "brier_large_training": 0.001846207706971921,
+  "calibration_by_label": {
+    "0": {
+      "brier": 0.00841142332402854,
+      "mean_predicted_probability": 0.2744490972864224,
+      "observed_fraction": 0.2623063683304647,
+      "rows": 2905
+    },
+    "1": {
+      "brier": 0.024583974737105475,
+      "mean_predicted_probability": 0.21573853144684202,
+      "observed_fraction": 0.23752151462994836,
+      "rows": 2905
+    },
+    "2": {
+      "brier": 0.018264729634158587,
+      "mean_predicted_probability": 0.3017207163585596,
+      "observed_fraction": 0.2884681583476764,
+      "rows": 2905
+    },
+    "3": {
+      "brier": 0.0017921366976442609,
+      "mean_predicted_probability": 0.1570568215171177,
+      "observed_fraction": 0.15938037865748708,
+      "rows": 2905
+    },
+    "4": {
+      "brier": 7.20109313866924e-05,
+      "mean_predicted_probability": 0.05103483339105826,
+      "observed_fraction": 0.05232358003442341,
+      "rows": 2905
+    }
+  },
+  "expected_calibration_error_large_training": 0.010488907264903001,
+  "log_loss": 0.10567154528993643,
+  "reliability_bins_large_training": [
+    {
+      "bin_index": 0,
+      "count": 2283,
+      "lower": 0.0,
+      "mean_predicted_probability": 0.002786695213286837,
+      "observed_fraction": 0.0,
+      "upper": 0.1
+    },
+    {
+      "bin_index": 1,
+      "count": 0,
+      "lower": 0.1,
+      "mean_predicted_probability": 0.0,
+      "observed_fraction": 0.0,
+      "upper": 0.2
+    },
+    {
+      "bin_index": 2,
+      "count": 2,
+      "lower": 0.2,
+      "mean_predicted_probability": 0.24783574032227917,
+      "observed_fraction": 1.0,
+      "upper": 0.30000000000000004
+    },
+    {
+      "bin_index": 3,
+      "count": 3,
+      "lower": 0.30000000000000004,
+      "mean_predicted_probability": 0.30455095537998655,
+      "observed_fraction": 0.0,
+      "upper": 0.4
+    },
+    {
+      "bin_index": 4,
+      "count": 0,
+      "lower": 0.4,
+      "mean_predicted_probability": 0.0,
+      "observed_fraction": 0.0,
+      "upper": 0.5
+    },
+    {
+      "bin_index": 5,
+      "count": 8,
+      "lower": 0.5,
+      "mean_predicted_probability": 0.5612793252894144,
+      "observed_fraction": 1.0,
+      "upper": 0.6000000000000001
+    },
+    {
+      "bin_index": 6,
+      "count": 4,
+      "lower": 0.6000000000000001,
+      "mean_predicted_probability": 0.6781471295808353,
+      "observed_fraction": 0.0,
+      "upper": 0.7000000000000001
+    },
+    {
+      "bin_index": 7,
+      "count": 0,
+      "lower": 0.7000000000000001,
+      "mean_predicted_probability": 0.0,
+      "observed_fraction": 0.0,
+      "upper": 0.8
+    },
+    {
+      "bin_index": 8,
+      "count": 4,
+      "lower": 0.8,
+      "mean_predicted_probability": 0.8676498355263033,
+      "observed_fraction": 1.0,
+      "upper": 0.9
+    },
+    {
+      "bin_index": 9,
+      "count": 601,
+      "lower": 0.9,
+      "mean_predicted_probability": 0.9751439027068034,
+      "observed_fraction": 1.0,
+      "upper": 1.0
+    }
+  ]
+}

data/model_runs/synthetic_v1_baseline/classification_report.json

@@ -0,0 +1,45 @@
+{
+  "0": {
+    "f1-score": 0.9915419648666233,
+    "precision": 0.983225806451613,
+    "recall": 1.0,
+    "support": 762.0
+  },
+  "1": {
+    "f1-score": 0.9450222882615156,
+    "precision": 0.9695121951219512,
+    "recall": 0.9217391304347826,
+    "support": 690.0
+  },
+  "2": {
+    "f1-score": 0.96047197640118,
+    "precision": 0.9498249708284714,
+    "recall": 0.9713603818615751,
+    "support": 838.0
+  },
+  "3": {
+    "f1-score": 0.9935344827586207,
+    "precision": 0.9913978494623656,
+    "recall": 0.9956803455723542,
+    "support": 463.0
+  },
+  "4": {
+    "f1-score": 1.0,
+    "precision": 1.0,
+    "recall": 1.0,
+    "support": 152.0
+  },
+  "accuracy": 0.9724612736660929,
+  "macro avg": {
+    "f1-score": 0.9781141424575879,
+    "precision": 0.9787921643728803,
+    "recall": 0.9777559715737423,
+    "support": 2905.0
+  },
+  "weighted avg": {
+    "f1-score": 0.9722899614011162,
+    "precision": 0.9725136003461651,
+    "recall": 0.9724612736660929,
+    "support": 2905.0
+  }
+}

data/model_runs/synthetic_v1_baseline/confusion_matrix.csv

@@ -0,0 +1,6 @@
+true_label,pred_0,pred_1,pred_2,pred_3,pred_4
+true_0,762,0,0,0,0
+true_1,13,636,41,0,0
+true_2,0,20,814,4,0
+true_3,0,0,2,461,0
+true_4,0,0,0,0,152

data/model_runs/synthetic_v1_baseline/evidence_audit_sample.csv

@@ -0,0 +1,51 @@
+audit_category,feature_row_id,split,latent_workload_class,true_label,predicted_label,p_large_training,severity_score,top_evidence,critical_missing_layers,integrity_warning,policy_compute_ratio,o2_max_concurrent_normalized_gpus,o2_allocation_duration_hours,o2_gpu_hours_policy_ratio,o4_gpu_util_p95,o4_gpu_util_duty_gt_70,o7_synchronized_fabric_footprint,o7_collective_periodicity_score,o8_rack_power_fraction_p95,o10_runtime_framework_class,o11_checkpoint_periodicity_score,o12_signed_ml_logs_present,o12_declared_parameter_count_b,o14_min_critical_coverage,o14_gap_fraction_critical,o13_confidential_compute_mode_fraction,o2_elastic_resize_count,o2_preemption_restart_count,o2_account_linkage_confidence,o4_hbm_pressure_duration_fraction,o4_power_cap_active_fraction,o7_account_flow_linkage_confidence,o10_runtime_metadata_confidence,o11_artifact_write_pattern_score,o11_dataloader_read_pattern_score,o12_log_delivery_delay_hours,o12_log_completeness_fraction,o4_missing_reason,o7_missing_reason,o12_missing_reason
+correct_label_0,feat_cc640fafc2b8e3abf770c579,test,idle_or_low_activity,0,0,0.0015660703107509954,0.030981720299867704,no strong positive evidence,,False,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.1549,none,0.0,False,,0.98,0.01,0.0,0,0,1.0,0.0,0.0,1.0,1.0,0.0,0.0,0.0,0.0,observed,observed,privacy_redacted
+correct_label_0,feat_b9e262daf17134352518a78d,test,idle_or_low_activity,0,0,0.0015660703107509954,0.030981720299867704,no strong positive evidence,,False,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.1549,none,0.0,False,,0.98,0.01,0.0,0,0,1.0,0.0,0.0,1.0,1.0,0.0,0.0,0.0,0.0,observed,observed,privacy_redacted
+correct_label_0,feat_f2d694eed3d39809a028e2b2,test,idle_or_low_activity,0,0,0.0015660703107509954,0.030981720299867704,no strong positive evidence,,False,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.1549,none,0.0,False,,0.98,0.01,0.0,0,0,1.0,0.0,0.0,1.0,1.0,0.0,0.0,0.0,0.0,observed,observed,privacy_redacted
+correct_label_0,feat_729feb643bec26839d5503c5,test,idle_or_low_activity,0,0,0.0015660703107509954,0.030981720299867704,no strong positive evidence,,False,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.1053,none,0.0,False,,0.98,0.01,0.0,0,0,1.0,0.0,0.0,1.0,1.0,0.0,0.0,0.0,0.0,observed,observed,privacy_redacted
+correct_label_0,feat_3d5fc07aac19d9a5f2f3b3c4,test,idle_or_low_activity,0,0,0.0015660703107509954,0.030981720299867704,no strong positive evidence,,False,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.1053,none,0.0,False,,0.98,0.01,0.0,0,0,1.0,0.0,0.0,1.0,1.0,0.0,0.0,0.0,0.0,observed,observed,privacy_redacted
+correct_label_0,feat_c7f33b2997f139238815270f,test,idle_or_low_activity,0,0,0.0015660703107509954,0.030981720299867704,no strong positive evidence,,False,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.1115,none,0.0,False,,0.98,0.01,0.0,0,0,1.0,0.0,0.0,1.0,1.0,0.0,0.0,0.0,0.0,observed,observed,privacy_redacted
+correct_label_0,feat_313de45b023c747cb085c198,test,idle_or_low_activity,0,0,0.0015660703107509954,0.030981720299867704,no strong positive evidence,,False,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.1115,none,0.0,False,,0.98,0.01,0.0,0,0,1.0,0.0,0.0,1.0,1.0,0.0,0.0,0.0,0.0,observed,observed,privacy_redacted
+correct_label_0,feat_ae25d41c049ce4e3cac1242a,test,idle_or_low_activity,0,0,0.0015660703107509954,0.030981720299867704,no strong positive evidence,,False,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.1053,none,0.0,False,,0.98,0.01,0.0,0,0,1.0,0.0,0.0,1.0,1.0,0.0,0.0,0.0,0.0,observed,observed,privacy_redacted
+correct_label_0,feat_a4957cf7cc0d15a01c6fb2e3,test,idle_or_low_activity,0,0,0.0015660703107509954,0.030981720299867704,no strong positive evidence,,False,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.1053,none,0.0,False,,0.98,0.01,0.0,0,0,1.0,0.0,0.0,1.0,1.0,0.0,0.0,0.0,0.0,observed,observed,privacy_redacted
+correct_label_0,feat_5f0c6f536a6333175af7d1ad,test,idle_or_low_activity,0,0,0.0015660703107509954,0.030981720299867704,no strong positive evidence,,False,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.1549,none,0.0,False,,0.98,0.01,0.0,0,0,1.0,0.0,0.0,1.0,1.0,0.0,0.0,0.0,0.0,observed,observed,privacy_redacted
+correct_label_2_hard_false_positive,feat_a13e07bfed9432972ef31ffd,test,large_batch_inference,2,2,0.011030947812113877,1.977255258500455,large allocation; long allocation duration; high GPU activity,,False,8.1915,1089.61,92.38,8.1915,56.3448,0.3667,162.0,0.1229,0.3835,large_batch_inference,0.0,False,,0.98,0.01,0.0,0,0,1.0,0.3024,0.0,1.0,1.0,0.0,0.0,0.0,0.0,observed,observed,privacy_redacted
+correct_label_2_hard_false_positive,feat_00e8cf11a42724292298701b,test,large_batch_inference,2,2,0.010478314009615335,1.975675486490715,large allocation; long allocation duration; high GPU activity,,False,8.1915,1089.61,92.38,8.1915,43.0872,0.2804,124.0,0.0809,0.406,large_batch_inference,0.0,False,,0.98,0.01,0.0,0,0,1.0,0.3024,0.0,1.0,1.0,0.0,0.0,0.0,0.0,observed,observed,privacy_redacted
+correct_label_2_hard_false_positive,feat_cd03bedba3159582ed1baa14,test,embedding_generation,2,2,0.010431098895779199,1.9626186151571463,large allocation; high GPU activity,,False,0.3629,713.64,6.25,0.3629,63.8316,0.3986,164.0,0.1905,0.3809,embedding_generation,0.0,False,,0.98,0.01,0.0,0,0,1.0,0.3328,0.0,1.0,1.0,0.0,0.3263,0.0,0.0,observed,observed,privacy_redacted
+correct_label_2_hard_false_positive,feat_34bca06c5eb77d0686eb0b00,test,large_batch_inference,2,2,0.010426946064660568,1.975794732799792,large allocation; long allocation duration; high GPU activity,,False,8.1915,1089.61,92.38,8.1915,43.0872,0.2804,124.0,0.094,0.406,large_batch_inference,0.0,False,,0.98,0.01,0.0,0,0,1.0,0.3024,0.0,1.0,1.0,0.0,0.0,0.0,0.0,observed,observed,privacy_redacted
+correct_label_2_hard_false_positive,feat_a9949f2525b28aadd5101e47,test,large_batch_inference,2,2,0.010426946064660568,1.975794732799792,large allocation; long allocation duration; high GPU activity,,False,8.1915,1089.61,92.38,8.1915,43.0872,0.2804,124.0,0.094,0.406,large_batch_inference,0.0,False,,0.98,0.01,0.0,0,0,1.0,0.3024,0.0,1.0,1.0,0.0,0.0,0.0,0.0,observed,observed,privacy_redacted
+correct_label_2_hard_false_positive,feat_33b02f78b085c861a75ce235,test,large_batch_inference,2,2,0.0103685521855723,1.9784666216612823,large allocation; long allocation duration; high GPU activity,,False,8.1915,1089.61,92.38,8.1915,82.86,0.5393,226.0,0.1808,0.5639,large_batch_inference,0.0,False,,0.98,0.01,0.0,0,0,1.0,0.3024,0.0,1.0,1.0,0.0,0.0,0.0,0.0,observed,observed,privacy_redacted
+correct_label_2_hard_false_positive,feat_db7205bdba2eac91416428d7,test,large_batch_inference,2,2,0.0103685521855723,1.9784666216612823,large allocation; long allocation duration; high GPU activity,,False,8.1915,1089.61,92.38,8.1915,82.86,0.5393,226.0,0.1808,0.5639,large_batch_inference,0.0,False,,0.98,0.01,0.0,0,0,1.0,0.3024,0.0,1.0,1.0,0.0,0.0,0.0,0.0,observed,observed,privacy_redacted
+correct_label_2_hard_false_positive,feat_9aa413faa9c554abf6d2bb64,test,large_batch_inference,2,2,0.010222344032340501,1.9762696989913593,large allocation; long allocation duration; high GPU activity,,False,8.1915,1089.61,92.38,8.1915,56.3448,0.3667,162.0,0.1057,0.3835,large_batch_inference,0.0,False,,0.98,0.01,0.0,0,0,1.0,0.3024,0.0,1.0,1.0,0.0,0.0,0.0,0.0,observed,observed,privacy_redacted
+correct_label_2_hard_false_positive,feat_d7444c225b8334e750e5a74e,test,large_batch_inference,2,2,0.010177838904019004,1.9763730138561455,large allocation; long allocation duration; high GPU activity,,False,8.1915,1089.61,92.38,8.1915,56.3448,0.3667,162.0,0.1229,0.3835,large_batch_inference,0.0,False,,0.98,0.01,0.0,0,0,1.0,0.3024,0.0,1.0,1.0,0.0,0.0,0.0,0.0,observed,observed,privacy_redacted
+correct_label_2_hard_false_positive,feat_123d069e43df582c24b12e43,test,large_batch_inference,2,2,0.010177838904019004,1.9763730138561455,large allocation; long allocation duration; high GPU activity,,False,8.1915,1089.61,92.38,8.1915,56.3448,0.3667,162.0,0.1229,0.3835,large_batch_inference,0.0,False,,0.98,0.01,0.0,0,0,1.0,0.3024,0.0,1.0,1.0,0.0,0.0,0.0,0.0,observed,observed,privacy_redacted
+correct_label_3_4_likely_training,feat_aa9a945cbc067309f88926db,test,cloud_training_redacted_runtime,3,3,0.9801384593906176,2.9538324770207036,large allocation; long allocation duration; high GPU activity; synchronized scale-out fabric; training runtime metadata; checkpoint cadence; signed ML logs,,False,2.4407,664.66,45.12,2.4407,82.83,0.6073,598.0,0.8189,0.4996,pytorch_distributed_training,0.5803,True,516.61,0.98,0.01,0.0,0,0,1.0,0.4217,0.0,1.0,0.25,0.5495,0.567,0.0,0.4157,observed,observed,privacy_redacted
+correct_label_3_4_likely_training,feat_45062add05716b1857d2d421,test,cloud_training_redacted_runtime,3,3,0.9801384593906176,2.9538324770207036,large allocation; long allocation duration; high GPU activity; synchronized scale-out fabric; training runtime metadata; checkpoint cadence; signed ML logs,,False,2.4407,664.66,45.12,2.4407,82.83,0.6073,598.0,0.8189,0.4996,pytorch_distributed_training,0.5803,True,516.61,0.98,0.01,0.0,0,0,1.0,0.4217,0.0,1.0,0.25,0.5495,0.567,0.0,0.4157,observed,observed,privacy_redacted
+correct_label_3_4_likely_training,feat_daa683c0ae4f6ebda93534f5,test,cloud_training_redacted_runtime,3,3,0.9801384593906176,2.9538324770207036,large allocation; long allocation duration; high GPU activity; synchronized scale-out fabric; training runtime metadata; checkpoint cadence; signed ML logs,,False,2.4407,664.66,45.12,2.4407,82.83,0.6073,598.0,0.8189,0.4996,pytorch_distributed_training,0.5803,True,516.61,0.98,0.01,0.0,0,0,1.0,0.4217,0.0,1.0,0.25,0.5495,0.567,0.0,0.4157,observed,observed,privacy_redacted
+correct_label_3_4_likely_training,feat_1a65e998609bc78ecf0dc1cb,test,cloud_training_redacted_runtime,3,3,0.9801384593906176,2.9538324770207036,large allocation; long allocation duration; high GPU activity; synchronized scale-out fabric; training runtime metadata; checkpoint cadence; signed ML logs,,False,2.4407,664.66,45.12,2.4407,82.83,0.6073,598.0,0.8189,0.4996,pytorch_distributed_training,0.5803,True,516.61,0.98,0.01,0.0,0,0,1.0,0.4217,0.0,1.0,0.25,0.5495,0.567,0.0,0.4157,observed,observed,privacy_redacted
+correct_label_3_4_likely_training,feat_e60eeafb2f9e0d9386d1c5d0,test,cloud_training_redacted_runtime,3,3,0.9801384593906176,2.9538324770207036,large allocation; long allocation duration; high GPU activity; synchronized scale-out fabric; training runtime metadata; checkpoint cadence; signed ML logs,,False,2.4407,664.66,45.12,2.4407,82.83,0.6073,598.0,0.8189,0.4996,pytorch_distributed_training,0.5803,True,516.61,0.98,0.01,0.0,0,0,1.0,0.4217,0.0,1.0,0.25,0.5495,0.567,0.0,0.4157,observed,observed,privacy_redacted
+correct_label_3_4_likely_training,feat_3fa8d05e390014d891252788,test,cloud_training_redacted_runtime,3,3,0.9801384593906176,2.9538324770207036,large allocation; long allocation duration; high GPU activity; synchronized scale-out fabric; training runtime metadata; checkpoint cadence; signed ML logs,,False,2.4407,664.66,45.12,2.4407,82.83,0.6073,598.0,0.8189,0.4996,pytorch_distributed_training,0.5803,True,516.61,0.98,0.01,0.0,0,0,1.0,0.4217,0.0,1.0,0.25,0.5495,0.567,0.0,0.4157,observed,observed,privacy_redacted
+correct_label_3_4_likely_training,feat_1f1405339cd56e6df182c550,test,cloud_training_redacted_runtime,3,3,0.9801384593906176,2.9538324770207036,large allocation; long allocation duration; high GPU activity; synchronized scale-out fabric; training runtime metadata; checkpoint cadence; signed ML logs,,False,2.4407,664.66,45.12,2.4407,82.83,0.6073,598.0,0.8189,0.4996,pytorch_distributed_training,0.5803,True,516.61,0.98,0.01,0.0,0,0,1.0,0.4217,0.0,1.0,0.25,0.5495,0.567,0.0,0.4157,observed,observed,privacy_redacted
+correct_label_3_4_likely_training,feat_2bf4e8f647710e486ecd057c,test,cloud_training_redacted_runtime,3,3,0.9801384593906176,2.9538324770207036,large allocation; long allocation duration; high GPU activity; synchronized scale-out fabric; training runtime metadata; checkpoint cadence; signed ML logs,,False,2.4407,664.66,45.12,2.4407,82.83,0.6073,598.0,0.8189,0.4996,pytorch_distributed_training,0.5803,True,516.61,0.98,0.01,0.0,0,0,1.0,0.4217,0.0,1.0,0.25,0.5495,0.567,0.0,0.4157,observed,observed,privacy_redacted
+correct_label_3_4_likely_training,feat_af8e945fc6fc7f9343da8288,test,cloud_training_redacted_runtime,3,3,0.9801384593906176,2.9538324770207036,large allocation; long allocation duration; high GPU activity; synchronized scale-out fabric; training runtime metadata; checkpoint cadence; signed ML logs,,False,2.4407,664.66,45.12,2.4407,82.83,0.6073,598.0,0.8189,0.4996,pytorch_distributed_training,0.5803,True,516.61,0.98,0.01,0.0,0,0,1.0,0.4217,0.0,1.0,0.25,0.5495,0.567,0.0,0.4157,observed,observed,privacy_redacted
+correct_label_3_4_likely_training,feat_2267762b2b19047a21c23936,test,training_without_semantic_logs,3,3,0.9800541368127997,2.9537374894493977,large allocation; long allocation duration; high GPU activity; training runtime metadata; checkpoint cadence,,False,2.5163,461.48,67.0,2.5163,71.44,0.8187,302.0,0.5238,0.38,pytorch_distributed_training,0.7481,False,,0.98,0.01,0.0,0,0,0.8407,0.4875,0.0,0.8407,0.55,0.6235,0.5428,0.0,0.0,observed,observed,privacy_redacted
+false_positive_large_training,feat_49c6f202eb6d2980105018df,test,underclocked_energy_capped_training,2,3,0.6981856840003529,2.6442344946513,large allocation; long allocation duration; high GPU activity; training runtime metadata; signed ML logs; low critical coverage; counter disabled by CC mode,O4:counter_disabled_by_cc_mode:coverage=0.29,True,9.2912,513.05,222.53,9.2912,53.4752,0.5516,267.0,0.4296,0.4283,pytorch_distributed_training,0.2781,True,316.04,0.2921,0.01,0.8046,0,0,1.0,0.3765,0.4654,1.0,0.7611,0.2456,0.7229,0.0,0.342,counter_disabled_by_cc_mode,observed,delayed_log_delivery
+false_positive_large_training,feat_a383ed6dabdf9145d8ee996e,test,underclocked_energy_capped_training,2,3,0.6714676114409962,2.6140027926406604,large allocation; long allocation duration; high GPU activity; training runtime metadata; signed ML logs; low critical coverage; counter disabled by CC mode,O4:counter_disabled_by_cc_mode:coverage=0.29,True,9.2912,513.05,222.53,9.2912,53.4752,0.5516,267.0,0.3695,0.4283,pytorch_distributed_training,0.1981,True,316.04,0.2921,0.01,0.8046,0,0,1.0,0.3765,0.4654,1.0,0.7611,0.2456,0.7229,0.0,0.342,counter_disabled_by_cc_mode,observed,delayed_log_delivery
+false_positive_large_training,feat_ee68609d2f2683b7bef8841b,test,underclocked_energy_capped_training,2,3,0.6714676114409962,2.6140027926406604,large allocation; long allocation duration; high GPU activity; training runtime metadata; signed ML logs; low critical coverage; counter disabled by CC mode,O4:counter_disabled_by_cc_mode:coverage=0.29,True,9.2912,513.05,222.53,9.2912,53.4752,0.5516,267.0,0.4296,0.4283,pytorch_distributed_training,0.1981,True,316.04,0.2921,0.01,0.8046,0,0,1.0,0.3765,0.4654,1.0,0.7611,0.2456,0.7229,0.0,0.342,counter_disabled_by_cc_mode,observed,delayed_log_delivery
+false_positive_large_training,feat_50f4faf6dbd38b6daae6b62f,test,underclocked_energy_capped_training,2,3,0.6714676114409962,2.6140027926406604,large allocation; long allocation duration; high GPU activity; training runtime metadata; signed ML logs; low critical coverage; counter disabled by CC mode,O4:counter_disabled_by_cc_mode:coverage=0.29,True,9.2912,513.05,222.53,9.2912,53.4752,0.5516,267.0,0.4296,0.4283,pytorch_distributed_training,0.1981,True,316.04,0.2921,0.01,0.8046,0,0,1.0,0.3765,0.4654,1.0,0.7611,0.2456,0.7229,0.0,0.342,counter_disabled_by_cc_mode,observed,delayed_log_delivery
+false_negative_large_training,feat_aa8009dde8ce7ceadcee314b,test,fragmented_training_linked,3,2,0.2474303218229664,2.2261162832074306,large allocation; long allocation duration; high GPU activity; training runtime metadata,,False,1.1744,260.88,55.32,1.1744,73.74,0.5307,211.0,0.5568,0.38,pytorch_distributed_training,0.4556,False,,0.98,0.01,0.0,0,0,0.2797,0.3251,0.0,0.2797,0.9268,0.3735,0.3108,0.0,0.0,observed,observed,privacy_redacted
+false_negative_large_training,feat_1d405c4b2a8bf7a3fa052e89,test,fragmented_training_linked,3,2,0.24824115882159198,2.226857273426679,large allocation; long allocation duration; high GPU activity; synchronized scale-out fabric; training runtime metadata,,False,1.1744,260.88,55.32,1.1744,73.74,0.5307,211.0,0.6474,0.38,pytorch_distributed_training,0.4556,False,,0.98,0.01,0.0,0,0,0.2797,0.3251,0.0,0.2797,0.9268,0.3735,0.3108,0.0,0.0,observed,observed,privacy_redacted
+high_uncertainty_or_coverage_edge,feat_3652e586595bc4b0f0a73268,test,training_without_semantic_logs,3,3,0.5362924911368747,2.402141567675965,large allocation; long allocation duration; high GPU activity; power corroboration; training runtime metadata,,False,2.8294,748.77,46.43,2.8294,59.602,0.4835,310.0,0.2928,0.6028,pytorch_distributed_training,0.2225,False,,0.98,0.01,0.0,0,0,1.0,0.533,0.0,1.0,0.7154,0.2446,0.5083,0.0,0.0,observed,observed,privacy_redacted
+high_uncertainty_or_coverage_edge,feat_9aff07df6b963210981b227a,test,training_without_semantic_logs,3,3,0.5362924911368747,2.402141567675965,large allocation; long allocation duration; high GPU activity; power corroboration; training runtime metadata,,False,2.8294,748.77,46.43,2.8294,59.602,0.4835,310.0,0.3405,0.6028,pytorch_distributed_training,0.2225,False,,0.98,0.01,0.0,0,0,1.0,0.533,0.0,1.0,0.7154,0.2446,0.5083,0.0,0.0,observed,observed,privacy_redacted
+high_uncertainty_or_coverage_edge,feat_4877f8beca7baf999dfc875b,test,training_without_semantic_logs,3,3,0.5362924911368747,2.402141567675965,large allocation; long allocation duration; high GPU activity; power corroboration; training runtime metadata,,False,2.8294,748.77,46.43,2.8294,59.602,0.4835,310.0,0.3405,0.6028,pytorch_distributed_training,0.2225,False,,0.98,0.01,0.0,0,0,1.0,0.533,0.0,1.0,0.7154,0.2446,0.5083,0.0,0.0,observed,observed,privacy_redacted
+high_uncertainty_or_coverage_edge,feat_48db5d097443aaf9625944d9,test,training_with_delayed_logs,3,3,0.5742738161131281,2.555844926309712,large allocation; long allocation duration; high GPU activity; synchronized scale-out fabric; training runtime metadata; signed ML logs,,False,26.8381,1974.23,167.05,26.8381,64.9128,0.5727,1278.0,0.6055,0.3937,pytorch_distributed_training,0.2932,True,254.18,0.98,0.01,0.0,0,0,1.0,0.3817,0.0,1.0,0.8011,0.2015,0.6239,47.3,0.4119,observed,observed,delayed_log_delivery
+high_uncertainty_or_coverage_edge,feat_9089822165b23e10b9ea605c,test,training_with_delayed_logs,3,3,0.575021844861732,2.5565689502386792,large allocation; long allocation duration; high GPU activity; synchronized scale-out fabric; training runtime metadata; signed ML logs,,False,26.8381,1974.23,167.05,26.8381,64.9128,0.5727,1278.0,0.5207,0.3937,pytorch_distributed_training,0.2932,True,254.18,0.98,0.01,0.0,0,0,1.0,0.3817,0.0,1.0,0.8011,0.2015,0.6239,47.3,0.4119,observed,observed,delayed_log_delivery
+high_uncertainty_or_coverage_edge,feat_d218ca927f7ec4d4efcd5cd2,test,training_with_delayed_logs,3,3,0.575021844861732,2.5565689502386792,large allocation; long allocation duration; high GPU activity; synchronized scale-out fabric; training runtime metadata; signed ML logs,,False,26.8381,1974.23,167.05,26.8381,64.9128,0.5727,1278.0,0.6055,0.3937,pytorch_distributed_training,0.2932,True,254.18,0.98,0.01,0.0,0,0,1.0,0.3817,0.0,1.0,0.8011,0.2015,0.6239,47.3,0.4119,observed,observed,delayed_log_delivery
+high_uncertainty_or_coverage_edge,feat_38e806129aa6f1882a2e6ec4,test,training_with_delayed_logs,3,3,0.575021844861732,2.5565689502386792,large allocation; long allocation duration; high GPU activity; synchronized scale-out fabric; training runtime metadata; signed ML logs,,False,26.8381,1974.23,167.05,26.8381,64.9128,0.5727,1278.0,0.6055,0.3937,pytorch_distributed_training,0.2932,True,254.18,0.98,0.01,0.0,0,0,1.0,0.3817,0.0,1.0,0.8011,0.2015,0.6239,47.3,0.4119,observed,observed,delayed_log_delivery
+high_uncertainty_or_coverage_edge,feat_f8462a0c8142dee9ac8578b1,test,training_without_semantic_logs,3,3,0.5820177782063665,2.458458270477721,large allocation; long allocation duration; high GPU activity; power corroboration; training runtime metadata,,False,2.8294,748.77,46.43,2.8294,59.602,0.4835,310.0,0.3405,0.6028,pytorch_distributed_training,0.3025,False,,0.98,0.01,0.0,0,0,1.0,0.533,0.0,1.0,0.7154,0.2446,0.5083,0.0,0.0,observed,observed,privacy_redacted
+high_uncertainty_or_coverage_edge,feat_414b3f8239f8d77cc1b8fe90,test,fragmented_training_linked,2,2,0.3074817930715641,2.284158029977732,large allocation; long allocation duration; high GPU activity; training runtime metadata,,False,2.8713,272.73,129.37,2.8713,73.56,0.8511,179.0,0.5256,0.38,pytorch_distributed_training,0.4202,False,,0.98,0.01,0.0,0,0,0.2112,0.3986,0.0,0.2112,0.8524,0.5538,0.3507,0.0,0.0,observed,observed,privacy_redacted
+high_uncertainty_or_coverage_edge,feat_ba27f8df9db4f33ee67bbb36,test,fragmented_training_linked,2,2,0.30308553653419773,2.280095247643677,large allocation; long allocation duration; high GPU activity; training runtime metadata,,False,2.8713,272.73,129.37,2.8713,73.56,0.8511,179.0,0.5256,0.38,pytorch_distributed_training,0.4202,False,,0.98,0.01,0.0,0,0,0.2112,0.3986,0.0,0.2112,0.8524,0.5538,0.3507,0.0,0.0,observed,observed,privacy_redacted
+high_uncertainty_or_coverage_edge,feat_6de8a0c0b3e26c7a626710a7,test,fragmented_training_linked,2,2,0.30308553653419773,2.280095247643677,large allocation; long allocation duration; high GPU activity; training runtime metadata,,False,2.8713,272.73,129.37,2.8713,73.56,0.8511,179.0,0.5256,0.38,pytorch_distributed_training,0.4202,False,,0.98,0.01,0.0,0,0,0.2112,0.3986,0.0,0.2112,0.8524,0.5538,0.3507,0.0,0.0,observed,observed,privacy_redacted
+high_uncertainty_or_coverage_edge,feat_b40bf45e96cf6cc4b679357e,test,training_with_delayed_logs,4,4,0.8674512750198456,3.7090949558285335,large allocation; long allocation duration; high GPU activity; synchronized scale-out fabric; training runtime metadata; signed ML logs,,False,14.8646,944.34,193.42,14.8646,56.7324,0.4636,611.0,0.5216,0.306,pytorch_distributed_training,0.2137,True,167.81,0.98,0.01,0.0,0,0,1.0,0.3747,0.0,1.0,0.7358,0.2546,0.4936,58.2,0.9151,observed,observed,delayed_log_delivery
+high_uncertainty_or_coverage_edge,feat_59636d473b9ba2c6c75a8909,test,training_with_delayed_logs,4,4,0.8674512750198456,3.7090949558285335,large allocation; long allocation duration; high GPU activity; synchronized scale-out fabric; training runtime metadata; signed ML logs,,False,14.8646,944.34,193.42,14.8646,56.7324,0.4636,611.0,0.6066,0.306,pytorch_distributed_training,0.2137,True,167.81,0.98,0.01,0.0,0,0,1.0,0.3747,0.0,1.0,0.7358,0.2546,0.4936,58.2,0.9151,observed,observed,delayed_log_delivery
+high_uncertainty_or_coverage_edge,feat_1fc099dcc832d0323269b33f,test,training_with_delayed_logs,4,4,0.8674512750198456,3.7090949558285335,large allocation; long allocation duration; high GPU activity; synchronized scale-out fabric; training runtime metadata; signed ML logs,,False,14.8646,944.34,193.42,14.8646,56.7324,0.4636,611.0,0.6066,0.306,pytorch_distributed_training,0.2137,True,167.81,0.98,0.01,0.0,0,0,1.0,0.3747,0.0,1.0,0.7358,0.2546,0.4936,58.2,0.9151,observed,observed,delayed_log_delivery

data/model_runs/synthetic_v1_baseline/excluded_columns.json

@@ -0,0 +1,76 @@
+{
+  "missing_requested_excluded_columns": [],
+  "notes": [
+    "Identifier, label, direct leakage, site, and synthetic-only audit columns are excluded.",
+    "scope_type and window_length_seconds are retained because they are valid deployment-time context.",
+    "Constant version metadata columns are excluded from the baseline."
+  ],
+  "present_excluded_columns": [
+    "feature_row_id",
+    "dataset_id",
+    "seed",
+    "site_id",
+    "scope_id_hash",
+    "window_start",
+    "window_end",
+    "episode_id",
+    "label_0_to_4",
+    "label_confidence",
+    "label_reason",
+    "label_source",
+    "raw_input_manifest_hash",
+    "latent_workload_class",
+    "scenario_family",
+    "scenario_variant",
+    "evidence_recipe_id",
+    "counterfactual_group_id",
+    "synthetic_counterfactual_role",
+    "data_quality_regime",
+    "privacy_tier",
+    "collector_profile",
+    "topology_class",
+    "temporal_phase",
+    "synthetic_hard_case_tags",
+    "synthetic_evidence_profile",
+    "capacity_evidence_only",
+    "integrity_evidence_only",
+    "physical_evidence_only",
+    "feature_pipeline_version",
+    "policy_threshold_version",
+    "hardware_normalization_version"
+  ],
+  "requested_excluded_columns": [
+    "feature_row_id",
+    "dataset_id",
+    "seed",
+    "site_id",
+    "scope_id_hash",
+    "window_start",
+    "window_end",
+    "episode_id",
+    "label_0_to_4",
+    "label_confidence",
+    "label_reason",
+    "label_source",
+    "raw_input_manifest_hash",
+    "latent_workload_class",
+    "scenario_family",
+    "scenario_variant",
+    "evidence_recipe_id",
+    "counterfactual_group_id",
+    "synthetic_counterfactual_role",
+    "data_quality_regime",
+    "privacy_tier",
+    "collector_profile",
+    "topology_class",
+    "temporal_phase",
+    "synthetic_hard_case_tags",
+    "synthetic_evidence_profile",
+    "capacity_evidence_only",
+    "integrity_evidence_only",
+    "physical_evidence_only",
+    "feature_pipeline_version",
+    "policy_threshold_version",
+    "hardware_normalization_version"
+  ]
+}

data/model_runs/synthetic_v1_baseline/feature_columns.json

@@ -0,0 +1,154 @@
+[
+  "scope_type",
+  "window_length_seconds",
+  "capacity_possible",
+  "policy_compute_ratio",
+  "o1_normalized_h100e_capacity",
+  "o1_largest_contiguous_domain_gpus",
+  "o1_homogeneous_high_end_fraction",
+  "o1_non_partitioned_fraction",
+  "o1_inventory_delta_rate",
+  "o17_external_capacity_conflict_score",
+  "o2_max_concurrent_normalized_gpus",
+  "o2_allocation_duration_hours",
+  "o2_gpu_hours_policy_ratio",
+  "o2_concurrency_fraction_domain",
+  "o2_topology_contiguity_score",
+  "o2_declared_workload_class",
+  "o2_reservation_exclusive_flag",
+  "o2_elastic_resize_count",
+  "o2_preemption_restart_count",
+  "o2_scheduler_queue_delay_hours",
+  "o2_account_linkage_confidence",
+  "o2_job_array_width",
+  "o2_reservation_reuse_count",
+  "o3_batch_provisioned_gpus",
+  "o3_capacity_reservation_duration_hours",
+  "o3_training_sku_fraction",
+  "o3_billing_continuity_score",
+  "o3_egress_tb",
+  "o4_gpu_util_p50",
+  "o4_gpu_util_p95",
+  "o4_gpu_util_duty_gt_70",
+  "o4_sm_tensor_active_p95",
+  "o4_hbm_used_fraction_p50",
+  "o4_hbm_bandwidth_active_p95",
+  "o4_gpu_power_fraction_p95",
+  "o4_error_spike_score",
+  "o4_gpu_util_cv",
+  "o4_gpu_idle_gap_p95_minutes",
+  "o4_hbm_pressure_duration_fraction",
+  "o4_power_cap_active_fraction",
+  "o4_thermal_throttle_fraction",
+  "o5_kernel_training_motif_score",
+  "o5_tensor_throughput_ratio",
+  "o5_profiler_available",
+  "o6_nvlink_util_p95",
+  "o6_nvlink_periodicity_score",
+  "o6_link_error_spike_score",
+  "o7_scaleout_port_util_p95",
+  "o7_synchronized_fabric_footprint",
+  "o7_collective_periodicity_score",
+  "o7_burst_duty_cycle",
+  "o7_rdma_congestion_score",
+  "o7_job_to_port_mapping_coverage",
+  "o7_flow_entropy_score",
+  "o7_cross_section_sync_score",
+  "o7_collective_jitter_score",
+  "o7_storage_traffic_fraction",
+  "o7_inference_fanout_score",
+  "o7_account_flow_linkage_confidence",
+  "o8_rack_power_fraction_p95",
+  "o8_facility_it_power_mw",
+  "o8_baseline_subtracted_energy_kwh",
+  "o8_power_continuity_days",
+  "o8_power_cv",
+  "o8_power_to_gpu_residual",
+  "o8_power_baseline_drift_score",
+  "o8_power_cap_or_curtailment_active",
+  "o8_unattributed_power_fraction",
+  "o9_gpu_hbm_temp_score",
+  "o9_thermal_delta_t_score",
+  "o9_cooling_flow_duty",
+  "o9_cooling_maintenance_active",
+  "o9_thermal_throttle_support_score",
+  "o10_world_size",
+  "o10_runtime_framework_class",
+  "o10_rank_stability_score",
+  "o10_same_image_gpu_count",
+  "o10_rendezvous_present",
+  "o10_runtime_metadata_confidence",
+  "o10_declared_vs_observed_mismatch_score",
+  "o11_data_staging_tb",
+  "o11_checkpoint_write_tb_per_event",
+  "o11_checkpoint_periodicity_score",
+  "o11_read_write_training_pattern_score",
+  "o11_checkpoint_jitter_score",
+  "o11_artifact_write_pattern_score",
+  "o11_dataloader_read_pattern_score",
+  "o11_backup_or_replication_pattern_score",
+  "o11_storage_cotraffic_score",
+  "o12_signed_ml_logs_present",
+  "o12_declared_parameter_count_b",
+  "o12_training_tokens_b",
+  "o12_step_count",
+  "o12_loss_curve_present",
+  "o12_optimizer_state_present",
+  "o12_log_delivery_delay_hours",
+  "o12_log_completeness_fraction",
+  "o12_declaration_consistency_score",
+  "o13_attestation_valid_fraction",
+  "o13_confidential_compute_mode_fraction",
+  "o13_collector_measurement_valid",
+  "o14_min_critical_coverage",
+  "o14_gap_fraction_critical",
+  "o14_clock_drift_max_ms",
+  "o14_counter_reset_count",
+  "o15_unapproved_physical_change_near_window",
+  "o15_firmware_bmc_change_near_window",
+  "o16_probe_throughput_ratio_min",
+  "o16_probe_latency_inflation_max",
+  "o16_vram_residency_conflict_score",
+  "o17_external_capacity_assertion",
+  "o17_energy_contract_alignment_score",
+  "o17_network_provider_utilization_score",
+  "o17_procurement_or_maintenance_explanation_score",
+  "o1_coverage_fraction",
+  "o1_missing_reason",
+  "o2_coverage_fraction",
+  "o2_missing_reason",
+  "o3_coverage_fraction",
+  "o3_missing_reason",
+  "o4_coverage_fraction",
+  "o4_missing_reason",
+  "o5_coverage_fraction",
+  "o5_missing_reason",
+  "o6_coverage_fraction",
+  "o6_missing_reason",
+  "o7_coverage_fraction",
+  "o7_missing_reason",
+  "o8_coverage_fraction",
+  "o8_missing_reason",
+  "o9_coverage_fraction",
+  "o9_missing_reason",
+  "o10_coverage_fraction",
+  "o10_missing_reason",
+  "o11_coverage_fraction",
+  "o11_missing_reason",
+  "o12_coverage_fraction",
+  "o12_missing_reason",
+  "o13_coverage_fraction",
+  "o13_missing_reason",
+  "o14_coverage_fraction",
+  "o14_missing_reason",
+  "o15_coverage_fraction",
+  "o15_missing_reason",
+  "o16_coverage_fraction",
+  "o16_missing_reason",
+  "o17_coverage_fraction",
+  "o17_missing_reason",
+  "scheduler_signature_status",
+  "gpu_telemetry_trust_level",
+  "fabric_telemetry_trust_level",
+  "power_meter_trust_level"
+]

data/model_runs/synthetic_v1_baseline/feature_importance.csv

@@ -0,0 +1,153 @@
+feature,importance_mean,importance_std,baseline_macro_f1,permuted_macro_f1_mean
+o12_log_completeness_fraction,0.23359158430990315,0.005373690197882257,0.9781141424575879,0.7445225581476848
+o2_allocation_duration_hours,0.14849344450367144,0.0052789099249230705,0.9781141424575879,0.8296206979539164
+o11_checkpoint_periodicity_score,0.09666000121374829,0.0011059737197036232,0.9781141424575879,0.8814541412438396
+o14_min_critical_coverage,0.08425447962391323,0.0028386015672915883,0.9781141424575879,0.8938596628336746
+o7_inference_fanout_score,0.023026962271348744,0.000352376812149185,0.9781141424575879,0.9550871801862391
+o7_synchronized_fabric_footprint,0.0216681347660882,0.0031004413375968374,0.9781141424575879,0.9564460076914997
+policy_compute_ratio,0.01888468408039194,0.002076870274795479,0.9781141424575879,0.959229458377196
+o9_thermal_delta_t_score,0.015522644977829625,0.0017361645292459213,0.9781141424575879,0.9625914974797581
+o8_rack_power_fraction_p95,0.013580628502308012,0.0017886779558558932,0.9781141424575879,0.9645335139552799
+o8_power_cv,0.010049244358529422,0.0007442207607013514,0.9781141424575879,0.9680648980990584
+o2_max_concurrent_normalized_gpus,0.00956190440682921,0.0005897138759531066,0.9781141424575879,0.9685522380507586
+o10_runtime_framework_class,0.009123255221792212,0.0002556343603386503,0.9781141424575879,0.9689908872357957
+o2_topology_contiguity_score,0.008465602515600726,0.000978973878068464,0.9781141424575879,0.9696485399419871
+o4_gpu_idle_gap_p95_minutes,0.008401230772478193,0.0007671011073014647,0.9781141424575879,0.9697129116851096
+o8_power_to_gpu_residual,0.006467490611914021,0.0012204702714004524,0.9781141424575879,0.9716466518456738
+o11_data_staging_tb,0.004774385170709716,0.0011077200740331708,0.9781141424575879,0.9733397572868782
+o4_gpu_util_cv,0.004515834867171553,0.00012814093735156415,0.9781141424575879,0.9735983075904163
+o11_dataloader_read_pattern_score,0.003564301181372668,0.00039712138629140767,0.9781141424575879,0.9745498412762151
+o7_collective_jitter_score,0.003338909258623716,0.00025581495381753373,0.9781141424575879,0.9747752331989642
+o7_collective_periodicity_score,0.0022545386038181534,0.0006384080942919812,0.9781141424575879,0.9758596038537698
+o4_hbm_bandwidth_active_p95,0.0021022174553009254,0.0008926825404043516,0.9781141424575879,0.976011925002287
+o7_burst_duty_cycle,0.001712656669582091,0.0003376920900513128,0.9781141424575879,0.9764014857880058
+o11_read_write_training_pattern_score,0.0014772421580005284,0.0007389848391526284,0.9781141424575879,0.9766369002995873
+o9_gpu_hbm_temp_score,0.001261581096622888,0.0003374158130753862,0.9781141424575879,0.976852561360965
+o7_cross_section_sync_score,0.0012615470192883043,0.0002551176083138864,0.9781141424575879,0.9768525954382996
+o7_coverage_fraction,0.0009957627772852657,0.0002559712480885151,0.9781141424575879,0.9771183796803026
+o4_gpu_power_fraction_p95,0.0009910051757533085,0.0001274863360027242,0.9781141424575879,0.9771231372818345
+o2_concurrency_fraction_domain,0.0009705866111717754,0.0004495525752025081,0.9781141424575879,0.9771435558464162
+o4_coverage_fraction,0.0008706275099251334,0.0002696962800440298,0.9781141424575879,0.9772435149476628
+o7_flow_entropy_score,0.0008107122703610026,0.0,0.9781141424575879,0.9773034301872269
+o11_backup_or_replication_pattern_score,0.0006612636677401454,0.0002701885364253015,0.9781141424575879,0.9774528787898477
+o4_hbm_used_fraction_p50,0.0006554675510399935,0.0004944170097273767,0.9781141424575879,0.9774586749065479
+o5_coverage_fraction,0.0005406101441621051,0.0005840293050570898,0.9781141424575879,0.9775735323134258
+o7_storage_traffic_fraction,0.0002716123882583095,0.00022176553850800082,0.9781141424575879,0.9778425300693296
+o7_account_flow_linkage_confidence,0.00011013595622607249,0.00015575576299984125,0.9781141424575879,0.9780040065013619
+o2_declared_workload_class,6.777792644607761e-05,0.0002861792488526281,0.9781141424575879,0.9780463645311418
+capacity_possible,0.0,0.0,0.9781141424575879,0.9781141424575879
+fabric_telemetry_trust_level,0.0,0.0,0.9781141424575879,0.978114142457588
+gpu_telemetry_trust_level,0.0,0.0,0.9781141424575879,0.978114142457588
+o10_coverage_fraction,0.0,0.0,0.9781141424575879,0.978114142457588
+o10_declared_vs_observed_mismatch_score,0.0,0.0,0.9781141424575879,0.9781141424575879
+o10_missing_reason,0.0,0.0,0.9781141424575879,0.978114142457588
+o10_rank_stability_score,0.0,0.0,0.9781141424575879,0.978114142457588
+o10_rendezvous_present,0.0,0.0,0.9781141424575879,0.978114142457588
+o10_runtime_metadata_confidence,0.0,0.0,0.9781141424575879,0.978114142457588
+o10_same_image_gpu_count,0.0,0.0,0.9781141424575879,0.978114142457588
+o10_world_size,0.0,0.0,0.9781141424575879,0.978114142457588
+o11_artifact_write_pattern_score,0.0,0.0,0.9781141424575879,0.978114142457588
+o11_coverage_fraction,0.0,0.0,0.9781141424575879,0.9781141424575879
+o11_missing_reason,0.0,0.0,0.9781141424575879,0.9781141424575879
+o11_storage_cotraffic_score,0.0,0.0,0.9781141424575879,0.978114142457588
+o12_coverage_fraction,0.0,0.0,0.9781141424575879,0.978114142457588
+o12_declaration_consistency_score,0.0,0.0,0.9781141424575879,0.978114142457588
+o12_declared_parameter_count_b,0.0,0.0,0.9781141424575879,0.978114142457588
+o12_log_delivery_delay_hours,0.0,0.0,0.9781141424575879,0.978114142457588
+o12_loss_curve_present,0.0,0.0,0.9781141424575879,0.978114142457588
+o12_missing_reason,0.0,0.0,0.9781141424575879,0.978114142457588
+o12_optimizer_state_present,0.0,0.0,0.9781141424575879,0.978114142457588
+o12_signed_ml_logs_present,0.0,0.0,0.9781141424575879,0.978114142457588
+o12_step_count,0.0,0.0,0.9781141424575879,0.978114142457588
+o12_training_tokens_b,0.0,0.0,0.9781141424575879,0.978114142457588
+o13_attestation_valid_fraction,0.0,0.0,0.9781141424575879,0.9781141424575879
+o13_collector_measurement_valid,0.0,0.0,0.9781141424575879,0.9781141424575879
+o13_confidential_compute_mode_fraction,0.0,0.0,0.9781141424575879,0.978114142457588
+o13_coverage_fraction,0.0,0.0,0.9781141424575879,0.9781141424575879
+o13_missing_reason,0.0,0.0,0.9781141424575879,0.9781141424575879
+o14_counter_reset_count,0.0,0.0,0.9781141424575879,0.978114142457588
+o14_coverage_fraction,0.0,0.0,0.9781141424575879,0.978114142457588
+o14_gap_fraction_critical,0.0,0.0,0.9781141424575879,0.978114142457588
+o14_missing_reason,0.0,0.0,0.9781141424575879,0.978114142457588
+o15_coverage_fraction,0.0,0.0,0.9781141424575879,0.9781141424575879
+o15_firmware_bmc_change_near_window,0.0,0.0,0.9781141424575879,0.978114142457588
+o15_missing_reason,0.0,0.0,0.9781141424575879,0.9781141424575879
+o15_unapproved_physical_change_near_window,0.0,0.0,0.9781141424575879,0.9781141424575879
+o16_coverage_fraction,0.0,0.0,0.9781141424575879,0.978114142457588
+o16_missing_reason,0.0,0.0,0.9781141424575879,0.978114142457588
+o16_probe_latency_inflation_max,0.0,0.0,0.9781141424575879,0.978114142457588
+o16_probe_throughput_ratio_min,0.0,0.0,0.9781141424575879,0.978114142457588
+o16_vram_residency_conflict_score,0.0,0.0,0.9781141424575879,0.978114142457588
+o17_coverage_fraction,0.0,0.0,0.9781141424575879,0.9781141424575879
+o17_energy_contract_alignment_score,0.0,0.0,0.9781141424575879,0.9781141424575879
+o17_external_capacity_assertion,0.0,0.0,0.9781141424575879,0.9781141424575879
+o17_external_capacity_conflict_score,0.0,0.0,0.9781141424575879,0.9781141424575879
+o17_missing_reason,0.0,0.0,0.9781141424575879,0.9781141424575879
+o17_network_provider_utilization_score,0.0,0.0,0.9781141424575879,0.9781141424575879
+o17_procurement_or_maintenance_explanation_score,0.0,0.0,0.9781141424575879,0.9781141424575879
+o1_coverage_fraction,0.0,0.0,0.9781141424575879,0.9781141424575879
+o1_homogeneous_high_end_fraction,0.0,0.0,0.9781141424575879,0.978114142457588
+o1_inventory_delta_rate,0.0,0.0,0.9781141424575879,0.9781141424575879
+o1_largest_contiguous_domain_gpus,0.0,0.0,0.9781141424575879,0.978114142457588
+o1_missing_reason,0.0,0.0,0.9781141424575879,0.9781141424575879
+o1_non_partitioned_fraction,0.0,0.0,0.9781141424575879,0.9781141424575879
+o1_normalized_h100e_capacity,0.0,0.0,0.9781141424575879,0.978114142457588
+o2_coverage_fraction,0.0,0.0,0.9781141424575879,0.9781141424575879
+o2_elastic_resize_count,0.0,0.0,0.9781141424575879,0.978114142457588
+o2_gpu_hours_policy_ratio,0.0,0.0,0.9781141424575879,0.978114142457588
+o2_job_array_width,0.0,0.0,0.9781141424575879,0.978114142457588
+o2_missing_reason,0.0,0.0,0.9781141424575879,0.9781141424575879
+o2_preemption_restart_count,0.0,0.0,0.9781141424575879,0.9781141424575879
+o2_reservation_exclusive_flag,0.0,0.0,0.9781141424575879,0.978114142457588
+o2_reservation_reuse_count,0.0,0.0,0.9781141424575879,0.978114142457588
+o2_scheduler_queue_delay_hours,0.0,0.0,0.9781141424575879,0.9781141424575879
+o3_batch_provisioned_gpus,0.0,0.0,0.9781141424575879,0.978114142457588
+o3_billing_continuity_score,0.0,0.0,0.9781141424575879,0.978114142457588
+o3_capacity_reservation_duration_hours,0.0,0.0,0.9781141424575879,0.978114142457588
+o3_coverage_fraction,0.0,0.0,0.9781141424575879,0.978114142457588
+o3_egress_tb,0.0,0.0,0.9781141424575879,0.978114142457588
+o3_missing_reason,0.0,0.0,0.9781141424575879,0.978114142457588
+o3_training_sku_fraction,0.0,0.0,0.9781141424575879,0.978114142457588
+o4_gpu_util_duty_gt_70,0.0,0.0,0.9781141424575879,0.978114142457588
+o4_gpu_util_p50,0.0,0.0,0.9781141424575879,0.978114142457588
+o4_gpu_util_p95,0.0,0.0,0.9781141424575879,0.978114142457588
+o4_hbm_pressure_duration_fraction,0.0,0.0,0.9781141424575879,0.978114142457588
+o4_missing_reason,0.0,0.0,0.9781141424575879,0.978114142457588
+o4_power_cap_active_fraction,0.0,0.0,0.9781141424575879,0.978114142457588
+o4_sm_tensor_active_p95,0.0,0.0,0.9781141424575879,0.978114142457588
+o4_thermal_throttle_fraction,0.0,0.0,0.9781141424575879,0.9781141424575879
+o5_missing_reason,0.0,0.0,0.9781141424575879,0.978114142457588
+o5_profiler_available,0.0,0.0,0.9781141424575879,0.978114142457588
+o5_tensor_throughput_ratio,0.0,0.0,0.9781141424575879,0.978114142457588
+o6_coverage_fraction,0.0,0.0,0.9781141424575879,0.9781141424575879
+o6_link_error_spike_score,0.0,0.0,0.9781141424575879,0.9781141424575879
+o6_missing_reason,0.0,0.0,0.9781141424575879,0.9781141424575879
+o6_nvlink_util_p95,0.0,0.0,0.9781141424575879,0.978114142457588
+o7_job_to_port_mapping_coverage,0.0,0.0,0.9781141424575879,0.978114142457588
+o7_missing_reason,0.0,0.0,0.9781141424575879,0.978114142457588
+o7_rdma_congestion_score,0.0,0.0,0.9781141424575879,0.978114142457588
+o7_scaleout_port_util_p95,0.0,0.0,0.9781141424575879,0.978114142457588
+o8_coverage_fraction,0.0,0.0,0.9781141424575879,0.978114142457588
+o8_facility_it_power_mw,0.0,0.0,0.9781141424575879,0.978114142457588
+o8_missing_reason,0.0,0.0,0.9781141424575879,0.978114142457588
+o8_power_baseline_drift_score,0.0,0.0,0.9781141424575879,0.978114142457588
+o8_power_cap_or_curtailment_active,0.0,0.0,0.9781141424575879,0.978114142457588
+o8_power_continuity_days,0.0,0.0,0.9781141424575879,0.978114142457588
+o8_unattributed_power_fraction,0.0,0.0,0.9781141424575879,0.978114142457588
+o9_cooling_flow_duty,0.0,0.0,0.9781141424575879,0.978114142457588
+o9_cooling_maintenance_active,0.0,0.0,0.9781141424575879,0.978114142457588
+o9_coverage_fraction,0.0,0.0,0.9781141424575879,0.9781141424575879
+o9_missing_reason,0.0,0.0,0.9781141424575879,0.9781141424575879
+o9_thermal_throttle_support_score,0.0,0.0,0.9781141424575879,0.978114142457588
+power_meter_trust_level,0.0,0.0,0.9781141424575879,0.9781141424575879
+scheduler_signature_status,0.0,0.0,0.9781141424575879,0.9781141424575879
+scope_type,0.0,0.0,0.9781141424575879,0.978114142457588
+window_length_seconds,0.0,0.0,0.9781141424575879,0.978114142457588
+o4_error_spike_score,-0.000112313549517018,0.00015883534496522904,0.9781141424575879,0.9782264560071049
+o5_kernel_training_motif_score,-0.000112313549517018,0.00015883534496522904,0.9781141424575879,0.9782264560071049
+o2_account_linkage_confidence,-0.00022690538180129374,0.00016047059544896947,0.9781141424575879,0.9783410478393891
+o11_checkpoint_jitter_score,-0.00024461273048711113,0.00041613647356600846,0.9781141424575879,0.9783587551880749
+o11_checkpoint_write_tb_per_event,-0.0003526063865893307,0.00026759207342307376,0.9781141424575879,0.9784667488441773
+o6_nvlink_periodicity_score,-0.0006442779638646812,0.0003427827988719719,0.9781141424575879,0.9787584204214527
+o14_clock_drift_max_ms,-0.001169240694518477,0.0009162828683296031,0.9781141424575879,0.9792833831521065
+o8_baseline_subtracted_energy_kwh,-0.0031618376880434562,0.0004309423062690692,0.9781141424575879,0.9812759801456313

data/model_runs/synthetic_v1_baseline/manifest.json

@@ -0,0 +1,127 @@
+{
+  "artifact_hashes": {
+    "README.md": "sha256:e1cca94649d1dd9f08cf2b1a772737fbd7362e3d24f0e5f49567f79fdced1130",
+    "calibration_metrics.json": "sha256:a1b673af40faa3ed6b614b7e3cf3753964ede2e3810bcd5354d6657153577897",
+    "classification_report.json": "sha256:c74bb9ef28966c94937dd3ddb30296937b8bca224dd8f6fcf781d680621eab0c",
+    "confusion_matrix.csv": "sha256:265fa42852a745220b1ea17ee8f502c1b5a24f74063a60f7de9acdd154a830af",
+    "evidence_audit_sample.csv": "sha256:a9efe77b1ccc902b0a8f28b2b8b0cb72c9524e39f0d21676b27bb20ec3ce683d",
+    "excluded_columns.json": "sha256:0811708d16c4a2b8444eb8324b378ea61cc7ad30931adc14312f2a62587c182b",
+    "feature_columns.json": "sha256:f8c411ac97a84df3e13f37163bfa210c2a4019208678b77d5131cb5579f3f976",
+    "feature_importance.csv": "sha256:96cb96808c2b5ad9e7a23ba08fc485ddb5e026eddbc54ffa8496e7bb8635c095",
+    "metrics.json": "sha256:d7d459e44ed94014a33fb39fbffca89deb69d40ec38658ffa703f009a511a317",
+    "model.joblib": "sha256:06c701b0d9c74cdfd0066fe3212013ad2393c3eca34ca625c78792631fa3f338",
+    "predictions_all.csv": "sha256:e2e66b8c5f743d13bb4a24b57656cf7f246cb423071efc1049a9f11019e8f549",
+    "predictions_test.csv": "sha256:e4fb8e249d438455967b6ff491d51e3db243002c326fe2f66c023f134c5160e1",
+    "preprocessing.joblib": "sha256:edf78f71a10335c7703a183a76b97add8ee6ecc8b0ca1068fdcdedab18bcf2de",
+    "split_manifest.json": "sha256:4ab606273d10f182b02a5154982a7943c0837b6e5725dee907c97c25e2daff92",
+    "validation_summary.md": "sha256:d3da3e62f5e957260990001209805854976b3b2dfe56b921104077a6395918c3"
+  },
+  "calibration_method": "sigmoid_on_validation_split_frozen_estimator",
+  "created_or_updated_at": "2026-05-11T00:23:31Z",
+  "features_path": "data/synthetic_v1/features/window_features_all.csv",
+  "limitations": [
+    "Trained on synthetic data only.",
+    "Performance numbers are not real-world deployment claims.",
+    "Episode-level group splitting is required because adjacent windows are correlated.",
+    "Synthetic labels encode generator assumptions.",
+    "Real deployment requires real telemetry calibration and controlled drills."
+  ],
+  "metrics_summary": {
+    "accuracy": 0.9724612736660929,
+    "brier_large_training": 0.001846207706971921,
+    "ece_large_training": 0.010488907264903001,
+    "label_3_4_precision": 0.993517017828201,
+    "label_3_4_recall": 0.9967479674796748,
+    "macro_f1": 0.9781141424575879,
+    "p_large_training_precision_at_0_5": 0.993517017828201,
+    "p_large_training_recall_at_0_5": 0.9967479674796748,
+    "weighted_f1": 0.9722899614011162
+  },
+  "model_run_id": "synthetic_v1_baseline",
+  "model_type": "CalibratedClassifierCV over HistGradientBoostingClassifier",
+  "required_outputs": [
+    "README.md",
+    "manifest.json",
+    "model.joblib",
+    "preprocessing.joblib",
+    "feature_columns.json",
+    "excluded_columns.json",
+    "split_manifest.json",
+    "metrics.json",
+    "calibration_metrics.json",
+    "confusion_matrix.csv",
+    "classification_report.json",
+    "predictions_test.csv",
+    "predictions_all.csv",
+    "feature_importance.csv",
+    "evidence_audit_sample.csv",
+    "validation_summary.md"
+  ],
+  "training_metadata": {
+    "calibration_method": "sigmoid_on_validation_split_frozen_estimator",
+    "dataset_validation_status": [
+      {
+        "command": "/Users/ifc24/miniforge3/bin/python /Users/ifc24/Develop/datacenter-verification/src/datacenter_verification_synthetic/validate_synthetic_dataset.py --dataset data/synthetic_v1",
+        "name": "synthetic_dataset_validator",
+        "returncode": 0,
+        "stderr": "",
+        "stdout": "PASS\nraw_records: 152457\nfeature_rows: 14712\nlabel_distribution: {0: 3861, 1: 3333, 2: 4203, 3: 2489, 4: 826}"
+      },
+      {
+        "command": "/Users/ifc24/miniforge3/bin/python -m src.datacenter_verification_validators --dataset data/synthetic_v1",
+        "name": "public_dataset_validator",
+        "returncode": 0,
+        "stderr": "",
+        "stdout": "Validated dataset: data/synthetic_v1\nErrors: 0\nWarnings: 0\nReport: data/synthetic_v1/validation/validation_report.md"
+      }
+    ],
+    "feature_count": 152,
+    "features_path": "data/synthetic_v1/features/window_features_all.csv",
+    "model_parameters": {
+      "categorical_features": "from_dtype",
+      "class_weight": "balanced",
+      "early_stopping": true,
+      "interaction_cst": null,
+      "l2_regularization": 0.03,
+      "learning_rate": 0.05,
+      "loss": "log_loss",
+      "max_bins": 255,
+      "max_depth": null,
+      "max_features": 1.0,
+      "max_iter": 350,
+      "max_leaf_nodes": 31,
+      "min_samples_leaf": 20,
+      "monotonic_cst": null,
+      "n_iter_no_change": 10,
+      "random_state": 20260510,
+      "scoring": "loss",
+      "tol": 1e-07,
+      "validation_fraction": 0.15,
+      "verbose": 0,
+      "warm_start": false
+    },
+    "model_type": "HistGradientBoostingClassifier",
+    "seed": 20260510,
+    "train_episodes": 458,
+    "train_rows": 8870,
+    "trained_at": "2026-05-11T00:21:12Z",
+    "validation_episodes": 152,
+    "validation_rows": 2937
+  },
+  "validation_status": [
+    {
+      "command": "/Users/ifc24/miniforge3/bin/python /Users/ifc24/Develop/datacenter-verification/src/datacenter_verification_synthetic/validate_synthetic_dataset.py --dataset data/synthetic_v1",
+      "name": "synthetic_dataset_validator",
+      "returncode": 0,
+      "stderr": "",
+      "stdout": "PASS\nraw_records: 152457\nfeature_rows: 14712\nlabel_distribution: {0: 3861, 1: 3333, 2: 4203, 3: 2489, 4: 826}"
+    },
+    {
+      "command": "/Users/ifc24/miniforge3/bin/python -m src.datacenter_verification_validators --dataset data/synthetic_v1",
+      "name": "public_dataset_validator",
+      "returncode": 0,
+      "stderr": "",
+      "stdout": "Validated dataset: data/synthetic_v1\nErrors: 0\nWarnings: 0\nReport: data/synthetic_v1/validation/validation_report.md"
+    }
+  ]
+}

data/model_runs/synthetic_v1_baseline/metrics.json

@@ -0,0 +1,931 @@
+{
+  "calibration": {
+    "brier_large_training": 0.001846207706971921,
+    "expected_calibration_error_large_training": 0.010488907264903001,
+    "log_loss": 0.10567154528993643
+  },
+  "dataset": {
+    "test_episodes": 150,
+    "test_label_distribution": {
+      "0": 762,
+      "1": 690,
+      "2": 838,
+      "3": 463,
+      "4": 152
+    },
+    "test_rows": 2905
+  },
+  "governance": {
+    "false_negative_scenarios_at_0_5": {
+      "fragmented_training_linked": 2
+    },
+    "false_positive_scenarios_at_0_5": {
+      "underclocked_energy_capped_training": 4
+    },
+    "label_0_missed_under_high_coverage": {
+      "missed_rate": 0.0,
+      "missed_rows": 0,
+      "rows": 762
+    },
+    "label_3_4_predicted_label": {
+      "f1": 0.9951298701298701,
+      "precision": 0.993517017828201,
+      "recall": 0.9967479674796748
+    },
+    "p_large_training_threshold_0_5": {
+      "f1": 0.9951298701298701,
+      "precision": 0.993517017828201,
+      "recall": 0.9967479674796748
+    }
+  },
+  "model": {
+    "accuracy": 0.9724612736660929,
+    "brier_multiclass_mean": 0.010624855064864712,
+    "log_loss": 0.10567154528993643,
+    "macro_f1": 0.9781141424575879,
+    "weighted_f1": 0.9722899614011162
+  },
+  "rule_baseline": {
+    "accuracy": 0.42857142857142855,
+    "label_3_4_predicted_label": {
+      "f1": 0.8418891170431212,
+      "precision": 0.7269503546099291,
+      "recall": 1.0
+    },
+    "macro_f1": 0.36363847151518797,
+    "weighted_f1": 0.3650992917466867
+  },
+  "subgroup_metrics": {
+    "capacity_possible": [
+      {
+        "accuracy": 0.9724612736660929,
+        "group": "True",
+        "large_training_false_negatives": 2,
+        "large_training_false_positives": 4,
+        "large_training_precision_at_0_5": 0.993517017828201,
+        "large_training_recall_at_0_5": 0.9967479674796748,
+        "macro_f1": 0.9781141424575879,
+        "rows": 2905
+      }
+    ],
+    "data_quality_regime": [
+      {
+        "accuracy": 0.9742895805142084,
+        "group": "full_observability",
+        "large_training_false_negatives": 2,
+        "large_training_false_positives": 0,
+        "large_training_precision_at_0_5": 1.0,
+        "large_training_recall_at_0_5": 0.995249406175772,
+        "macro_f1": 0.97854699410172,
+        "rows": 2217
+      },
+      {
+        "accuracy": 1.0,
+        "group": "routine_privacy_redaction",
+        "large_training_false_negatives": 0,
+        "large_training_false_positives": 0,
+        "large_training_precision_at_0_5": 1.0,
+        "large_training_recall_at_0_5": 1.0,
+        "macro_f1": 0.4,
+        "rows": 126
+      },
+      {
+        "accuracy": 1.0,
+        "group": "maintenance_observability_loss",
+        "large_training_false_negatives": 0,
+        "large_training_false_positives": 0,
+        "large_training_precision_at_0_5": 0.0,
+        "large_training_recall_at_0_5": 0.0,
+        "macro_f1": 0.4,
+        "rows": 112
+      },
+      {
+        "accuracy": 1.0,
+        "group": "delayed_log_delivery",
+        "large_training_false_negatives": 0,
+        "large_training_false_positives": 0,
+        "large_training_precision_at_0_5": 1.0,
+        "large_training_recall_at_0_5": 1.0,
+        "macro_f1": 0.6,
+        "rows": 111
+      },
+      {
+        "accuracy": 0.8020833333333334,
+        "group": "collector_gap_long",
+        "large_training_false_negatives": 0,
+        "large_training_false_positives": 0,
+        "large_training_precision_at_0_5": 0.0,
+        "large_training_recall_at_0_5": 0.0,
+        "macro_f1": 0.31288358208955225,
+        "rows": 96
+      },
+      {
+        "accuracy": 0.9393939393939394,
+        "group": "confidential_compute_counters_disabled",
+        "large_training_false_negatives": 0,
+        "large_training_false_positives": 4,
+        "large_training_precision_at_0_5": 0.0,
+        "large_training_recall_at_0_5": 0.0,
+        "macro_f1": 0.19375,
+        "rows": 66
+      },
+      {
+        "accuracy": 1.0,
+        "group": "source_not_deployed_for_site",
+        "large_training_false_negatives": 0,
+        "large_training_false_positives": 0,
+        "large_training_precision_at_0_5": 1.0,
+        "large_training_recall_at_0_5": 1.0,
+        "macro_f1": 0.6,
+        "rows": 62
+      },
+      {
+        "accuracy": 1.0,
+        "group": "partial_fabric_mapping",
+        "large_training_false_negatives": 0,
+        "large_training_false_positives": 0,
+        "large_training_precision_at_0_5": 0.0,
+        "large_training_recall_at_0_5": 0.0,
+        "macro_f1": 0.4,
+        "rows": 55
+      },
+      {
+        "accuracy": 1.0,
+        "group": "collector_gap_short",
+        "large_training_false_negatives": 0,
+        "large_training_false_positives": 0,
+        "large_training_precision_at_0_5": 0.0,
+        "large_training_recall_at_0_5": 0.0,
+        "macro_f1": 0.4,
+        "rows": 44
+      },
+      {
+        "accuracy": 1.0,
+        "group": "clock_drift_episode",
+        "large_training_false_negatives": 0,
+        "large_training_false_positives": 0,
+        "large_training_precision_at_0_5": 0.0,
+        "large_training_recall_at_0_5": 0.0,
+        "macro_f1": 0.2,
+        "rows": 16
+      }
+    ],
+    "integrity_warning": [
+      {
+        "accuracy": 0.9765335529024289,
+        "group": "False",
+        "large_training_false_negatives": 2,
+        "large_training_false_positives": 0,
+        "large_training_precision_at_0_5": 1.0,
+        "large_training_recall_at_0_5": 0.9967479674796748,
+        "macro_f1": 0.978471713929574,
+        "rows": 2429
+      },
+      {
+        "accuracy": 0.9516806722689075,
+        "group": "True",
+        "large_training_false_negatives": 0,
+        "large_training_false_positives": 4,
+        "large_training_precision_at_0_5": 0.0,
+        "large_training_recall_at_0_5": 0.0,
+        "macro_f1": 0.5816990094283327,
+        "rows": 476
+      }
+    ],
+    "latent_workload_class": [
+      {
+        "accuracy": 1.0,
+        "group": "idle_or_low_activity",
+        "large_training_false_negatives": 0,
+        "large_training_false_positives": 0,
+        "large_training_precision_at_0_5": 0.0,
+        "large_training_recall_at_0_5": 0.0,
+        "macro_f1": 0.4,
+        "rows": 807
+      },
+      {
+        "accuracy": 1.0,
+        "group": "hpc_mpi_collective",
+        "large_training_false_negatives": 0,
+        "large_training_false_positives": 0,
+        "large_training_precision_at_0_5": 0.0,
+        "large_training_recall_at_0_5": 0.0,
+        "macro_f1": 0.2,
+        "rows": 171
+      },
+      {
+        "accuracy": 0.9872611464968153,
+        "group": "fragmented_training_linked",
+        "large_training_false_negatives": 2,
+        "large_training_false_positives": 0,
+        "large_training_precision_at_0_5": 1.0,
+        "large_training_recall_at_0_5": 0.9672131147540983,
+        "macro_f1": 0.3946048109965636,
+        "rows": 157
+      },
+      {
+        "accuracy": 1.0,
+        "group": "training_without_semantic_logs",
+        "large_training_false_negatives": 0,
+        "large_training_false_positives": 0,
+        "large_training_precision_at_0_5": 1.0,
+        "large_training_recall_at_0_5": 1.0,
+        "macro_f1": 0.4,
+        "rows": 144
+      },
+      {
+        "accuracy": 1.0,
+        "group": "large_etl_or_data_movement",
+        "large_training_false_negatives": 0,
+        "large_training_false_positives": 0,
+        "large_training_precision_at_0_5": 0.0,
+        "large_training_recall_at_0_5": 0.0,
+        "macro_f1": 0.2,
+        "rows": 136
+      },
+      {
+        "accuracy": 1.0,
+        "group": "reserved_but_unused_capacity",
+        "large_training_false_negatives": 0,
+        "large_training_false_positives": 0,
+        "large_training_precision_at_0_5": 0.0,
+        "large_training_recall_at_0_5": 0.0,
+        "macro_f1": 0.2,
+        "rows": 122
+      },
+      {
+        "accuracy": 1.0,
+        "group": "model_parallel_inference",
+        "large_training_false_negatives": 0,
+        "large_training_false_positives": 0,
+        "large_training_precision_at_0_5": 0.0,
+        "large_training_recall_at_0_5": 0.0,
+        "macro_f1": 0.4,
+        "rows": 104
+      },
+      {
+        "accuracy": 1.0,
+        "group": "training_with_low_fabric_high_checkpoint",
+        "large_training_false_negatives": 0,
+        "large_training_false_positives": 0,
+        "large_training_precision_at_0_5": 1.0,
+        "large_training_recall_at_0_5": 1.0,
+        "macro_f1": 0.4,
+        "rows": 104
+      },
+      {
+        "accuracy": 1.0,
+        "group": "hardware_burn_in_or_thermal_soak",
+        "large_training_false_negatives": 0,
+        "large_training_false_positives": 0,
+        "large_training_precision_at_0_5": 0.0,
+        "large_training_recall_at_0_5": 0.0,
+        "macro_f1": 0.2,
+        "rows": 103
+      },
+      {
+        "accuracy": 0.6534653465346535,
+        "group": "normal_inference",
+        "large_training_false_negatives": 0,
+        "large_training_false_positives": 0,
+        "large_training_precision_at_0_5": 0.0,
+        "large_training_recall_at_0_5": 0.0,
+        "macro_f1": 0.277555239695051,
+        "rows": 101
+      },
+      {
+        "accuracy": 0.7956989247311828,
+        "group": "large_batch_inference",
+        "large_training_false_negatives": 0,
+        "large_training_false_positives": 0,
+        "large_training_precision_at_0_5": 0.0,
+        "large_training_recall_at_0_5": 0.0,
+        "macro_f1": 0.3176703552708212,
+        "rows": 93
+      },
+      {
+        "accuracy": 1.0,
+        "group": "training_with_delayed_logs",
+        "large_training_false_negatives": 0,
+        "large_training_false_positives": 0,
+        "large_training_precision_at_0_5": 1.0,
+        "large_training_recall_at_0_5": 1.0,
+        "macro_f1": 0.6,
+        "rows": 91
+      },
+      {
+        "accuracy": 0.9550561797752809,
+        "group": "underclocked_energy_capped_training",
+        "large_training_false_negatives": 0,
+        "large_training_false_positives": 4,
+        "large_training_precision_at_0_5": 0.9384615384615385,
+        "large_training_recall_at_0_5": 1.0,
+        "macro_f1": 0.37826617826617825,
+        "rows": 89
+      },
+      {
+        "accuracy": 1.0,
+        "group": "capacity_or_integrity_only_warning",
+        "large_training_false_negatives": 0,
+        "large_training_false_positives": 0,
+        "large_training_precision_at_0_5": 0.0,
+        "large_training_recall_at_0_5": 0.0,
+        "macro_f1": 0.2,
+        "rows": 87
+      },
+      {
+        "accuracy": 1.0,
+        "group": "cloud_training_redacted_runtime",
+        "large_training_false_negatives": 0,
+        "large_training_false_positives": 0,
+        "large_training_precision_at_0_5": 1.0,
+        "large_training_recall_at_0_5": 1.0,
+        "macro_f1": 0.4,
+        "rows": 86
+      },
+      {
+        "accuracy": 1.0,
+        "group": "pretraining_standard",
+        "large_training_false_negatives": 0,
+        "large_training_false_positives": 0,
+        "large_training_precision_at_0_5": 1.0,
+        "large_training_recall_at_0_5": 1.0,
+        "macro_f1": 0.2,
+        "rows": 86
+      },
+      {
+        "accuracy": 1.0,
+        "group": "multi_tenant_fragmented_nontraining",
+        "large_training_false_negatives": 0,
+        "large_training_false_positives": 0,
+        "large_training_precision_at_0_5": 0.0,
+        "large_training_recall_at_0_5": 0.0,
+        "macro_f1": 0.2,
+        "rows": 84
+      },
+      {
+        "accuracy": 1.0,
+        "group": "storage_rebuild_or_replication",
+        "large_training_false_negatives": 0,
+        "large_training_false_positives": 0,
+        "large_training_precision_at_0_5": 0.0,
+        "large_training_recall_at_0_5": 0.0,
+        "macro_f1": 0.4,
+        "rows": 82
+      },
+      {
+        "accuracy": 1.0,
+        "group": "nccl_extended_benchmark",
+        "large_training_false_negatives": 0,
+        "large_training_false_positives": 0,
+        "large_training_precision_at_0_5": 0.0,
+        "large_training_recall_at_0_5": 0.0,
+        "macro_f1": 0.2,
+        "rows": 69
+      },
+      {
+        "accuracy": 0.6296296296296297,
+        "group": "embedding_generation",
+        "large_training_false_negatives": 0,
+        "large_training_false_positives": 0,
+        "large_training_precision_at_0_5": 0.0,
+        "large_training_recall_at_0_5": 0.0,
+        "macro_f1": 0.15454545454545454,
+        "rows": 54
+      },
+      {
+        "accuracy": 1.0,
+        "group": "large_fine_tune_standard",
+        "large_training_false_negatives": 0,
+        "large_training_false_positives": 0,
+        "large_training_precision_at_0_5": 1.0,
+        "large_training_recall_at_0_5": 1.0,
+        "macro_f1": 0.4,
+        "rows": 42
+      },
+      {
+        "accuracy": 1.0,
+        "group": "multi_stage_training_pipeline",
+        "large_training_false_negatives": 0,
+        "large_training_false_positives": 0,
+        "large_training_precision_at_0_5": 1.0,
+        "large_training_recall_at_0_5": 1.0,
+        "macro_f1": 0.4,
+        "rows": 24
+      },
+      {
+        "accuracy": 1.0,
+        "group": "counter_suppressed_candidate_window",
+        "large_training_false_negatives": 0,
+        "large_training_false_positives": 0,
+        "large_training_precision_at_0_5": 0.0,
+        "large_training_recall_at_0_5": 0.0,
+        "macro_f1": 0.2,
+        "rows": 22
+      },
+      {
+        "accuracy": 1.0,
+        "group": "distributed_database_or_graph_analytics",
+        "large_training_false_negatives": 0,
+        "large_training_false_positives": 0,
+        "large_training_precision_at_0_5": 0.0,
+        "large_training_recall_at_0_5": 0.0,
+        "macro_f1": 0.2,
+        "rows": 18
+      },
+      {
+        "accuracy": 1.0,
+        "group": "maintenance_with_collector_gaps",
+        "large_training_false_negatives": 0,
+        "large_training_false_positives": 0,
+        "large_training_precision_at_0_5": 0.0,
+        "large_training_recall_at_0_5": 0.0,
+        "macro_f1": 0.2,
+        "rows": 15
+      },
+      {
+        "accuracy": 1.0,
+        "group": "synthetic_data_generation_gpu_heavy",
+        "large_training_false_negatives": 0,
+        "large_training_false_positives": 0,
+        "large_training_precision_at_0_5": 0.0,
+        "large_training_recall_at_0_5": 0.0,
+        "macro_f1": 0.2,
+        "rows": 14
+      }
+    ],
+    "o12_missing_reason": [
+      {
+        "accuracy": 0.968040370058873,
+        "group": "privacy_redacted",
+        "large_training_false_negatives": 2,
+        "large_training_false_positives": 0,
+        "large_training_precision_at_0_5": 1.0,
+        "large_training_recall_at_0_5": 0.991701244813278,
+        "macro_f1": 0.777350004737448,
+        "rows": 2378
+      },
+      {
+        "accuracy": 0.9913978494623656,
+        "group": "delayed_log_delivery",
+        "large_training_false_negatives": 0,
+        "large_training_false_positives": 4,
+        "large_training_precision_at_0_5": 0.9889196675900277,
+        "large_training_recall_at_0_5": 1.0,
+        "macro_f1": 0.5942940479445811,
+        "rows": 465
+      },
+      {
+        "accuracy": 1.0,
+        "group": "source_not_deployed",
+        "large_training_false_negatives": 0,
+        "large_training_false_positives": 0,
+        "large_training_precision_at_0_5": 1.0,
+        "large_training_recall_at_0_5": 1.0,
+        "macro_f1": 0.6,
+        "rows": 62
+      }
+    ],
+    "o4_missing_reason": [
+      {
+        "accuracy": 0.9777950915465524,
+        "group": "observed",
+        "large_training_false_negatives": 2,
+        "large_training_false_positives": 0,
+        "large_training_precision_at_0_5": 1.0,
+        "large_training_recall_at_0_5": 0.9967479674796748,
+        "macro_f1": 0.9808162711257303,
+        "rows": 2567
+      },
+      {
+        "accuracy": 0.9301470588235294,
+        "group": "collector_gap",
+        "large_training_false_negatives": 0,
+        "large_training_false_positives": 0,
+        "large_training_precision_at_0_5": 0.0,
+        "large_training_recall_at_0_5": 0.0,
+        "macro_f1": 0.3720493787098257,
+        "rows": 272
+      },
+      {
+        "accuracy": 0.9393939393939394,
+        "group": "counter_disabled_by_cc_mode",
+        "large_training_false_negatives": 0,
+        "large_training_false_positives": 4,
+        "large_training_precision_at_0_5": 0.0,
+        "large_training_recall_at_0_5": 0.0,
+        "macro_f1": 0.19375,
+        "rows": 66
+      }
+    ],
+    "o7_missing_reason": [
+      {
+        "accuracy": 0.9763382467028704,
+        "group": "observed",
+        "large_training_false_negatives": 2,
+        "large_training_false_positives": 4,
+        "large_training_precision_at_0_5": 0.993517017828201,
+        "large_training_recall_at_0_5": 0.9967479674796748,
+        "macro_f1": 0.9793065248203424,
+        "rows": 2578
+      },
+      {
+        "accuracy": 0.9301470588235294,
+        "group": "collector_gap",
+        "large_training_false_negatives": 0,
+        "large_training_false_positives": 0,
+        "large_training_precision_at_0_5": 0.0,
+        "large_training_recall_at_0_5": 0.0,
+        "macro_f1": 0.3720493787098257,
+        "rows": 272
+      },
+      {
+        "accuracy": 1.0,
+        "group": "privacy_redacted",
+        "large_training_false_negatives": 0,
+        "large_training_false_positives": 0,
+        "large_training_precision_at_0_5": 0.0,
+        "large_training_recall_at_0_5": 0.0,
+        "macro_f1": 0.4,
+        "rows": 55
+      }
+    ],
+    "scenario_family": [
+      {
+        "accuracy": 1.0,
+        "group": "idle_or_low_activity",
+        "large_training_false_negatives": 0,
+        "large_training_false_positives": 0,
+        "large_training_precision_at_0_5": 0.0,
+        "large_training_recall_at_0_5": 0.0,
+        "macro_f1": 0.4,
+        "rows": 807
+      },
+      {
+        "accuracy": 1.0,
+        "group": "hpc_mpi_collective",
+        "large_training_false_negatives": 0,
+        "large_training_false_positives": 0,
+        "large_training_precision_at_0_5": 0.0,
+        "large_training_recall_at_0_5": 0.0,
+        "macro_f1": 0.2,
+        "rows": 171
+      },
+      {
+        "accuracy": 0.9872611464968153,
+        "group": "fragmented_training_linked",
+        "large_training_false_negatives": 2,
+        "large_training_false_positives": 0,
+        "large_training_precision_at_0_5": 1.0,
+        "large_training_recall_at_0_5": 0.9672131147540983,
+        "macro_f1": 0.3946048109965636,
+        "rows": 157
+      },
+      {
+        "accuracy": 1.0,
+        "group": "training_without_semantic_logs",
+        "large_training_false_negatives": 0,
+        "large_training_false_positives": 0,
+        "large_training_precision_at_0_5": 1.0,
+        "large_training_recall_at_0_5": 1.0,
+        "macro_f1": 0.4,
+        "rows": 144
+      },
+      {
+        "accuracy": 1.0,
+        "group": "large_etl_or_data_movement",
+        "large_training_false_negatives": 0,
+        "large_training_false_positives": 0,
+        "large_training_precision_at_0_5": 0.0,
+        "large_training_recall_at_0_5": 0.0,
+        "macro_f1": 0.2,
+        "rows": 136
+      },
+      {
+        "accuracy": 1.0,
+        "group": "reserved_but_unused_capacity",
+        "large_training_false_negatives": 0,
+        "large_training_false_positives": 0,
+        "large_training_precision_at_0_5": 0.0,
+        "large_training_recall_at_0_5": 0.0,
+        "macro_f1": 0.2,
+        "rows": 122
+      },
+      {
+        "accuracy": 1.0,
+        "group": "model_parallel_inference",
+        "large_training_false_negatives": 0,
+        "large_training_false_positives": 0,
+        "large_training_precision_at_0_5": 0.0,
+        "large_training_recall_at_0_5": 0.0,
+        "macro_f1": 0.4,
+        "rows": 104
+      },
+      {
+        "accuracy": 1.0,
+        "group": "training_with_low_fabric_high_checkpoint",
+        "large_training_false_negatives": 0,
+        "large_training_false_positives": 0,
+        "large_training_precision_at_0_5": 1.0,
+        "large_training_recall_at_0_5": 1.0,
+        "macro_f1": 0.4,
+        "rows": 104
+      },
+      {
+        "accuracy": 1.0,
+        "group": "hardware_burn_in_or_thermal_soak",
+        "large_training_false_negatives": 0,
+        "large_training_false_positives": 0,
+        "large_training_precision_at_0_5": 0.0,
+        "large_training_recall_at_0_5": 0.0,
+        "macro_f1": 0.2,
+        "rows": 103
+      },
+      {
+        "accuracy": 0.6534653465346535,
+        "group": "normal_inference",
+        "large_training_false_negatives": 0,
+        "large_training_false_positives": 0,
+        "large_training_precision_at_0_5": 0.0,
+        "large_training_recall_at_0_5": 0.0,
+        "macro_f1": 0.277555239695051,
+        "rows": 101
+      },
+      {
+        "accuracy": 0.7956989247311828,
+        "group": "large_batch_inference",
+        "large_training_false_negatives": 0,
+        "large_training_false_positives": 0,
+        "large_training_precision_at_0_5": 0.0,
+        "large_training_recall_at_0_5": 0.0,
+        "macro_f1": 0.3176703552708212,
+        "rows": 93
+      },
+      {
+        "accuracy": 1.0,
+        "group": "training_with_delayed_logs",
+        "large_training_false_negatives": 0,
+        "large_training_false_positives": 0,
+        "large_training_precision_at_0_5": 1.0,
+        "large_training_recall_at_0_5": 1.0,
+        "macro_f1": 0.6,
+        "rows": 91
+      },
+      {
+        "accuracy": 0.9550561797752809,
+        "group": "underclocked_energy_capped_training",
+        "large_training_false_negatives": 0,
+        "large_training_false_positives": 4,
+        "large_training_precision_at_0_5": 0.9384615384615385,
+        "large_training_recall_at_0_5": 1.0,
+        "macro_f1": 0.37826617826617825,
+        "rows": 89
+      },
+      {
+        "accuracy": 1.0,
+        "group": "capacity_or_integrity_only_warning",
+        "large_training_false_negatives": 0,
+        "large_training_false_positives": 0,
+        "large_training_precision_at_0_5": 0.0,
+        "large_training_recall_at_0_5": 0.0,
+        "macro_f1": 0.2,
+        "rows": 87
+      },
+      {
+        "accuracy": 1.0,
+        "group": "cloud_training_redacted_runtime",
+        "large_training_false_negatives": 0,
+        "large_training_false_positives": 0,
+        "large_training_precision_at_0_5": 1.0,
+        "large_training_recall_at_0_5": 1.0,
+        "macro_f1": 0.4,
+        "rows": 86
+      },
+      {
+        "accuracy": 1.0,
+        "group": "pretraining_standard",
+        "large_training_false_negatives": 0,
+        "large_training_false_positives": 0,
+        "large_training_precision_at_0_5": 1.0,
+        "large_training_recall_at_0_5": 1.0,
+        "macro_f1": 0.2,
+        "rows": 86
+      },
+      {
+        "accuracy": 1.0,
+        "group": "multi_tenant_fragmented_nontraining",
+        "large_training_false_negatives": 0,
+        "large_training_false_positives": 0,
+        "large_training_precision_at_0_5": 0.0,
+        "large_training_recall_at_0_5": 0.0,
+        "macro_f1": 0.2,
+        "rows": 84
+      },
+      {
+        "accuracy": 1.0,
+        "group": "storage_rebuild_or_replication",
+        "large_training_false_negatives": 0,
+        "large_training_false_positives": 0,
+        "large_training_precision_at_0_5": 0.0,
+        "large_training_recall_at_0_5": 0.0,
+        "macro_f1": 0.4,
+        "rows": 82
+      },
+      {
+        "accuracy": 1.0,
+        "group": "nccl_extended_benchmark",
+        "large_training_false_negatives": 0,
+        "large_training_false_positives": 0,
+        "large_training_precision_at_0_5": 0.0,
+        "large_training_recall_at_0_5": 0.0,
+        "macro_f1": 0.2,
+        "rows": 69
+      },
+      {
+        "accuracy": 0.6296296296296297,
+        "group": "embedding_generation",
+        "large_training_false_negatives": 0,
+        "large_training_false_positives": 0,
+        "large_training_precision_at_0_5": 0.0,
+        "large_training_recall_at_0_5": 0.0,
+        "macro_f1": 0.15454545454545454,
+        "rows": 54
+      },
+      {
+        "accuracy": 1.0,
+        "group": "large_fine_tune_standard",
+        "large_training_false_negatives": 0,
+        "large_training_false_positives": 0,
+        "large_training_precision_at_0_5": 1.0,
+        "large_training_recall_at_0_5": 1.0,
+        "macro_f1": 0.4,
+        "rows": 42
+      },
+      {
+        "accuracy": 1.0,
+        "group": "multi_stage_training_pipeline",
+        "large_training_false_negatives": 0,
+        "large_training_false_positives": 0,
+        "large_training_precision_at_0_5": 1.0,
+        "large_training_recall_at_0_5": 1.0,
+        "macro_f1": 0.4,
+        "rows": 24
+      },
+      {
+        "accuracy": 1.0,
+        "group": "counter_suppressed_candidate_window",
+        "large_training_false_negatives": 0,
+        "large_training_false_positives": 0,
+        "large_training_precision_at_0_5": 0.0,
+        "large_training_recall_at_0_5": 0.0,
+        "macro_f1": 0.2,
+        "rows": 22
+      },
+      {
+        "accuracy": 1.0,
+        "group": "distributed_database_or_graph_analytics",
+        "large_training_false_negatives": 0,
+        "large_training_false_positives": 0,
+        "large_training_precision_at_0_5": 0.0,
+        "large_training_recall_at_0_5": 0.0,
+        "macro_f1": 0.2,
+        "rows": 18
+      },
+      {
+        "accuracy": 1.0,
+        "group": "maintenance_with_collector_gaps",
+        "large_training_false_negatives": 0,
+        "large_training_false_positives": 0,
+        "large_training_precision_at_0_5": 0.0,
+        "large_training_recall_at_0_5": 0.0,
+        "macro_f1": 0.2,
+        "rows": 15
+      },
+      {
+        "accuracy": 1.0,
+        "group": "synthetic_data_generation_gpu_heavy",
+        "large_training_false_negatives": 0,
+        "large_training_false_positives": 0,
+        "large_training_precision_at_0_5": 0.0,
+        "large_training_recall_at_0_5": 0.0,
+        "macro_f1": 0.2,
+        "rows": 14
+      }
+    ],
+    "site_id": [
+      {
+        "accuracy": 0.9966101694915255,
+        "group": "site_b",
+        "large_training_false_negatives": 2,
+        "large_training_false_positives": 0,
+        "large_training_precision_at_0_5": 1.0,
+        "large_training_recall_at_0_5": 0.9878787878787879,
+        "macro_f1": 0.9975238356540229,
+        "rows": 590
+      },
+      {
+        "accuracy": 1.0,
+        "group": "site_a",
+        "large_training_false_negatives": 0,
+        "large_training_false_positives": 0,
+        "large_training_precision_at_0_5": 1.0,
+        "large_training_recall_at_0_5": 1.0,
+        "macro_f1": 1.0,
+        "rows": 464
+      },
+      {
+        "accuracy": 1.0,
+        "group": "site_d",
+        "large_training_false_negatives": 0,
+        "large_training_false_positives": 0,
+        "large_training_precision_at_0_5": 1.0,
+        "large_training_recall_at_0_5": 1.0,
+        "macro_f1": 0.8,
+        "rows": 454
+      },
+      {
+        "accuracy": 0.9016786570743405,
+        "group": "site_g",
+        "large_training_false_negatives": 0,
+        "large_training_false_positives": 0,
+        "large_training_precision_at_0_5": 1.0,
+        "large_training_recall_at_0_5": 1.0,
+        "macro_f1": 0.9040527108006021,
+        "rows": 417
+      },
+      {
+        "accuracy": 0.948051948051948,
+        "group": "site_e",
+        "large_training_false_negatives": 0,
+        "large_training_false_positives": 0,
+        "large_training_precision_at_0_5": 1.0,
+        "large_training_recall_at_0_5": 1.0,
+        "macro_f1": 0.7622607921673342,
+        "rows": 385
+      },
+      {
+        "accuracy": 0.9872611464968153,
+        "group": "site_c",
+        "large_training_false_negatives": 0,
+        "large_training_false_positives": 4,
+        "large_training_precision_at_0_5": 0.8333333333333334,
+        "large_training_recall_at_0_5": 1.0,
+        "macro_f1": 0.776940133037694,
+        "rows": 314
+      },
+      {
+        "accuracy": 0.9537366548042705,
+        "group": "site_f",
+        "large_training_false_negatives": 0,
+        "large_training_false_positives": 0,
+        "large_training_precision_at_0_5": 1.0,
+        "large_training_recall_at_0_5": 1.0,
+        "macro_f1": 0.7762378716744914,
+        "rows": 281
+      }
+    ],
+    "temporal_phase": [],
+    "window_length_seconds": [
+      {
+        "accuracy": 0.9710467706013363,
+        "group": "900",
+        "large_training_false_negatives": 1,
+        "large_training_false_positives": 1,
+        "large_training_precision_at_0_5": 0.9942528735632183,
+        "large_training_recall_at_0_5": 0.9942528735632183,
+        "macro_f1": 0.976753637450509,
+        "rows": 898
+      },
+      {
+        "accuracy": 0.9720044792833147,
+        "group": "3600",
+        "large_training_false_negatives": 1,
+        "large_training_false_positives": 1,
+        "large_training_precision_at_0_5": 0.9942196531791907,
+        "large_training_recall_at_0_5": 0.9942196531791907,
+        "macro_f1": 0.9774669561641736,
+        "rows": 893
+      },
+      {
+        "accuracy": 0.9731903485254692,
+        "group": "21600",
+        "large_training_false_negatives": 0,
+        "large_training_false_positives": 1,
+        "large_training_precision_at_0_5": 0.9942528735632183,
+        "large_training_recall_at_0_5": 1.0,
+        "macro_f1": 0.9793116710307792,
+        "rows": 746
+      },
+      {
+        "accuracy": 0.9755434782608695,
+        "group": "86400",
+        "large_training_false_negatives": 0,
+        "large_training_false_positives": 1,
+        "large_training_precision_at_0_5": 0.9895833333333334,
+        "large_training_recall_at_0_5": 1.0,
+        "macro_f1": 0.9804646860359316,
+        "rows": 368
+      }
+    ]
+  }
+}

data/model_runs/synthetic_v1_baseline/model.joblib

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:06c701b0d9c74cdfd0066fe3212013ad2393c3eca34ca625c78792631fa3f338
+size 2178380

data/model_runs/synthetic_v1_baseline/predictions_all.csv

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e2e66b8c5f743d13bb4a24b57656cf7f246cb423071efc1049a9f11019e8f549
+size 12768446

data/model_runs/synthetic_v1_baseline/preprocessing.joblib

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:edf78f71a10335c7703a183a76b97add8ee6ecc8b0ca1068fdcdedab18bcf2de
+size 23460

data/model_runs/synthetic_v1_baseline/stress_metrics.json

@@ -0,0 +1,707 @@
+{
+  "exclusion_metadata": {
+    "missing_requested_excluded_columns": [],
+    "notes": [
+      "Identifier, label, direct leakage, site, and synthetic-only audit columns are excluded.",
+      "scope_type and window_length_seconds are retained because they are valid deployment-time context.",
+      "Constant version metadata columns are excluded from the baseline."
+    ],
+    "present_excluded_columns": [
+      "feature_row_id",
+      "dataset_id",
+      "seed",
+      "site_id",
+      "scope_id_hash",
+      "window_start",
+      "window_end",
+      "episode_id",
+      "label_0_to_4",
+      "label_confidence",
+      "label_reason",
+      "label_source",
+      "raw_input_manifest_hash",
+      "latent_workload_class",
+      "scenario_family",
+      "scenario_variant",
+      "evidence_recipe_id",
+      "counterfactual_group_id",
+      "synthetic_counterfactual_role",
+      "data_quality_regime",
+      "privacy_tier",
+      "collector_profile",
+      "topology_class",
+      "temporal_phase",
+      "synthetic_hard_case_tags",
+      "synthetic_evidence_profile",
+      "capacity_evidence_only",
+      "integrity_evidence_only",
+      "physical_evidence_only",
+      "feature_pipeline_version",
+      "policy_threshold_version",
+      "hardware_normalization_version"
+    ],
+    "requested_excluded_columns": [
+      "feature_row_id",
+      "dataset_id",
+      "seed",
+      "site_id",
+      "scope_id_hash",
+      "window_start",
+      "window_end",
+      "episode_id",
+      "label_0_to_4",
+      "label_confidence",
+      "label_reason",
+      "label_source",
+      "raw_input_manifest_hash",
+      "latent_workload_class",
+      "scenario_family",
+      "scenario_variant",
+      "evidence_recipe_id",
+      "counterfactual_group_id",
+      "synthetic_counterfactual_role",
+      "data_quality_regime",
+      "privacy_tier",
+      "collector_profile",
+      "topology_class",
+      "temporal_phase",
+      "synthetic_hard_case_tags",
+      "synthetic_evidence_profile",
+      "capacity_evidence_only",
+      "integrity_evidence_only",
+      "physical_evidence_only",
+      "feature_pipeline_version",
+      "policy_threshold_version",
+      "hardware_normalization_version"
+    ]
+  },
+  "features_path": "data/synthetic_v1/features/window_features_all.csv",
+  "max_iter": 90,
+  "model_feature_count": 152,
+  "scenario_column": "scenario_family",
+  "seed": 20260510,
+  "splits": {
+    "episode_grouped_random": {
+      "accuracy": 0.9776247848537005,
+      "false_negative_scenarios": {
+        "fragmented_training_linked": 2
+      },
+      "false_positive_scenarios": {
+        "underclocked_energy_capped_training": 4
+      },
+      "large_training": {
+        "f1": 0.9951298701298701,
+        "precision": 0.993517017828201,
+        "recall": 0.9967479674796748
+      },
+      "large_training_false_negative_count": 2,
+      "large_training_false_positive_count": 4,
+      "log_loss": 0.09684510600740838,
+      "macro_f1": 0.9820361510992616,
+      "middle_probability_bins": {
+        "0.1_to_0.3": 18,
+        "0.3_to_0.7": 2,
+        "0.7_to_0.9": 0
+      },
+      "test_episodes": 150,
+      "test_label_distribution": {
+        "0": 762,
+        "1": 690,
+        "2": 838,
+        "3": 463,
+        "4": 152
+      },
+      "test_rows": 2905,
+      "train_episodes": 610,
+      "train_rows": 11807
+    },
+    "scenario_family_holdout": {
+      "fragmented_training_linked": {
+        "accuracy": 0.8805774278215223,
+        "false_negative_scenarios": {
+          "fragmented_training_linked": 91
+        },
+        "false_positive_scenarios": {},
+        "large_training": {
+          "f1": 0.8465430016863407,
+          "precision": 1.0,
+          "recall": 0.7339181286549707
+        },
+        "large_training_false_negative_count": 91,
+        "large_training_false_positive_count": 0,
+        "log_loss": 0.23491922435876197,
+        "macro_f1": 0.3501482128238128,
+        "middle_probability_bins": {
+          "0.1_to_0.3": 6,
+          "0.3_to_0.7": 81,
+          "0.7_to_0.9": 0
+        },
+        "test_episodes": 35,
+        "test_label_distribution": {
+          "2": 420,
+          "3": 342
+        },
+        "test_rows": 762,
+        "train_episodes": 725,
+        "train_rows": 13950
+      },
+      "hpc_mpi_collective": {
+        "accuracy": 0.9530516431924883,
+        "false_negative_scenarios": {},
+        "false_positive_scenarios": {},
+        "large_training": {
+          "f1": 0.0,
+          "precision": 0.0,
+          "recall": 0.0
+        },
+        "large_training_false_negative_count": 0,
+        "large_training_false_positive_count": 0,
+        "log_loss": 0.24695636309738603,
+        "macro_f1": 0.19744680851063828,
+        "middle_probability_bins": {
+          "0.1_to_0.3": 58,
+          "0.3_to_0.7": 0,
+          "0.7_to_0.9": 0
+        },
+        "test_episodes": 42,
+        "test_label_distribution": {
+          "0": 34,
+          "2": 818
+        },
+        "test_rows": 852,
+        "train_episodes": 718,
+        "train_rows": 13860
+      },
+      "model_parallel_inference": {
+        "accuracy": 0.8814655172413793,
+        "false_negative_scenarios": {},
+        "false_positive_scenarios": {},
+        "large_training": {
+          "f1": 0.0,
+          "precision": 0.0,
+          "recall": 0.0
+        },
+        "large_training_false_negative_count": 0,
+        "large_training_false_positive_count": 0,
+        "log_loss": 0.32601787797092746,
+        "macro_f1": 0.32858292230734254,
+        "middle_probability_bins": {
+          "0.1_to_0.3": 0,
+          "0.3_to_0.7": 0,
+          "0.7_to_0.9": 0
+        },
+        "test_episodes": 23,
+        "test_label_distribution": {
+          "1": 125,
+          "2": 339
+        },
+        "test_rows": 464,
+        "train_episodes": 737,
+        "train_rows": 14248
+      },
+      "multi_tenant_fragmented_nontraining": {
+        "accuracy": 1.0,
+        "false_negative_scenarios": {},
+        "false_positive_scenarios": {},
+        "large_training": {
+          "f1": 0.0,
+          "precision": 0.0,
+          "recall": 0.0
+        },
+        "large_training_false_negative_count": 0,
+        "large_training_false_positive_count": 0,
+        "log_loss": 0.0001384917381362078,
+        "macro_f1": 0.2,
+        "middle_probability_bins": {
+          "0.1_to_0.3": 0,
+          "0.3_to_0.7": 0,
+          "0.7_to_0.9": 0
+        },
+        "test_episodes": 22,
+        "test_label_distribution": {
+          "2": 461
+        },
+        "test_rows": 461,
+        "train_episodes": 738,
+        "train_rows": 14251
+      },
+      "underclocked_energy_capped_training": {
+        "accuracy": 0.979539641943734,
+        "false_negative_scenarios": {},
+        "false_positive_scenarios": {
+          "underclocked_energy_capped_training": 8
+        },
+        "large_training": {
+          "f1": 0.9873015873015873,
+          "precision": 0.9749216300940439,
+          "recall": 1.0
+        },
+        "large_training_false_negative_count": 0,
+        "large_training_false_positive_count": 8,
+        "log_loss": 0.1272874809495267,
+        "macro_f1": 0.38693400167084374,
+        "middle_probability_bins": {
+          "0.1_to_0.3": 0,
+          "0.3_to_0.7": 0,
+          "0.7_to_0.9": 0
+        },
+        "test_episodes": 18,
+        "test_label_distribution": {
+          "2": 80,
+          "3": 311
+        },
+        "test_rows": 391,
+        "train_episodes": 742,
+        "train_rows": 14321
+      }
+    },
+    "site_holdout": {
+      "site_a": {
+        "accuracy": 0.9651113913408995,
+        "false_negative_scenarios": {
+          "fragmented_training_linked": 5
+        },
+        "false_positive_scenarios": {},
+        "large_training": {
+          "f1": 0.9958643507030603,
+          "precision": 1.0,
+          "recall": 0.9917627677100495
+        },
+        "large_training_false_negative_count": 5,
+        "large_training_false_positive_count": 0,
+        "log_loss": 0.12471327669067636,
+        "macro_f1": 0.969965539074414,
+        "middle_probability_bins": {
+          "0.1_to_0.3": 3,
+          "0.3_to_0.7": 6,
+          "0.7_to_0.9": 0
+        },
+        "test_episodes": 119,
+        "test_label_distribution": {
+          "0": 523,
+          "1": 498,
+          "2": 751,
+          "3": 428,
+          "4": 179
+        },
+        "test_rows": 2379,
+        "train_episodes": 641,
+        "train_rows": 12333
+      },
+      "site_b": {
+        "accuracy": 0.9817596566523605,
+        "false_negative_scenarios": {},
+        "false_positive_scenarios": {},
+        "large_training": {
+          "f1": 1.0,
+          "precision": 1.0,
+          "recall": 1.0
+        },
+        "large_training_false_negative_count": 0,
+        "large_training_false_positive_count": 0,
+        "log_loss": 0.12083701002956747,
+        "macro_f1": 0.9823840732636331,
+        "middle_probability_bins": {
+          "0.1_to_0.3": 59,
+          "0.3_to_0.7": 0,
+          "0.7_to_0.9": 0
+        },
+        "test_episodes": 143,
+        "test_label_distribution": {
+          "0": 766,
+          "1": 442,
+          "2": 923,
+          "3": 472,
+          "4": 193
+        },
+        "test_rows": 2796,
+        "train_episodes": 617,
+        "train_rows": 11916
+      },
+      "site_c": {
+        "accuracy": 0.9793233082706767,
+        "false_negative_scenarios": {},
+        "false_positive_scenarios": {
+          "underclocked_energy_capped_training": 4
+        },
+        "large_training": {
+          "f1": 0.993485342019544,
+          "precision": 0.9870550161812298,
+          "recall": 1.0
+        },
+        "large_training_false_negative_count": 0,
+        "large_training_false_positive_count": 4,
+        "log_loss": 0.12938808464320045,
+        "macro_f1": 0.9812786466966241,
+        "middle_probability_bins": {
+          "0.1_to_0.3": 0,
+          "0.3_to_0.7": 0,
+          "0.7_to_0.9": 0
+        },
+        "test_episodes": 83,
+        "test_label_distribution": {
+          "0": 544,
+          "1": 314,
+          "2": 433,
+          "3": 216,
+          "4": 89
+        },
+        "test_rows": 1596,
+        "train_episodes": 677,
+        "train_rows": 13116
+      },
+      "site_d": {
+        "accuracy": 0.9671179883945842,
+        "false_negative_scenarios": {},
+        "false_positive_scenarios": {},
+        "large_training": {
+          "f1": 1.0,
+          "precision": 1.0,
+          "recall": 1.0
+        },
+        "large_training_false_negative_count": 0,
+        "large_training_false_positive_count": 0,
+        "log_loss": 0.1630732533805434,
+        "macro_f1": 0.9761029329988823,
+        "middle_probability_bins": {
+          "0.1_to_0.3": 20,
+          "0.3_to_0.7": 0,
+          "0.7_to_0.9": 0
+        },
+        "test_episodes": 134,
+        "test_label_distribution": {
+          "0": 657,
+          "1": 691,
+          "2": 751,
+          "3": 415,
+          "4": 71
+        },
+        "test_rows": 2585,
+        "train_episodes": 626,
+        "train_rows": 12127
+      },
+      "site_e": {
+        "accuracy": 0.9577039274924471,
+        "false_negative_scenarios": {
+          "training_without_semantic_logs": 4
+        },
+        "false_positive_scenarios": {},
+        "large_training": {
+          "f1": 0.9952718676122931,
+          "precision": 1.0,
+          "recall": 0.9905882352941177
+        },
+        "large_training_false_negative_count": 4,
+        "large_training_false_positive_count": 0,
+        "log_loss": 0.10221061129077724,
+        "macro_f1": 0.9672000210392337,
+        "middle_probability_bins": {
+          "0.1_to_0.3": 0,
+          "0.3_to_0.7": 0,
+          "0.7_to_0.9": 0
+        },
+        "test_episodes": 105,
+        "test_label_distribution": {
+          "0": 501,
+          "1": 505,
+          "2": 555,
+          "3": 323,
+          "4": 102
+        },
+        "test_rows": 1986,
+        "train_episodes": 655,
+        "train_rows": 12726
+      },
+      "site_f": {
+        "accuracy": 0.974571586511885,
+        "false_negative_scenarios": {
+          "fragmented_training_linked": 4
+        },
+        "false_positive_scenarios": {},
+        "large_training": {
+          "f1": 0.9951338199513382,
+          "precision": 1.0,
+          "recall": 0.9903147699757869
+        },
+        "large_training_false_negative_count": 4,
+        "large_training_false_positive_count": 0,
+        "log_loss": 0.12939604463893833,
+        "macro_f1": 0.979930256900869,
+        "middle_probability_bins": {
+          "0.1_to_0.3": 2,
+          "0.3_to_0.7": 4,
+          "0.7_to_0.9": 0
+        },
+        "test_episodes": 94,
+        "test_label_distribution": {
+          "0": 512,
+          "1": 449,
+          "2": 435,
+          "3": 330,
+          "4": 83
+        },
+        "test_rows": 1809,
+        "train_episodes": 666,
+        "train_rows": 12903
+      },
+      "site_g": {
+        "accuracy": 0.9647661755285074,
+        "false_negative_scenarios": {
+          "training_without_semantic_logs": 3
+        },
+        "false_positive_scenarios": {},
+        "large_training": {
+          "f1": 0.9963636363636363,
+          "precision": 1.0,
+          "recall": 0.9927536231884058
+        },
+        "large_training_false_negative_count": 3,
+        "large_training_false_positive_count": 0,
+        "log_loss": 0.05853045642772131,
+        "macro_f1": 0.9717372783794571,
+        "middle_probability_bins": {
+          "0.1_to_0.3": 0,
+          "0.3_to_0.7": 7,
+          "0.7_to_0.9": 5
+        },
+        "test_episodes": 82,
+        "test_label_distribution": {
+          "0": 358,
+          "1": 434,
+          "2": 355,
+          "3": 305,
+          "4": 109
+        },
+        "test_rows": 1561,
+        "train_episodes": 678,
+        "train_rows": 13151
+      }
+    },
+    "source_ablations": {
+      "source_ablation_drop_fabric": {
+        "accuracy": 0.9865748709122203,
+        "dropped_prefixes": [
+          "o6_",
+          "o7_",
+          "fabric_telemetry_trust_level"
+        ],
+        "false_negative_scenarios": {},
+        "false_positive_scenarios": {
+          "underclocked_energy_capped_training": 4
+        },
+        "feature_count": 132,
+        "large_training": {
+          "f1": 0.9967585089141004,
+          "precision": 0.9935379644588045,
+          "recall": 1.0
+        },
+        "large_training_false_negative_count": 0,
+        "large_training_false_positive_count": 4,
+        "log_loss": 0.08731589431092471,
+        "macro_f1": 0.9891740593379061,
+        "middle_probability_bins": {
+          "0.1_to_0.3": 0,
+          "0.3_to_0.7": 0,
+          "0.7_to_0.9": 0
+        },
+        "test_episodes": 150,
+        "test_label_distribution": {
+          "0": 762,
+          "1": 690,
+          "2": 838,
+          "3": 463,
+          "4": 152
+        },
+        "test_rows": 2905,
+        "train_episodes": 610,
+        "train_rows": 11807
+      },
+      "source_ablation_drop_gpu_telemetry": {
+        "accuracy": 0.9645438898450946,
+        "dropped_prefixes": [
+          "o4_",
+          "o5_",
+          "gpu_telemetry_trust_level"
+        ],
+        "false_negative_scenarios": {
+          "fragmented_training_linked": 2
+        },
+        "false_positive_scenarios": {
+          "underclocked_energy_capped_training": 4
+        },
+        "feature_count": 131,
+        "large_training": {
+          "f1": 0.9951298701298701,
+          "precision": 0.993517017828201,
+          "recall": 0.9967479674796748
+        },
+        "large_training_false_negative_count": 2,
+        "large_training_false_positive_count": 4,
+        "log_loss": 0.09299517347739551,
+        "macro_f1": 0.9719602384218333,
+        "middle_probability_bins": {
+          "0.1_to_0.3": 0,
+          "0.3_to_0.7": 0,
+          "0.7_to_0.9": 0
+        },
+        "test_episodes": 150,
+        "test_label_distribution": {
+          "0": 762,
+          "1": 690,
+          "2": 838,
+          "3": 463,
+          "4": 152
+        },
+        "test_rows": 2905,
+        "train_episodes": 610,
+        "train_rows": 11807
+      },
+      "source_ablation_drop_power": {
+        "accuracy": 0.9648881239242685,
+        "dropped_prefixes": [
+          "o8_",
+          "o9_",
+          "power_meter_trust_level"
+        ],
+        "false_negative_scenarios": {
+          "training_with_delayed_logs": 4
+        },
+        "false_positive_scenarios": {
+          "underclocked_energy_capped_training": 4
+        },
+        "feature_count": 133,
+        "large_training": {
+          "f1": 0.9934959349593496,
+          "precision": 0.9934959349593496,
+          "recall": 0.9934959349593496
+        },
+        "large_training_false_negative_count": 4,
+        "large_training_false_positive_count": 4,
+        "log_loss": 0.14389758581141518,
+        "macro_f1": 0.9719275856055797,
+        "middle_probability_bins": {
+          "0.1_to_0.3": 4,
+          "0.3_to_0.7": 0,
+          "0.7_to_0.9": 0
+        },
+        "test_episodes": 150,
+        "test_label_distribution": {
+          "0": 762,
+          "1": 690,
+          "2": 838,
+          "3": 463,
+          "4": 152
+        },
+        "test_rows": 2905,
+        "train_episodes": 610,
+        "train_rows": 11807
+      },
+      "source_ablation_drop_runtime_and_ml_logs": {
+        "accuracy": 0.9380378657487092,
+        "dropped_prefixes": [
+          "o10_",
+          "o12_"
+        ],
+        "false_negative_scenarios": {},
+        "false_positive_scenarios": {
+          "training_with_delayed_logs": 4,
+          "underclocked_energy_capped_training": 4
+        },
+        "feature_count": 132,
+        "large_training": {
+          "f1": 0.9935379644588045,
+          "precision": 0.9871589085072231,
+          "recall": 1.0
+        },
+        "large_training_false_negative_count": 0,
+        "large_training_false_positive_count": 8,
+        "log_loss": 0.35327183914707877,
+        "macro_f1": 0.8925854563387956,
+        "middle_probability_bins": {
+          "0.1_to_0.3": 4,
+          "0.3_to_0.7": 4,
+          "0.7_to_0.9": 44
+        },
+        "test_episodes": 150,
+        "test_label_distribution": {
+          "0": 762,
+          "1": 690,
+          "2": 838,
+          "3": 463,
+          "4": 152
+        },
+        "test_rows": 2905,
+        "train_episodes": 610,
+        "train_rows": 11807
+      },
+      "source_ablation_drop_storage": {
+        "accuracy": 0.9655765920826161,
+        "dropped_prefixes": [
+          "o11_"
+        ],
+        "false_negative_scenarios": {
+          "fragmented_training_linked": 21
+        },
+        "false_positive_scenarios": {
+          "fragmented_training_linked": 7
+        },
+        "feature_count": 141,
+        "large_training": {
+          "f1": 0.9769736842105263,
+          "precision": 0.9883527454242929,
+          "recall": 0.9658536585365853
+        },
+        "large_training_false_negative_count": 21,
+        "large_training_false_positive_count": 7,
+        "log_loss": 0.12576959836354257,
+        "macro_f1": 0.9712103257601914,
+        "middle_probability_bins": {
+          "0.1_to_0.3": 13,
+          "0.3_to_0.7": 15,
+          "0.7_to_0.9": 8
+        },
+        "test_episodes": 150,
+        "test_label_distribution": {
+          "0": 762,
+          "1": 690,
+          "2": 838,
+          "3": 463,
+          "4": 152
+        },
+        "test_rows": 2905,
+        "train_episodes": 610,
+        "train_rows": 11807
+      }
+    },
+    "time_holdout": {
+      "accuracy": 0.980971797485559,
+      "false_negative_scenarios": {},
+      "false_positive_scenarios": {},
+      "large_training": {
+        "f1": 1.0,
+        "precision": 1.0,
+        "recall": 1.0
+      },
+      "large_training_false_negative_count": 0,
+      "large_training_false_positive_count": 0,
+      "log_loss": 0.12262942519580222,
+      "macro_f1": 0.9862673370214863,
+      "middle_probability_bins": {
+        "0.1_to_0.3": 4,
+        "0.3_to_0.7": 0,
+        "0.7_to_0.9": 0
+      },
+      "test_episodes": 179,
+      "test_label_distribution": {
+        "0": 715,
+        "1": 857,
+        "2": 776,
+        "3": 486,
+        "4": 109
+      },
+      "test_rows": 2943,
+      "train_episodes": 622,
+      "train_rows": 11769
+    }
+  }
+}

data/model_runs/synthetic_v1_baseline/stress_summary.md

@@ -0,0 +1,50 @@
+# Synthetic v1 Stress Summary
+
+Feature table: `data/synthetic_v1/features/window_features_all.csv`
+Seed: `20260510`
+Lightweight stress model max_iter: `90`
+
+## Core Splits
+
+- episode_grouped_random: rows=2905, macro F1=0.982, large precision=0.994, large recall=0.997, FP=4, FN=2, middle p bins={'0.1_to_0.3': 18, '0.3_to_0.7': 2, '0.7_to_0.9': 0}
+- time_holdout: rows=2943, macro F1=0.986, large precision=1.000, large recall=1.000, FP=0, FN=0, middle p bins={'0.1_to_0.3': 4, '0.3_to_0.7': 0, '0.7_to_0.9': 0}
+
+## Site Holdout
+
+- site_a: rows=2379, macro F1=0.970, large precision=1.000, large recall=0.992, FP=0, FN=5, middle p bins={'0.1_to_0.3': 3, '0.3_to_0.7': 6, '0.7_to_0.9': 0}
+- site_b: rows=2796, macro F1=0.982, large precision=1.000, large recall=1.000, FP=0, FN=0, middle p bins={'0.1_to_0.3': 59, '0.3_to_0.7': 0, '0.7_to_0.9': 0}
+- site_c: rows=1596, macro F1=0.981, large precision=0.987, large recall=1.000, FP=4, FN=0, middle p bins={'0.1_to_0.3': 0, '0.3_to_0.7': 0, '0.7_to_0.9': 0}
+- site_d: rows=2585, macro F1=0.976, large precision=1.000, large recall=1.000, FP=0, FN=0, middle p bins={'0.1_to_0.3': 20, '0.3_to_0.7': 0, '0.7_to_0.9': 0}
+- site_e: rows=1986, macro F1=0.967, large precision=1.000, large recall=0.991, FP=0, FN=4, middle p bins={'0.1_to_0.3': 0, '0.3_to_0.7': 0, '0.7_to_0.9': 0}
+- site_f: rows=1809, macro F1=0.980, large precision=1.000, large recall=0.990, FP=0, FN=4, middle p bins={'0.1_to_0.3': 2, '0.3_to_0.7': 4, '0.7_to_0.9': 0}
+- site_g: rows=1561, macro F1=0.972, large precision=1.000, large recall=0.993, FP=0, FN=3, middle p bins={'0.1_to_0.3': 0, '0.3_to_0.7': 7, '0.7_to_0.9': 5}
+
+## Scenario Family Holdout
+
+- underclocked_energy_capped_training: rows=391, macro F1=0.387, large precision=0.975, large recall=1.000, FP=8, FN=0, middle p bins={'0.1_to_0.3': 0, '0.3_to_0.7': 0, '0.7_to_0.9': 0}
+  false positives: {'underclocked_energy_capped_training': 8}
+- fragmented_training_linked: rows=762, macro F1=0.350, large precision=1.000, large recall=0.734, FP=0, FN=91, middle p bins={'0.1_to_0.3': 6, '0.3_to_0.7': 81, '0.7_to_0.9': 0}
+  false negatives: {'fragmented_training_linked': 91}
+- multi_tenant_fragmented_nontraining: rows=461, macro F1=0.200, large precision=0.000, large recall=0.000, FP=0, FN=0, middle p bins={'0.1_to_0.3': 0, '0.3_to_0.7': 0, '0.7_to_0.9': 0}
+- model_parallel_inference: rows=464, macro F1=0.329, large precision=0.000, large recall=0.000, FP=0, FN=0, middle p bins={'0.1_to_0.3': 0, '0.3_to_0.7': 0, '0.7_to_0.9': 0}
+- hpc_mpi_collective: rows=852, macro F1=0.197, large precision=0.000, large recall=0.000, FP=0, FN=0, middle p bins={'0.1_to_0.3': 58, '0.3_to_0.7': 0, '0.7_to_0.9': 0}
+
+## Source Ablations
+
+- source_ablation_drop_fabric: rows=2905, macro F1=0.989, large precision=0.994, large recall=1.000, FP=4, FN=0, middle p bins={'0.1_to_0.3': 0, '0.3_to_0.7': 0, '0.7_to_0.9': 0}
+  macro F1 delta vs random baseline: +0.007
+- source_ablation_drop_runtime_and_ml_logs: rows=2905, macro F1=0.893, large precision=0.987, large recall=1.000, FP=8, FN=0, middle p bins={'0.1_to_0.3': 4, '0.3_to_0.7': 4, '0.7_to_0.9': 44}
+  macro F1 delta vs random baseline: -0.089
+- source_ablation_drop_gpu_telemetry: rows=2905, macro F1=0.972, large precision=0.994, large recall=0.997, FP=4, FN=2, middle p bins={'0.1_to_0.3': 0, '0.3_to_0.7': 0, '0.7_to_0.9': 0}
+  macro F1 delta vs random baseline: -0.010
+- source_ablation_drop_power: rows=2905, macro F1=0.972, large precision=0.993, large recall=0.993, FP=4, FN=4, middle p bins={'0.1_to_0.3': 4, '0.3_to_0.7': 0, '0.7_to_0.9': 0}
+  macro F1 delta vs random baseline: -0.010
+- source_ablation_drop_storage: rows=2905, macro F1=0.971, large precision=0.988, large recall=0.966, FP=7, FN=21, middle p bins={'0.1_to_0.3': 13, '0.3_to_0.7': 15, '0.7_to_0.9': 8}
+  macro F1 delta vs random baseline: -0.011
+
+## Interpretation
+
+- These are stress diagnostics, not tuned headline metrics.
+- Family holdouts identify hard families that still fail under scenario generalization.
+- Source ablations should degrade performance when the dropped layer carries independent evidence.
+- Middle probability bins indicate whether v1 has less all-or-nothing large-training probability mass than v0.

data/model_runs/synthetic_v1_baseline/validation_summary.md

@@ -0,0 +1,43 @@
+# Validation Summary
+
+Feature table: `data/synthetic_v1/features/window_features_all.csv`
+
+## Dataset Validation Before Training
+
+- `synthetic_dataset_validator`: return code 0; PASS
+- `public_dataset_validator`: return code 0; PASS
+
+## Required Artifacts
+
+- `README.md`: present
+- `manifest.json`: present
+- `model.joblib`: present
+- `preprocessing.joblib`: present
+- `feature_columns.json`: present
+- `excluded_columns.json`: present
+- `split_manifest.json`: present
+- `metrics.json`: present
+- `calibration_metrics.json`: present
+- `confusion_matrix.csv`: present
+- `classification_report.json`: present
+- `predictions_test.csv`: present
+- `predictions_all.csv`: present
+- `feature_importance.csv`: present
+- `evidence_audit_sample.csv`: present
+- `validation_summary.md`: present
+
+## Test Metrics
+
+- Accuracy: 0.9725
+- Macro F1: 0.9781
+- Label 3/4 precision: 0.9935
+- Label 3/4 recall: 0.9967
+- `p_large_training >= 0.5` precision: 0.9935
+- `p_large_training >= 0.5` recall: 0.9967
+
+## Governance Checks
+
+- Capacity gate applied to post-processed probabilities.
+- Negative certification confidence is `p_label_0 * min_critical_coverage`.
+- Integrity warnings are reported separately from positive training evidence.
+- Raw model probabilities are retained as `raw_p_label_0` through `raw_p_label_4`.

src/datacenter_verification_api/__init__.py

@@ -0,0 +1,5 @@
+"""Live inference API for the datacenter verification demo."""
+
+__all__ = ["__version__"]
+
+__version__ = "0.1.0"

src/datacenter_verification_api/app.py

@@ -0,0 +1,111 @@
+"""FastAPI app for live datacenter verification model inference."""
+
+from __future__ import annotations
+
+import os
+from contextlib import asynccontextmanager
+from typing import AsyncIterator
+
+from fastapi import FastAPI, HTTPException
+from fastapi.middleware.cors import CORSMiddleware
+
+from src.datacenter_verification_api import __version__
+from src.datacenter_verification_api.model_service import ModelService
+from src.datacenter_verification_api.schemas import (
+    BatchPredictRequest,
+    BatchPredictResponse,
+    HealthResponse,
+    MetadataResponse,
+    PredictRequest,
+    PredictResponse,
+)
+
+
+DEFAULT_ALLOWED_ORIGINS = [
+    "https://idacy.github.io",
+    "http://localhost:8000",
+    "http://localhost:5173",
+    "http://localhost:7860",
+    "http://127.0.0.1:8000",
+    "http://127.0.0.1:5173",
+    "http://127.0.0.1:7860",
+]
+
+service: ModelService | None = None
+load_error: str | None = None
+
+
+def parse_allowed_origins() -> list[str]:
+    raw = os.getenv("DCV_ALLOWED_ORIGINS", "")
+    if not raw.strip():
+        return DEFAULT_ALLOWED_ORIGINS
+    return [item.strip() for item in raw.split(",") if item.strip()]
+
+
+@asynccontextmanager
+async def lifespan(_: FastAPI) -> AsyncIterator[None]:
+    global service, load_error
+    try:
+        service = ModelService.from_env()
+        load_error = None
+    except Exception as exc:  # pragma: no cover - exercised by deployment misconfiguration
+        service = None
+        load_error = str(exc)
+    yield
+
+
+app = FastAPI(
+    title="Datacenter Verification Live Inference API",
+    version=__version__,
+    lifespan=lifespan,
+)
+
+allowed_origins = parse_allowed_origins()
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=["*"] if "*" in allowed_origins else allowed_origins,
+    allow_credentials=False,
+    allow_methods=["GET", "POST", "OPTIONS"],
+    allow_headers=["*"],
+)
+
+
+def require_service() -> ModelService:
+    if service is None:
+        detail = "model service is not loaded"
+        if load_error:
+            detail = f"{detail}: {load_error}"
+        raise HTTPException(status_code=503, detail=detail)
+    return service
+
+
+@app.get("/health", response_model=HealthResponse)
+def health() -> HealthResponse:
+    loaded = service is not None
+    return HealthResponse(
+        status="ok" if loaded else "error",
+        model_loaded=loaded,
+        api_version=__version__,
+        model_run_id=service.model_run_id if service else None,
+        dataset_id=service.dataset_id if service else None,
+        feature_count=len(service.feature_columns) if service else 0,
+        base_row_lookup_enabled=bool(service.feature_lookup) if service else False,
+        error=load_error,
+    )
+
+
+@app.get("/metadata", response_model=MetadataResponse)
+def metadata() -> MetadataResponse:
+    return require_service().metadata()
+
+
+@app.post("/predict", response_model=PredictResponse)
+def predict(request: PredictRequest) -> PredictResponse:
+    return require_service().predict(request)
+
+
+@app.post("/predict/batch", response_model=BatchPredictResponse)
+def predict_batch(request: BatchPredictRequest) -> BatchPredictResponse:
+    loaded = require_service()
+    predictions = [loaded.predict(item) for item in request.requests]
+    return BatchPredictResponse(model_run_id=loaded.model_run_id, predictions=predictions)

src/datacenter_verification_api/feature_completion.py

@@ -0,0 +1,298 @@
+"""Feature normalization and dependent-feature completion for live inference."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any
+
+import math
+
+import numpy as np
+import pandas as pd
+
+
+NULL_STRINGS = {"", "nan", "none", "null", "undefined", "na", "n/a", "<na>"}
+
+FRACTION_COLUMNS = {
+    "o1_homogeneous_high_end_fraction",
+    "o1_non_partitioned_fraction",
+    "o2_concurrency_fraction_domain",
+    "o3_training_sku_fraction",
+    "o3_billing_continuity_score",
+    "o4_gpu_util_duty_gt_70",
+    "o4_sm_tensor_active_p95",
+    "o4_hbm_used_fraction_p50",
+    "o4_hbm_bandwidth_active_p95",
+    "o4_gpu_power_fraction_p95",
+    "o4_hbm_pressure_duration_fraction",
+    "o4_power_cap_active_fraction",
+    "o4_thermal_throttle_fraction",
+    "o5_kernel_training_motif_score",
+    "o5_tensor_throughput_ratio",
+    "o6_nvlink_util_p95",
+    "o6_nvlink_periodicity_score",
+    "o7_scaleout_port_util_p95",
+    "o7_collective_periodicity_score",
+    "o7_burst_duty_cycle",
+    "o7_job_to_port_mapping_coverage",
+    "o7_flow_entropy_score",
+    "o7_cross_section_sync_score",
+    "o7_collective_jitter_score",
+    "o7_storage_traffic_fraction",
+    "o7_inference_fanout_score",
+    "o7_account_flow_linkage_confidence",
+    "o8_rack_power_fraction_p95",
+    "o8_power_cv",
+    "o8_power_cap_or_curtailment_active",
+    "o8_unattributed_power_fraction",
+    "o9_cooling_flow_duty",
+    "o10_rank_stability_score",
+    "o10_runtime_metadata_confidence",
+    "o11_checkpoint_periodicity_score",
+    "o11_read_write_training_pattern_score",
+    "o11_checkpoint_jitter_score",
+    "o11_artifact_write_pattern_score",
+    "o11_dataloader_read_pattern_score",
+    "o11_backup_or_replication_pattern_score",
+    "o11_storage_cotraffic_score",
+    "o12_log_completeness_fraction",
+    "o12_declaration_consistency_score",
+    "o13_attestation_valid_fraction",
+    "o13_confidential_compute_mode_fraction",
+    "o13_collector_measurement_valid",
+    "o14_min_critical_coverage",
+    "o14_gap_fraction_critical",
+    "o16_probe_throughput_ratio_min",
+    "o17_energy_contract_alignment_score",
+    "o17_network_provider_utilization_score",
+    "o17_procurement_or_maintenance_explanation_score",
+}
+
+for index in range(1, 18):
+    FRACTION_COLUMNS.add(f"o{index}_coverage_fraction")
+
+
+@dataclass
+class FeatureSchema:
+    """Column type hints inferred from the training feature table."""
+
+    feature_columns: list[str]
+    numeric_columns: set[str] = field(default_factory=set)
+    boolean_columns: set[str] = field(default_factory=set)
+
+    @classmethod
+    def from_frame(cls, feature_columns: list[str], frame: pd.DataFrame | None) -> "FeatureSchema":
+        numeric_columns: set[str] = set()
+        boolean_columns: set[str] = set()
+        if frame is not None:
+            for column in feature_columns:
+                if column not in frame.columns:
+                    continue
+                dtype = frame[column].dtype
+                if pd.api.types.is_bool_dtype(dtype):
+                    boolean_columns.add(column)
+                    numeric_columns.add(column)
+                elif pd.api.types.is_numeric_dtype(dtype):
+                    numeric_columns.add(column)
+        return cls(
+            feature_columns=list(feature_columns),
+            numeric_columns=numeric_columns,
+            boolean_columns=boolean_columns,
+        )
+
+
+def is_missing(value: Any) -> bool:
+    if value is None or value is pd.NA:
+        return True
+    if isinstance(value, str):
+        return value.strip().lower() in NULL_STRINGS
+    if isinstance(value, (float, np.floating)):
+        return math.isnan(float(value))
+    try:
+        return bool(pd.isna(value))
+    except (TypeError, ValueError):
+        return False
+
+
+def jsonable(value: Any) -> Any:
+    if is_missing(value):
+        return None
+    if isinstance(value, (np.integer,)):
+        return int(value)
+    if isinstance(value, (np.floating,)):
+        return float(value)
+    if isinstance(value, (np.bool_,)):
+        return bool(value)
+    if isinstance(value, pd.Timestamp):
+        return value.isoformat()
+    return value
+
+
+def normalize_value(value: Any, column: str | None = None, schema: FeatureSchema | None = None) -> Any:
+    if is_missing(value):
+        return None
+    if isinstance(value, str):
+        stripped = value.strip()
+        lowered = stripped.lower()
+        if lowered in {"true", "t", "yes", "y"}:
+            return True
+        if lowered in {"false", "f", "no", "n"}:
+            return False
+        if lowered in {"1", "0"} and schema and column in schema.boolean_columns:
+            return lowered == "1"
+        if schema and column in schema.numeric_columns:
+            try:
+                number = float(stripped)
+            except ValueError:
+                return stripped
+            return int(number) if number.is_integer() else number
+        return stripped
+    if schema and column in schema.boolean_columns:
+        if isinstance(value, (bool, np.bool_)):
+            return bool(value)
+        if isinstance(value, (int, float, np.integer, np.floating)) and not is_missing(value):
+            return bool(value)
+    if schema and column in schema.numeric_columns:
+        try:
+            number = float(value)
+        except (TypeError, ValueError):
+            return value
+        if math.isnan(number):
+            return None
+        return int(number) if number.is_integer() else number
+    return jsonable(value)
+
+
+def normalize_mapping(values: dict[str, Any] | None, schema: FeatureSchema) -> dict[str, Any]:
+    if not values:
+        return {}
+    return {str(key): normalize_value(value, str(key), schema) for key, value in values.items()}
+
+
+def number_or_none(value: Any) -> float | None:
+    if is_missing(value):
+        return None
+    try:
+        number = float(value)
+    except (TypeError, ValueError):
+        return None
+    if math.isnan(number):
+        return None
+    return number
+
+
+def bool_or_false(value: Any) -> bool:
+    if is_missing(value):
+        return False
+    if isinstance(value, (bool, np.bool_)):
+        return bool(value)
+    text = str(value).strip().lower()
+    if text in {"true", "t", "1", "yes", "y"}:
+        return True
+    if text in {"false", "f", "0", "no", "n"}:
+        return False
+    return False
+
+
+def set_if_changed(row: dict[str, Any], key: str, value: Any, warnings: list[str], derived: list[str]) -> None:
+    old = row.get(key)
+    normalized = jsonable(value)
+    old_normalized = jsonable(old)
+    if old_normalized != normalized:
+        row[key] = normalized
+        derived.append(key)
+
+
+def clamp_fraction(value: Any) -> float | None:
+    number = number_or_none(value)
+    if number is None:
+        return None
+    return min(1.0, max(0.0, number))
+
+
+def complete_features(
+    row: dict[str, Any],
+    changed_keys: set[str],
+    *,
+    has_base_row: bool,
+    derive: bool,
+) -> tuple[dict[str, Any], list[str]]:
+    """Return a completed row and warnings for coherent live inference.
+
+    Base-row predictions must remain byte-for-byte close to the exported model
+    replay, so dependent features are recomputed for base rows only when an
+    edited input can affect them. Rows without a base use every available input
+    to derive minimal context.
+    """
+
+    out = dict(row)
+    warnings: list[str] = []
+    derived: list[str] = []
+
+    if not has_base_row:
+        if is_missing(out.get("scope_type")):
+            out["scope_type"] = "site"
+        if is_missing(out.get("window_length_seconds")):
+            out["window_length_seconds"] = 3600
+        if is_missing(out.get("capacity_possible")):
+            out["capacity_possible"] = False
+
+    if not derive:
+        return out, warnings
+
+    should_derive_all = not has_base_row
+
+    allocation = number_or_none(out.get("o2_max_concurrent_normalized_gpus"))
+    duration = number_or_none(out.get("o2_allocation_duration_hours"))
+    capacity = number_or_none(out.get("o1_normalized_h100e_capacity"))
+    contiguous = number_or_none(out.get("o1_largest_contiguous_domain_gpus"))
+
+    ratio_inputs_changed = bool(
+        {"o2_max_concurrent_normalized_gpus", "o2_allocation_duration_hours"} & changed_keys
+    )
+    if (should_derive_all or ratio_inputs_changed) and allocation is not None and duration is not None:
+        ratio = (allocation * duration) / (512 * 24)
+        set_if_changed(out, "o2_gpu_hours_policy_ratio", ratio, warnings, derived)
+        set_if_changed(out, "policy_compute_ratio", ratio, warnings, derived)
+
+    concurrency_inputs_changed = bool(
+        {"o2_max_concurrent_normalized_gpus", "o1_normalized_h100e_capacity"} & changed_keys
+    )
+    if (should_derive_all or concurrency_inputs_changed) and allocation is not None and capacity is not None:
+        concurrency = clamp_fraction(allocation / capacity) if capacity > 0 else 0.0
+        set_if_changed(out, "o2_concurrency_fraction_domain", concurrency, warnings, derived)
+
+    if (should_derive_all or "o2_max_concurrent_normalized_gpus" in changed_keys) and allocation is not None:
+        rounded_allocation = int(round(allocation))
+        set_if_changed(out, "o10_world_size", rounded_allocation, warnings, derived)
+        set_if_changed(out, "o10_same_image_gpu_count", rounded_allocation, warnings, derived)
+
+    capacity_inputs_changed = bool(
+        {"o1_normalized_h100e_capacity", "o1_largest_contiguous_domain_gpus"} & changed_keys
+    )
+    if should_derive_all or capacity_inputs_changed:
+        if capacity is not None and contiguous is not None:
+            set_if_changed(out, "capacity_possible", capacity >= 512 and contiguous >= 512, warnings, derived)
+        elif not has_base_row:
+            set_if_changed(out, "capacity_possible", False, warnings, derived)
+
+    for key in sorted(FRACTION_COLUMNS & set(out)):
+        if should_derive_all or key in changed_keys:
+            clamped = clamp_fraction(out.get(key))
+            if clamped is not None:
+                set_if_changed(out, key, clamped, warnings, derived)
+
+    if derived:
+        warnings.append(f"derived fields updated from edited inputs: {', '.join(sorted(set(derived)))}")
+
+    allocation = number_or_none(out.get("o2_max_concurrent_normalized_gpus"))
+    capacity = number_or_none(out.get("o1_normalized_h100e_capacity"))
+    fabric = number_or_none(out.get("o7_synchronized_fabric_footprint"))
+    if capacity is not None and allocation is not None and allocation > capacity + 1:
+        warnings.append("allocated GPUs exceed monitored H100e capacity")
+    if capacity is not None and fabric is not None and fabric > capacity + 1:
+        warnings.append("fabric footprint exceeds monitored H100e capacity")
+    if allocation is not None and allocation > 0 and fabric is not None and fabric > allocation + 1:
+        warnings.append("fabric footprint exceeds allocated GPUs")
+
+    return out, warnings

src/datacenter_verification_api/model_service.py

@@ -0,0 +1,249 @@
+"""Sklearn model loading and one-row inference for the live API."""
+
+from __future__ import annotations
+
+import os
+from pathlib import Path
+from typing import Any
+
+import joblib
+import numpy as np
+import pandas as pd
+
+from src.datacenter_verification_api import __version__
+from src.datacenter_verification_api.feature_completion import (
+    FeatureSchema,
+    complete_features,
+    jsonable,
+    normalize_mapping,
+)
+from src.datacenter_verification_api.schemas import MetadataResponse, PredictRequest, PredictResponse
+from src.datacenter_verification_modeling.common import (
+    LABELS,
+    PROB_COLUMNS,
+    add_governance_outputs,
+    model_input_frame,
+    probability_frame,
+    read_json,
+)
+
+
+REPO_ROOT = Path(__file__).resolve().parents[2]
+DEFAULT_MODEL_RUN_DIR = Path("data/model_runs/synthetic_v1_baseline")
+DEFAULT_FEATURE_TABLE = Path("data/synthetic_v1/features/window_features_all.csv")
+
+
+def resolve_repo_path(value: str | Path | None, default: Path) -> Path:
+    path = Path(value) if value else default
+    if path.is_absolute():
+        return path
+    return (REPO_ROOT / path).resolve()
+
+
+def repo_relative(path: Path | None) -> str | None:
+    if path is None:
+        return None
+    try:
+        return path.resolve().relative_to(REPO_ROOT).as_posix()
+    except ValueError:
+        return path.name
+
+
+def read_json_if_exists(path: Path) -> Any:
+    return read_json(path) if path.exists() else {}
+
+
+def split_semicolon(value: Any) -> list[str]:
+    if value is None:
+        return []
+    try:
+        if pd.isna(value):
+            return []
+    except (TypeError, ValueError):
+        pass
+    return [item.strip() for item in str(value).split(";") if item.strip()]
+
+
+class ModelService:
+    """Loaded model artifacts plus base-row lookup for prediction requests."""
+
+    def __init__(self, model_run_dir: Path, feature_table_path: Path | None = None) -> None:
+        self.model_run_dir = model_run_dir
+        self.feature_table_path = feature_table_path
+        self.manifest = read_json_if_exists(model_run_dir / "manifest.json")
+        self.metrics = read_json_if_exists(model_run_dir / "metrics.json")
+        self.feature_columns: list[str] = read_json(model_run_dir / "feature_columns.json")
+        self.preprocessor = joblib.load(model_run_dir / "preprocessing.joblib")
+        self.model = joblib.load(model_run_dir / "model.joblib")
+        self.feature_table = self._load_feature_table(feature_table_path)
+        self.feature_schema = FeatureSchema.from_frame(self.feature_columns, self.feature_table)
+        self.feature_lookup = self._build_feature_lookup(self.feature_table)
+        self.model_run_id = str(self.manifest.get("model_run_id") or model_run_dir.name)
+
+    @classmethod
+    def from_env(cls) -> "ModelService":
+        model_run_dir = resolve_repo_path(os.getenv("DCV_MODEL_RUN_DIR"), DEFAULT_MODEL_RUN_DIR)
+        feature_table = resolve_repo_path(os.getenv("DCV_FEATURE_TABLE"), DEFAULT_FEATURE_TABLE)
+        return cls(model_run_dir=model_run_dir, feature_table_path=feature_table)
+
+    def _load_feature_table(self, path: Path | None) -> pd.DataFrame | None:
+        if path is None or not path.exists():
+            return None
+        frame = pd.read_csv(path)
+        if "feature_row_id" not in frame.columns:
+            raise ValueError(f"feature table lacks feature_row_id column: {path}")
+        return frame
+
+    def _build_feature_lookup(self, frame: pd.DataFrame | None) -> dict[str, dict[str, Any]]:
+        if frame is None:
+            return {}
+        rows: dict[str, dict[str, Any]] = {}
+        for record in frame.to_dict(orient="records"):
+            row_id = record.get("feature_row_id")
+            if row_id is not None:
+                rows[str(row_id)] = dict(record)
+        return rows
+
+    @property
+    def dataset_id(self) -> str | None:
+        training_metadata = self.manifest.get("training_metadata", {})
+        if isinstance(training_metadata, dict):
+            path = training_metadata.get("features_path") or self.manifest.get("features_path")
+        else:
+            path = self.manifest.get("features_path")
+        if isinstance(path, str) and "synthetic_v1" in path:
+            return "synthetic_v1"
+        if self.feature_table is not None and "dataset_id" in self.feature_table.columns:
+            values = self.feature_table["dataset_id"].dropna().unique()
+            if len(values):
+                return str(values[0])
+        return None
+
+    @property
+    def dataset_scale(self) -> str | None:
+        if self.feature_table is not None and "dataset_id" in self.feature_table.columns:
+            values = self.feature_table["dataset_id"].dropna().unique()
+            if len(values):
+                return str(values[0])
+        return None
+
+    @property
+    def model_type(self) -> str | None:
+        value = self.manifest.get("model_type")
+        if isinstance(value, str):
+            return value
+        training_metadata = self.manifest.get("training_metadata", {})
+        if isinstance(training_metadata, dict):
+            inner = training_metadata.get("model_type")
+            if isinstance(inner, str):
+                return inner
+        return type(self.model).__name__
+
+    def metadata(self) -> MetadataResponse:
+        metrics_summary = self.manifest.get("metrics_summary")
+        if not isinstance(metrics_summary, dict):
+            metrics_summary = {
+                "model": self.metrics.get("model", {}),
+                "governance": self.metrics.get("governance", {}),
+                "calibration": self.metrics.get("calibration", {}),
+            }
+        return MetadataResponse(
+            api_version=__version__,
+            model_run_id=self.model_run_id,
+            model_run_dir=repo_relative(self.model_run_dir) or self.model_run_dir.name,
+            feature_table=repo_relative(self.feature_table_path),
+            dataset_id=self.dataset_id,
+            dataset_scale=self.dataset_scale,
+            model_type=self.model_type,
+            metrics_summary=metrics_summary,
+            feature_count=len(self.feature_columns),
+            feature_columns=self.feature_columns,
+            supported_labels=LABELS,
+            base_row_lookup_enabled=bool(self.feature_lookup),
+        )
+
+    def build_feature_row(self, request: PredictRequest) -> tuple[dict[str, Any], list[str]]:
+        warnings: list[str] = []
+        base_row = None
+        if request.feature_row_id:
+            base_row = self.feature_lookup.get(request.feature_row_id)
+            if base_row is None:
+                warnings.append(f"feature_row_id not found in base lookup: {request.feature_row_id}")
+
+        row: dict[str, Any]
+        has_base_row = base_row is not None
+        if base_row is not None:
+            row = dict(base_row)
+        else:
+            row = {column: None for column in self.feature_columns}
+            if request.feature_row_id:
+                row["feature_row_id"] = request.feature_row_id
+
+        context = normalize_mapping(request.context, self.feature_schema)
+        features = normalize_mapping(request.features, self.feature_schema)
+        changed_keys: set[str] = set()
+
+        for source_name, values in [("context", context), ("features", features)]:
+            for key, value in values.items():
+                row[key] = value
+                changed_keys.add(key)
+                if key not in self.feature_columns and key not in {"feature_row_id"}:
+                    warnings.append(f"{source_name} field is not a model feature and was kept only as metadata: {key}")
+
+        for column in self.feature_columns:
+            row.setdefault(column, None)
+
+        row, completion_warnings = complete_features(
+            row,
+            changed_keys,
+            has_base_row=has_base_row,
+            derive=request.derive,
+        )
+        warnings.extend(completion_warnings)
+
+        if not has_base_row:
+            null_count = sum(1 for column in self.feature_columns if row.get(column) is None)
+            if null_count > len(self.feature_columns) // 3:
+                warnings.append(
+                    f"no base row supplied; {null_count} of {len(self.feature_columns)} model features are null"
+                )
+
+        return row, warnings
+
+    def predict(self, request: PredictRequest) -> PredictResponse:
+        row, warnings = self.build_feature_row(request)
+        record = {
+            column: (np.nan if row.get(column) is None else row.get(column))
+            for column in sorted(set(row) | set(self.feature_columns))
+        }
+        frame = pd.DataFrame([record])
+        model_frame = model_input_frame(frame, self.feature_columns)
+        transformed = self.preprocessor.transform(model_frame)
+        raw_probabilities = probability_frame(self.model, transformed)
+        governance = add_governance_outputs(frame, raw_probabilities)
+        raw = raw_probabilities.iloc[0]
+        post = governance.iloc[0]
+        probabilities = [float(post[f"p_label_{label}"]) for label in LABELS]
+        completed_features = {
+            column: jsonable(row.get(column))
+            for column in self.feature_columns
+            if request.return_completed_features
+        }
+        return PredictResponse(
+            model_run_id=self.model_run_id,
+            feature_row_id=request.feature_row_id or jsonable(row.get("feature_row_id")),
+            predicted_label=int(post["predicted_label"]),
+            p_large_training=float(post["p_large_training"]),
+            severity_score=float(post["severity_score"]),
+            negative_certification_confidence=float(post["negative_certification_confidence"]),
+            integrity_warning=bool(post["integrity_warning"]),
+            capacity_possible=bool(post["capacity_possible"]),
+            min_critical_coverage=float(post["min_critical_coverage"]),
+            probabilities=probabilities,
+            probability_by_label={str(label): float(post[f"p_label_{label}"]) for label in LABELS},
+            raw_probability_by_label={str(label): float(raw[f"p_label_{label}"]) for label in LABELS},
+            top_evidence=split_semicolon(post["top_evidence"]),
+            critical_missing_layers=split_semicolon(post["critical_missing_layers"]),
+            input_warnings=warnings,
+            completed_features=completed_features,
+        )

src/datacenter_verification_api/requirements.txt

@@ -0,0 +1,7 @@
+# Python >= 3.10
+fastapi
+uvicorn[standard]
+pandas
+numpy
+scikit-learn
+joblib

src/datacenter_verification_api/schemas.py

@@ -0,0 +1,80 @@
+"""Pydantic schemas for the live datacenter verification inference API."""
+
+from __future__ import annotations
+
+from typing import Any
+
+from pydantic import BaseModel, ConfigDict, Field
+
+
+class PredictRequest(BaseModel):
+    """One-row prediction request.
+
+    The feature dictionary intentionally accepts arbitrary keys because the
+    public demo exposes only a subset of the model columns and future UI
+    controls may add new fields before the API schema changes.
+    """
+
+    model_config = ConfigDict(extra="allow")
+
+    feature_row_id: str | None = None
+    features: dict[str, Any] = Field(default_factory=dict)
+    context: dict[str, Any] = Field(default_factory=dict)
+    derive: bool = True
+    return_completed_features: bool = False
+
+
+class BatchPredictRequest(BaseModel):
+    requests: list[PredictRequest]
+
+
+class HealthResponse(BaseModel):
+    status: str
+    model_loaded: bool
+    api_version: str
+    model_run_id: str | None = None
+    dataset_id: str | None = None
+    feature_count: int = 0
+    base_row_lookup_enabled: bool = False
+    error: str | None = None
+
+
+class MetadataResponse(BaseModel):
+    api_version: str
+    model_run_id: str
+    model_run_dir: str
+    feature_table: str | None = None
+    dataset_id: str | None = None
+    dataset_scale: str | None = None
+    model_type: str | None = None
+    metrics_summary: dict[str, Any] = Field(default_factory=dict)
+    feature_count: int
+    feature_columns: list[str]
+    supported_labels: list[int]
+    base_row_lookup_enabled: bool
+
+
+class PredictResponse(BaseModel):
+    mode: str = "live_model_inference"
+    model_run_id: str
+    feature_row_id: str | None = None
+    predicted_label: int
+    p_large_training: float
+    severity_score: float
+    negative_certification_confidence: float
+    integrity_warning: bool
+    capacity_possible: bool
+    min_critical_coverage: float
+    probabilities: list[float]
+    probability_by_label: dict[str, float]
+    raw_probability_by_label: dict[str, float] = Field(default_factory=dict)
+    top_evidence: list[str]
+    critical_missing_layers: list[str]
+    input_warnings: list[str] = Field(default_factory=list)
+    completed_features: dict[str, Any] = Field(default_factory=dict)
+
+
+class BatchPredictResponse(BaseModel):
+    mode: str = "live_model_inference_batch"
+    model_run_id: str
+    predictions: list[PredictResponse]

src/datacenter_verification_api/smoke_test.py

@@ -0,0 +1,96 @@
+"""Smoke checks for the live sklearn inference service.
+
+Run from the repository root:
+
+    python -m src.datacenter_verification_api.smoke_test
+"""
+
+from __future__ import annotations
+
+import argparse
+from pathlib import Path
+
+import pandas as pd
+
+from src.datacenter_verification_api.model_service import DEFAULT_MODEL_RUN_DIR, REPO_ROOT, ModelService, resolve_repo_path
+from src.datacenter_verification_api.schemas import PredictRequest
+from src.datacenter_verification_modeling.common import LABELS
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--model-run", type=Path, default=DEFAULT_MODEL_RUN_DIR)
+    parser.add_argument("--tolerance", type=float, default=1e-8)
+    return parser.parse_args()
+
+
+def selected_prediction_rows(predictions: pd.DataFrame) -> pd.DataFrame:
+    rows = []
+    for label in LABELS:
+        part = predictions[predictions["predicted_label"].astype(int) == label]
+        if len(part):
+            rows.append(part.iloc[0])
+    if len(rows) < len(LABELS):
+        for _, row in predictions.head(len(LABELS) - len(rows)).iterrows():
+            rows.append(row)
+    return pd.DataFrame(rows).drop_duplicates("feature_row_id")
+
+
+def main() -> int:
+    args = parse_args()
+    model_run_dir = resolve_repo_path(args.model_run, DEFAULT_MODEL_RUN_DIR)
+    service = ModelService(model_run_dir=model_run_dir, feature_table_path=REPO_ROOT / "data/synthetic_v1/features/window_features_all.csv")
+    predictions_path = model_run_dir / "predictions_all.csv"
+    predictions = pd.read_csv(predictions_path)
+    sample = selected_prediction_rows(predictions)
+
+    checked = 0
+    for _, expected in sample.iterrows():
+        row_id = str(expected["feature_row_id"])
+        actual = service.predict(PredictRequest(feature_row_id=row_id))
+        if actual.predicted_label != int(expected["predicted_label"]):
+            raise AssertionError(
+                f"{row_id}: predicted label mismatch {actual.predicted_label} != {int(expected['predicted_label'])}"
+            )
+        for label in LABELS:
+            expected_probability = float(expected[f"p_label_{label}"])
+            actual_probability = actual.probability_by_label[str(label)]
+            delta = abs(actual_probability - expected_probability)
+            if delta > args.tolerance:
+                raise AssertionError(
+                    f"{row_id}: p_label_{label} mismatch {actual_probability} != {expected_probability} "
+                    f"(delta={delta})"
+                )
+        checked += 1
+
+    edit_row_id = str(sample.iloc[-1]["feature_row_id"])
+    edited = service.predict(
+        PredictRequest(
+            feature_row_id=edit_row_id,
+            features={
+                "o2_max_concurrent_normalized_gpus": 1024,
+                "o2_allocation_duration_hours": 48,
+                "o4_gpu_util_p95": 82,
+            },
+            context={"scope_type": "topology_domain", "window_length_seconds": 3600},
+            derive=True,
+            return_completed_features=True,
+        )
+    )
+    if len(edited.probabilities) != len(LABELS):
+        raise AssertionError("edited prediction did not return five label probabilities")
+    if abs(sum(edited.probabilities) - 1.0) > 1e-8:
+        raise AssertionError("edited prediction probabilities do not sum to 1")
+    if not edited.completed_features:
+        raise AssertionError("edited prediction did not return completed_features")
+
+    print(f"PASS live inference smoke test: matched {checked} exported rows from {predictions_path}")
+    print(
+        "PASS edited prediction schema: "
+        f"row={edit_row_id} label={edited.predicted_label} p_large={edited.p_large_training:.6f}"
+    )
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

src/datacenter_verification_modeling/README.md

@@ -0,0 +1,111 @@
+# datacenter verification modeling
+
+this package trains and evaluates the first public baseline model for the synthetic v0 datacenter training-run verification dataset.
+
+the training unit is one row from:
+
+```text
+data/synthetic_v0/features/window_features_all.csv
+```
+
+the model does not read raw telemetry samples, events, or snapshots directly
+
+## outputs
+
+the default model run is written to:
+
+```text
+data/model_runs/synthetic_v0_baseline/
+```
+
+it contains the fitted calibrated model, preprocessing pipeline, split manifest, feature metadata, predictions, metrics, calibration diagnostics, feature importance, and an evidence audit sample
+
+## leakage controls
+
+rows are split by `episode_id`, not randomly. This is required because the same latent episode appears in multiple adjacent windows. The default split is scenario-stratified at the episode level with seed `20260510`:
+
+```text
+train: 60%
+validation/calibration: 20%
+test: 20%
+```
+
+the supervised model excludes identifiers, direct labels, site id, episode id, raw manifest hashes, and synthetic-only audit columns such as
+`latent_workload_class` and `synthetic_evidence_profile`
+
+## model
+
+the supervised baseline is:
+
+```text
+SimpleImputer + OneHotEncoder preprocessing
+HistGradientBoostingClassifier
+CalibratedClassifierCV on the validation split
+```
+
+the package also includes `rule_baseline.py`, a deterministic evidence-rule baseline that encodes broad study logic:
+
+- capacity below threshold with strong coverage is negative evidence
+- capacity alone does not prove training
+- missing data is not zero activity
+- integrity anomalies are warnings, not positive proof
+- labels 3 and 4 require coherent multi-layer evidence
+
+## commands
+
+train, evaluate, and generate all default artifacts:
+
+```bash
+python src/datacenter_verification_modeling/train_model.py \
+  --features data/synthetic_v0/features/window_features_all.csv \
+  --output data/model_runs/synthetic_v0_baseline \
+  --seed 20260510
+```
+
+Evaluate an existing run:
+
+```bash
+python src/datacenter_verification_modeling/evaluate_model.py \
+  --model-run data/model_runs/synthetic_v0_baseline \
+  --features data/synthetic_v0/features/window_features_all.csv
+```
+
+Generate predictions from an existing run:
+
+```bash
+python src/datacenter_verification_modeling/predict.py \
+  --model-run data/model_runs/synthetic_v0_baseline \
+  --features data/synthetic_v0/features/window_features_all.csv \
+  --output data/model_runs/synthetic_v0_baseline/predictions_all.csv
+```
+
+Prepare split and feature metadata without training:
+
+```bash
+python src/datacenter_verification_modeling/prepare_features.py \
+  --features data/synthetic_v0/features/window_features_all.csv \
+  --output data/model_runs/synthetic_v0_baseline \
+  --seed 20260510
+```
+
+## governance outputs
+
+prediction files include:
+
+- `p_label_0` through `p_label_4`
+- `raw_p_label_0` through `raw_p_label_4`
+- `p_large_training`
+- `severity_score`
+- `capacity_possible`
+- `negative_certification_confidence`
+- `integrity_warning`
+- `critical_missing_layers`
+- `top_evidence`
+
+the `p_label_*` columns are post-processed with the capacity gate; the `raw_p_label_*` columns preserve the calibrated model probabilities before that
+gate
+
+## limitations
+
+This is a synthetic v0 prototype ! Metrics are useful for testing the pipeline, not for deployment claims. Real datacenter use would require real telemetry, controlled drills, operational calibration, and independent review
+

src/datacenter_verification_modeling/__init__.py

@@ -0,0 +1,6 @@
+"""Modeling pipeline for datacenter training-run verification prototypes."""
+
+__all__ = ["__version__"]
+
+__version__ = "0.1.0"
+

src/datacenter_verification_modeling/common.py

@@ -0,0 +1,611 @@
+"""Shared utilities for synthetic datacenter verification modeling baselines."""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import math
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any
+
+import numpy as np
+import pandas as pd
+from sklearn.compose import ColumnTransformer
+from sklearn.impute import SimpleImputer
+from sklearn.pipeline import Pipeline
+from sklearn.preprocessing import OneHotEncoder
+
+
+DEFAULT_SEED = 20260510
+LABELS = [0, 1, 2, 3, 4]
+SPLITS = ["train", "validation", "test"]
+PROB_COLUMNS = [f"p_label_{label}" for label in LABELS]
+RAW_PROB_COLUMNS = [f"raw_p_label_{label}" for label in LABELS]
+
+BASE_EXCLUDED_COLUMNS = [
+    "feature_row_id",
+    "dataset_id",
+    "seed",
+    "site_id",
+    "scope_id_hash",
+    "window_start",
+    "window_end",
+    "episode_id",
+    "label_0_to_4",
+    "label_confidence",
+    "label_reason",
+    "label_source",
+    "raw_input_manifest_hash",
+    "latent_workload_class",
+    "scenario_family",
+    "scenario_variant",
+    "evidence_recipe_id",
+    "counterfactual_group_id",
+    "synthetic_counterfactual_role",
+    "data_quality_regime",
+    "privacy_tier",
+    "collector_profile",
+    "topology_class",
+    "temporal_phase",
+    "synthetic_hard_case_tags",
+    "synthetic_evidence_profile",
+    "capacity_evidence_only",
+    "integrity_evidence_only",
+    "physical_evidence_only",
+]
+
+VERSION_COLUMNS_EXCLUDED = [
+    "feature_pipeline_version",
+    "policy_threshold_version",
+    "hardware_normalization_version",
+]
+
+CRITICAL_COVERAGE_COLUMNS = [
+    "o1_coverage_fraction",
+    "o2_coverage_fraction",
+    "o4_coverage_fraction",
+    "o7_coverage_fraction",
+    "o8_coverage_fraction",
+    "o14_coverage_fraction",
+]
+
+CRITICAL_MISSING_REASON_COLUMNS = {
+    "o1_coverage_fraction": "o1_missing_reason",
+    "o2_coverage_fraction": "o2_missing_reason",
+    "o4_coverage_fraction": "o4_missing_reason",
+    "o7_coverage_fraction": "o7_missing_reason",
+    "o8_coverage_fraction": "o8_missing_reason",
+    "o14_coverage_fraction": "o14_missing_reason",
+}
+
+SELECTED_AUDIT_FEATURES = [
+    "policy_compute_ratio",
+    "o2_max_concurrent_normalized_gpus",
+    "o2_allocation_duration_hours",
+    "o2_gpu_hours_policy_ratio",
+    "o4_gpu_util_p95",
+    "o4_gpu_util_duty_gt_70",
+    "o7_synchronized_fabric_footprint",
+    "o7_collective_periodicity_score",
+    "o8_rack_power_fraction_p95",
+    "o10_runtime_framework_class",
+    "o11_checkpoint_periodicity_score",
+    "o12_signed_ml_logs_present",
+    "o12_declared_parameter_count_b",
+    "o14_min_critical_coverage",
+    "o14_gap_fraction_critical",
+    "o13_confidential_compute_mode_fraction",
+    "o2_elastic_resize_count",
+    "o2_preemption_restart_count",
+    "o2_account_linkage_confidence",
+    "o4_hbm_pressure_duration_fraction",
+    "o4_power_cap_active_fraction",
+    "o7_account_flow_linkage_confidence",
+    "o10_runtime_metadata_confidence",
+    "o11_artifact_write_pattern_score",
+    "o11_dataloader_read_pattern_score",
+    "o12_log_delivery_delay_hours",
+    "o12_log_completeness_fraction",
+    "o4_missing_reason",
+    "o7_missing_reason",
+    "o12_missing_reason",
+]
+
+
+def utc_now_iso() -> str:
+    return datetime.now(timezone.utc).replace(microsecond=0).isoformat().replace("+00:00", "Z")
+
+
+def ensure_dir(path: Path) -> Path:
+    path.mkdir(parents=True, exist_ok=True)
+    return path
+
+
+def to_jsonable(value: Any) -> Any:
+    if isinstance(value, (np.integer,)):
+        return int(value)
+    if isinstance(value, (np.floating,)):
+        if math.isnan(float(value)):
+            return None
+        return float(value)
+    if isinstance(value, (np.bool_,)):
+        return bool(value)
+    if isinstance(value, np.ndarray):
+        return [to_jsonable(item) for item in value.tolist()]
+    if isinstance(value, pd.Series):
+        return [to_jsonable(item) for item in value.tolist()]
+    if isinstance(value, pd.Timestamp):
+        return value.isoformat()
+    if value is pd.NA:
+        return None
+    return value
+
+
+def write_json(path: Path, payload: Any) -> None:
+    path.write_text(json.dumps(payload, indent=2, sort_keys=True, default=to_jsonable) + "\n", encoding="utf-8")
+
+
+def read_json(path: Path) -> Any:
+    return json.loads(path.read_text(encoding="utf-8"))
+
+
+def sha256_file(path: Path) -> str:
+    digest = hashlib.sha256()
+    with path.open("rb") as handle:
+        for chunk in iter(lambda: handle.read(1024 * 1024), b""):
+            digest.update(chunk)
+    return f"sha256:{digest.hexdigest()}"
+
+
+def as_bool(value: Any, default: bool = False) -> bool:
+    if value is None:
+        return default
+    try:
+        if pd.isna(value):
+            return default
+    except TypeError:
+        pass
+    if isinstance(value, (bool, np.bool_)):
+        return bool(value)
+    text = str(value).strip().lower()
+    if text in {"true", "t", "1", "yes", "y"}:
+        return True
+    if text in {"false", "f", "0", "no", "n"}:
+        return False
+    return default
+
+
+def as_float(value: Any, default: float = 0.0) -> float:
+    if value is None:
+        return default
+    try:
+        if pd.isna(value):
+            return default
+    except TypeError:
+        pass
+    try:
+        return float(value)
+    except (TypeError, ValueError):
+        return default
+
+
+def row_float(row: pd.Series, column: str, default: float = 0.0) -> float:
+    return as_float(row[column], default) if column in row else default
+
+
+def row_bool(row: pd.Series, column: str, default: bool = False) -> bool:
+    return as_bool(row[column], default) if column in row else default
+
+
+def load_feature_table(path: Path) -> pd.DataFrame:
+    df = pd.read_csv(path)
+    required = {
+        "feature_row_id",
+        "episode_id",
+        "latent_workload_class",
+        "label_0_to_4",
+        "capacity_possible",
+    }
+    missing = sorted(required - set(df.columns))
+    if missing:
+        raise ValueError(f"feature table missing required columns: {missing}")
+    df["label_0_to_4"] = df["label_0_to_4"].astype(int)
+    return df
+
+
+def derive_dataset_dir(features_path: Path) -> Path:
+    path = features_path.resolve()
+    if path.parent.name == "features":
+        return path.parent.parent
+    return path.parent
+
+
+def split_summary(df: pd.DataFrame) -> dict[str, Any]:
+    summary: dict[str, Any] = {}
+    for split in SPLITS:
+        part = df[df["split"] == split]
+        summary[split] = {
+            "rows": int(len(part)),
+            "episodes": int(part["episode_id"].nunique()),
+            "label_distribution": {
+                str(label): int(count)
+                for label, count in part["label_0_to_4"].value_counts().sort_index().items()
+            },
+            "scenario_distribution": {
+                str(name): int(count)
+                for name, count in part["latent_workload_class"].value_counts().sort_index().items()
+            },
+        }
+    return summary
+
+
+def make_episode_split(df: pd.DataFrame, seed: int = DEFAULT_SEED) -> tuple[pd.DataFrame, dict[str, Any]]:
+    """Create a deterministic scenario-stratified grouped split by episode_id."""
+    episode_df = (
+        df.groupby("episode_id", as_index=False)
+        .agg(
+            latent_workload_class=("latent_workload_class", "first"),
+            label_mode=("label_0_to_4", lambda values: int(values.mode().iloc[0])),
+            label_min=("label_0_to_4", "min"),
+            label_max=("label_0_to_4", "max"),
+            row_count=("feature_row_id", "count"),
+            site_count=("site_id", "nunique") if "site_id" in df.columns else ("episode_id", "count"),
+        )
+        .sort_values(["latent_workload_class", "episode_id"])
+        .reset_index(drop=True)
+    )
+
+    rng = np.random.default_rng(seed)
+    assignments: dict[str, str] = {}
+    scenario_allocations: dict[str, dict[str, int]] = {}
+    for scenario, scenario_rows in episode_df.groupby("latent_workload_class", sort=True):
+        episode_ids = scenario_rows["episode_id"].to_numpy().copy()
+        rng.shuffle(episode_ids)
+        count = len(episode_ids)
+        if count >= 3:
+            validation_count = max(1, round(0.20 * count))
+            test_count = max(1, round(0.20 * count))
+            train_count = count - validation_count - test_count
+            if train_count < 1:
+                train_count = 1
+                if validation_count > test_count:
+                    validation_count -= 1
+                else:
+                    test_count -= 1
+        elif count == 2:
+            train_count, validation_count, test_count = 1, 1, 0
+        else:
+            train_count, validation_count, test_count = 1, 0, 0
+
+        for episode_id in episode_ids[:train_count]:
+            assignments[str(episode_id)] = "train"
+        for episode_id in episode_ids[train_count : train_count + validation_count]:
+            assignments[str(episode_id)] = "validation"
+        for episode_id in episode_ids[train_count + validation_count :]:
+            assignments[str(episode_id)] = "test"
+        scenario_allocations[str(scenario)] = {
+            "episodes": int(count),
+            "train": int(train_count),
+            "validation": int(validation_count),
+            "test": int(test_count),
+        }
+
+    split_df = df.copy()
+    split_df["split"] = split_df["episode_id"].map(assignments)
+    if split_df["split"].isna().any():
+        raise ValueError("episode split assignment failed for some rows")
+
+    episode_assignments = []
+    split_lookup = dict(zip(episode_df["episode_id"], episode_df.index, strict=False))
+    for episode_id, split in sorted(assignments.items()):
+        row = episode_df.iloc[split_lookup[episode_id]]
+        episode_assignments.append(
+            {
+                "episode_id": episode_id,
+                "split": split,
+                "latent_workload_class": row["latent_workload_class"],
+                "label_mode": int(row["label_mode"]),
+                "label_min": int(row["label_min"]),
+                "label_max": int(row["label_max"]),
+                "row_count": int(row["row_count"]),
+            }
+        )
+
+    manifest = {
+        "seed": int(seed),
+        "method": "scenario_stratified_grouped_by_episode_id",
+        "split_fractions_requested": {"train": 0.60, "validation": 0.20, "test": 0.20},
+        "leakage_prevention": "All rows from the same episode_id are assigned to exactly one split.",
+        "scenario_allocations": scenario_allocations,
+        "summary": split_summary(split_df),
+        "episode_assignments": episode_assignments,
+    }
+    return split_df, manifest
+
+
+def apply_split_manifest(df: pd.DataFrame, split_manifest: dict[str, Any]) -> pd.DataFrame:
+    assignments = {
+        item["episode_id"]: item["split"]
+        for item in split_manifest.get("episode_assignments", [])
+        if "episode_id" in item and "split" in item
+    }
+    out = df.copy()
+    out["split"] = out["episode_id"].map(assignments).fillna("unassigned")
+    return out
+
+
+def determine_feature_columns(df: pd.DataFrame) -> tuple[list[str], dict[str, Any]]:
+    requested_exclusions = BASE_EXCLUDED_COLUMNS + VERSION_COLUMNS_EXCLUDED
+    present_exclusions = [column for column in requested_exclusions if column in df.columns]
+    missing_requested = [column for column in requested_exclusions if column not in df.columns]
+    feature_columns = [column for column in df.columns if column not in set(present_exclusions + ["split"])]
+    feature_columns = [column for column in feature_columns if not column.startswith("raw_")]
+    metadata = {
+        "requested_excluded_columns": requested_exclusions,
+        "present_excluded_columns": present_exclusions,
+        "missing_requested_excluded_columns": missing_requested,
+        "notes": [
+            "Identifier, label, direct leakage, site, and synthetic-only audit columns are excluded.",
+            "scope_type and window_length_seconds are retained because they are valid deployment-time context.",
+            "Constant version metadata columns are excluded from the baseline.",
+        ],
+    }
+    return feature_columns, metadata
+
+
+def categorize_feature_columns(df: pd.DataFrame, feature_columns: list[str]) -> dict[str, list[str]]:
+    numeric_columns: list[str] = []
+    categorical_columns: list[str] = []
+    for column in feature_columns:
+        dtype = df[column].dtype
+        if pd.api.types.is_numeric_dtype(dtype) or pd.api.types.is_bool_dtype(dtype):
+            numeric_columns.append(column)
+        else:
+            categorical_columns.append(column)
+    return {"numeric": numeric_columns, "categorical": categorical_columns}
+
+
+def make_preprocessor(df: pd.DataFrame, feature_columns: list[str]) -> ColumnTransformer:
+    types = categorize_feature_columns(df, feature_columns)
+    transformers: list[tuple[str, Pipeline, list[str]]] = []
+    if types["numeric"]:
+        transformers.append(
+            (
+                "numeric",
+                Pipeline(
+                    steps=[
+                        ("imputer", SimpleImputer(strategy="median", add_indicator=True)),
+                    ]
+                ),
+                types["numeric"],
+            )
+        )
+    if types["categorical"]:
+        try:
+            encoder = OneHotEncoder(handle_unknown="ignore", sparse_output=False, dtype=np.float64)
+        except TypeError:  # pragma: no cover - compatibility for older scikit-learn
+            encoder = OneHotEncoder(handle_unknown="ignore", sparse=False, dtype=np.float64)
+        transformers.append(
+            (
+                "categorical",
+                Pipeline(
+                    steps=[
+                        ("imputer", SimpleImputer(strategy="constant", fill_value="__missing__")),
+                        ("onehot", encoder),
+                    ]
+                ),
+                types["categorical"],
+            )
+        )
+    return ColumnTransformer(transformers=transformers, remainder="drop", verbose_feature_names_out=True)
+
+
+def model_input_frame(df: pd.DataFrame, feature_columns: list[str]) -> pd.DataFrame:
+    missing = [column for column in feature_columns if column not in df.columns]
+    if missing:
+        raise ValueError(f"feature table missing model feature columns: {missing}")
+    return df.loc[:, feature_columns].copy()
+
+
+def probability_frame(model: Any, transformed_features: Any, prefix: str = "p_label") -> pd.DataFrame:
+    raw = model.predict_proba(transformed_features)
+    classes = [int(label) for label in model.classes_]
+    data = np.zeros((raw.shape[0], len(LABELS)), dtype=float)
+    for source_index, label in enumerate(classes):
+        if label in LABELS:
+            data[:, LABELS.index(label)] = raw[:, source_index]
+    data = normalize_probability_array(data)
+    return pd.DataFrame(data, columns=[f"{prefix}_{label}" for label in LABELS])
+
+
+def normalize_probability_array(probs: np.ndarray) -> np.ndarray:
+    probs = np.clip(np.asarray(probs, dtype=float), 0.0, 1.0)
+    row_sums = probs.sum(axis=1)
+    empty = row_sums <= 0
+    if np.any(empty):
+        probs[empty, :] = 1.0 / len(LABELS)
+        row_sums = probs.sum(axis=1)
+    return probs / row_sums[:, None]
+
+
+def minimum_critical_coverage(df: pd.DataFrame) -> pd.Series:
+    coverage_values = []
+    for column in CRITICAL_COVERAGE_COLUMNS:
+        if column in df.columns:
+            coverage_values.append(pd.to_numeric(df[column], errors="coerce").fillna(0.0).clip(0.0, 1.0))
+        else:
+            coverage_values.append(pd.Series(np.zeros(len(df)), index=df.index))
+    return pd.concat(coverage_values, axis=1).min(axis=1)
+
+
+def integrity_warning_series(df: pd.DataFrame) -> pd.Series:
+    warnings = pd.Series(False, index=df.index)
+    checks = {
+        "o14_gap_fraction_critical": (0.05, "gt"),
+        "o14_min_critical_coverage": (0.80, "lt"),
+        "o13_attestation_valid_fraction": (0.90, "lt"),
+        "o13_confidential_compute_mode_fraction": (0.50, "gt"),
+        "o14_counter_reset_count": (0.0, "gt"),
+    }
+    for column, (threshold, direction) in checks.items():
+        if column not in df.columns:
+            continue
+        values = pd.to_numeric(df[column], errors="coerce").fillna(0.0)
+        if direction == "gt":
+            warnings |= values > threshold
+        else:
+            warnings |= values < threshold
+    if "o15_unapproved_physical_change_near_window" in df.columns:
+        warnings |= df["o15_unapproved_physical_change_near_window"].map(as_bool)
+    return warnings.astype(bool)
+
+
+def apply_capacity_gate(df: pd.DataFrame, probabilities: pd.DataFrame, high_label_cap: float = 0.02) -> pd.DataFrame:
+    probs = probabilities.loc[:, PROB_COLUMNS].to_numpy(copy=True)
+    if "capacity_possible" not in df.columns:
+        return pd.DataFrame(normalize_probability_array(probs), columns=PROB_COLUMNS, index=probabilities.index)
+    capacity_possible = df["capacity_possible"].map(as_bool).to_numpy()
+    if "o17_external_capacity_conflict_score" in df.columns:
+        external_conflict = pd.to_numeric(df["o17_external_capacity_conflict_score"], errors="coerce").fillna(0.0).to_numpy()
+    else:
+        external_conflict = np.zeros(len(df))
+    gate_mask = (~capacity_possible) & (external_conflict < 0.5)
+    for row_index in np.where(gate_mask)[0]:
+        high = probs[row_index, 2:5].sum()
+        capped_high = min(float(high), high_label_cap)
+        if high > 0:
+            probs[row_index, 2:5] *= capped_high / high
+        low_target = 1.0 - capped_high
+        low = probs[row_index, 0:2].sum()
+        if low > 0:
+            probs[row_index, 0:2] *= low_target / low
+        else:
+            probs[row_index, 0] = low_target
+            probs[row_index, 1] = 0.0
+    return pd.DataFrame(normalize_probability_array(probs), columns=PROB_COLUMNS, index=probabilities.index)
+
+
+def critical_missing_layers_for_row(row: pd.Series) -> str:
+    layers: list[str] = []
+    for coverage_column, missing_column in CRITICAL_MISSING_REASON_COLUMNS.items():
+        coverage = row_float(row, coverage_column, 0.0)
+        reason = str(row[missing_column]) if missing_column in row and not pd.isna(row[missing_column]) else "unknown"
+        if coverage < 0.80 or reason not in {"observed", ""}:
+            observable = coverage_column.split("_", 1)[0].upper()
+            layers.append(f"{observable}:{reason}:coverage={coverage:.2f}")
+    return "; ".join(layers)
+
+
+def top_evidence_for_row(row: pd.Series) -> str:
+    evidence: list[str] = []
+    capacity = row_bool(row, "capacity_possible")
+    external_conflict = row_float(row, "o17_external_capacity_conflict_score", 0.0)
+    allocation_gpus = row_float(row, "o2_max_concurrent_normalized_gpus")
+    allocation_hours = row_float(row, "o2_allocation_duration_hours")
+    gpu_hours_ratio = row_float(row, "o2_gpu_hours_policy_ratio")
+    gpu_util = row_float(row, "o4_gpu_util_p95")
+    tensor = row_float(row, "o4_sm_tensor_active_p95")
+    fabric_footprint = row_float(row, "o7_synchronized_fabric_footprint")
+    fabric_periodicity = row_float(row, "o7_collective_periodicity_score")
+    rack_power = row_float(row, "o8_rack_power_fraction_p95")
+    checkpoint = row_float(row, "o11_checkpoint_periodicity_score")
+    signed_logs = row_bool(row, "o12_signed_ml_logs_present")
+    min_coverage = row_float(row, "o14_min_critical_coverage", 1.0)
+    gap_fraction = row_float(row, "o14_gap_fraction_critical")
+    cc_fraction = row_float(row, "o13_confidential_compute_mode_fraction")
+    runtime = str(row["o10_runtime_framework_class"]) if "o10_runtime_framework_class" in row else ""
+
+    if not capacity and external_conflict < 0.5:
+        evidence.append("capacity below policy threshold")
+    if external_conflict >= 0.5:
+        evidence.append("external capacity conflict")
+    if allocation_gpus >= 512 or gpu_hours_ratio >= 1.0:
+        evidence.append("large allocation")
+    elif allocation_gpus >= 128:
+        evidence.append("moderate allocation")
+    if allocation_hours >= 24:
+        evidence.append("long allocation duration")
+    if gpu_util >= 70 or tensor >= 0.60:
+        evidence.append("high GPU activity")
+    if fabric_footprint >= 512 or fabric_periodicity >= 0.60:
+        evidence.append("synchronized scale-out fabric")
+    if rack_power >= 0.60:
+        evidence.append("power corroboration")
+    if "training" in runtime or "fine_tune" in runtime:
+        evidence.append("training runtime metadata")
+    if checkpoint >= 0.55:
+        evidence.append("checkpoint cadence")
+    if signed_logs:
+        evidence.append("signed ML logs")
+    if min_coverage < 0.80 or gap_fraction > 0.05:
+        evidence.append("low critical coverage")
+    if cc_fraction > 0.50 or str(row.get("o4_missing_reason", "")) == "counter_disabled_by_cc_mode":
+        evidence.append("counter disabled by CC mode")
+    if not evidence:
+        evidence.append("no strong positive evidence")
+    return "; ".join(evidence[:8])
+
+
+def add_governance_outputs(df: pd.DataFrame, raw_probabilities: pd.DataFrame) -> pd.DataFrame:
+    post_probs = apply_capacity_gate(df, raw_probabilities.loc[:, PROB_COLUMNS])
+    out = post_probs.copy()
+    prob_values = out.loc[:, PROB_COLUMNS].to_numpy()
+    predicted = np.asarray(LABELS)[np.argmax(prob_values, axis=1)]
+    out["predicted_label"] = predicted.astype(int)
+    out["p_large_training"] = out["p_label_3"] + out["p_label_4"]
+    out["severity_score"] = sum(label * out[f"p_label_{label}"] for label in LABELS)
+    min_coverage = minimum_critical_coverage(df)
+    out["min_critical_coverage"] = min_coverage
+    out["negative_certification_confidence"] = out["p_label_0"] * min_coverage
+    out["capacity_possible"] = df["capacity_possible"].map(as_bool) if "capacity_possible" in df.columns else False
+    out["integrity_warning"] = integrity_warning_series(df)
+    out["critical_missing_layers"] = df.apply(critical_missing_layers_for_row, axis=1)
+    out["top_evidence"] = df.apply(top_evidence_for_row, axis=1)
+    return out
+
+
+def build_prediction_frame(
+    df: pd.DataFrame,
+    raw_probabilities: pd.DataFrame,
+    governance_probabilities: pd.DataFrame,
+) -> pd.DataFrame:
+    base_columns = [
+        "split",
+        "episode_id",
+        "feature_row_id",
+        "site_id",
+        "scope_type",
+        "scope_id_hash",
+        "window_start",
+        "window_end",
+        "window_length_seconds",
+        "latent_workload_class",
+        "scenario_family",
+        "scenario_variant",
+        "data_quality_regime",
+        "counterfactual_group_id",
+        "synthetic_hard_case_tags",
+        "label_0_to_4",
+    ]
+    present_base_columns = [column for column in base_columns if column in df.columns]
+    out = df.loc[:, present_base_columns].copy()
+    raw = raw_probabilities.loc[:, PROB_COLUMNS].copy()
+    raw.columns = RAW_PROB_COLUMNS
+    out = pd.concat([out.reset_index(drop=True), raw.reset_index(drop=True)], axis=1)
+    governance_columns = PROB_COLUMNS + [
+        "predicted_label",
+        "p_large_training",
+        "severity_score",
+        "capacity_possible",
+        "negative_certification_confidence",
+        "integrity_warning",
+        "critical_missing_layers",
+        "top_evidence",
+        "min_critical_coverage",
+    ]
+    out = pd.concat([out, governance_probabilities.loc[:, governance_columns].reset_index(drop=True)], axis=1)
+    for column in SELECTED_AUDIT_FEATURES:
+        if column in df.columns and column not in out.columns:
+            out[column] = df[column].to_numpy()
+    return out

src/datacenter_verification_modeling/evaluate_model.py

@@ -0,0 +1,704 @@
+"""Evaluate a trained synthetic datacenter verification modeling run."""
+
+from __future__ import annotations
+
+import argparse
+from pathlib import Path
+from typing import Any
+
+import joblib
+import numpy as np
+import pandas as pd
+from sklearn.metrics import (
+    accuracy_score,
+    brier_score_loss,
+    classification_report,
+    confusion_matrix,
+    f1_score,
+    log_loss,
+    precision_recall_fscore_support,
+)
+
+try:
+    from .common import (
+        DEFAULT_SEED,
+        LABELS,
+        PROB_COLUMNS,
+        RAW_PROB_COLUMNS,
+        SELECTED_AUDIT_FEATURES,
+        add_governance_outputs,
+        apply_split_manifest,
+        build_prediction_frame,
+        load_feature_table,
+        minimum_critical_coverage,
+        model_input_frame,
+        probability_frame,
+        read_json,
+        sha256_file,
+        utc_now_iso,
+        write_json,
+    )
+    from .rule_baseline import predict_rule_labels
+except ImportError:  # pragma: no cover - direct script execution
+    from common import (
+        DEFAULT_SEED,
+        LABELS,
+        PROB_COLUMNS,
+        RAW_PROB_COLUMNS,
+        SELECTED_AUDIT_FEATURES,
+        add_governance_outputs,
+        apply_split_manifest,
+        build_prediction_frame,
+        load_feature_table,
+        minimum_critical_coverage,
+        model_input_frame,
+        probability_frame,
+        read_json,
+        sha256_file,
+        utc_now_iso,
+        write_json,
+    )
+    from rule_baseline import predict_rule_labels
+
+
+HARD_FALSE_POSITIVE_SCENARIOS = {
+    "large_batch_inference",
+    "synthetic_data_generation",
+    "hpc_mpi_simulation",
+    "nccl_benchmark",
+    "hardware_burn_in",
+    "storage_rebuild",
+    "large_etl_data_movement",
+    "reserved_but_unused_capacity",
+    "maintenance_window",
+    "large_batch_inference",
+    "model_parallel_inference",
+    "embedding_generation",
+    "synthetic_data_generation_gpu_heavy",
+    "hpc_mpi_collective",
+    "nccl_extended_benchmark",
+    "hardware_burn_in_or_thermal_soak",
+    "storage_rebuild_or_replication",
+    "large_etl_or_data_movement",
+    "distributed_database_or_graph_analytics",
+    "reserved_but_unused_capacity",
+    "maintenance_with_collector_gaps",
+    "multi_tenant_fragmented_nontraining",
+    "capacity_or_integrity_only_warning",
+}
+
+TRAINING_SCENARIOS = {
+    "large_fine_tune",
+    "pretraining",
+    "cloud_reservation_used_for_training",
+    "adversarial_fragmented_training",
+    "underclocked_long_duration_training",
+    "pretraining_standard",
+    "large_fine_tune_standard",
+    "cloud_training_redacted_runtime",
+    "training_without_semantic_logs",
+    "underclocked_energy_capped_training",
+    "elastic_preempted_training",
+    "fragmented_training_linked",
+    "sparse_or_moe_bursty_training",
+    "training_with_low_fabric_high_checkpoint",
+    "training_with_delayed_logs",
+    "multi_stage_training_pipeline",
+}
+
+REQUIRED_OUTPUTS = [
+    "README.md",
+    "manifest.json",
+    "model.joblib",
+    "preprocessing.joblib",
+    "feature_columns.json",
+    "excluded_columns.json",
+    "split_manifest.json",
+    "metrics.json",
+    "calibration_metrics.json",
+    "confusion_matrix.csv",
+    "classification_report.json",
+    "predictions_test.csv",
+    "predictions_all.csv",
+    "feature_importance.csv",
+    "evidence_audit_sample.csv",
+    "validation_summary.md",
+]
+
+
+def predict_for_model_run(model_run_dir: Path, features_path: Path) -> tuple[pd.DataFrame, pd.DataFrame]:
+    df = load_feature_table(features_path)
+    split_manifest_path = model_run_dir / "split_manifest.json"
+    if split_manifest_path.exists():
+        df = apply_split_manifest(df, read_json(split_manifest_path))
+    else:
+        df = df.copy()
+        df["split"] = "unassigned"
+
+    feature_columns = read_json(model_run_dir / "feature_columns.json")
+    preprocessor = joblib.load(model_run_dir / "preprocessing.joblib")
+    model = joblib.load(model_run_dir / "model.joblib")
+
+    model_frame = model_input_frame(df, feature_columns)
+    transformed = preprocessor.transform(model_frame)
+    raw_probabilities = probability_frame(model, transformed)
+    governance = add_governance_outputs(df, raw_probabilities)
+    predictions = build_prediction_frame(df, raw_probabilities, governance)
+    return df, predictions
+
+
+def binary_prf(true_binary: pd.Series | np.ndarray, predicted_binary: pd.Series | np.ndarray) -> dict[str, float]:
+    precision, recall, f1, _ = precision_recall_fscore_support(
+        true_binary.astype(bool),
+        predicted_binary.astype(bool),
+        average="binary",
+        zero_division=0,
+    )
+    return {"precision": float(precision), "recall": float(recall), "f1": float(f1)}
+
+
+def scenario_counts(frame: pd.DataFrame) -> dict[str, int]:
+    if len(frame) == 0:
+        return {}
+    return {str(key): int(value) for key, value in frame["latent_workload_class"].value_counts().sort_values(ascending=False).items()}
+
+
+def calibration_by_label(test_predictions: pd.DataFrame) -> dict[str, Any]:
+    y_true = test_predictions["label_0_to_4"].astype(int).to_numpy()
+    out: dict[str, Any] = {}
+    for label in LABELS:
+        observed = (y_true == label).astype(int)
+        predicted = test_predictions[f"p_label_{label}"].to_numpy()
+        out[str(label)] = {
+            "rows": int(len(test_predictions)),
+            "observed_fraction": float(observed.mean()) if len(observed) else 0.0,
+            "mean_predicted_probability": float(predicted.mean()) if len(predicted) else 0.0,
+            "brier": float(brier_score_loss(observed, predicted)) if len(np.unique(observed)) > 1 else float(np.mean((predicted - observed) ** 2)),
+        }
+    return out
+
+
+def reliability_bins(probabilities: np.ndarray, observed: np.ndarray, bin_count: int = 10) -> tuple[list[dict[str, Any]], float]:
+    bins: list[dict[str, Any]] = []
+    expected_calibration_error = 0.0
+    total = len(probabilities)
+    edges = np.linspace(0.0, 1.0, bin_count + 1)
+    for index in range(bin_count):
+        lower = edges[index]
+        upper = edges[index + 1]
+        if index == bin_count - 1:
+            mask = (probabilities >= lower) & (probabilities <= upper)
+        else:
+            mask = (probabilities >= lower) & (probabilities < upper)
+        count = int(mask.sum())
+        if count:
+            mean_predicted = float(probabilities[mask].mean())
+            observed_fraction = float(observed[mask].mean())
+            expected_calibration_error += (count / total) * abs(mean_predicted - observed_fraction)
+        else:
+            mean_predicted = 0.0
+            observed_fraction = 0.0
+        bins.append(
+            {
+                "bin_index": index,
+                "lower": float(lower),
+                "upper": float(upper),
+                "count": count,
+                "mean_predicted_probability": mean_predicted,
+                "observed_fraction": observed_fraction,
+            }
+        )
+    return bins, float(expected_calibration_error)
+
+
+def compute_calibration_metrics(test_predictions: pd.DataFrame) -> dict[str, Any]:
+    y_true = test_predictions["label_0_to_4"].astype(int).to_numpy()
+    probabilities = test_predictions[PROB_COLUMNS].to_numpy()
+    large_observed = (y_true >= 3).astype(int)
+    large_probability = test_predictions["p_large_training"].to_numpy()
+    bins, ece = reliability_bins(large_probability, large_observed)
+    return {
+        "log_loss": float(log_loss(y_true, probabilities, labels=LABELS)),
+        "brier_large_training": float(brier_score_loss(large_observed, large_probability)),
+        "expected_calibration_error_large_training": ece,
+        "reliability_bins_large_training": bins,
+        "calibration_by_label": calibration_by_label(test_predictions),
+    }
+
+
+def subgroup_metrics(test_predictions: pd.DataFrame, group_column: str) -> list[dict[str, Any]]:
+    if group_column not in test_predictions.columns:
+        return []
+    rows: list[dict[str, Any]] = []
+    for value, part in test_predictions.groupby(group_column, dropna=False):
+        y_true = part["label_0_to_4"].astype(int)
+        y_pred = part["predicted_label"].astype(int)
+        large_true = y_true >= 3
+        large_pred = part["p_large_training"] >= 0.5
+        prf = binary_prf(large_true.to_numpy(), large_pred.to_numpy())
+        rows.append(
+            {
+                "group": str(value),
+                "rows": int(len(part)),
+                "accuracy": float(accuracy_score(y_true, y_pred)),
+                "macro_f1": float(f1_score(y_true, y_pred, labels=LABELS, average="macro", zero_division=0)),
+                "large_training_precision_at_0_5": prf["precision"],
+                "large_training_recall_at_0_5": prf["recall"],
+                "large_training_false_positives": int(((~large_true) & large_pred).sum()),
+                "large_training_false_negatives": int((large_true & (~large_pred)).sum()),
+            }
+        )
+    return sorted(rows, key=lambda item: (-item["rows"], item["group"]))
+
+
+def compute_metrics(test_predictions: pd.DataFrame, test_features: pd.DataFrame) -> tuple[dict[str, Any], dict[str, Any]]:
+    y_true = test_predictions["label_0_to_4"].astype(int)
+    y_pred = test_predictions["predicted_label"].astype(int)
+    probabilities = test_predictions[PROB_COLUMNS].to_numpy()
+    large_true = y_true >= 3
+    large_pred_by_label = y_pred >= 3
+    large_pred_by_threshold = test_predictions["p_large_training"] >= 0.5
+    large_label_prf = binary_prf(large_true.to_numpy(), large_pred_by_label.to_numpy())
+    large_threshold_prf = binary_prf(large_true.to_numpy(), large_pred_by_threshold.to_numpy())
+
+    y_one_hot = np.zeros_like(probabilities)
+    y_one_hot[np.arange(len(y_true)), y_true.to_numpy()] = 1.0
+
+    rule_labels = predict_rule_labels(test_features)
+    rule_large = rule_labels >= 3
+    rule_prf = binary_prf(large_true.to_numpy(), rule_large.to_numpy())
+
+    false_positives = test_predictions[(y_true < 3) & large_pred_by_threshold]
+    false_negatives = test_predictions[(y_true >= 3) & (~large_pred_by_threshold)]
+    high_coverage_label0 = test_predictions[
+        (y_true == 0) & (pd.to_numeric(test_predictions["min_critical_coverage"], errors="coerce").fillna(0.0) >= 0.95)
+    ]
+    label0_missed = high_coverage_label0[high_coverage_label0["predicted_label"] != 0]
+
+    metrics: dict[str, Any] = {
+        "dataset": {
+            "test_rows": int(len(test_predictions)),
+            "test_episodes": int(test_predictions["episode_id"].nunique()),
+            "test_label_distribution": {
+                str(label): int(count) for label, count in y_true.value_counts().sort_index().items()
+            },
+        },
+        "model": {
+            "accuracy": float(accuracy_score(y_true, y_pred)),
+            "macro_f1": float(f1_score(y_true, y_pred, labels=LABELS, average="macro", zero_division=0)),
+            "weighted_f1": float(f1_score(y_true, y_pred, labels=LABELS, average="weighted", zero_division=0)),
+            "log_loss": float(log_loss(y_true, probabilities, labels=LABELS)),
+            "brier_multiclass_mean": float(np.mean((probabilities - y_one_hot) ** 2)),
+        },
+        "governance": {
+            "label_3_4_predicted_label": large_label_prf,
+            "p_large_training_threshold_0_5": large_threshold_prf,
+            "false_positive_scenarios_at_0_5": scenario_counts(false_positives),
+            "false_negative_scenarios_at_0_5": scenario_counts(false_negatives),
+            "label_0_missed_under_high_coverage": {
+                "rows": int(len(high_coverage_label0)),
+                "missed_rows": int(len(label0_missed)),
+                "missed_rate": float(len(label0_missed) / len(high_coverage_label0)) if len(high_coverage_label0) else 0.0,
+            },
+        },
+        "rule_baseline": {
+            "accuracy": float(accuracy_score(y_true, rule_labels)),
+            "macro_f1": float(f1_score(y_true, rule_labels, labels=LABELS, average="macro", zero_division=0)),
+            "weighted_f1": float(f1_score(y_true, rule_labels, labels=LABELS, average="weighted", zero_division=0)),
+            "label_3_4_predicted_label": rule_prf,
+        },
+        "subgroup_metrics": {
+            column: subgroup_metrics(test_predictions, column)
+            for column in [
+                "latent_workload_class",
+                "scenario_family",
+                "data_quality_regime",
+                "temporal_phase",
+                "site_id",
+                "window_length_seconds",
+                "o4_missing_reason",
+                "o7_missing_reason",
+                "o12_missing_reason",
+                "capacity_possible",
+                "integrity_warning",
+            ]
+        },
+    }
+    calibration = compute_calibration_metrics(test_predictions)
+    metrics["calibration"] = {
+        "log_loss": calibration["log_loss"],
+        "brier_large_training": calibration["brier_large_training"],
+        "expected_calibration_error_large_training": calibration["expected_calibration_error_large_training"],
+    }
+    return metrics, calibration
+
+
+def write_confusion_matrix(path: Path, test_predictions: pd.DataFrame) -> None:
+    y_true = test_predictions["label_0_to_4"].astype(int)
+    y_pred = test_predictions["predicted_label"].astype(int)
+    matrix = confusion_matrix(y_true, y_pred, labels=LABELS)
+    frame = pd.DataFrame(matrix, index=[f"true_{label}" for label in LABELS], columns=[f"pred_{label}" for label in LABELS])
+    frame.to_csv(path, index_label="true_label")
+
+
+def write_classification_report(path: Path, test_predictions: pd.DataFrame) -> None:
+    report = classification_report(
+        test_predictions["label_0_to_4"].astype(int),
+        test_predictions["predicted_label"].astype(int),
+        labels=LABELS,
+        output_dict=True,
+        zero_division=0,
+    )
+    write_json(path, report)
+
+
+def compute_feature_importance(
+    model_run_dir: Path,
+    test_features: pd.DataFrame,
+    test_predictions: pd.DataFrame,
+    seed: int = DEFAULT_SEED,
+    repeats: int = 3,
+) -> pd.DataFrame:
+    feature_columns = read_json(model_run_dir / "feature_columns.json")
+    preprocessor = joblib.load(model_run_dir / "preprocessing.joblib")
+    model = joblib.load(model_run_dir / "model.joblib")
+    x_test = model_input_frame(test_features, feature_columns)
+    y_true = test_predictions["label_0_to_4"].astype(int)
+    baseline_pred = np.asarray(LABELS)[
+        np.argmax(probability_frame(model, preprocessor.transform(x_test))[PROB_COLUMNS].to_numpy(), axis=1)
+    ]
+    baseline_score = f1_score(y_true, baseline_pred, labels=LABELS, average="macro", zero_division=0)
+    rng = np.random.default_rng(seed)
+    rows: list[dict[str, Any]] = []
+    for column in feature_columns:
+        scores: list[float] = []
+        if x_test[column].nunique(dropna=False) <= 1:
+            importances = [0.0] * repeats
+        else:
+            importances = []
+            for _ in range(repeats):
+                permuted = x_test.copy()
+                values = permuted[column].to_numpy(copy=True)
+                rng.shuffle(values)
+                permuted[column] = values
+                permuted_pred = np.asarray(LABELS)[
+                    np.argmax(
+                        probability_frame(model, preprocessor.transform(permuted))[PROB_COLUMNS].to_numpy(),
+                        axis=1,
+                    )
+                ]
+                score = f1_score(y_true, permuted_pred, labels=LABELS, average="macro", zero_division=0)
+                scores.append(float(score))
+                importances.append(float(baseline_score - score))
+        rows.append(
+            {
+                "feature": column,
+                "importance_mean": float(np.mean(importances)),
+                "importance_std": float(np.std(importances)),
+                "baseline_macro_f1": float(baseline_score),
+                "permuted_macro_f1_mean": float(np.mean(scores)) if scores else float(baseline_score),
+            }
+        )
+    return pd.DataFrame(rows).sort_values(["importance_mean", "feature"], ascending=[False, True])
+
+
+def evidence_audit_sample(test_predictions: pd.DataFrame, min_rows: int = 50) -> pd.DataFrame:
+    selected_indices: set[int] = set()
+    samples: list[pd.DataFrame] = []
+
+    def add_sample(name: str, mask: pd.Series, count: int, sort_column: str | None = None, ascending: bool = False) -> None:
+        candidates = test_predictions[mask & (~test_predictions.index.isin(selected_indices))].copy()
+        if sort_column and sort_column in candidates.columns:
+            candidates = candidates.sort_values(sort_column, ascending=ascending)
+        candidates = candidates.head(count)
+        if len(candidates):
+            selected_indices.update(int(index) for index in candidates.index)
+            candidates["audit_category"] = name
+            samples.append(candidates)
+
+    true_label = test_predictions["label_0_to_4"].astype(int)
+    predicted_label = test_predictions["predicted_label"].astype(int)
+    large_pred = test_predictions["p_large_training"] >= 0.5
+    add_sample("correct_label_0", (true_label == 0) & (predicted_label == 0), 10, "negative_certification_confidence", False)
+    add_sample(
+        "correct_label_2_hard_false_positive",
+        (true_label == 2)
+        & (predicted_label == 2)
+        & test_predictions["latent_workload_class"].isin(HARD_FALSE_POSITIVE_SCENARIOS),
+        10,
+        "p_large_training",
+        False,
+    )
+    add_sample("correct_label_3_4_likely_training", (true_label >= 3) & (predicted_label >= 3), 10, "p_large_training", False)
+    add_sample("false_positive_large_training", (true_label < 3) & large_pred, 10, "p_large_training", False)
+    add_sample("false_negative_large_training", (true_label >= 3) & (~large_pred), 10, "p_large_training", True)
+
+    if samples:
+        sample = pd.concat(samples, axis=0)
+    else:
+        sample = test_predictions.iloc[0:0].copy()
+    if len(sample) < min_rows:
+        filler = test_predictions[~test_predictions.index.isin(selected_indices)].copy()
+        filler["uncertainty_distance"] = (filler["p_large_training"] - 0.5).abs()
+        filler = filler.sort_values(["uncertainty_distance", "severity_score"], ascending=[True, False]).head(min_rows - len(sample))
+        filler["audit_category"] = "high_uncertainty_or_coverage_edge"
+        sample = pd.concat([sample, filler], axis=0)
+
+    columns = [
+        "audit_category",
+        "feature_row_id",
+        "split",
+        "latent_workload_class",
+        "label_0_to_4",
+        "predicted_label",
+        "p_large_training",
+        "severity_score",
+        "top_evidence",
+        "critical_missing_layers",
+        "integrity_warning",
+    ]
+    columns += [column for column in SELECTED_AUDIT_FEATURES if column in sample.columns]
+    return sample.loc[:, [column for column in columns if column in sample.columns]].rename(columns={"label_0_to_4": "true_label"})
+
+
+def top_scenarios_text(scenarios: dict[str, int], limit: int = 5) -> str:
+    if not scenarios:
+        return "none"
+    return ", ".join(f"{name}: {count}" for name, count in list(scenarios.items())[:limit])
+
+
+def write_run_readme(model_run_dir: Path, features_path: Path, metrics: dict[str, Any], calibration: dict[str, Any]) -> None:
+    model_metrics = metrics["model"]
+    governance = metrics["governance"]
+    split_manifest = read_json(model_run_dir / "split_manifest.json")
+    dataset_dir = features_path.parent.parent if features_path.parent.name == "features" else features_path.parent
+    dataset_name = dataset_dir.name
+    model_run_name = model_run_dir.name
+    readme = f"""# {model_run_name} model run
+
+This directory contains a public runnable baseline for the `{dataset_name}` datacenter training-run verification dataset.
+
+## Dataset
+
+- Feature table: `{features_path}`
+- Rows: {sum(split['rows'] for split in split_manifest['summary'].values())}
+- Episode split: grouped by `episode_id`, scenario-stratified, seed `{split_manifest['seed']}`
+- Split rows: train {split_manifest['summary']['train']['rows']}, validation {split_manifest['summary']['validation']['rows']}, test {split_manifest['summary']['test']['rows']}
+- Split episodes: train {split_manifest['summary']['train']['episodes']}, validation {split_manifest['summary']['validation']['episodes']}, test {split_manifest['summary']['test']['episodes']}
+
+## Model
+
+- Supervised model: calibrated scikit-learn histogram gradient boosting classifier
+- Calibration: validation split only, held-out test evaluated once
+- Rule baseline: deterministic evidence rules in `src/datacenter_verification_modeling/rule_baseline.py`
+- Leakage exclusions: identifiers, labels, site id, episode id, raw manifest hash, scenario metadata, counterfactual metadata, and synthetic-only audit columns
+
+## Headline Test Metrics
+
+- Accuracy: {model_metrics['accuracy']:.4f}
+- Macro F1: {model_metrics['macro_f1']:.4f}
+- Weighted F1: {model_metrics['weighted_f1']:.4f}
+- Log loss: {model_metrics['log_loss']:.4f}
+- Label 3/4 precision by predicted label: {governance['label_3_4_predicted_label']['precision']:.4f}
+- Label 3/4 recall by predicted label: {governance['label_3_4_predicted_label']['recall']:.4f}
+- `p_large_training >= 0.5` precision: {governance['p_large_training_threshold_0_5']['precision']:.4f}
+- `p_large_training >= 0.5` recall: {governance['p_large_training_threshold_0_5']['recall']:.4f}
+- Rule baseline macro F1: {metrics['rule_baseline']['macro_f1']:.4f}
+
+## Error Scenarios
+
+- Largest false-positive scenarios at `p_large_training >= 0.5`: {top_scenarios_text(governance['false_positive_scenarios_at_0_5'])}
+- Largest false-negative scenarios at `p_large_training >= 0.5`: {top_scenarios_text(governance['false_negative_scenarios_at_0_5'])}
+
+## Calibration
+
+- Brier score for `p_large_training`: {calibration['brier_large_training']:.4f}
+- Expected calibration error for `p_large_training`: {calibration['expected_calibration_error_large_training']:.4f}
+
+## Reproduce
+
+```bash
+python src/datacenter_verification_modeling/train_model.py \\
+  --features {features_path} \\
+  --output {model_run_dir} \\
+  --seed {split_manifest['seed']}
+```
+
+```bash
+python src/datacenter_verification_modeling/evaluate_model.py \\
+  --model-run {model_run_dir} \\
+  --features {features_path}
+```
+
+```bash
+python src/datacenter_verification_modeling/predict.py \\
+  --model-run {model_run_dir} \\
+  --features {features_path} \\
+  --output {model_run_dir / 'predictions_all.csv'}
+```
+
+## Limitations
+
+- This model is trained on synthetic data only.
+- Performance numbers are not real-world deployment claims.
+- Adjacent windows are correlated, so group splitting by `episode_id` is mandatory.
+- Synthetic labels are generated from rules and latent scenarios, so the model may learn generator assumptions.
+- Real datacenter deployment would require calibration on real telemetry and controlled drills.
+- The model should assist audit triage; it should not be treated as sole proof of a violation.
+"""
+    (model_run_dir / "README.md").write_text(readme, encoding="utf-8")
+
+
+def write_validation_summary(
+    model_run_dir: Path,
+    features_path: Path,
+    metrics: dict[str, Any],
+    validation_status: list[dict[str, Any]] | None,
+) -> None:
+    if validation_status is None and (model_run_dir / "manifest.json").exists():
+        existing_manifest = read_json(model_run_dir / "manifest.json")
+        validation_status = existing_manifest.get("validation_status") or None
+    required_rows = []
+    for filename in REQUIRED_OUTPUTS:
+        path = model_run_dir / filename
+        exists = path.exists() or filename == "validation_summary.md"
+        required_rows.append(f"- `{filename}`: {'present' if exists else 'missing'}")
+    validation_lines = []
+    if validation_status:
+        for item in validation_status:
+            validation_lines.append(
+                f"- `{item['name']}`: return code {item['returncode']}; "
+                f"{'PASS' if item['returncode'] == 0 else 'FAIL'}"
+            )
+    else:
+        validation_lines.append("- Dataset validation was not rerun by this evaluation command; see `manifest.json` for any recorded training validation status.")
+
+    model_metrics = metrics["model"]
+    governance = metrics["governance"]
+    text = f"""# Validation Summary
+
+Feature table: `{features_path}`
+
+## Dataset Validation Before Training
+
+{chr(10).join(validation_lines)}
+
+## Required Artifacts
+
+{chr(10).join(required_rows)}
+
+## Test Metrics
+
+- Accuracy: {model_metrics['accuracy']:.4f}
+- Macro F1: {model_metrics['macro_f1']:.4f}
+- Label 3/4 precision: {governance['label_3_4_predicted_label']['precision']:.4f}
+- Label 3/4 recall: {governance['label_3_4_predicted_label']['recall']:.4f}
+- `p_large_training >= 0.5` precision: {governance['p_large_training_threshold_0_5']['precision']:.4f}
+- `p_large_training >= 0.5` recall: {governance['p_large_training_threshold_0_5']['recall']:.4f}
+
+## Governance Checks
+
+- Capacity gate applied to post-processed probabilities.
+- Negative certification confidence is `p_label_0 * min_critical_coverage`.
+- Integrity warnings are reported separately from positive training evidence.
+- Raw model probabilities are retained as `raw_p_label_0` through `raw_p_label_4`.
+"""
+    (model_run_dir / "validation_summary.md").write_text(text, encoding="utf-8")
+
+
+def write_manifest(
+    model_run_dir: Path,
+    features_path: Path,
+    metrics: dict[str, Any],
+    calibration: dict[str, Any],
+    validation_status: list[dict[str, Any]] | None,
+    training_metadata: dict[str, Any] | None,
+) -> None:
+    existing: dict[str, Any] = {}
+    manifest_path = model_run_dir / "manifest.json"
+    if manifest_path.exists():
+        existing = read_json(manifest_path)
+    artifact_hashes = {
+        filename: sha256_file(model_run_dir / filename)
+        for filename in REQUIRED_OUTPUTS
+        if (model_run_dir / filename).exists() and filename != "manifest.json"
+    }
+    manifest = {
+        **existing,
+        "created_or_updated_at": utc_now_iso(),
+        "model_run_id": model_run_dir.name,
+        "features_path": str(features_path),
+        "model_type": "CalibratedClassifierCV over HistGradientBoostingClassifier",
+        "calibration_method": existing.get("calibration_method", "sigmoid_on_validation_split"),
+        "metrics_summary": {
+            "accuracy": metrics["model"]["accuracy"],
+            "macro_f1": metrics["model"]["macro_f1"],
+            "weighted_f1": metrics["model"]["weighted_f1"],
+            "label_3_4_precision": metrics["governance"]["label_3_4_predicted_label"]["precision"],
+            "label_3_4_recall": metrics["governance"]["label_3_4_predicted_label"]["recall"],
+            "p_large_training_precision_at_0_5": metrics["governance"]["p_large_training_threshold_0_5"]["precision"],
+            "p_large_training_recall_at_0_5": metrics["governance"]["p_large_training_threshold_0_5"]["recall"],
+            "brier_large_training": calibration["brier_large_training"],
+            "ece_large_training": calibration["expected_calibration_error_large_training"],
+        },
+        "validation_status": validation_status or existing.get("validation_status", []),
+        "training_metadata": training_metadata or existing.get("training_metadata", {}),
+        "required_outputs": REQUIRED_OUTPUTS,
+        "artifact_hashes": artifact_hashes,
+        "limitations": [
+            "Trained on synthetic data only.",
+            "Performance numbers are not real-world deployment claims.",
+            "Episode-level group splitting is required because adjacent windows are correlated.",
+            "Synthetic labels encode generator assumptions.",
+            "Real deployment requires real telemetry calibration and controlled drills.",
+        ],
+    }
+    write_json(manifest_path, manifest)
+
+
+def evaluate_model_run(
+    model_run_dir: Path,
+    features_path: Path,
+    validation_status: list[dict[str, Any]] | None = None,
+    training_metadata: dict[str, Any] | None = None,
+) -> dict[str, Any]:
+    model_run_dir.mkdir(parents=True, exist_ok=True)
+    feature_df, predictions = predict_for_model_run(model_run_dir, features_path)
+    predictions.to_csv(model_run_dir / "predictions_all.csv", index=False)
+    test_predictions = predictions[predictions["split"] == "test"].copy()
+    test_features = feature_df[feature_df["split"] == "test"].copy()
+    if test_predictions.empty:
+        raise ValueError("no test split rows available for evaluation")
+    test_predictions.to_csv(model_run_dir / "predictions_test.csv", index=False)
+
+    metrics, calibration = compute_metrics(test_predictions, test_features)
+    write_json(model_run_dir / "metrics.json", metrics)
+    write_json(model_run_dir / "calibration_metrics.json", calibration)
+    write_confusion_matrix(model_run_dir / "confusion_matrix.csv", test_predictions)
+    write_classification_report(model_run_dir / "classification_report.json", test_predictions)
+    importance = compute_feature_importance(model_run_dir, test_features, test_predictions)
+    importance.to_csv(model_run_dir / "feature_importance.csv", index=False)
+    audit = evidence_audit_sample(test_predictions)
+    audit.to_csv(model_run_dir / "evidence_audit_sample.csv", index=False)
+    write_run_readme(model_run_dir, features_path, metrics, calibration)
+    write_manifest(model_run_dir, features_path, metrics, calibration, validation_status, training_metadata)
+    write_validation_summary(model_run_dir, features_path, metrics, validation_status)
+    write_manifest(model_run_dir, features_path, metrics, calibration, validation_status, training_metadata)
+    return metrics
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--model-run", type=Path, required=True)
+    parser.add_argument("--features", type=Path, required=True)
+    args = parser.parse_args(argv)
+    metrics = evaluate_model_run(args.model_run, args.features)
+    print(f"accuracy: {metrics['model']['accuracy']:.4f}")
+    print(f"macro_f1: {metrics['model']['macro_f1']:.4f}")
+    print(f"label_3_4_precision: {metrics['governance']['label_3_4_predicted_label']['precision']:.4f}")
+    print(f"label_3_4_recall: {metrics['governance']['label_3_4_predicted_label']['recall']:.4f}")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

src/datacenter_verification_modeling/evaluate_stress_splits.py

@@ -0,0 +1,366 @@
+"""Evaluate stress splits for synthetic datacenter verification datasets."""
+
+from __future__ import annotations
+
+import argparse
+from collections import Counter
+from pathlib import Path
+from typing import Any
+
+import numpy as np
+import pandas as pd
+from sklearn.ensemble import HistGradientBoostingClassifier
+from sklearn.metrics import accuracy_score, f1_score, log_loss, precision_recall_fscore_support
+
+try:
+    from .common import (
+        DEFAULT_SEED,
+        LABELS,
+        PROB_COLUMNS,
+        determine_feature_columns,
+        ensure_dir,
+        load_feature_table,
+        make_episode_split,
+        make_preprocessor,
+        model_input_frame,
+        probability_frame,
+        write_json,
+    )
+except ImportError:  # pragma: no cover - direct script execution
+    from common import (
+        DEFAULT_SEED,
+        LABELS,
+        PROB_COLUMNS,
+        determine_feature_columns,
+        ensure_dir,
+        load_feature_table,
+        make_episode_split,
+        make_preprocessor,
+        model_input_frame,
+        probability_frame,
+        write_json,
+    )
+
+
+FAMILY_HOLDOUTS = [
+    "underclocked_energy_capped_training",
+    "fragmented_training_linked",
+    "multi_tenant_fragmented_nontraining",
+    "model_parallel_inference",
+    "hpc_mpi_collective",
+]
+
+SOURCE_ABLATIONS = {
+    "source_ablation_drop_fabric": ["o6_", "o7_", "fabric_telemetry_trust_level"],
+    "source_ablation_drop_runtime_and_ml_logs": ["o10_", "o12_"],
+    "source_ablation_drop_gpu_telemetry": ["o4_", "o5_", "gpu_telemetry_trust_level"],
+    "source_ablation_drop_power": ["o8_", "o9_", "power_meter_trust_level"],
+    "source_ablation_drop_storage": ["o11_"],
+}
+
+
+def _base_classifier(seed: int, max_iter: int) -> HistGradientBoostingClassifier:
+    kwargs: dict[str, Any] = {
+        "learning_rate": 0.06,
+        "max_iter": max_iter,
+        "max_leaf_nodes": 31,
+        "l2_regularization": 0.04,
+        "early_stopping": True,
+        "validation_fraction": 0.15,
+        "random_state": seed,
+    }
+    try:
+        return HistGradientBoostingClassifier(class_weight="balanced", **kwargs)
+    except TypeError:  # pragma: no cover
+        return HistGradientBoostingClassifier(**kwargs)
+
+
+def _binary_prf(y_true: np.ndarray, p_large: np.ndarray) -> dict[str, float]:
+    precision, recall, f1, _ = precision_recall_fscore_support(
+        y_true >= 3,
+        p_large >= 0.5,
+        average="binary",
+        zero_division=0,
+    )
+    return {"precision": float(precision), "recall": float(recall), "f1": float(f1)}
+
+
+def _scenario_counts(df: pd.DataFrame, mask: pd.Series, scenario_column: str) -> dict[str, int]:
+    if not mask.any():
+        return {}
+    return {str(key): int(value) for key, value in df.loc[mask, scenario_column].value_counts().sort_values(ascending=False).items()}
+
+
+def _middle_probability_bins(p_large: np.ndarray) -> dict[str, int]:
+    return {
+        "0.1_to_0.3": int(((p_large >= 0.1) & (p_large < 0.3)).sum()),
+        "0.3_to_0.7": int(((p_large >= 0.3) & (p_large < 0.7)).sum()),
+        "0.7_to_0.9": int(((p_large >= 0.7) & (p_large < 0.9)).sum()),
+    }
+
+
+def _fit_predict_split(
+    df: pd.DataFrame,
+    train_mask: pd.Series,
+    test_mask: pd.Series,
+    feature_columns: list[str],
+    seed: int,
+    max_iter: int,
+) -> tuple[pd.DataFrame, dict[str, Any]]:
+    train_df = df[train_mask].copy()
+    test_df = df[test_mask].copy()
+    if train_df.empty or test_df.empty:
+        raise ValueError("stress split has empty train or test set")
+    if train_df["label_0_to_4"].nunique() < 2:
+        raise ValueError("stress split train set has fewer than two labels")
+
+    preprocessor = make_preprocessor(train_df, feature_columns)
+    x_train = preprocessor.fit_transform(model_input_frame(train_df, feature_columns))
+    y_train = train_df["label_0_to_4"].astype(int).to_numpy()
+    x_test = preprocessor.transform(model_input_frame(test_df, feature_columns))
+    y_test = test_df["label_0_to_4"].astype(int).to_numpy()
+
+    model = _base_classifier(seed, max_iter)
+    model.fit(x_train, y_train)
+    probabilities = probability_frame(model, x_test)
+    p_values = probabilities[PROB_COLUMNS].to_numpy()
+    y_pred = np.asarray(LABELS)[np.argmax(p_values, axis=1)]
+    p_large = probabilities["p_label_3"].to_numpy() + probabilities["p_label_4"].to_numpy()
+    out = test_df[["feature_row_id", "episode_id", "label_0_to_4"]].copy()
+    for column in ["scenario_family", "latent_workload_class", "site_id", "window_start"]:
+        if column in test_df.columns:
+            out[column] = test_df[column].to_numpy()
+    out["predicted_label"] = y_pred
+    out["p_large_training"] = p_large
+    for column in PROB_COLUMNS:
+        out[column] = probabilities[column].to_numpy()
+
+    large_true = y_test >= 3
+    large_pred = p_large >= 0.5
+    scenario_column = "scenario_family" if "scenario_family" in test_df.columns else "latent_workload_class"
+    metrics = {
+        "train_rows": int(len(train_df)),
+        "test_rows": int(len(test_df)),
+        "train_episodes": int(train_df["episode_id"].nunique()),
+        "test_episodes": int(test_df["episode_id"].nunique()),
+        "test_label_distribution": {str(k): int(v) for k, v in Counter(y_test).items()},
+        "accuracy": float(accuracy_score(y_test, y_pred)),
+        "macro_f1": float(f1_score(y_test, y_pred, labels=LABELS, average="macro", zero_division=0)),
+        "large_training": _binary_prf(y_test, p_large),
+        "large_training_false_positive_count": int(((~large_true) & large_pred).sum()),
+        "large_training_false_negative_count": int((large_true & (~large_pred)).sum()),
+        "false_positive_scenarios": _scenario_counts(test_df, pd.Series((~large_true) & large_pred, index=test_df.index), scenario_column),
+        "false_negative_scenarios": _scenario_counts(test_df, pd.Series(large_true & (~large_pred), index=test_df.index), scenario_column),
+        "middle_probability_bins": _middle_probability_bins(p_large),
+    }
+    try:
+        metrics["log_loss"] = float(log_loss(y_test, p_values, labels=LABELS))
+    except ValueError:
+        metrics["log_loss"] = None
+    return out, metrics
+
+
+def _drop_features(feature_columns: list[str], prefixes: list[str]) -> list[str]:
+    dropped = []
+    for column in feature_columns:
+        if any(column == prefix or column.startswith(prefix) for prefix in prefixes):
+            continue
+        dropped.append(column)
+    return dropped
+
+
+def evaluate_stress_splits(features_path: Path, output_dir: Path, seed: int = DEFAULT_SEED, max_iter: int = 120) -> dict[str, Any]:
+    ensure_dir(output_dir)
+    df = load_feature_table(features_path)
+    scenario_column = "scenario_family" if "scenario_family" in df.columns else "latent_workload_class"
+    feature_columns, exclusion_meta = determine_feature_columns(df.copy())
+    metrics: dict[str, Any] = {
+        "features_path": str(features_path),
+        "seed": int(seed),
+        "max_iter": int(max_iter),
+        "scenario_column": scenario_column,
+        "model_feature_count": int(len(feature_columns)),
+        "exclusion_metadata": exclusion_meta,
+        "splits": {},
+    }
+
+    split_df, split_manifest = make_episode_split(df, seed=seed)
+    random_train = split_df["split"].isin(["train", "validation"])
+    random_test = split_df["split"] == "test"
+    _, metrics["splits"]["episode_grouped_random"] = _fit_predict_split(
+        split_df, random_train, random_test, feature_columns, seed, max_iter
+    )
+
+    sorted_times = pd.to_datetime(df["window_start"], utc=True)
+    cutoff = sorted_times.quantile(0.80)
+    _, metrics["splits"]["time_holdout"] = _fit_predict_split(
+        df, sorted_times < cutoff, sorted_times >= cutoff, feature_columns, seed + 1, max_iter
+    )
+
+    site_results = {}
+    for idx, site_id in enumerate(sorted(df["site_id"].dropna().unique())):
+        test_mask = df["site_id"] == site_id
+        train_mask = ~test_mask
+        try:
+            _, site_results[str(site_id)] = _fit_predict_split(df, train_mask, test_mask, feature_columns, seed + 10 + idx, max_iter)
+        except ValueError as exc:
+            site_results[str(site_id)] = {"error": str(exc)}
+    metrics["splits"]["site_holdout"] = site_results
+
+    family_results = {}
+    for idx, family in enumerate(FAMILY_HOLDOUTS):
+        if family not in set(df[scenario_column].astype(str)):
+            family_results[family] = {"error": "family not present"}
+            continue
+        test_mask = df[scenario_column].astype(str) == family
+        train_mask = ~test_mask
+        try:
+            _, family_results[family] = _fit_predict_split(df, train_mask, test_mask, feature_columns, seed + 30 + idx, max_iter)
+        except ValueError as exc:
+            family_results[family] = {"error": str(exc)}
+    metrics["splits"]["scenario_family_holdout"] = family_results
+
+    ablation_results = {}
+    for idx, (name, prefixes) in enumerate(SOURCE_ABLATIONS.items()):
+        ablated_features = _drop_features(feature_columns, prefixes)
+        try:
+            _, ablation_results[name] = _fit_predict_split(
+                split_df, random_train, random_test, ablated_features, seed + 50 + idx, max_iter
+            )
+            ablation_results[name]["feature_count"] = int(len(ablated_features))
+            ablation_results[name]["dropped_prefixes"] = prefixes
+        except ValueError as exc:
+            ablation_results[name] = {"error": str(exc)}
+    metrics["splits"]["source_ablations"] = ablation_results
+
+    write_json(output_dir / "stress_metrics.json", metrics)
+    write_stress_summary(output_dir / "stress_summary.md", metrics)
+    update_model_run_readme_with_stress(output_dir, metrics)
+    return metrics
+
+
+def _split_line(name: str, item: dict[str, Any]) -> str:
+    if "error" in item:
+        return f"- {name}: ERROR: {item['error']}"
+    return (
+        f"- {name}: rows={item['test_rows']}, macro F1={item['macro_f1']:.3f}, "
+        f"large precision={item['large_training']['precision']:.3f}, "
+        f"large recall={item['large_training']['recall']:.3f}, "
+        f"FP={item['large_training_false_positive_count']}, FN={item['large_training_false_negative_count']}, "
+        f"middle p bins={item['middle_probability_bins']}"
+    )
+
+
+def write_stress_summary(path: Path, metrics: dict[str, Any]) -> None:
+    splits = metrics["splits"]
+    lines = [
+        "# Synthetic v1 Stress Summary",
+        "",
+        f"Feature table: `{metrics['features_path']}`",
+        f"Seed: `{metrics['seed']}`",
+        f"Lightweight stress model max_iter: `{metrics['max_iter']}`",
+        "",
+        "## Core Splits",
+        "",
+        _split_line("episode_grouped_random", splits["episode_grouped_random"]),
+        _split_line("time_holdout", splits["time_holdout"]),
+        "",
+        "## Site Holdout",
+        "",
+    ]
+    for site_id, item in splits["site_holdout"].items():
+        lines.append(_split_line(site_id, item))
+    lines.extend(["", "## Scenario Family Holdout", ""])
+    for family, item in splits["scenario_family_holdout"].items():
+        lines.append(_split_line(family, item))
+        if "false_negative_scenarios" in item and item["false_negative_scenarios"]:
+            lines.append(f"  false negatives: {item['false_negative_scenarios']}")
+        if "false_positive_scenarios" in item and item["false_positive_scenarios"]:
+            lines.append(f"  false positives: {item['false_positive_scenarios']}")
+    lines.extend(["", "## Source Ablations", ""])
+    baseline = splits["episode_grouped_random"]
+    baseline_f1 = baseline.get("macro_f1") if "error" not in baseline else None
+    for name, item in splits["source_ablations"].items():
+        lines.append(_split_line(name, item))
+        if baseline_f1 is not None and "macro_f1" in item:
+            lines.append(f"  macro F1 delta vs random baseline: {item['macro_f1'] - baseline_f1:+.3f}")
+    lines.extend(
+        [
+            "",
+            "## Interpretation",
+            "",
+            "- These are stress diagnostics, not tuned headline metrics.",
+            "- Family holdouts identify hard families that still fail under scenario generalization.",
+            "- Source ablations should degrade performance when the dropped layer carries independent evidence.",
+            "- Middle probability bins indicate whether v1 has less all-or-nothing large-training probability mass than v0.",
+        ]
+    )
+    path.write_text("\n".join(lines) + "\n", encoding="utf-8")
+
+
+def update_model_run_readme_with_stress(model_run_dir: Path, metrics: dict[str, Any]) -> None:
+    readme_path = model_run_dir / "README.md"
+    if not readme_path.exists():
+        return
+    splits = metrics["splits"]
+    baseline = splits["episode_grouped_random"]
+    fragmented = splits["scenario_family_holdout"].get("fragmented_training_linked", {})
+    underclocked = splits["scenario_family_holdout"].get("underclocked_energy_capped_training", {})
+    runtime_drop = splits["source_ablations"].get("source_ablation_drop_runtime_and_ml_logs", {})
+    baseline_f1 = baseline.get("macro_f1", 0.0)
+    runtime_delta = runtime_drop.get("macro_f1", baseline_f1) - baseline_f1 if "macro_f1" in runtime_drop else 0.0
+    section = f"""## Stress Splits
+
+Stress outputs are in:
+
+- `stress_metrics.json`
+- `stress_summary.md`
+
+Key stress checks from the lightweight retraining script:
+
+- Episode-grouped random split macro F1: {baseline.get('macro_f1', 0.0):.4f}; large-training recall: {baseline.get('large_training', {}).get('recall', 0.0):.4f}
+- Time holdout macro F1: {splits['time_holdout'].get('macro_f1', 0.0):.4f}; large-training recall: {splits['time_holdout'].get('large_training', {}).get('recall', 0.0):.4f}
+- Underclocked holdout large-training recall: {underclocked.get('large_training', {}).get('recall', 0.0):.4f}; false negatives: {underclocked.get('large_training_false_negative_count', 0)}
+- Fragmented linked training holdout remains hard: {fragmented.get('large_training_false_negative_count', 0)} large-training false negatives
+- Source ablation dropping runtime and ML logs changed macro F1 by {runtime_delta:+.3f} versus the random stress baseline
+- Middle `p_large_training` bins are populated in stress checks: {baseline.get('middle_probability_bins', {})}
+
+"""
+    readme = readme_path.read_text(encoding="utf-8")
+    start = readme.find("## Stress Splits\n")
+    if start != -1:
+        end = readme.find("## Reproduce\n", start)
+        if end == -1:
+            end = readme.find("## Limitations\n", start)
+        if end == -1:
+            readme = readme[:start].rstrip() + "\n\n" + section
+        else:
+            readme = readme[:start].rstrip() + "\n\n" + section + readme[end:]
+    else:
+        marker = "## Reproduce\n"
+        idx = readme.find(marker)
+        if idx == -1:
+            readme = readme.rstrip() + "\n\n" + section
+        else:
+            readme = readme[:idx].rstrip() + "\n\n" + section + readme[idx:]
+    readme_path.write_text(readme, encoding="utf-8")
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--features", type=Path, required=True)
+    parser.add_argument("--model-run", type=Path, required=True, help="Directory to write stress outputs into.")
+    parser.add_argument("--seed", type=int, default=DEFAULT_SEED)
+    parser.add_argument("--max-iter", type=int, default=120)
+    args = parser.parse_args(argv)
+    metrics = evaluate_stress_splits(args.features, args.model_run, seed=args.seed, max_iter=args.max_iter)
+    random_metrics = metrics["splits"]["episode_grouped_random"]
+    print(f"episode_grouped_random_macro_f1: {random_metrics['macro_f1']:.4f}")
+    print(f"episode_grouped_random_large_recall: {random_metrics['large_training']['recall']:.4f}")
+    print(f"stress_metrics: {args.model_run / 'stress_metrics.json'}")
+    print(f"stress_summary: {args.model_run / 'stress_summary.md'}")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

src/datacenter_verification_modeling/predict.py

@@ -0,0 +1,30 @@
+"""Generate calibrated and governance post-processed predictions."""
+
+from __future__ import annotations
+
+import argparse
+from pathlib import Path
+
+try:
+    from .evaluate_model import predict_for_model_run
+except ImportError:  # pragma: no cover - direct script execution
+    from evaluate_model import predict_for_model_run
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--model-run", type=Path, required=True)
+    parser.add_argument("--features", type=Path, required=True)
+    parser.add_argument("--output", type=Path, required=True)
+    args = parser.parse_args(argv)
+    _, predictions = predict_for_model_run(args.model_run, args.features)
+    args.output.parent.mkdir(parents=True, exist_ok=True)
+    predictions.to_csv(args.output, index=False)
+    print(f"wrote_predictions: {args.output}")
+    print(f"rows: {len(predictions)}")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())
+

src/datacenter_verification_modeling/prepare_features.py

@@ -0,0 +1,49 @@
+"""Prepare leakage-safe feature metadata for the synthetic v0 modeling run."""
+
+from __future__ import annotations
+
+import argparse
+from pathlib import Path
+
+try:
+    from .common import DEFAULT_SEED, determine_feature_columns, ensure_dir, load_feature_table, make_episode_split, write_json
+except ImportError:  # pragma: no cover - direct script execution
+    from common import DEFAULT_SEED, determine_feature_columns, ensure_dir, load_feature_table, make_episode_split, write_json
+
+
+def prepare_features(features_path: Path, output_dir: Path, seed: int = DEFAULT_SEED) -> dict[str, object]:
+    df = load_feature_table(features_path)
+    split_df, split_manifest = make_episode_split(df, seed=seed)
+    feature_columns, excluded_columns = determine_feature_columns(split_df)
+
+    ensure_dir(output_dir)
+    write_json(output_dir / "split_manifest.json", split_manifest)
+    write_json(output_dir / "feature_columns.json", feature_columns)
+    write_json(output_dir / "excluded_columns.json", excluded_columns)
+
+    return {
+        "rows": int(len(split_df)),
+        "episodes": int(split_df["episode_id"].nunique()),
+        "feature_count": int(len(feature_columns)),
+        "split_summary": split_manifest["summary"],
+    }
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--features", type=Path, required=True)
+    parser.add_argument("--output", type=Path, required=True)
+    parser.add_argument("--seed", type=int, default=DEFAULT_SEED)
+    args = parser.parse_args(argv)
+    summary = prepare_features(args.features, args.output, seed=args.seed)
+    print(f"rows: {summary['rows']}")
+    print(f"episodes: {summary['episodes']}")
+    print(f"model_features: {summary['feature_count']}")
+    for split, values in summary["split_summary"].items():
+        print(f"{split}: rows={values['rows']} episodes={values['episodes']}")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())
+

src/datacenter_verification_modeling/requirements.txt

@@ -0,0 +1,5 @@
+# python >= 3.10
+pandas
+numpy
+scikit-learn
+joblib

src/datacenter_verification_modeling/rule_baseline.py

@@ -0,0 +1,192 @@
+"""Deterministic evidence-rule baseline for synthetic v0.
+
+The rule baseline: not intended to replace the supervised model; it provides a leakage-free point of comparison and encodes the study's governance constraints around capacity, missingness, and multi-layer evidence
+"""
+
+from __future__ import annotations
+
+import argparse
+from pathlib import Path
+from typing import Any
+
+import numpy as np
+import pandas as pd
+
+try:
+    from .common import LABELS, PROB_COLUMNS, as_bool, load_feature_table, normalize_probability_array, row_bool, row_float
+except ImportError:  # pragma: no cover - direct script execution
+    from common import LABELS, PROB_COLUMNS, as_bool, load_feature_table, normalize_probability_array, row_bool, row_float
+
+
+FALSE_POSITIVE_RUNTIME_MARKERS = [
+    "inference",
+    "hpc",
+    "mpi",
+    "nccl",
+    "benchmark",
+    "burn_in",
+    "storage",
+    "etl",
+    "synthetic_data",
+]
+
+FALSE_POSITIVE_DECLARED_CLASSES = {
+    "inference",
+    "hpc",
+    "benchmark",
+    "burn_in",
+    "data",
+    "reserved",
+    "none",
+}
+
+
+def evidence_flags(row: pd.Series) -> dict[str, bool]:
+    capacity = row_bool(row, "capacity_possible")
+    external_conflict = row_float(row, "o17_external_capacity_conflict_score")
+    allocation_gpus = row_float(row, "o2_max_concurrent_normalized_gpus")
+    allocation_hours = row_float(row, "o2_allocation_duration_hours")
+    gpu_hours_ratio = row_float(row, "o2_gpu_hours_policy_ratio")
+    policy_ratio = row_float(row, "policy_compute_ratio")
+    gpu_util = row_float(row, "o4_gpu_util_p95")
+    gpu_duty = row_float(row, "o4_gpu_util_duty_gt_70")
+    tensor = row_float(row, "o4_sm_tensor_active_p95")
+    fabric_footprint = row_float(row, "o7_synchronized_fabric_footprint")
+    fabric_periodicity = row_float(row, "o7_collective_periodicity_score")
+    fabric_util = row_float(row, "o7_scaleout_port_util_p95")
+    rack_power = row_float(row, "o8_rack_power_fraction_p95")
+    checkpoint = row_float(row, "o11_checkpoint_periodicity_score")
+    checkpoint_tb = row_float(row, "o11_checkpoint_write_tb_per_event")
+    storage_pattern = row_float(row, "o11_read_write_training_pattern_score")
+    signed_logs = row_bool(row, "o12_signed_ml_logs_present")
+    declared_params = row_float(row, "o12_declared_parameter_count_b")
+    training_tokens = row_float(row, "o12_training_tokens_b")
+    optimizer_state = row_bool(row, "o12_optimizer_state_present")
+    min_coverage = row_float(row, "o14_min_critical_coverage", 1.0)
+    gap_fraction = row_float(row, "o14_gap_fraction_critical")
+    cc_fraction = row_float(row, "o13_confidential_compute_mode_fraction")
+    counter_resets = row_float(row, "o14_counter_reset_count")
+    runtime = str(row["o10_runtime_framework_class"]).lower() if "o10_runtime_framework_class" in row else ""
+    declared_class = str(row["o2_declared_workload_class"]).lower() if "o2_declared_workload_class" in row else ""
+
+    allocation = allocation_gpus >= 512 or gpu_hours_ratio >= 1.0 or policy_ratio >= 1.0 or (
+        allocation_gpus >= 256 and allocation_hours >= 12
+    )
+    gpu_activity = gpu_util >= 70 or gpu_duty >= 0.45 or tensor >= 0.60
+    fabric_sync = fabric_footprint >= 256 or fabric_periodicity >= 0.55 or fabric_util >= 0.65
+    physical_support = rack_power >= 0.55
+    storage_semantic = checkpoint >= 0.55 or checkpoint_tb >= 0.25 or storage_pattern >= 0.60
+    runtime_semantic = "training" in runtime or "fine_tune" in runtime or "pytorch_distributed" in runtime
+    ml_semantic = signed_logs or declared_params >= 50 or training_tokens >= 100 or optimizer_state
+    integrity = (
+        gap_fraction > 0.05
+        or min_coverage < 0.80
+        or cc_fraction > 0.50
+        or counter_resets > 0
+        or row_bool(row, "o15_unapproved_physical_change_near_window")
+        or str(row.get("o4_missing_reason", "")) == "counter_disabled_by_cc_mode"
+    )
+    false_positive_pattern = any(marker in runtime for marker in FALSE_POSITIVE_RUNTIME_MARKERS) or declared_class in (
+        FALSE_POSITIVE_DECLARED_CLASSES
+    )
+    reserved_without_activity = (
+        allocation_gpus >= 512
+        and gpu_util < 30
+        and fabric_footprint < 64
+        and fabric_periodicity < 0.20
+        and not storage_semantic
+        and not ml_semantic
+    )
+    return {
+        "capacity": capacity,
+        "external_conflict": external_conflict >= 0.5,
+        "allocation": allocation,
+        "gpu_activity": gpu_activity,
+        "fabric_sync": fabric_sync,
+        "physical_support": physical_support,
+        "storage_semantic": storage_semantic,
+        "runtime_semantic": runtime_semantic,
+        "ml_semantic": ml_semantic,
+        "signed_logs": signed_logs,
+        "integrity": integrity,
+        "false_positive_pattern": false_positive_pattern,
+        "reserved_without_activity": reserved_without_activity,
+        "strong_coverage": min_coverage >= 0.90 and gap_fraction <= 0.05,
+        "policy_scale": policy_ratio >= 1.0 or gpu_hours_ratio >= 1.0 or allocation_gpus >= 512,
+    }
+
+
+def predict_rule_label(row: pd.Series) -> int:
+    flags = evidence_flags(row)
+    primary_count = int(flags["allocation"]) + int(flags["gpu_activity"]) + int(flags["fabric_sync"])
+    semantic_count = int(flags["runtime_semantic"]) + int(flags["storage_semantic"]) + int(flags["ml_semantic"])
+
+    if not flags["capacity"] and not flags["external_conflict"]:
+        return 0 if flags["strong_coverage"] else 1
+
+    if flags["signed_logs"] and flags["policy_scale"] and primary_count >= 2:
+        return 4
+
+    if (
+        flags["allocation"]
+        and flags["gpu_activity"]
+        and flags["fabric_sync"]
+        and flags["physical_support"]
+        and semantic_count >= 2
+        and flags["policy_scale"]
+    ):
+        return 4
+
+    if flags["reserved_without_activity"]:
+        return 1
+
+    if flags["false_positive_pattern"] and (flags["gpu_activity"] or flags["fabric_sync"] or flags["physical_support"]):
+        return 2
+
+    if flags["integrity"] and not flags["ml_semantic"] and (primary_count >= 1 or flags["physical_support"]):
+        return 2
+
+    if primary_count >= 2 and (flags["physical_support"] or semantic_count >= 1):
+        return 3
+
+    if primary_count >= 1 or flags["physical_support"] or semantic_count >= 1:
+        return 2
+
+    return 1 if flags["capacity"] else 0
+
+
+def predict_rule_labels(df: pd.DataFrame) -> pd.Series:
+    return df.apply(predict_rule_label, axis=1).astype(int)
+
+
+def predict_rule_probabilities(df: pd.DataFrame, confidence: float = 0.82) -> pd.DataFrame:
+    labels = predict_rule_labels(df)
+    off_probability = (1.0 - confidence) / (len(LABELS) - 1)
+    probabilities = np.full((len(df), len(LABELS)), off_probability, dtype=float)
+    for row_index, label in enumerate(labels):
+        probabilities[row_index, LABELS.index(int(label))] = confidence
+    probabilities = normalize_probability_array(probabilities)
+    return pd.DataFrame(probabilities, columns=PROB_COLUMNS, index=df.index)
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--features", type=Path, required=True)
+    parser.add_argument("--output", type=Path)
+    args = parser.parse_args(argv)
+
+    df = load_feature_table(args.features)
+    labels = predict_rule_labels(df)
+    out = df[["feature_row_id", "episode_id", "latent_workload_class", "label_0_to_4"]].copy()
+    out["rule_predicted_label"] = labels
+    if args.output:
+        args.output.parent.mkdir(parents=True, exist_ok=True)
+        out.to_csv(args.output, index=False)
+    else:
+        print(out.head(20).to_string(index=False))
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())
+

src/datacenter_verification_modeling/train_model.py

@@ -0,0 +1,226 @@
+"""Train the calibrated supervised baseline for synthetic datacenter verification data."""
+
+from __future__ import annotations
+
+import argparse
+import subprocess
+import sys
+from pathlib import Path
+from typing import Any
+
+import joblib
+import numpy as np
+from sklearn.calibration import CalibratedClassifierCV
+from sklearn.ensemble import HistGradientBoostingClassifier
+
+try:
+    from .common import (
+        DEFAULT_SEED,
+        determine_feature_columns,
+        derive_dataset_dir,
+        ensure_dir,
+        load_feature_table,
+        make_episode_split,
+        make_preprocessor,
+        model_input_frame,
+        utc_now_iso,
+        write_json,
+    )
+    from .evaluate_model import evaluate_model_run
+except ImportError:  # pragma: no cover - direct script execution
+    from common import (
+        DEFAULT_SEED,
+        determine_feature_columns,
+        derive_dataset_dir,
+        ensure_dir,
+        load_feature_table,
+        make_episode_split,
+        make_preprocessor,
+        model_input_frame,
+        utc_now_iso,
+        write_json,
+    )
+    from evaluate_model import evaluate_model_run
+
+
+def repo_root() -> Path:
+    return Path(__file__).resolve().parents[2]
+
+
+def run_dataset_validations(features_path: Path) -> list[dict[str, Any]]:
+    root = repo_root()
+    dataset_dir = derive_dataset_dir(features_path)
+    try:
+        dataset_arg = dataset_dir.relative_to(root)
+    except ValueError:
+        dataset_arg = dataset_dir
+    commands = [
+        {
+            "name": "synthetic_dataset_validator",
+            "command": [
+                sys.executable,
+                str(root / "src/datacenter_verification_synthetic/validate_synthetic_dataset.py"),
+                "--dataset",
+                str(dataset_arg),
+            ],
+        },
+        {
+            "name": "public_dataset_validator",
+            "command": [
+                sys.executable,
+                "-m",
+                "src.datacenter_verification_validators",
+                "--dataset",
+                str(dataset_arg),
+            ],
+        },
+    ]
+    results: list[dict[str, Any]] = []
+    for item in commands:
+        completed = subprocess.run(
+            item["command"],
+            cwd=root,
+            check=False,
+            text=True,
+            capture_output=True,
+        )
+        result = {
+            "name": item["name"],
+            "command": " ".join(item["command"]),
+            "returncode": int(completed.returncode),
+            "stdout": completed.stdout.strip(),
+            "stderr": completed.stderr.strip(),
+        }
+        results.append(result)
+        if completed.returncode != 0:
+            raise RuntimeError(
+                f"dataset validation failed for {item['name']} with return code {completed.returncode}\n"
+                f"stdout:\n{completed.stdout}\n\nstderr:\n{completed.stderr}"
+            )
+    return results
+
+
+def make_base_classifier(seed: int) -> HistGradientBoostingClassifier:
+    kwargs: dict[str, Any] = {
+        "learning_rate": 0.05,
+        "max_iter": 350,
+        "max_leaf_nodes": 31,
+        "l2_regularization": 0.03,
+        "early_stopping": True,
+        "validation_fraction": 0.15,
+        "random_state": seed,
+    }
+    try:
+        return HistGradientBoostingClassifier(class_weight="balanced", **kwargs)
+    except TypeError:  # pragma: no cover - older scikit-learn fallback
+        return HistGradientBoostingClassifier(**kwargs)
+
+
+def calibrate_prefit_model(base_model: Any, x_validation: np.ndarray, y_validation: np.ndarray) -> tuple[Any, str]:
+    try:
+        from sklearn.frozen import FrozenEstimator
+
+        calibrator = CalibratedClassifierCV(FrozenEstimator(base_model), method="sigmoid")
+        method = "sigmoid_on_validation_split_frozen_estimator"
+    except Exception:  # pragma: no cover - older scikit-learn fallback
+        calibrator = CalibratedClassifierCV(base_model, method="sigmoid", cv="prefit")
+        method = "sigmoid_on_validation_split_prefit"
+    calibrator.fit(x_validation, y_validation)
+    return calibrator, method
+
+
+def train_model(features_path: Path, output_dir: Path, seed: int = DEFAULT_SEED, skip_dataset_validation: bool = False) -> dict[str, Any]:
+    ensure_dir(output_dir)
+    validation_status = [] if skip_dataset_validation else run_dataset_validations(features_path)
+
+    df = load_feature_table(features_path)
+    split_df, split_manifest = make_episode_split(df, seed=seed)
+    feature_columns, excluded_columns = determine_feature_columns(split_df)
+    write_json(output_dir / "split_manifest.json", split_manifest)
+    write_json(output_dir / "feature_columns.json", feature_columns)
+    write_json(output_dir / "excluded_columns.json", excluded_columns)
+
+    train_df = split_df[split_df["split"] == "train"].copy()
+    validation_df = split_df[split_df["split"] == "validation"].copy()
+    if train_df.empty or validation_df.empty:
+        raise ValueError("train and validation splits must both be non-empty")
+
+    preprocessor = make_preprocessor(train_df, feature_columns)
+    x_train = preprocessor.fit_transform(model_input_frame(train_df, feature_columns))
+    y_train = train_df["label_0_to_4"].astype(int).to_numpy()
+    x_validation = preprocessor.transform(model_input_frame(validation_df, feature_columns))
+    y_validation = validation_df["label_0_to_4"].astype(int).to_numpy()
+
+    base_model = make_base_classifier(seed)
+    base_model.fit(x_train, y_train)
+    calibrated_model, calibration_method = calibrate_prefit_model(base_model, x_validation, y_validation)
+
+    joblib.dump(calibrated_model, output_dir / "model.joblib")
+    joblib.dump(preprocessor, output_dir / "preprocessing.joblib")
+
+    training_metadata = {
+        "trained_at": utc_now_iso(),
+        "seed": int(seed),
+        "features_path": str(features_path),
+        "model_type": "HistGradientBoostingClassifier",
+        "model_parameters": base_model.get_params(),
+        "calibration_method": calibration_method,
+        "train_rows": int(len(train_df)),
+        "validation_rows": int(len(validation_df)),
+        "train_episodes": int(train_df["episode_id"].nunique()),
+        "validation_episodes": int(validation_df["episode_id"].nunique()),
+        "feature_count": int(len(feature_columns)),
+        "dataset_validation_status": validation_status,
+    }
+    write_json(
+        output_dir / "manifest.json",
+        {
+            "model_run_id": output_dir.name,
+            "created_or_updated_at": utc_now_iso(),
+            "features_path": str(features_path),
+            "model_type": "CalibratedClassifierCV over HistGradientBoostingClassifier",
+            "calibration_method": calibration_method,
+            "training_metadata": training_metadata,
+            "validation_status": validation_status,
+        },
+    )
+
+    metrics = evaluate_model_run(
+        output_dir,
+        features_path,
+        validation_status=validation_status,
+        training_metadata=training_metadata,
+    )
+    return {
+        "metrics": metrics,
+        "split_summary": split_manifest["summary"],
+        "validation_status": validation_status,
+        "training_metadata": training_metadata,
+    }
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--features", type=Path, required=True)
+    parser.add_argument("--output", type=Path, required=True)
+    parser.add_argument("--seed", type=int, default=DEFAULT_SEED)
+    parser.add_argument(
+        "--skip-dataset-validation",
+        action="store_true",
+        help="Skip pre-training dataset validation. Intended only for development.",
+    )
+    args = parser.parse_args(argv)
+    result = train_model(args.features, args.output, seed=args.seed, skip_dataset_validation=args.skip_dataset_validation)
+    metrics = result["metrics"]
+    print(f"train_rows: {result['split_summary']['train']['rows']}")
+    print(f"validation_rows: {result['split_summary']['validation']['rows']}")
+    print(f"test_rows: {result['split_summary']['test']['rows']}")
+    print(f"accuracy: {metrics['model']['accuracy']:.4f}")
+    print(f"macro_f1: {metrics['model']['macro_f1']:.4f}")
+    print(f"label_3_4_precision: {metrics['governance']['label_3_4_predicted_label']['precision']:.4f}")
+    print(f"label_3_4_recall: {metrics['governance']['label_3_4_predicted_label']['recall']:.4f}")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())