Add BOOM benchmark results

Add the complete 7,413-configuration in-corpus BOOM result and comparator leaderboard.

README.md

@@ -24,6 +24,7 @@ tags:
 - gift-eval
 - fev-bench
 - time-benchmark
+- boom
 - github
 - mimo-2-6-pro
 - deepseek-v4-1-flash
@@ -120,6 +121,25 @@ model-index:
     source:
       name: FEV-Bench pull request 189
       url: https://github.com/autogluon/fev/pull/189
+  - task:
+      type: time-series-forecasting
+      name: Time Series Forecasting
+    dataset:
+      type: boom
+      name: BOOM in-corpus protocol
+    metrics:
+    - type: scaled_mase
+      value: 0.723
+      name: Scaled MASE
+    - type: scaled_crps
+      value: 0.434
+      name: Scaled CRPS
+    - type: rank
+      value: 11.808
+      name: Mean per-dataset rank of 24
+    source:
+      name: Official BOOM leaderboard comparison
+      url: https://huggingface.co/spaces/Datadog/BOOM
 ---
 
 # Fracast-0
@@ -281,6 +301,19 @@ The full disclosure is public in
 observations after official task slicing. The adapter repeats the most recent
 finite value for those windows; it does not use labels to change forecasts.
 
+## BOOM Benchmark
+
+Fracast-0 completed all 7,413 BOOM dataset-term configurations and all
+623,602 forecasts with the FP32 checkpoint and no task failures. In the
+official 24-model comparison, it obtains scaled MASE `0.723`, scaled CRPS
+`0.434`, and mean per-dataset rank `11.808`. The corresponding ordinal
+positions are `12 / 24` by MASE and `11 / 24` by CRPS.
+
+`Datadog/BOOM` is also part of the complete Fracast-0 pretraining recipe, so
+this result is in-corpus and must not be presented as zero-shot. The raw
+per-configuration result, manifest, and comparator leaderboard are archived in
+the [GitHub repository](https://github.com/ztxtech/fracast-0/tree/main/script/boom_bench).
+
 ## Scenario Highlights
 
 These slices use per-configuration or per-task error ratios against Seasonal
@@ -322,6 +355,9 @@ disclosures above still apply.
 | `benchmark/time_benchmark/TIME_pareto.csv` | 2,885 B | `8c33c9f3fc0bc592103653d40f10fdd79600eb83b7b563cdff049660b875b44e` |
 | `benchmark/time_benchmark/TIME_pareto_summary.json` | 2,812 B | `44ba218c32a89b26ccfb1df642465de858554c3a13ff33a3b5ad055169244d64` |
 | `benchmark/time_benchmark/run-manifest.json` | 3,820 B | `c7295d31a403c1bfa94dc0c1f2a41e0f76758066961cda6fe41d909bf1c03f06` |
+| `benchmark/boom_bench/all_results.csv` | 2,025,022 B | `0928a3e54f8f74b3bbf0f2f93745a3a43ee4865fb779ae3d20330c746cbd32a2` |
+| `benchmark/boom_bench/run-manifest.json` | 1,406 B | `e264bca59e3bd7fcc38f7c0b36778fac8644b448938c1d33c4138f2058d6ac9d` |
+| `benchmark/boom_bench/leaderboard.csv` | 1,335 B | `6156f9d10fa76bfcbd006fec85c2f41cd6198151c3ca22871cae744187175958` |
 
 ## Training Data
 

benchmark/boom_bench/leaderboard.csv

@@ -0,0 +1,25 @@
+model,MASE-6750-scaled,CRPS-6750-scaled,Rank-6750-scaled,eval_metrics/MAE[0.5]-663-unscaled,CRPS-663-unscaled,Rank-663-unscaled
+toto_2.0_2.5b,0.581,0.349,3.973,0.000,0.001,4.621
+toto_2.0_1b,0.582,0.349,4.060,0.000,0.001,7.158
+toto_2.0_313m,0.585,0.351,4.353,0.000,0.001,6.193
+toto_2.0_22m,0.601,0.363,5.645,0.000,0.001,8.777
+toto,0.617,0.375,7.101,0.001,0.025,15.087
+toto_2.0_4m,0.624,0.377,7.324,0.000,0.001,6.005
+chronos_2,0.641,0.392,7.561,0.000,0.002,12.733
+timesfm_2_5_200m,0.632,0.382,7.849,0.000,0.001,8.781
+moirai_1.1_base,0.710,0.428,10.838,0.000,0.003,13.084
+moirai_1.1_large,0.720,0.436,11.209,0.001,0.005,14.451
+moirai_1.1_small,0.729,0.442,11.727,0.000,0.006,15.017
+fracast-0,0.723,0.434,11.808,0.001,0.002,13.125
+moirai_2_small,0.672,0.427,11.821,0.001,0.002,13.644
+timesfm_2_0_500m,0.725,0.447,12.469,0.014,0.091,18.763
+chronos_bolt_base,0.726,0.451,12.932,0.003,0.019,16.146
+chronos_bolt_small,0.733,0.455,13.350,0.003,0.022,16.302
+autoarima,0.824,0.736,18.020,0.000,0.001,7.166
+time_moe_50M,0.806,0.649,18.441,0.001,0.005,13.632
+time_moe_200M,0.881,0.643,18.461,0.000,0.005,16.057
+timer,0.796,0.639,18.533,0.001,0.005,11.997
+visionts,0.988,0.673,19.668,0.001,0.009,17.448
+autoets,0.842,1.975,20.283,0.000,0.030,14.078
+autotheta,1.123,1.018,21.196,0.001,0.002,12.195
+seasonalnaive,1.000,1.000,21.376,0.000,0.006,17.538

benchmark/boom_bench/run-manifest.json

@@ -0,0 +1,51 @@
+{
+  "schema_version": 1,
+  "benchmark": {
+    "name": "BOOM",
+    "repository": "https://huggingface.co/datasets/Datadog/BOOM",
+    "dataset_revision": "69325b544c45ff0d6c43c7a99c49a6601a01725b",
+    "protocol_revision": "1527c41589189ad1bc3883ed4d3d97b3e5a3b47c",
+    "protocol": "official rolling evaluation"
+  },
+  "model": {
+    "name": "Fracast-0",
+    "parameter_count": 85001,
+    "weights": "fp32",
+    "weights_sha256": "0d678f123f3d1f91c5065cca2b5d40eccc469360afc2381b27290486011ded2c",
+    "context_length": 2048,
+    "quantiles": [
+      0.1,
+      0.2,
+      0.3,
+      0.4,
+      0.5,
+      0.6,
+      0.7,
+      0.8,
+      0.9
+    ]
+  },
+  "evaluation": {
+    "device": "mps",
+    "batch_size": 64,
+    "configurations_completed": 7413,
+    "configurations_total": 7413,
+    "forecasts_completed": 623602,
+    "task_failures": 0
+  },
+  "results": {
+    "scaled_mase": 0.723,
+    "scaled_crps": 0.434,
+    "scaled_average_rank": 11.808,
+    "scaled_mase_rank": 12,
+    "scaled_crps_rank": 11,
+    "scaled_average_rank_ordinal": 12,
+    "models_compared": 24,
+    "scaled_configurations": 6750,
+    "zero_inflated_configurations": 663
+  },
+  "artifacts": {
+    "results/Fracast-0/all_results.csv": "0928a3e54f8f74b3bbf0f2f93745a3a43ee4865fb779ae3d20330c746cbd32a2",
+    "leaderboards/BOOM_leaderboard_with_fracast.csv": "6156f9d10fa76bfcbd006fec85c2f41cd6198151c3ca22871cae744187175958"
+  }
+}