Update leaderboard results from local eval

results/aggregates/live_metadata.json

@@ -1,5 +1,5 @@
 {
-  "generated_at": "2026-07-29T08:26:43.498590+00:00",
+  "generated_at": "2026-07-29T08:38:57.853569+00:00",
   "scope": "Canonical metrics-only release history when available; otherwise legacy release history plus latest allowlisted snapshots",
   "release_deduplication": "One row per model, dataset, data source timestamp, and prediction length",
   "dataset_aggregation": "Arithmetic mean over releases, then equal-weight mean over datasets",

results/aggregates/metadata.json

@@ -1,5 +1,5 @@
 {
-  "generated_at": "2026-07-29T08:26:41.451502+00:00",
+  "generated_at": "2026-07-29T08:38:55.778248+00:00",
   "baseline_model": "Seasonal-Naive",
   "normalization": "Per-config metric / Seasonal-Naive metric",
   "metric_aggregation": "Geometric mean over dataset configurations",
@@ -10,7 +10,7 @@
     "frequency",
     "prediction_length"
   ],
-  "num_configs": 13,
+  "num_configs": 12,
   "num_models": 12,
   "num_rows_missing_prediction_length": 0
 }

results/aggregates/results_by_domain.csv

@@ -47,18 +47,18 @@ Hydrology,Chronos-2,0.9878412392964167,1.0111200784587953,9,1,1,1.0
 Hydrology,TiRex,0.9880750420668056,1.0113745182170464,10,1,1,1.0
 Hydrology,Chronos-Bolt,0.9959839316857422,1.0129095028376935,11,1,1,1.0
 Hydrology,ARIMA,0.9850111992171071,1.0608896634781397,12,1,1,1.0
-Ocean,Chronos-2,0.07501412876756498,0.44114594529050166,1,2,2,1.0
-Ocean,TiRex,0.8911453966828544,0.803792414958183,2,2,2,1.0
-Ocean,Moving-Average,0.8708420095854966,0.9104023711080581,3,2,2,1.0
-Ocean,TimesFM-2.5,0.3998624728680003,0.5786475456177379,4,2,2,1.0
-Ocean,ETS,1.1184971796901257,1.006608632559059,5,2,2,1.0
-Ocean,ARIMA,1.2792272250801207,1.1314964099503113,6,2,2,1.0
-Ocean,Moirai-2.0,0.5349808934300746,0.7202186365541717,7,2,2,1.0
-Ocean,Seasonal-Naive,1.0,1.0,8,2,2,1.0
-Ocean,Chronos-Bolt,1.8911055039013083,1.4225017843200154,9,2,2,1.0
-Ocean,Toto-1.0,1.7517259894667812,1.0276238080615925,10,2,2,1.0
-Ocean,Sundial,1.255772243598817,0.9990726066446702,11,2,2,1.0
-Ocean,TabPFN-TS,2.964534107417528,1.7918933603456857,12,2,2,1.0
+Ocean,Chronos-2,0.004065646210503269,0.18017789189149058,1,1,1,1.0
+Ocean,TimesFM-2.5,0.11897587453818394,0.30185471840281924,2,1,1,1.0
+Ocean,Moirai-2.0,0.15529767214638196,0.43989214409973665,3,1,1,1.0
+Ocean,TiRex,0.6376297045685118,0.6707335479358393,4,1,1,1.0
+Ocean,Sundial,0.9959918135802209,0.7833070865839833,5,1,1,1.0
+Ocean,Moving-Average,0.9775155990789853,0.9475054070613252,6,1,1,1.0
+Ocean,Seasonal-Naive,1.0,1.0,7,1,1,1.0
+Ocean,Toto-1.0,1.9088894806295245,1.0227024399756848,8,1,1,1.0
+Ocean,ARIMA,2.540064093386627,1.6083878608681195,9,1,1,1.0
+Ocean,ETS,3.412470602278523,1.8188179208652995,10,1,1,1.0
+Ocean,TabPFN-TS,5.662366765178726,2.4144987331689447,11,1,1,1.0
+Ocean,Chronos-Bolt,8.296209105081665,2.741122381016384,12,1,1,1.0
 Transportation,Seasonal-Naive,1.0,1.0,1,1,1,1.0
 Transportation,TabPFN-TS,1.0,1.0,2,1,1,1.0
 Transportation,TimesFM-2.5,1.0,1.0,3,1,1,1.0

results/aggregates/results_by_frequency.csv

@@ -1,16 +1,4 @@
 frequency,model,MSE,CRPS,Rank,n_configs,n_group_configs,coverage
-10min,ETS,0.3666070969635438,0.5570986119712091,1,1,1,1.0
-10min,Chronos-Bolt,0.4310739979655584,0.7382053936764865,2,1,1,1.0
-10min,ARIMA,0.6442444888090875,0.7960045937174731,3,1,1,1.0
-10min,Moving-Average,0.7758094155974984,0.8747522400846098,4,1,1,1.0
-10min,TiRex,1.245456590775428,0.9632472512111627,5,1,1,1.0
-10min,Seasonal-Naive,1.0,1.0,6,1,1,1.0
-10min,Toto-1.0,1.607502148925569,1.0325688583671666,7,1,1,1.0
-10min,Chronos-2,1.384065219501788,1.0800978022511833,8,1,1,1.0
-10min,TimesFM-2.5,1.343885874583737,1.1092521058511466,9,1,1,1.0
-10min,Moirai-2.0,1.8429417027285986,1.1791865151434529,10,1,1,1.0
-10min,Sundial,1.5833101299543884,1.2742717261766512,11,1,1,1.0
-10min,TabPFN-TS,1.5520828724990656,1.3298337127875746,12,1,1,1.0
 15min,Seasonal-Naive,1.0,1.0,1,1,1,1.0
 15min,Toto-1.0,0.9943660646221287,1.0009272600088706,2,1,1,1.0
 15min,ETS,0.9906410276164291,1.0010085634811112,3,1,1,1.0

results/aggregates/results_by_prediction_length.csv

@@ -59,15 +59,3 @@ prediction_length,model,MSE,CRPS,Rank,n_configs,n_group_configs,coverage
 60,ETS,3.412470602278523,1.8188179208652995,10,1,1,1.0
 60,TabPFN-TS,5.662366765178726,2.4144987331689447,11,1,1,1.0
 60,Chronos-Bolt,8.296209105081665,2.741122381016384,12,1,1,1.0
-72,ETS,0.3666070969635438,0.5570986119712091,1,1,1,1.0
-72,Chronos-Bolt,0.4310739979655584,0.7382053936764865,2,1,1,1.0
-72,ARIMA,0.6442444888090875,0.7960045937174731,3,1,1,1.0
-72,Moving-Average,0.7758094155974984,0.8747522400846098,4,1,1,1.0
-72,TiRex,1.245456590775428,0.9632472512111627,5,1,1,1.0
-72,Seasonal-Naive,1.0,1.0,6,1,1,1.0
-72,Toto-1.0,1.607502148925569,1.0325688583671666,7,1,1,1.0
-72,Chronos-2,1.384065219501788,1.0800978022511833,8,1,1,1.0
-72,TimesFM-2.5,1.343885874583737,1.1092521058511466,9,1,1,1.0
-72,Moirai-2.0,1.8429417027285986,1.1791865151434529,10,1,1,1.0
-72,Sundial,1.5833101299543884,1.2742717261766512,11,1,1,1.0
-72,TabPFN-TS,1.5520828724990656,1.3298337127875746,12,1,1,1.0

results/aggregates/results_overall.csv

@@ -1,13 +1,13 @@
 overall,model,MSE,CRPS,Rank,n_configs,n_group_configs,coverage
-Overall,Chronos-2,1.22353684168191,6.292242495202032,1,13,13,1.0
-Overall,TiRex,1.1689162120919134,6.08521001296559,2,13,13,1.0
-Overall,Moirai-2.0,1.7017793606796376,10.408496738053778,3,13,13,1.0
-Overall,Sundial,3.5541279308684763,9.212774515164686,4,13,13,1.0
-Overall,TimesFM-2.5,1.1618735561040594,3.841847803108096,5,13,13,1.0
-Overall,Toto-1.0,82.01979314816886,18.504684863909027,6,13,13,1.0
-Overall,Seasonal-Naive,1.0,1.0,7,13,13,1.0
-Overall,Chronos-Bolt,2.3493667785713606,14.796532883778605,8,13,13,1.0
-Overall,Moving-Average,7.7608860943607105,3.111828431450043,9,13,13,1.0
-Overall,ARIMA,8.24570539024492,20.708201114313958,10,13,13,1.0
-Overall,ETS,17.373220028815215,12.42520728997576,11,13,13,1.0
-Overall,TabPFN-TS,2.859794462583278,5.1542661264590395,12,13,13,1.0
+Overall,Chronos-2,1.211031460396387,7.2875908551911435,1,12,12,1.0
+Overall,Moirai-2.0,1.6905157582194856,12.479728463263214,2,12,12,1.0
+Overall,TiRex,1.1627542822323917,7.095566162007958,3,12,12,1.0
+Overall,Sundial,3.801867511089004,10.863867865059087,4,12,12,1.0
+Overall,TimesFM-2.5,1.1478678899654162,4.260880104494603,5,12,12,1.0
+Overall,Toto-1.0,113.82367285862311,23.535779284371472,6,12,12,1.0
+Overall,Seasonal-Naive,1.0,1.0,7,12,12,1.0
+Overall,Chronos-Bolt,2.7059352878135767,18.99584101515884,8,12,12,1.0
+Overall,Moving-Average,9.402810089605852,3.4589413079363243,9,12,12,1.0
+Overall,ARIMA,10.197473194256876,27.169268191967188,10,12,12,1.0
+Overall,TabPFN-TS,3.00921450895394,5.7702898647453456,11,12,12,1.0
+Overall,ETS,23.961846088404652,16.09414612028139,12,12,12,1.0

results/dataset_properties.csv

@@ -5,7 +5,6 @@ data_gov_sg_singapore_rainfall_5min_rainfall_5min_total,Climate,5min,1
 data_gov_sg_singapore_taxi_availability_available_taxi_count,Transportation,5min,1
 nasa_power_shanghai_hourly_t2m,Weather,1h,1
 noaa_coops_sf_water_level_water_level,Ocean,6min,1
-noaa_ndbc_46013_realtime_wave_height,Ocean,10min,1
 nws_ksfo_observations_temperature,Weather,1h,1
 open_meteo_shanghai_air_quality_hourly_pm2_5,Air Quality,1h,1
 open_meteo_shanghai_hourly_temperature_2m,Weather,1h,1

results/online_status.json

@@ -1,17 +1,23 @@
 {
-    "status": "ok",
+    "status": "partial",
     "evaluation_protocol": "issue-now-score-later",
     "metric_version": "livehouse-canonical-v1",
-    "started_at": "2026-07-29T08:26:35.139725+00:00",
-    "finished_at": "2026-07-29T08:26:43.510663+00:00",
+    "started_at": "2026-07-29T08:38:50.545226+00:00",
+    "finished_at": "2026-07-29T08:38:57.865550+00:00",
     "data_source": "ts_bench",
     "data_config": "configs/datasets/ts_bench.yaml",
-    "task_count": 2,
+    "task_count": 4,
     "issued_tasks": 0,
     "issued_forecasts": 0,
     "resolved_forecasts": 0,
     "pending_tasks": 13,
-    "failed_forecasts": [],
+    "failed_forecasts": [
+        {
+            "model": "Seasonal-Naive",
+            "dataset": "nws_ksfo_observations_temperature/1h/short",
+            "error": "Mean forecast length 10 < 24"
+        }
+    ],
     "models": {
         "Chronos-2": {
             "model": "Chronos-2",
@@ -19,7 +25,7 @@
             "admitted_at": "2026-06-01T00:00:00Z",
             "evaluation_protocol": "issue-now-score-later",
             "metric_version": "livehouse-canonical-v1",
-            "evaluated_at": "2026-07-29T08:26:39.383927+00:00",
+            "evaluated_at": "2026-07-29T08:38:54.276408+00:00",
             "resolved_tasks": []
         },
         "TiRex": {
@@ -28,7 +34,7 @@
             "admitted_at": "2026-06-01T00:00:00Z",
             "evaluation_protocol": "issue-now-score-later",
             "metric_version": "livehouse-canonical-v1",
-            "evaluated_at": "2026-07-29T08:26:39.383944+00:00",
+            "evaluated_at": "2026-07-29T08:38:54.276415+00:00",
             "resolved_tasks": []
         },
         "TimesFM-2.5": {
@@ -37,7 +43,7 @@
             "admitted_at": "2026-06-01T00:00:00Z",
             "evaluation_protocol": "issue-now-score-later",
             "metric_version": "livehouse-canonical-v1",
-            "evaluated_at": "2026-07-29T08:26:39.383947+00:00",
+            "evaluated_at": "2026-07-29T08:38:54.276417+00:00",
             "resolved_tasks": []
         },
         "Toto-1.0": {
@@ -46,7 +52,7 @@
             "admitted_at": "2026-06-01T00:00:00Z",
             "evaluation_protocol": "issue-now-score-later",
             "metric_version": "livehouse-canonical-v1",
-            "evaluated_at": "2026-07-29T08:26:39.383950+00:00",
+            "evaluated_at": "2026-07-29T08:38:54.276418+00:00",
             "resolved_tasks": []
         },
         "Moirai-2.0": {
@@ -55,7 +61,7 @@
             "admitted_at": "2026-06-01T00:00:00Z",
             "evaluation_protocol": "issue-now-score-later",
             "metric_version": "livehouse-canonical-v1",
-            "evaluated_at": "2026-07-29T08:26:39.383952+00:00",
+            "evaluated_at": "2026-07-29T08:38:54.276419+00:00",
             "resolved_tasks": []
         },
         "Chronos-Bolt": {
@@ -64,7 +70,7 @@
             "admitted_at": "2026-06-01T00:00:00Z",
             "evaluation_protocol": "issue-now-score-later",
             "metric_version": "livehouse-canonical-v1",
-            "evaluated_at": "2026-07-29T08:26:39.383957+00:00",
+            "evaluated_at": "2026-07-29T08:38:54.276420+00:00",
             "resolved_tasks": []
         },
         "TabPFN-TS": {
@@ -73,7 +79,7 @@
             "admitted_at": "2026-06-01T00:00:00Z",
             "evaluation_protocol": "issue-now-score-later",
             "metric_version": "livehouse-canonical-v1",
-            "evaluated_at": "2026-07-29T08:26:39.383960+00:00",
+            "evaluated_at": "2026-07-29T08:38:54.276421+00:00",
             "resolved_tasks": []
         },
         "Sundial": {
@@ -82,7 +88,7 @@
             "admitted_at": "2026-06-01T00:00:00Z",
             "evaluation_protocol": "issue-now-score-later",
             "metric_version": "livehouse-canonical-v1",
-            "evaluated_at": "2026-07-29T08:26:39.383962+00:00",
+            "evaluated_at": "2026-07-29T08:38:54.276421+00:00",
             "resolved_tasks": []
         },
         "Moving-Average": {
@@ -91,7 +97,7 @@
             "admitted_at": "2026-06-01T00:00:00Z",
             "evaluation_protocol": "issue-now-score-later",
             "metric_version": "livehouse-canonical-v1",
-            "evaluated_at": "2026-07-29T08:26:39.383964+00:00",
+            "evaluated_at": "2026-07-29T08:38:54.276422+00:00",
             "resolved_tasks": []
         },
         "ETS": {
@@ -100,7 +106,7 @@
             "admitted_at": "2026-06-01T00:00:00Z",
             "evaluation_protocol": "issue-now-score-later",
             "metric_version": "livehouse-canonical-v1",
-            "evaluated_at": "2026-07-29T08:26:39.383967+00:00",
+            "evaluated_at": "2026-07-29T08:38:54.276423+00:00",
             "resolved_tasks": []
         },
         "ARIMA": {
@@ -109,7 +115,7 @@
             "admitted_at": "2026-06-01T00:00:00Z",
             "evaluation_protocol": "issue-now-score-later",
             "metric_version": "livehouse-canonical-v1",
-            "evaluated_at": "2026-07-29T08:26:39.383969+00:00",
+            "evaluated_at": "2026-07-29T08:38:54.276424+00:00",
             "resolved_tasks": []
         },
         "Seasonal-Naive": {
@@ -118,7 +124,7 @@
             "admitted_at": "2026-06-01T00:00:00Z",
             "evaluation_protocol": "issue-now-score-later",
             "metric_version": "livehouse-canonical-v1",
-            "evaluated_at": "2026-07-29T08:26:39.383971+00:00",
+            "evaluated_at": "2026-07-29T08:38:54.276425+00:00",
             "resolved_tasks": []
         }
     },
@@ -142,7 +148,5 @@
         "live_overall_30d": "aggregates/live_overall_30d.csv",
         "live_metadata": "aggregates/live_metadata.json"
     },
-    "push_status": "ok",
-    "pushed_at": "2026-07-29T08:27:51.945544+00:00",
-    "push_detail": "HF Hub API"
+    "push_status": "pending"
 }