Update leaderboard results from local eval

results/aggregates/live_metadata.json

@@ -1,5 +1,5 @@
 {
-  "generated_at": "2026-07-29T02:34:30.170237+00:00",
+  "generated_at": "2026-07-29T02:48:10.213133+00:00",
   "scope": "Canonical metrics-only release history when available; otherwise legacy release history plus latest allowlisted snapshots",
   "release_deduplication": "One row per model, dataset, data source timestamp, and prediction length",
   "dataset_aggregation": "Arithmetic mean over releases, then equal-weight mean over datasets",

results/aggregates/metadata.json

@@ -1,5 +1,5 @@
 {
-  "generated_at": "2026-07-29T02:34:28.032024+00:00",
+  "generated_at": "2026-07-29T02:48:08.314805+00:00",
   "baseline_model": "Seasonal-Naive",
   "normalization": "Per-config metric / Seasonal-Naive metric",
   "metric_aggregation": "Geometric mean over dataset configurations",
@@ -10,7 +10,7 @@
     "frequency",
     "prediction_length"
   ],
-  "num_configs": 13,
+  "num_configs": 12,
   "num_models": 12,
   "num_rows_missing_prediction_length": 0
 }

results/aggregates/results_by_domain.csv

@@ -35,18 +35,6 @@ Hazards,Toto-1.0,0.5447067827362504,0.6198857623270299,9,1,1,1.0
 Hazards,TimesFM-2.5,0.5159030493533491,0.6552873663892604,10,1,1,1.0
 Hazards,TabPFN-TS,0.5234081583087049,0.7154825875538315,11,1,1,1.0
 Hazards,Seasonal-Naive,1.0,1.0,12,1,1,1.0
-Hydrology,Moving-Average,1.0214140625,0.6370328425821091,1,1,1,1.0
-Hydrology,TabPFN-TS,0.999726629540324,0.6371976997142226,2,1,1,1.0
-Hydrology,Chronos-Bolt,0.9985341234579681,0.6392717717863047,3,1,1,1.0
-Hydrology,Chronos-2,1.001236886292696,0.6392940292272022,4,1,1,1.0
-Hydrology,TimesFM-2.5,0.9991976121589539,0.6429353154529154,5,1,1,1.0
-Hydrology,Sundial,1.12480897089839,0.6844293199066155,6,1,1,1.0
-Hydrology,Moirai-2.0,1.0619692907109857,0.7372901890583003,7,1,1,1.0
-Hydrology,TiRex,1.0007208641320466,0.7887214842091093,8,1,1,1.0
-Hydrology,Toto-1.0,1.033309433609247,0.8128151639806623,9,1,1,1.0
-Hydrology,ETS,0.8679554287684873,0.9135102558601378,10,1,1,1.0
-Hydrology,Seasonal-Naive,1.0,1.0,11,1,1,1.0
-Hydrology,ARIMA,2.8471547283782317,3.1754261339537115,12,1,1,1.0
 Ocean,ETS,0.09518516095471151,0.2761720625285491,1,2,2,1.0
 Ocean,TimesFM-2.5,0.07281852879570394,0.2570119454529022,2,2,2,1.0
 Ocean,Chronos-2,0.05541782384319625,0.2784674272092755,3,2,2,1.0

results/aggregates/results_by_frequency.csv

@@ -11,18 +11,6 @@ frequency,model,MSE,CRPS,Rank,n_configs,n_group_configs,coverage
 10min,Moirai-2.0,0.9931170314987676,0.9506171711084153,10,1,1,1.0
 10min,Seasonal-Naive,1.0,1.0,11,1,1,1.0
 10min,TabPFN-TS,1.0965296750513482,1.1700878991909816,12,1,1,1.0
-15min,Moving-Average,1.0214140625,0.6370328425821091,1,1,1,1.0
-15min,TabPFN-TS,0.999726629540324,0.6371976997142226,2,1,1,1.0
-15min,Chronos-Bolt,0.9985341234579681,0.6392717717863047,3,1,1,1.0
-15min,Chronos-2,1.001236886292696,0.6392940292272022,4,1,1,1.0
-15min,TimesFM-2.5,0.9991976121589539,0.6429353154529154,5,1,1,1.0
-15min,Sundial,1.12480897089839,0.6844293199066155,6,1,1,1.0
-15min,Moirai-2.0,1.0619692907109857,0.7372901890583003,7,1,1,1.0
-15min,TiRex,1.0007208641320466,0.7887214842091093,8,1,1,1.0
-15min,Toto-1.0,1.033309433609247,0.8128151639806623,9,1,1,1.0
-15min,ETS,0.8679554287684873,0.9135102558601378,10,1,1,1.0
-15min,Seasonal-Naive,1.0,1.0,11,1,1,1.0
-15min,ARIMA,2.8471547283782317,3.1754261339537115,12,1,1,1.0
 1D,ARIMA,0.35081770342317864,0.47944164778388026,1,1,1,1.0
 1D,Moirai-2.0,0.40785680526551005,0.5118677129116019,2,1,1,1.0
 1D,Sundial,0.303769640015456,0.5282879649026628,3,1,1,1.0

results/aggregates/results_by_prediction_length.csv

@@ -23,18 +23,18 @@ prediction_length,model,MSE,CRPS,Rank,n_configs,n_group_configs,coverage
 12,ETS,0.8706436180201554,439.02846065451104,10,2,2,1.0
 12,Toto-1.0,255690.8555968103,13030.420401776826,11,2,2,1.0
 12,Moving-Average,0.9617678019724286,471.56119324419365,12,2,2,1.0
-24,Chronos-2,0.33112343830775287,0.6408591087639055,1,7,7,1.0
-24,Chronos-Bolt,0.44303153510032106,0.6530382053012764,2,7,7,1.0
-24,TiRex,0.2907103671635409,0.6572440134253928,3,7,7,1.0
-24,Sundial,0.5403622341038394,0.6078867021056567,4,7,7,1.0
-24,Moirai-2.0,0.3567187274513251,0.6919534774596541,5,7,7,1.0
-24,TimesFM-2.5,0.3059580583366679,0.622723097736219,6,7,7,1.0
-24,Toto-1.0,1.1680858370126643,0.6735050933105378,7,7,7,1.0
-24,TabPFN-TS,0.49750143180707923,0.7055098464842646,8,7,7,1.0
-24,Moving-Average,1.5163338622973692,1.2480242202276746,9,7,7,1.0
-24,Seasonal-Naive,1.0,1.0,10,7,7,1.0
-24,ARIMA,1.6111643626474184,1.5559883799961705,11,7,7,1.0
-24,ETS,4.232993404975467,2.235937000348479,12,7,7,1.0
+24,TiRex,0.23658386029479467,0.637568913552396,1,6,6,1.0
+24,Chronos-2,0.27535867933395397,0.641120327610135,2,6,6,1.0
+24,Sundial,0.4782107758197922,0.5959890984516459,3,6,6,1.0
+24,Chronos-Bolt,0.3869129467061027,0.6553612620801093,4,6,6,1.0
+24,Toto-1.0,1.1921991800605538,0.6527282343390408,5,6,6,1.0
+24,Moirai-2.0,0.29741411006123314,0.6846731596793509,6,6,6,1.0
+24,TimesFM-2.5,0.2511868969740782,0.6194167190839796,7,6,6,1.0
+24,TabPFN-TS,0.4428736560186536,0.7175869951826023,8,6,6,1.0
+24,ARIMA,1.465304815202299,1.3815726858440398,9,6,6,1.0
+24,Seasonal-Naive,1.0,1.0,10,6,6,1.0
+24,Moving-Average,1.6195474144088537,1.3960464827885333,11,6,6,1.0
+24,ETS,5.51238175363501,2.595676619701423,12,6,6,1.0
 36,Seasonal-Naive,1.0,1.0,1,1,1,1.0
 36,Moving-Average,4301413138.721198,1.0,2,1,1,1.0
 36,ETS,4301412560.972201,123967.05458766423,3,1,1,1.0

results/aggregates/results_overall.csv

@@ -1,13 +1,13 @@
 overall,model,MSE,CRPS,Rank,n_configs,n_group_configs,coverage
-Overall,TiRex,0.8017260142696447,5.60310165565298,1,13,13,1.0
-Overall,Chronos-2,1.0990823535166132,6.922901496899398,2,13,13,1.0
-Overall,Sundial,2.336405644992367,7.308038988236886,3,13,13,1.0
-Overall,TimesFM-2.5,0.7678796232570124,3.0811081667098477,4,13,13,1.0
-Overall,Moirai-2.0,1.1656250122658105,8.650226658565176,5,13,13,1.0
-Overall,Chronos-Bolt,1.6865390808253753,11.552018828811239,6,13,13,1.0
-Overall,Moving-Average,6.300088763429552,2.7326347968300655,7,13,13,1.0
-Overall,Toto-1.0,50.696070929765014,15.5992336705747,8,13,13,1.0
-Overall,ARIMA,5.582983997675871,18.10276290637925,9,13,13,1.0
-Overall,TabPFN-TS,1.9088862726566063,4.025370192511457,10,13,13,1.0
-Overall,ETS,7.646102057230824,7.590098954379924,11,13,13,1.0
-Overall,Seasonal-Naive,1.0,1.0,12,13,13,1.0
+Overall,TiRex,0.7870495244413689,6.5976190835324715,1,12,12,1.0
+Overall,Chronos-2,1.1076554394122438,8.443135859020373,2,12,12,1.0
+Overall,Sundial,2.4831561845443395,8.902395336265226,3,12,12,1.0
+Overall,Moirai-2.0,1.174706665535206,10.620457857208972,4,12,12,1.0
+Overall,TimesFM-2.5,0.7512133364392474,3.5109010023776324,5,12,12,1.0
+Overall,Toto-1.0,70.12447572941844,19.95396560223523,6,12,12,1.0
+Overall,Chronos-Bolt,1.761837572480876,14.70298739069732,7,12,12,1.0
+Overall,ARIMA,5.90524144580455,20.928596302908005,8,12,12,1.0
+Overall,Moving-Average,7.331487042370404,3.0851994256541686,9,12,12,1.0
+Overall,ETS,9.166115233130736,9.054725150573324,10,12,12,1.0
+Overall,Seasonal-Naive,1.0,1.0,11,12,12,1.0
+Overall,TabPFN-TS,2.014597486146719,4.693715857687908,12,12,12,1.0

results/dataset_properties.csv

@@ -10,5 +10,4 @@ nws_ksfo_observations_temperature,Weather,1h,1
 open_meteo_shanghai_air_quality_hourly_pm2_5,Air Quality,1h,1
 open_meteo_shanghai_hourly_temperature_2m,Weather,1h,1
 usgs_earthquake_all_week_hourly_earthquake_count,Hazards,1h,1
-usgs_potomac_iv_usgs_00060,Hydrology,15min,1
 wikimedia_time_series_hourly_pageviews,Web,1D,1

results/online_status.json

@@ -2,14 +2,14 @@
     "status": "ok",
     "evaluation_protocol": "issue-now-score-later",
     "metric_version": "livehouse-canonical-v1",
-    "started_at": "2026-07-29T02:31:17.528395+00:00",
-    "finished_at": "2026-07-29T02:34:30.180926+00:00",
+    "started_at": "2026-07-29T02:48:01.381934+00:00",
+    "finished_at": "2026-07-29T02:48:10.224006+00:00",
     "data_source": "ts_bench",
     "data_config": "configs/datasets/ts_bench.yaml",
     "task_count": 3,
-    "issued_tasks": 1,
-    "issued_forecasts": 12,
-    "resolved_forecasts": 12,
+    "issued_tasks": 0,
+    "issued_forecasts": 0,
+    "resolved_forecasts": 0,
     "pending_tasks": 13,
     "failed_forecasts": [],
     "models": {
@@ -19,18 +19,8 @@
             "admitted_at": "2026-06-01T00:00:00Z",
             "evaluation_protocol": "issue-now-score-later",
             "metric_version": "livehouse-canonical-v1",
-            "evaluated_at": "2026-07-29T02:34:25.596308+00:00",
-            "resolved_tasks": [
-                {
-                    "dataset": "data_gov_sg_singapore_taxi_availability_available_taxi_count/5min/short",
-                    "task_id": "future-c056385aaaf66931e9d4",
-                    "issued_at": "2026-07-28T19:38:05.211084+00:00",
-                    "resolved_at": "2026-07-29T02:31:22.783151+00:00",
-                    "target_end": "2026-07-27T17:35:00+00:00",
-                    "prediction_length": 12,
-                    "metric_version": "livehouse-canonical-v1"
-                }
-            ]
+            "evaluated_at": "2026-07-29T02:48:06.653850+00:00",
+            "resolved_tasks": []
         },
         "TiRex": {
             "model": "TiRex",
@@ -38,18 +28,8 @@
             "admitted_at": "2026-06-01T00:00:00Z",
             "evaluation_protocol": "issue-now-score-later",
             "metric_version": "livehouse-canonical-v1",
-            "evaluated_at": "2026-07-29T02:34:25.624948+00:00",
-            "resolved_tasks": [
-                {
-                    "dataset": "data_gov_sg_singapore_taxi_availability_available_taxi_count/5min/short",
-                    "task_id": "future-c056385aaaf66931e9d4",
-                    "issued_at": "2026-07-28T19:38:05.211084+00:00",
-                    "resolved_at": "2026-07-29T02:31:23.239323+00:00",
-                    "target_end": "2026-07-27T17:35:00+00:00",
-                    "prediction_length": 12,
-                    "metric_version": "livehouse-canonical-v1"
-                }
-            ]
+            "evaluated_at": "2026-07-29T02:48:06.653857+00:00",
+            "resolved_tasks": []
         },
         "TimesFM-2.5": {
             "model": "TimesFM-2.5",
@@ -57,18 +37,8 @@
             "admitted_at": "2026-06-01T00:00:00Z",
             "evaluation_protocol": "issue-now-score-later",
             "metric_version": "livehouse-canonical-v1",
-            "evaluated_at": "2026-07-29T02:34:25.634421+00:00",
-            "resolved_tasks": [
-                {
-                    "dataset": "data_gov_sg_singapore_taxi_availability_available_taxi_count/5min/short",
-                    "task_id": "future-c056385aaaf66931e9d4",
-                    "issued_at": "2026-07-28T19:38:05.211084+00:00",
-                    "resolved_at": "2026-07-29T02:31:23.196597+00:00",
-                    "target_end": "2026-07-27T17:35:00+00:00",
-                    "prediction_length": 12,
-                    "metric_version": "livehouse-canonical-v1"
-                }
-            ]
+            "evaluated_at": "2026-07-29T02:48:06.653859+00:00",
+            "resolved_tasks": []
         },
         "Toto-1.0": {
             "model": "Toto-1.0",
@@ -76,18 +46,8 @@
             "admitted_at": "2026-06-01T00:00:00Z",
             "evaluation_protocol": "issue-now-score-later",
             "metric_version": "livehouse-canonical-v1",
-            "evaluated_at": "2026-07-29T02:34:25.642155+00:00",
-            "resolved_tasks": [
-                {
-                    "dataset": "data_gov_sg_singapore_taxi_availability_available_taxi_count/5min/short",
-                    "task_id": "future-c056385aaaf66931e9d4",
-                    "issued_at": "2026-07-28T19:38:05.211084+00:00",
-                    "resolved_at": "2026-07-29T02:31:23.267028+00:00",
-                    "target_end": "2026-07-27T17:35:00+00:00",
-                    "prediction_length": 12,
-                    "metric_version": "livehouse-canonical-v1"
-                }
-            ]
+            "evaluated_at": "2026-07-29T02:48:06.653860+00:00",
+            "resolved_tasks": []
         },
         "Moirai-2.0": {
             "model": "Moirai-2.0",
@@ -95,18 +55,8 @@
             "admitted_at": "2026-06-01T00:00:00Z",
             "evaluation_protocol": "issue-now-score-later",
             "metric_version": "livehouse-canonical-v1",
-            "evaluated_at": "2026-07-29T02:34:25.651854+00:00",
-            "resolved_tasks": [
-                {
-                    "dataset": "data_gov_sg_singapore_taxi_availability_available_taxi_count/5min/short",
-                    "task_id": "future-c056385aaaf66931e9d4",
-                    "issued_at": "2026-07-28T19:38:05.211084+00:00",
-                    "resolved_at": "2026-07-29T02:31:22.938113+00:00",
-                    "target_end": "2026-07-27T17:35:00+00:00",
-                    "prediction_length": 12,
-                    "metric_version": "livehouse-canonical-v1"
-                }
-            ]
+            "evaluated_at": "2026-07-29T02:48:06.653860+00:00",
+            "resolved_tasks": []
         },
         "Chronos-Bolt": {
             "model": "Chronos-Bolt",
@@ -114,18 +64,8 @@
             "admitted_at": "2026-06-01T00:00:00Z",
             "evaluation_protocol": "issue-now-score-later",
             "metric_version": "livehouse-canonical-v1",
-            "evaluated_at": "2026-07-29T02:34:25.654828+00:00",
-            "resolved_tasks": [
-                {
-                    "dataset": "data_gov_sg_singapore_taxi_availability_available_taxi_count/5min/short",
-                    "task_id": "future-c056385aaaf66931e9d4",
-                    "issued_at": "2026-07-28T19:38:05.211084+00:00",
-                    "resolved_at": "2026-07-29T02:31:22.830152+00:00",
-                    "target_end": "2026-07-27T17:35:00+00:00",
-                    "prediction_length": 12,
-                    "metric_version": "livehouse-canonical-v1"
-                }
-            ]
+            "evaluated_at": "2026-07-29T02:48:06.653862+00:00",
+            "resolved_tasks": []
         },
         "TabPFN-TS": {
             "model": "TabPFN-TS",
@@ -133,18 +73,8 @@
             "admitted_at": "2026-06-01T00:00:00Z",
             "evaluation_protocol": "issue-now-score-later",
             "metric_version": "livehouse-canonical-v1",
-            "evaluated_at": "2026-07-29T02:34:25.661364+00:00",
-            "resolved_tasks": [
-                {
-                    "dataset": "data_gov_sg_singapore_taxi_availability_available_taxi_count/5min/short",
-                    "task_id": "future-c056385aaaf66931e9d4",
-                    "issued_at": "2026-07-28T19:38:05.211084+00:00",
-                    "resolved_at": "2026-07-29T02:31:23.166139+00:00",
-                    "target_end": "2026-07-27T17:35:00+00:00",
-                    "prediction_length": 12,
-                    "metric_version": "livehouse-canonical-v1"
-                }
-            ]
+            "evaluated_at": "2026-07-29T02:48:06.653863+00:00",
+            "resolved_tasks": []
         },
         "Sundial": {
             "model": "Sundial",
@@ -152,18 +82,8 @@
             "admitted_at": "2026-06-01T00:00:00Z",
             "evaluation_protocol": "issue-now-score-later",
             "metric_version": "livehouse-canonical-v1",
-            "evaluated_at": "2026-07-29T02:34:25.685937+00:00",
-            "resolved_tasks": [
-                {
-                    "dataset": "data_gov_sg_singapore_taxi_availability_available_taxi_count/5min/short",
-                    "task_id": "future-c056385aaaf66931e9d4",
-                    "issued_at": "2026-07-28T19:38:05.211084+00:00",
-                    "resolved_at": "2026-07-29T02:31:23.126081+00:00",
-                    "target_end": "2026-07-27T17:35:00+00:00",
-                    "prediction_length": 12,
-                    "metric_version": "livehouse-canonical-v1"
-                }
-            ]
+            "evaluated_at": "2026-07-29T02:48:06.653864+00:00",
+            "resolved_tasks": []
         },
         "Moving-Average": {
             "model": "Moving-Average",
@@ -171,18 +91,8 @@
             "admitted_at": "2026-06-01T00:00:00Z",
             "evaluation_protocol": "issue-now-score-later",
             "metric_version": "livehouse-canonical-v1",
-            "evaluated_at": "2026-07-29T02:34:25.687760+00:00",
-            "resolved_tasks": [
-                {
-                    "dataset": "data_gov_sg_singapore_taxi_availability_available_taxi_count/5min/short",
-                    "task_id": "future-c056385aaaf66931e9d4",
-                    "issued_at": "2026-07-28T19:38:05.211084+00:00",
-                    "resolved_at": "2026-07-29T02:31:23.003115+00:00",
-                    "target_end": "2026-07-27T17:35:00+00:00",
-                    "prediction_length": 12,
-                    "metric_version": "livehouse-canonical-v1"
-                }
-            ]
+            "evaluated_at": "2026-07-29T02:48:06.653865+00:00",
+            "resolved_tasks": []
         },
         "ETS": {
             "model": "ETS",
@@ -190,18 +100,8 @@
             "admitted_at": "2026-06-01T00:00:00Z",
             "evaluation_protocol": "issue-now-score-later",
             "metric_version": "livehouse-canonical-v1",
-            "evaluated_at": "2026-07-29T02:34:25.693113+00:00",
-            "resolved_tasks": [
-                {
-                    "dataset": "data_gov_sg_singapore_taxi_availability_available_taxi_count/5min/short",
-                    "task_id": "future-c056385aaaf66931e9d4",
-                    "issued_at": "2026-07-28T19:38:05.211084+00:00",
-                    "resolved_at": "2026-07-29T02:31:22.864048+00:00",
-                    "target_end": "2026-07-27T17:35:00+00:00",
-                    "prediction_length": 12,
-                    "metric_version": "livehouse-canonical-v1"
-                }
-            ]
+            "evaluated_at": "2026-07-29T02:48:06.653865+00:00",
+            "resolved_tasks": []
         },
         "ARIMA": {
             "model": "ARIMA",
@@ -209,18 +109,8 @@
             "admitted_at": "2026-06-01T00:00:00Z",
             "evaluation_protocol": "issue-now-score-later",
             "metric_version": "livehouse-canonical-v1",
-            "evaluated_at": "2026-07-29T02:34:25.696398+00:00",
-            "resolved_tasks": [
-                {
-                    "dataset": "data_gov_sg_singapore_taxi_availability_available_taxi_count/5min/short",
-                    "task_id": "future-c056385aaaf66931e9d4",
-                    "issued_at": "2026-07-28T19:38:05.211084+00:00",
-                    "resolved_at": "2026-07-29T02:31:22.692674+00:00",
-                    "target_end": "2026-07-27T17:35:00+00:00",
-                    "prediction_length": 12,
-                    "metric_version": "livehouse-canonical-v1"
-                }
-            ]
+            "evaluated_at": "2026-07-29T02:48:06.653866+00:00",
+            "resolved_tasks": []
         },
         "Seasonal-Naive": {
             "model": "Seasonal-Naive",
@@ -228,18 +118,8 @@
             "admitted_at": "2026-06-01T00:00:00Z",
             "evaluation_protocol": "issue-now-score-later",
             "metric_version": "livehouse-canonical-v1",
-            "evaluated_at": "2026-07-29T02:34:25.720777+00:00",
-            "resolved_tasks": [
-                {
-                    "dataset": "data_gov_sg_singapore_taxi_availability_available_taxi_count/5min/short",
-                    "task_id": "future-c056385aaaf66931e9d4",
-                    "issued_at": "2026-07-28T19:38:05.211084+00:00",
-                    "resolved_at": "2026-07-29T02:31:23.038085+00:00",
-                    "target_end": "2026-07-27T17:35:00+00:00",
-                    "prediction_length": 12,
-                    "metric_version": "livehouse-canonical-v1"
-                }
-            ]
+            "evaluated_at": "2026-07-29T02:48:06.653867+00:00",
+            "resolved_tasks": []
         }
     },
     "skipped_models": [],
@@ -262,7 +142,5 @@
         "live_overall_30d": "aggregates/live_overall_30d.csv",
         "live_metadata": "aggregates/live_metadata.json"
     },
-    "push_status": "ok",
-    "pushed_at": "2026-07-29T02:35:31.675395+00:00",
-    "push_detail": "HF Hub API"
+    "push_status": "pending"
 }