Update leaderboard results from local eval

results/aggregates/live_metadata.json

@@ -1,5 +1,5 @@
 {
-  "generated_at": "2026-07-30T02:33:53.918333+00:00",
+  "generated_at": "2026-07-30T02:45:20.109668+00:00",
   "scope": "Canonical metrics-only release history when available; otherwise legacy release history plus latest allowlisted snapshots",
   "release_deduplication": "One row per model, dataset, data source timestamp, and prediction length",
   "dataset_aggregation": "Arithmetic mean over releases, then equal-weight mean over datasets",

results/aggregates/metadata.json

@@ -1,5 +1,5 @@
 {
-  "generated_at": "2026-07-30T02:33:51.743237+00:00",
+  "generated_at": "2026-07-30T02:45:17.948618+00:00",
   "baseline_model": "Seasonal-Naive",
   "normalization": "Per-config metric / Seasonal-Naive metric",
   "metric_aggregation": "Geometric mean over dataset configurations",
@@ -10,7 +10,7 @@
     "frequency",
     "prediction_length"
   ],
-  "num_configs": 13,
+  "num_configs": 12,
   "num_models": 12,
   "num_rows_missing_prediction_length": 0
 }

results/aggregates/results_by_domain.csv

@@ -71,15 +71,3 @@ Transportation,Moving-Average,1.0,250217.138218906,9,1,1,1.0
 Transportation,Moirai-2.0,1.0,410051.5180343537,10,1,1,1.0
 Transportation,Chronos-Bolt,1.094103907762285,3214861.8499168884,11,1,1,1.0
 Transportation,Toto-1.0,1028380083468.5435,276946616.8568997,12,1,1,1.0
-Web,ARIMA,0.35081770342317864,0.47944164778388026,1,1,1,1.0
-Web,Moirai-2.0,0.40785680526551005,0.5118677129116019,2,1,1,1.0
-Web,Sundial,0.303769640015456,0.5282879649026628,3,1,1,1.0
-Web,ETS,0.4243826259793997,0.5449399343905824,4,1,1,1.0
-Web,Moving-Average,0.4370340438348723,0.54802384805273,5,1,1,1.0
-Web,Chronos-2,0.4498860462918392,0.5739884284864399,6,1,1,1.0
-Web,TiRex,0.46006192226374054,0.5949360110321206,7,1,1,1.0
-Web,TimesFM-2.5,0.32382897462809757,0.6278956823267848,8,1,1,1.0
-Web,Chronos-Bolt,0.3947163650820193,0.6425428458659833,9,1,1,1.0
-Web,Toto-1.0,1.1828445386062165,0.7008146109363338,10,1,1,1.0
-Web,Seasonal-Naive,1.0,1.0,11,1,1,1.0
-Web,TabPFN-TS,0.8629829581417852,1.004036040514957,12,1,1,1.0

results/aggregates/results_by_frequency.csv

@@ -23,18 +23,6 @@ frequency,model,MSE,CRPS,Rank,n_configs,n_group_configs,coverage
 15min,Sundial,1.3977594282011032,1.074296938071852,10,1,1,1.0
 15min,TabPFN-TS,1.1276707612981798,1.1435318896536943,11,1,1,1.0
 15min,Moving-Average,1.0229049999998812,1.1718693936132067,12,1,1,1.0
-1D,ARIMA,0.35081770342317864,0.47944164778388026,1,1,1,1.0
-1D,Moirai-2.0,0.40785680526551005,0.5118677129116019,2,1,1,1.0
-1D,Sundial,0.303769640015456,0.5282879649026628,3,1,1,1.0
-1D,ETS,0.4243826259793997,0.5449399343905824,4,1,1,1.0
-1D,Moving-Average,0.4370340438348723,0.54802384805273,5,1,1,1.0
-1D,Chronos-2,0.4498860462918392,0.5739884284864399,6,1,1,1.0
-1D,TiRex,0.46006192226374054,0.5949360110321206,7,1,1,1.0
-1D,TimesFM-2.5,0.32382897462809757,0.6278956823267848,8,1,1,1.0
-1D,Chronos-Bolt,0.3947163650820193,0.6425428458659833,9,1,1,1.0
-1D,Toto-1.0,1.1828445386062165,0.7008146109363338,10,1,1,1.0
-1D,Seasonal-Naive,1.0,1.0,11,1,1,1.0
-1D,TabPFN-TS,0.8629829581417852,1.004036040514957,12,1,1,1.0
 1h,Chronos-2,0.4700571173602689,0.738760910451211,1,6,6,1.0
 1h,Sundial,0.8266958160973342,0.7589339078501757,2,6,6,1.0
 1h,Toto-1.0,1.2070389322819421,0.7621521414652723,3,6,6,1.0

results/aggregates/results_by_prediction_length.csv

@@ -1,16 +1,4 @@
 prediction_length,model,MSE,CRPS,Rank,n_configs,n_group_configs,coverage
-7,ARIMA,0.35081770342317864,0.47944164778388026,1,1,1,1.0
-7,Moirai-2.0,0.40785680526551005,0.5118677129116019,2,1,1,1.0
-7,Sundial,0.303769640015456,0.5282879649026628,3,1,1,1.0
-7,ETS,0.4243826259793997,0.5449399343905824,4,1,1,1.0
-7,Moving-Average,0.4370340438348723,0.54802384805273,5,1,1,1.0
-7,Chronos-2,0.4498860462918392,0.5739884284864399,6,1,1,1.0
-7,TiRex,0.46006192226374054,0.5949360110321206,7,1,1,1.0
-7,TimesFM-2.5,0.32382897462809757,0.6278956823267848,8,1,1,1.0
-7,Chronos-Bolt,0.3947163650820193,0.6425428458659833,9,1,1,1.0
-7,Toto-1.0,1.1828445386062165,0.7008146109363338,10,1,1,1.0
-7,Seasonal-Naive,1.0,1.0,11,1,1,1.0
-7,TabPFN-TS,0.8629829581417852,1.004036040514957,12,1,1,1.0
 12,Chronos-2,0.8294476545156683,38.946376459967496,1,2,2,1.0
 12,TiRex,0.847802672121473,30.22591777828443,2,2,2,1.0
 12,Moirai-2.0,0.7551088053055054,483.94979926515697,3,2,2,1.0

results/aggregates/results_overall.csv

@@ -1,13 +1,13 @@
 overall,model,MSE,CRPS,Rank,n_configs,n_group_configs,coverage
-Overall,Chronos-2,0.43268228989411894,2.1361019300883823,1,13,13,1.0
-Overall,TiRex,0.40995577193139365,2.344352317307399,2,13,13,1.0
-Overall,Sundial,0.5863450480593485,3.4406355318900266,3,13,13,1.0
-Overall,Moirai-2.0,0.5286975323240499,5.343807856551616,4,13,13,1.0
-Overall,TimesFM-2.5,0.4487953726978033,0.7310069040248663,5,13,13,1.0
-Overall,Chronos-Bolt,0.5934827454627915,5.853165573506846,6,13,13,1.0
-Overall,Toto-1.0,72.27252382299581,14.897341447673787,7,13,13,1.0
-Overall,ARIMA,1.4160569065190673,7.882303531315528,8,13,13,1.0
-Overall,Seasonal-Naive,1.0,1.0,9,13,13,1.0
-Overall,TabPFN-TS,0.8709943017830731,0.9811288865190397,10,13,13,1.0
-Overall,Moving-Average,1.401098006741648,8.005311788166683,11,13,13,1.0
-Overall,ETS,2.4684777060273344,10.6118711698599,12,13,13,1.0
+Overall,Chronos-2,0.431278692803954,2.383317354396122,1,12,12,1.0
+Overall,TiRex,0.40603524360779164,2.628162805248538,2,12,12,1.0
+Overall,Sundial,0.6193754722478689,4.022098166553945,3,12,12,1.0
+Overall,TimesFM-2.5,0.46116827775238495,0.7403282307219852,4,12,12,1.0
+Overall,Moirai-2.0,0.5402551483958009,6.49743473183667,5,12,12,1.0
+Overall,Chronos-Bolt,0.613999961769014,7.036362455455267,6,12,12,1.0
+Overall,Toto-1.0,101.8146681953687,19.219152844707423,7,12,12,1.0
+Overall,ARIMA,1.5906720142287076,9.953586716581292,8,12,12,1.0
+Overall,Seasonal-Naive,1.0,1.0,9,12,12,1.0
+Overall,TabPFN-TS,0.8716652613348104,0.9792437141631307,10,12,12,1.0
+Overall,Moving-Average,1.543943081618297,10.009824983346837,11,12,12,1.0
+Overall,ETS,2.858589826212619,13.590817479646768,12,12,12,1.0

results/dataset_properties.csv

@@ -11,4 +11,3 @@ open_meteo_shanghai_air_quality_hourly_pm2_5,Air Quality,1h,1
 open_meteo_shanghai_hourly_temperature_2m,Weather,1h,1
 usgs_earthquake_all_week_hourly_earthquake_count,Hazards,1h,1
 usgs_potomac_iv_usgs_00060,Hydrology,15min,1
-wikimedia_time_series_hourly_pageviews,Web,1D,1

results/online_status.json

@@ -1,23 +1,17 @@
 {
-    "status": "partial",
+    "status": "ok",
     "evaluation_protocol": "issue-now-score-later",
     "metric_version": "livehouse-canonical-v1",
-    "started_at": "2026-07-30T02:31:22.552849+00:00",
-    "finished_at": "2026-07-30T02:33:53.931151+00:00",
+    "started_at": "2026-07-30T02:44:42.758985+00:00",
+    "finished_at": "2026-07-30T02:45:20.124321+00:00",
     "data_source": "ts_bench",
     "data_config": "configs/datasets/ts_bench.yaml",
     "task_count": 4,
-    "issued_tasks": 1,
-    "issued_forecasts": 12,
-    "resolved_forecasts": 12,
+    "issued_tasks": 0,
+    "issued_forecasts": 1,
+    "resolved_forecasts": 0,
     "pending_tasks": 13,
-    "failed_forecasts": [
-        {
-            "model": "Seasonal-Naive",
-            "dataset": "nws_ksfo_observations_temperature/1h/short",
-            "error": "Mean forecast length 10 < 24"
-        }
-    ],
+    "failed_forecasts": [],
     "models": {
         "Chronos-2": {
             "model": "Chronos-2",
@@ -25,18 +19,8 @@
             "admitted_at": "2026-06-01T00:00:00Z",
             "evaluation_protocol": "issue-now-score-later",
             "metric_version": "livehouse-canonical-v1",
-            "evaluated_at": "2026-07-30T02:33:49.809680+00:00",
-            "resolved_tasks": [
-                {
-                    "dataset": "binance_btcusdt_1h_close/1h/short",
-                    "task_id": "future-9f0a77a1edc9a1c492a8",
-                    "issued_at": "2026-07-29T02:13:56.017653+00:00",
-                    "resolved_at": "2026-07-30T02:31:24.460581+00:00",
-                    "target_end": "2026-07-30T02:00:00+00:00",
-                    "prediction_length": 24,
-                    "metric_version": "livehouse-canonical-v1"
-                }
-            ]
+            "evaluated_at": "2026-07-30T02:45:16.188641+00:00",
+            "resolved_tasks": []
         },
         "TiRex": {
             "model": "TiRex",
@@ -44,18 +28,8 @@
             "admitted_at": "2026-06-01T00:00:00Z",
             "evaluation_protocol": "issue-now-score-later",
             "metric_version": "livehouse-canonical-v1",
-            "evaluated_at": "2026-07-30T02:33:49.828322+00:00",
-            "resolved_tasks": [
-                {
-                    "dataset": "binance_btcusdt_1h_close/1h/short",
-                    "task_id": "future-9f0a77a1edc9a1c492a8",
-                    "issued_at": "2026-07-29T02:13:56.017653+00:00",
-                    "resolved_at": "2026-07-30T02:31:24.720552+00:00",
-                    "target_end": "2026-07-30T02:00:00+00:00",
-                    "prediction_length": 24,
-                    "metric_version": "livehouse-canonical-v1"
-                }
-            ]
+            "evaluated_at": "2026-07-30T02:45:16.188659+00:00",
+            "resolved_tasks": []
         },
         "TimesFM-2.5": {
             "model": "TimesFM-2.5",
@@ -63,18 +37,8 @@
             "admitted_at": "2026-06-01T00:00:00Z",
             "evaluation_protocol": "issue-now-score-later",
             "metric_version": "livehouse-canonical-v1",
-            "evaluated_at": "2026-07-30T02:33:49.834434+00:00",
-            "resolved_tasks": [
-                {
-                    "dataset": "binance_btcusdt_1h_close/1h/short",
-                    "task_id": "future-9f0a77a1edc9a1c492a8",
-                    "issued_at": "2026-07-29T02:13:56.017653+00:00",
-                    "resolved_at": "2026-07-30T02:31:24.672628+00:00",
-                    "target_end": "2026-07-30T02:00:00+00:00",
-                    "prediction_length": 24,
-                    "metric_version": "livehouse-canonical-v1"
-                }
-            ]
+            "evaluated_at": "2026-07-30T02:45:16.188663+00:00",
+            "resolved_tasks": []
         },
         "Toto-1.0": {
             "model": "Toto-1.0",
@@ -82,18 +46,8 @@
             "admitted_at": "2026-06-01T00:00:00Z",
             "evaluation_protocol": "issue-now-score-later",
             "metric_version": "livehouse-canonical-v1",
-            "evaluated_at": "2026-07-30T02:33:49.837580+00:00",
-            "resolved_tasks": [
-                {
-                    "dataset": "binance_btcusdt_1h_close/1h/short",
-                    "task_id": "future-9f0a77a1edc9a1c492a8",
-                    "issued_at": "2026-07-29T02:13:56.017653+00:00",
-                    "resolved_at": "2026-07-30T02:31:24.759946+00:00",
-                    "target_end": "2026-07-30T02:00:00+00:00",
-                    "prediction_length": 24,
-                    "metric_version": "livehouse-canonical-v1"
-                }
-            ]
+            "evaluated_at": "2026-07-30T02:45:16.188665+00:00",
+            "resolved_tasks": []
         },
         "Moirai-2.0": {
             "model": "Moirai-2.0",
@@ -101,18 +55,8 @@
             "admitted_at": "2026-06-01T00:00:00Z",
             "evaluation_protocol": "issue-now-score-later",
             "metric_version": "livehouse-canonical-v1",
-            "evaluated_at": "2026-07-30T02:33:49.844359+00:00",
-            "resolved_tasks": [
-                {
-                    "dataset": "binance_btcusdt_1h_close/1h/short",
-                    "task_id": "future-9f0a77a1edc9a1c492a8",
-                    "issued_at": "2026-07-29T02:13:56.017653+00:00",
-                    "resolved_at": "2026-07-30T02:31:24.556996+00:00",
-                    "target_end": "2026-07-30T02:00:00+00:00",
-                    "prediction_length": 24,
-                    "metric_version": "livehouse-canonical-v1"
-                }
-            ]
+            "evaluated_at": "2026-07-30T02:45:16.188668+00:00",
+            "resolved_tasks": []
         },
         "Chronos-Bolt": {
             "model": "Chronos-Bolt",
@@ -120,18 +64,8 @@
             "admitted_at": "2026-06-01T00:00:00Z",
             "evaluation_protocol": "issue-now-score-later",
             "metric_version": "livehouse-canonical-v1",
-            "evaluated_at": "2026-07-30T02:33:49.857920+00:00",
-            "resolved_tasks": [
-                {
-                    "dataset": "binance_btcusdt_1h_close/1h/short",
-                    "task_id": "future-9f0a77a1edc9a1c492a8",
-                    "issued_at": "2026-07-29T02:13:56.017653+00:00",
-                    "resolved_at": "2026-07-30T02:31:24.489620+00:00",
-                    "target_end": "2026-07-30T02:00:00+00:00",
-                    "prediction_length": 24,
-                    "metric_version": "livehouse-canonical-v1"
-                }
-            ]
+            "evaluated_at": "2026-07-30T02:45:16.188671+00:00",
+            "resolved_tasks": []
         },
         "TabPFN-TS": {
             "model": "TabPFN-TS",
@@ -139,18 +73,8 @@
             "admitted_at": "2026-06-01T00:00:00Z",
             "evaluation_protocol": "issue-now-score-later",
             "metric_version": "livehouse-canonical-v1",
-            "evaluated_at": "2026-07-30T02:33:49.870832+00:00",
-            "resolved_tasks": [
-                {
-                    "dataset": "binance_btcusdt_1h_close/1h/short",
-                    "task_id": "future-9f0a77a1edc9a1c492a8",
-                    "issued_at": "2026-07-29T02:13:56.017653+00:00",
-                    "resolved_at": "2026-07-30T02:31:24.647908+00:00",
-                    "target_end": "2026-07-30T02:00:00+00:00",
-                    "prediction_length": 24,
-                    "metric_version": "livehouse-canonical-v1"
-                }
-            ]
+            "evaluated_at": "2026-07-30T02:45:16.188674+00:00",
+            "resolved_tasks": []
         },
         "Sundial": {
             "model": "Sundial",
@@ -158,18 +82,8 @@
             "admitted_at": "2026-06-01T00:00:00Z",
             "evaluation_protocol": "issue-now-score-later",
             "metric_version": "livehouse-canonical-v1",
-            "evaluated_at": "2026-07-30T02:33:49.886934+00:00",
-            "resolved_tasks": [
-                {
-                    "dataset": "binance_btcusdt_1h_close/1h/short",
-                    "task_id": "future-9f0a77a1edc9a1c492a8",
-                    "issued_at": "2026-07-29T02:13:56.017653+00:00",
-                    "resolved_at": "2026-07-30T02:31:24.629916+00:00",
-                    "target_end": "2026-07-30T02:00:00+00:00",
-                    "prediction_length": 24,
-                    "metric_version": "livehouse-canonical-v1"
-                }
-            ]
+            "evaluated_at": "2026-07-30T02:45:16.188676+00:00",
+            "resolved_tasks": []
         },
         "Moving-Average": {
             "model": "Moving-Average",
@@ -177,18 +91,8 @@
             "admitted_at": "2026-06-01T00:00:00Z",
             "evaluation_protocol": "issue-now-score-later",
             "metric_version": "livehouse-canonical-v1",
-            "evaluated_at": "2026-07-30T02:33:49.891404+00:00",
-            "resolved_tasks": [
-                {
-                    "dataset": "binance_btcusdt_1h_close/1h/short",
-                    "task_id": "future-9f0a77a1edc9a1c492a8",
-                    "issued_at": "2026-07-29T02:13:56.017653+00:00",
-                    "resolved_at": "2026-07-30T02:31:24.580203+00:00",
-                    "target_end": "2026-07-30T02:00:00+00:00",
-                    "prediction_length": 24,
-                    "metric_version": "livehouse-canonical-v1"
-                }
-            ]
+            "evaluated_at": "2026-07-30T02:45:16.188679+00:00",
+            "resolved_tasks": []
         },
         "ETS": {
             "model": "ETS",
@@ -196,18 +100,8 @@
             "admitted_at": "2026-06-01T00:00:00Z",
             "evaluation_protocol": "issue-now-score-later",
             "metric_version": "livehouse-canonical-v1",
-            "evaluated_at": "2026-07-30T02:33:49.898563+00:00",
-            "resolved_tasks": [
-                {
-                    "dataset": "binance_btcusdt_1h_close/1h/short",
-                    "task_id": "future-9f0a77a1edc9a1c492a8",
-                    "issued_at": "2026-07-29T02:13:56.017653+00:00",
-                    "resolved_at": "2026-07-30T02:31:24.525125+00:00",
-                    "target_end": "2026-07-30T02:00:00+00:00",
-                    "prediction_length": 24,
-                    "metric_version": "livehouse-canonical-v1"
-                }
-            ]
+            "evaluated_at": "2026-07-30T02:45:16.188682+00:00",
+            "resolved_tasks": []
         },
         "ARIMA": {
             "model": "ARIMA",
@@ -215,18 +109,8 @@
             "admitted_at": "2026-06-01T00:00:00Z",
             "evaluation_protocol": "issue-now-score-later",
             "metric_version": "livehouse-canonical-v1",
-            "evaluated_at": "2026-07-30T02:33:49.934478+00:00",
-            "resolved_tasks": [
-                {
-                    "dataset": "binance_btcusdt_1h_close/1h/short",
-                    "task_id": "future-9f0a77a1edc9a1c492a8",
-                    "issued_at": "2026-07-29T02:13:56.017653+00:00",
-                    "resolved_at": "2026-07-30T02:31:24.429726+00:00",
-                    "target_end": "2026-07-30T02:00:00+00:00",
-                    "prediction_length": 24,
-                    "metric_version": "livehouse-canonical-v1"
-                }
-            ]
+            "evaluated_at": "2026-07-30T02:45:16.188685+00:00",
+            "resolved_tasks": []
         },
         "Seasonal-Naive": {
             "model": "Seasonal-Naive",
@@ -234,18 +118,8 @@
             "admitted_at": "2026-06-01T00:00:00Z",
             "evaluation_protocol": "issue-now-score-later",
             "metric_version": "livehouse-canonical-v1",
-            "evaluated_at": "2026-07-30T02:33:49.972570+00:00",
-            "resolved_tasks": [
-                {
-                    "dataset": "binance_btcusdt_1h_close/1h/short",
-                    "task_id": "future-9f0a77a1edc9a1c492a8",
-                    "issued_at": "2026-07-29T02:13:56.017653+00:00",
-                    "resolved_at": "2026-07-30T02:31:24.610872+00:00",
-                    "target_end": "2026-07-30T02:00:00+00:00",
-                    "prediction_length": 24,
-                    "metric_version": "livehouse-canonical-v1"
-                }
-            ]
+            "evaluated_at": "2026-07-30T02:45:16.188687+00:00",
+            "resolved_tasks": []
         }
     },
     "skipped_models": [],
@@ -268,7 +142,5 @@
         "live_overall_30d": "aggregates/live_overall_30d.csv",
         "live_metadata": "aggregates/live_metadata.json"
     },
-    "push_status": "ok",
-    "pushed_at": "2026-07-30T02:34:37.188716+00:00",
-    "push_detail": "HF Hub API"
+    "push_status": "pending"
 }