Update leaderboard results from local eval

results/aggregates/live_metadata.json

@@ -1,5 +1,5 @@
 {
-  "generated_at": "2026-07-28T20:39:38.529995+00:00",
+  "generated_at": "2026-07-28T22:56:37.586274+00:00",
   "scope": "Canonical metrics-only release history when available; otherwise legacy release history plus latest allowlisted snapshots",
   "release_deduplication": "One row per model, dataset, data source timestamp, and prediction length",
   "dataset_aggregation": "Arithmetic mean over releases, then equal-weight mean over datasets",

results/aggregates/metadata.json

@@ -1,5 +1,5 @@
 {
-  "generated_at": "2026-07-28T20:39:36.352965+00:00",
+  "generated_at": "2026-07-28T22:56:35.560484+00:00",
   "baseline_model": "Seasonal-Naive",
   "normalization": "Per-config metric / Seasonal-Naive metric",
   "metric_aggregation": "Geometric mean over dataset configurations",
@@ -10,7 +10,7 @@
     "frequency",
     "prediction_length"
   ],
-  "num_configs": 13,
+  "num_configs": 12,
   "num_models": 12,
   "num_rows_missing_prediction_length": 0
 }

results/aggregates/results_by_domain.csv

@@ -71,15 +71,3 @@ Transportation,Moving-Average,1.0,250217.138218906,9,1,1,1.0
 Transportation,Moirai-2.0,1.0,410051.5180343537,10,1,1,1.0
 Transportation,Chronos-Bolt,1.094103907762285,3214861.8499168884,11,1,1,1.0
 Transportation,Toto-1.0,18677488.633820288,280673160.4289003,12,1,1,1.0
-Web,ARIMA,0.35081770342317864,0.47944164778388026,1,1,1,1.0
-Web,Moirai-2.0,0.40785680526551005,0.5118677129116019,2,1,1,1.0
-Web,Sundial,0.303769640015456,0.5282879649026628,3,1,1,1.0
-Web,ETS,0.4243826259793997,0.5449399343905824,4,1,1,1.0
-Web,Moving-Average,0.4370340438348723,0.54802384805273,5,1,1,1.0
-Web,Chronos-2,0.4498860462918392,0.5739884284864399,6,1,1,1.0
-Web,TiRex,0.46006192226374054,0.5949360110321206,7,1,1,1.0
-Web,TimesFM-2.5,0.32382897462809757,0.6278956823267848,8,1,1,1.0
-Web,Chronos-Bolt,0.3947163650820193,0.6425428458659833,9,1,1,1.0
-Web,Toto-1.0,1.1828445386062165,0.7008146109363338,10,1,1,1.0
-Web,Seasonal-Naive,1.0,1.0,11,1,1,1.0
-Web,TabPFN-TS,0.8629829581417852,1.004036040514957,12,1,1,1.0

results/aggregates/results_by_frequency.csv

@@ -23,18 +23,6 @@ frequency,model,MSE,CRPS,Rank,n_configs,n_group_configs,coverage
 15min,Chronos-2,1.0073362506270407,1.4859340309570945,10,1,1,1.0
 15min,TiRex,1.0012405403812725,1.4861053101432065,11,1,1,1.0
 15min,Sundial,1.440734424960613,1.5535414274488537,12,1,1,1.0
-1D,ARIMA,0.35081770342317864,0.47944164778388026,1,1,1,1.0
-1D,Moirai-2.0,0.40785680526551005,0.5118677129116019,2,1,1,1.0
-1D,Sundial,0.303769640015456,0.5282879649026628,3,1,1,1.0
-1D,ETS,0.4243826259793997,0.5449399343905824,4,1,1,1.0
-1D,Moving-Average,0.4370340438348723,0.54802384805273,5,1,1,1.0
-1D,Chronos-2,0.4498860462918392,0.5739884284864399,6,1,1,1.0
-1D,TiRex,0.46006192226374054,0.5949360110321206,7,1,1,1.0
-1D,TimesFM-2.5,0.32382897462809757,0.6278956823267848,8,1,1,1.0
-1D,Chronos-Bolt,0.3947163650820193,0.6425428458659833,9,1,1,1.0
-1D,Toto-1.0,1.1828445386062165,0.7008146109363338,10,1,1,1.0
-1D,Seasonal-Naive,1.0,1.0,11,1,1,1.0
-1D,TabPFN-TS,0.8629829581417852,1.004036040514957,12,1,1,1.0
 1h,TiRex,0.3508509668853035,0.7055885069620663,1,6,6,1.0
 1h,Chronos-2,0.37951076735916145,0.6934037615919085,2,6,6,1.0
 1h,Chronos-Bolt,0.5094193160936651,0.704481403367288,3,6,6,1.0

results/aggregates/results_by_prediction_length.csv

@@ -1,16 +1,4 @@
 prediction_length,model,MSE,CRPS,Rank,n_configs,n_group_configs,coverage
-7,ARIMA,0.35081770342317864,0.47944164778388026,1,1,1,1.0
-7,Moirai-2.0,0.40785680526551005,0.5118677129116019,2,1,1,1.0
-7,Sundial,0.303769640015456,0.5282879649026628,3,1,1,1.0
-7,ETS,0.4243826259793997,0.5449399343905824,4,1,1,1.0
-7,Moving-Average,0.4370340438348723,0.54802384805273,5,1,1,1.0
-7,Chronos-2,0.4498860462918392,0.5739884284864399,6,1,1,1.0
-7,TiRex,0.46006192226374054,0.5949360110321206,7,1,1,1.0
-7,TimesFM-2.5,0.32382897462809757,0.6278956823267848,8,1,1,1.0
-7,Chronos-Bolt,0.3947163650820193,0.6425428458659833,9,1,1,1.0
-7,Toto-1.0,1.1828445386062165,0.7008146109363338,10,1,1,1.0
-7,Seasonal-Naive,1.0,1.0,11,1,1,1.0
-7,TabPFN-TS,0.8629829581417852,1.004036040514957,12,1,1,1.0
 12,Chronos-2,0.8294476545156683,38.946376459967496,1,2,2,1.0
 12,TiRex,0.847802672121473,30.22591777828443,2,2,2,1.0
 12,Moirai-2.0,0.7551088053055054,483.94979926515697,3,2,2,1.0

results/aggregates/results_overall.csv

@@ -1,13 +1,13 @@
 overall,model,MSE,CRPS,Rank,n_configs,n_group_configs,coverage
-Overall,Chronos-2,0.1343403384720884,1.0212108671061753,1,13,13,1.0
-Overall,TiRex,0.12136897746563971,1.0087373287357833,2,13,13,1.0
-Overall,Moirai-2.0,0.1641530620565157,1.1991207340384495,3,13,13,1.0
-Overall,Toto-1.0,3.60779514491457,2.0020771961529915,4,13,13,1.0
-Overall,Sundial,0.36200337604088695,1.1438317036382775,5,13,13,1.0
-Overall,Chronos-Bolt,0.21286492814165073,1.8566289422565119,6,13,13,1.0
-Overall,ARIMA,0.6454634541730412,1.486398876025313,7,13,13,1.0
-Overall,TimesFM-2.5,0.1582783902689042,0.484951446122198,8,13,13,1.0
-Overall,ETS,0.7846172559779437,0.3821256743173788,9,13,13,1.0
-Overall,TabPFN-TS,0.18518759165386173,0.48116127498304256,10,13,13,1.0
-Overall,Moving-Average,0.9262789806686684,2.7900602219910033,11,13,13,1.0
-Overall,Seasonal-Naive,1.0,1.0,12,13,13,1.0
+Overall,Toto-1.0,3.959144807171818,2.185096517903194,1,12,12,1.0
+Overall,TiRex,0.1086130417957728,1.0541127797728755,2,12,12,1.0
+Overall,Chronos-2,0.1214688997582712,1.071436536306947,3,12,12,1.0
+Overall,Chronos-Bolt,0.2021881479559937,2.0282759208606773,4,12,12,1.0
+Overall,Moirai-2.0,0.15216359533435747,1.2872760141754034,5,12,12,1.0
+Overall,Sundial,0.36733302503728377,1.2198874052997883,6,12,12,1.0
+Overall,TimesFM-2.5,0.1491124083764099,0.47462341034676336,7,12,12,1.0
+Overall,ARIMA,0.6791059008593225,1.6333728627003652,8,12,12,1.0
+Overall,TabPFN-TS,0.1628968510702974,0.45255265181200255,9,12,12,1.0
+Overall,ETS,0.8258469012616879,0.37098898626911986,10,12,12,1.0
+Overall,Moving-Average,0.9861145211003569,3.19532291572967,11,12,12,1.0
+Overall,Seasonal-Naive,1.0,1.0,12,12,12,1.0

results/dataset_eval_intervals.json

@@ -5,6 +5,7 @@
     "data_gov_sg_singapore_taxi_availability_available_taxi_count": 300,
     "nasa_power_shanghai_hourly_T2M": 3600,
     "noaa_coops_sf_water_level_water_level": 360,
+    "noaa_ndbc_46013_realtime_wave_height": 600,
     "nws_ksfo_observations_temperature": 3600,
     "open_meteo_shanghai_air_quality_hourly_pm2_5": 3600,
     "open_meteo_shanghai_hourly_temperature_2m": 3600,

results/dataset_properties.csv

@@ -11,4 +11,3 @@ open_meteo_shanghai_air_quality_hourly_pm2_5,Air Quality,1h,1
 open_meteo_shanghai_hourly_temperature_2m,Weather,1h,1
 usgs_earthquake_all_week_hourly_earthquake_count,Hazards,1h,1
 usgs_potomac_iv_usgs_00060,Hydrology,15min,1
-wikimedia_time_series_hourly_pageviews,Web,1D,1

results/online_status.json

@@ -1,17 +1,23 @@
 {
-    "status": "ok",
+    "status": "partial",
     "evaluation_protocol": "issue-now-score-later",
     "metric_version": "livehouse-canonical-v1",
-    "started_at": "2026-07-28T19:37:59.635627+00:00",
-    "finished_at": "2026-07-28T20:39:38.543021+00:00",
+    "started_at": "2026-07-28T22:56:28.040831+00:00",
+    "finished_at": "2026-07-28T22:56:37.597741+00:00",
     "data_source": "ts_bench",
     "data_config": "configs/datasets/ts_bench.yaml",
-    "task_count": 2,
-    "issued_tasks": 1,
-    "issued_forecasts": 12,
-    "resolved_forecasts": 12,
+    "task_count": 4,
+    "issued_tasks": 0,
+    "issued_forecasts": 0,
+    "resolved_forecasts": 0,
     "pending_tasks": 13,
-    "failed_forecasts": [],
+    "failed_forecasts": [
+        {
+            "model": "Seasonal-Naive",
+            "dataset": "nws_ksfo_observations_temperature/1h/short",
+            "error": "Mean forecast length 10 < 24"
+        }
+    ],
     "models": {
         "Chronos-2": {
             "model": "Chronos-2",
@@ -19,18 +25,8 @@
             "admitted_at": "2026-06-01T00:00:00Z",
             "evaluation_protocol": "issue-now-score-later",
             "metric_version": "livehouse-canonical-v1",
-            "evaluated_at": "2026-07-28T20:39:34.711585+00:00",
-            "resolved_tasks": [
-                {
-                    "dataset": "data_gov_sg_singapore_taxi_availability_available_taxi_count/5min/short",
-                    "task_id": "future-48cd6c99a9ffcddbf31a",
-                    "issued_at": "2026-07-28T17:42:40.359220+00:00",
-                    "resolved_at": "2026-07-28T19:38:04.276069+00:00",
-                    "target_end": "2026-07-27T15:50:00+00:00",
-                    "prediction_length": 12,
-                    "metric_version": "livehouse-canonical-v1"
-                }
-            ]
+            "evaluated_at": "2026-07-28T22:56:33.262512+00:00",
+            "resolved_tasks": []
         },
         "TiRex": {
             "model": "TiRex",
@@ -38,18 +34,8 @@
             "admitted_at": "2026-06-01T00:00:00Z",
             "evaluation_protocol": "issue-now-score-later",
             "metric_version": "livehouse-canonical-v1",
-            "evaluated_at": "2026-07-28T20:39:34.722048+00:00",
-            "resolved_tasks": [
-                {
-                    "dataset": "data_gov_sg_singapore_taxi_availability_available_taxi_count/5min/short",
-                    "task_id": "future-48cd6c99a9ffcddbf31a",
-                    "issued_at": "2026-07-28T17:42:40.359220+00:00",
-                    "resolved_at": "2026-07-28T19:38:04.719991+00:00",
-                    "target_end": "2026-07-27T15:50:00+00:00",
-                    "prediction_length": 12,
-                    "metric_version": "livehouse-canonical-v1"
-                }
-            ]
+            "evaluated_at": "2026-07-28T22:56:33.262520+00:00",
+            "resolved_tasks": []
         },
         "TimesFM-2.5": {
             "model": "TimesFM-2.5",
@@ -57,18 +43,8 @@
             "admitted_at": "2026-06-01T00:00:00Z",
             "evaluation_protocol": "issue-now-score-later",
             "metric_version": "livehouse-canonical-v1",
-            "evaluated_at": "2026-07-28T20:39:34.725921+00:00",
-            "resolved_tasks": [
-                {
-                    "dataset": "data_gov_sg_singapore_taxi_availability_available_taxi_count/5min/short",
-                    "task_id": "future-48cd6c99a9ffcddbf31a",
-                    "issued_at": "2026-07-28T17:42:40.359220+00:00",
-                    "resolved_at": "2026-07-28T19:38:04.666035+00:00",
-                    "target_end": "2026-07-27T15:50:00+00:00",
-                    "prediction_length": 12,
-                    "metric_version": "livehouse-canonical-v1"
-                }
-            ]
+            "evaluated_at": "2026-07-28T22:56:33.262521+00:00",
+            "resolved_tasks": []
         },
         "Toto-1.0": {
             "model": "Toto-1.0",
@@ -76,18 +52,8 @@
             "admitted_at": "2026-06-01T00:00:00Z",
             "evaluation_protocol": "issue-now-score-later",
             "metric_version": "livehouse-canonical-v1",
-            "evaluated_at": "2026-07-28T20:39:34.728711+00:00",
-            "resolved_tasks": [
-                {
-                    "dataset": "data_gov_sg_singapore_taxi_availability_available_taxi_count/5min/short",
-                    "task_id": "future-48cd6c99a9ffcddbf31a",
-                    "issued_at": "2026-07-28T17:42:40.359220+00:00",
-                    "resolved_at": "2026-07-28T19:38:04.763051+00:00",
-                    "target_end": "2026-07-27T15:50:00+00:00",
-                    "prediction_length": 12,
-                    "metric_version": "livehouse-canonical-v1"
-                }
-            ]
+            "evaluated_at": "2026-07-28T22:56:33.262523+00:00",
+            "resolved_tasks": []
         },
         "Moirai-2.0": {
             "model": "Moirai-2.0",
@@ -95,18 +61,8 @@
             "admitted_at": "2026-06-01T00:00:00Z",
             "evaluation_protocol": "issue-now-score-later",
             "metric_version": "livehouse-canonical-v1",
-            "evaluated_at": "2026-07-28T20:39:34.738172+00:00",
-            "resolved_tasks": [
-                {
-                    "dataset": "data_gov_sg_singapore_taxi_availability_available_taxi_count/5min/short",
-                    "task_id": "future-48cd6c99a9ffcddbf31a",
-                    "issued_at": "2026-07-28T17:42:40.359220+00:00",
-                    "resolved_at": "2026-07-28T19:38:04.382007+00:00",
-                    "target_end": "2026-07-27T15:50:00+00:00",
-                    "prediction_length": 12,
-                    "metric_version": "livehouse-canonical-v1"
-                }
-            ]
+            "evaluated_at": "2026-07-28T22:56:33.262524+00:00",
+            "resolved_tasks": []
         },
         "Chronos-Bolt": {
             "model": "Chronos-Bolt",
@@ -114,18 +70,8 @@
             "admitted_at": "2026-06-01T00:00:00Z",
             "evaluation_protocol": "issue-now-score-later",
             "metric_version": "livehouse-canonical-v1",
-            "evaluated_at": "2026-07-28T20:39:34.744780+00:00",
-            "resolved_tasks": [
-                {
-                    "dataset": "data_gov_sg_singapore_taxi_availability_available_taxi_count/5min/short",
-                    "task_id": "future-48cd6c99a9ffcddbf31a",
-                    "issued_at": "2026-07-28T17:42:40.359220+00:00",
-                    "resolved_at": "2026-07-28T19:38:04.308155+00:00",
-                    "target_end": "2026-07-27T15:50:00+00:00",
-                    "prediction_length": 12,
-                    "metric_version": "livehouse-canonical-v1"
-                }
-            ]
+            "evaluated_at": "2026-07-28T22:56:33.262525+00:00",
+            "resolved_tasks": []
         },
         "TabPFN-TS": {
             "model": "TabPFN-TS",
@@ -133,18 +79,8 @@
             "admitted_at": "2026-06-01T00:00:00Z",
             "evaluation_protocol": "issue-now-score-later",
             "metric_version": "livehouse-canonical-v1",
-            "evaluated_at": "2026-07-28T20:39:34.751889+00:00",
-            "resolved_tasks": [
-                {
-                    "dataset": "data_gov_sg_singapore_taxi_availability_available_taxi_count/5min/short",
-                    "task_id": "future-48cd6c99a9ffcddbf31a",
-                    "issued_at": "2026-07-28T17:42:40.359220+00:00",
-                    "resolved_at": "2026-07-28T19:38:04.602994+00:00",
-                    "target_end": "2026-07-27T15:50:00+00:00",
-                    "prediction_length": 12,
-                    "metric_version": "livehouse-canonical-v1"
-                }
-            ]
+            "evaluated_at": "2026-07-28T22:56:33.262525+00:00",
+            "resolved_tasks": []
         },
         "Sundial": {
             "model": "Sundial",
@@ -152,18 +88,8 @@
             "admitted_at": "2026-06-01T00:00:00Z",
             "evaluation_protocol": "issue-now-score-later",
             "metric_version": "livehouse-canonical-v1",
-            "evaluated_at": "2026-07-28T20:39:34.757331+00:00",
-            "resolved_tasks": [
-                {
-                    "dataset": "data_gov_sg_singapore_taxi_availability_available_taxi_count/5min/short",
-                    "task_id": "future-48cd6c99a9ffcddbf31a",
-                    "issued_at": "2026-07-28T17:42:40.359220+00:00",
-                    "resolved_at": "2026-07-28T19:38:04.532044+00:00",
-                    "target_end": "2026-07-27T15:50:00+00:00",
-                    "prediction_length": 12,
-                    "metric_version": "livehouse-canonical-v1"
-                }
-            ]
+            "evaluated_at": "2026-07-28T22:56:33.262526+00:00",
+            "resolved_tasks": []
         },
         "Moving-Average": {
             "model": "Moving-Average",
@@ -171,18 +97,8 @@
             "admitted_at": "2026-06-01T00:00:00Z",
             "evaluation_protocol": "issue-now-score-later",
             "metric_version": "livehouse-canonical-v1",
-            "evaluated_at": "2026-07-28T20:39:34.765401+00:00",
-            "resolved_tasks": [
-                {
-                    "dataset": "data_gov_sg_singapore_taxi_availability_available_taxi_count/5min/short",
-                    "task_id": "future-48cd6c99a9ffcddbf31a",
-                    "issued_at": "2026-07-28T17:42:40.359220+00:00",
-                    "resolved_at": "2026-07-28T19:38:04.431187+00:00",
-                    "target_end": "2026-07-27T15:50:00+00:00",
-                    "prediction_length": 12,
-                    "metric_version": "livehouse-canonical-v1"
-                }
-            ]
+            "evaluated_at": "2026-07-28T22:56:33.262527+00:00",
+            "resolved_tasks": []
         },
         "ETS": {
             "model": "ETS",
@@ -190,18 +106,8 @@
             "admitted_at": "2026-06-01T00:00:00Z",
             "evaluation_protocol": "issue-now-score-later",
             "metric_version": "livehouse-canonical-v1",
-            "evaluated_at": "2026-07-28T20:39:34.770227+00:00",
-            "resolved_tasks": [
-                {
-                    "dataset": "data_gov_sg_singapore_taxi_availability_available_taxi_count/5min/short",
-                    "task_id": "future-48cd6c99a9ffcddbf31a",
-                    "issued_at": "2026-07-28T17:42:40.359220+00:00",
-                    "resolved_at": "2026-07-28T19:38:04.346964+00:00",
-                    "target_end": "2026-07-27T15:50:00+00:00",
-                    "prediction_length": 12,
-                    "metric_version": "livehouse-canonical-v1"
-                }
-            ]
+            "evaluated_at": "2026-07-28T22:56:33.262528+00:00",
+            "resolved_tasks": []
         },
         "ARIMA": {
             "model": "ARIMA",
@@ -209,18 +115,8 @@
             "admitted_at": "2026-06-01T00:00:00Z",
             "evaluation_protocol": "issue-now-score-later",
             "metric_version": "livehouse-canonical-v1",
-            "evaluated_at": "2026-07-28T20:39:34.782147+00:00",
-            "resolved_tasks": [
-                {
-                    "dataset": "data_gov_sg_singapore_taxi_availability_available_taxi_count/5min/short",
-                    "task_id": "future-48cd6c99a9ffcddbf31a",
-                    "issued_at": "2026-07-28T17:42:40.359220+00:00",
-                    "resolved_at": "2026-07-28T19:38:04.168245+00:00",
-                    "target_end": "2026-07-27T15:50:00+00:00",
-                    "prediction_length": 12,
-                    "metric_version": "livehouse-canonical-v1"
-                }
-            ]
+            "evaluated_at": "2026-07-28T22:56:33.262529+00:00",
+            "resolved_tasks": []
         },
         "Seasonal-Naive": {
             "model": "Seasonal-Naive",
@@ -228,18 +124,8 @@
             "admitted_at": "2026-06-01T00:00:00Z",
             "evaluation_protocol": "issue-now-score-later",
             "metric_version": "livehouse-canonical-v1",
-            "evaluated_at": "2026-07-28T20:39:34.789983+00:00",
-            "resolved_tasks": [
-                {
-                    "dataset": "data_gov_sg_singapore_taxi_availability_available_taxi_count/5min/short",
-                    "task_id": "future-48cd6c99a9ffcddbf31a",
-                    "issued_at": "2026-07-28T17:42:40.359220+00:00",
-                    "resolved_at": "2026-07-28T19:38:04.480914+00:00",
-                    "target_end": "2026-07-27T15:50:00+00:00",
-                    "prediction_length": 12,
-                    "metric_version": "livehouse-canonical-v1"
-                }
-            ]
+            "evaluated_at": "2026-07-28T22:56:33.262530+00:00",
+            "resolved_tasks": []
         }
     },
     "skipped_models": [],
@@ -262,7 +148,5 @@
         "live_overall_30d": "aggregates/live_overall_30d.csv",
         "live_metadata": "aggregates/live_metadata.json"
     },
-    "push_status": "ok",
-    "pushed_at": "2026-07-28T20:56:18.861848+00:00",
-    "push_detail": "HF Hub API"
+    "push_status": "pending"
 }