Update leaderboard results from local eval

results/aggregates/live_metadata.json

@@ -1,5 +1,5 @@
 {
-  "generated_at": "2026-07-27T10:39:50.934572+00:00",
+  "generated_at": "2026-07-27T10:51:33.114175+00:00",
   "scope": "Canonical metrics-only release history when available; otherwise legacy release history plus latest allowlisted snapshots",
   "release_deduplication": "One row per model, dataset, data source timestamp, and prediction length",
   "dataset_aggregation": "Arithmetic mean over releases, then equal-weight mean over datasets",

results/aggregates/metadata.json

@@ -1,5 +1,5 @@
 {
-  "generated_at": "2026-07-27T10:39:48.222374+00:00",
+  "generated_at": "2026-07-27T10:51:30.937614+00:00",
   "baseline_model": "Seasonal-Naive",
   "normalization": "Per-config metric / Seasonal-Naive metric",
   "metric_aggregation": "Geometric mean over dataset configurations",
@@ -10,7 +10,7 @@
     "frequency",
     "prediction_length"
   ],
-  "num_configs": 14,
+  "num_configs": 12,
   "num_models": 12,
   "num_rows_missing_prediction_length": 0
 }

results/aggregates/results_by_domain.csv

@@ -11,18 +11,18 @@ Climate,Chronos-Bolt,47.4655458843892,96.98926440492649,9,5,5,1.0
 Climate,ARIMA,498.03148316002745,9.29051171563987,10,5,5,1.0
 Climate,TimesFM-2.5,19.702239675462305,97.57289238633813,11,5,5,1.0
 Climate,ETS,2730.709743546208,63.60664425761785,12,5,5,1.0
-Finance,TiRex,0.511127609443142,0.6365436647590029,1,3,3,1.0
-Finance,Chronos-2,0.6756273588989797,0.7474684518753827,2,3,3,1.0
-Finance,Moirai-2.0,0.8939506322509885,0.8084053655303516,3,3,3,1.0
-Finance,Chronos-Bolt,0.7502482432292896,0.8303214097324793,4,3,3,1.0
-Finance,Toto-1.0,0.8948723732629463,0.8783617899756722,5,3,3,1.0
-Finance,Sundial,0.6307937874132711,0.8016412757672429,6,3,3,1.0
-Finance,ARIMA,1.090517359648809,1.1187187338654994,7,3,3,1.0
-Finance,TimesFM-2.5,0.7139757250034728,0.8742752788997603,8,3,3,1.0
-Finance,Seasonal-Naive,1.0,1.0,9,3,3,1.0
-Finance,TabPFN-TS,0.7471649627495729,0.9604369019880863,10,3,3,1.0
-Finance,ETS,1.1311048184754804,1.0949766947182327,11,3,3,1.0
-Finance,Moving-Average,1.0978870037380029,1.1315175601021201,12,3,3,1.0
+Finance,TiRex,0.3931124718915428,0.5540640832588316,1,2,2,1.0
+Finance,Chronos-Bolt,0.4094997273505915,0.5971475845353197,2,2,2,1.0
+Finance,Chronos-2,0.4792565171069634,0.5968336898677795,3,2,2,1.0
+Finance,Moirai-2.0,0.6022456428440649,0.6277441482199546,4,2,2,1.0
+Finance,Sundial,0.3850621616392369,0.5665205003918359,5,2,2,1.0
+Finance,Toto-1.0,0.5732008210305747,0.6524422824227847,6,2,2,1.0
+Finance,TimesFM-2.5,0.5243224958692184,0.6419690325196108,7,2,2,1.0
+Finance,ETS,1.017222403089634,0.9244745823211508,8,2,2,1.0
+Finance,ARIMA,1.0429819994006044,1.3044306584906822,9,2,2,1.0
+Finance,TabPFN-TS,0.647027083008818,0.7923082880130374,10,2,2,1.0
+Finance,Seasonal-Naive,1.0,1.0,11,2,2,1.0
+Finance,Moving-Average,1.047028420685017,1.0104050688948565,12,2,2,1.0
 Hazards,ARIMA,0.7462605338771628,0.6602389941338824,1,1,1,1.0
 Hazards,Moirai-2.0,0.7281972824447458,0.7036277604367918,2,1,1,1.0
 Hazards,Moving-Average,0.7963258085146442,0.7048427412162565,3,1,1,1.0
@@ -71,15 +71,3 @@ Transportation,Moving-Average,1.0,250217.138218906,9,1,1,1.0
 Transportation,Moirai-2.0,1.0,410051.5180343537,10,1,1,1.0
 Transportation,Chronos-Bolt,1.094103907762285,3214861.8499168884,11,1,1,1.0
 Transportation,Toto-1.0,139539233137.999,281471356.81470025,12,1,1,1.0
-Web,ARIMA,0.35081770342317864,0.47944164778388026,1,1,1,1.0
-Web,Moirai-2.0,0.40785680526551005,0.5118677129116019,2,1,1,1.0
-Web,Sundial,0.303769640015456,0.5282879649026628,3,1,1,1.0
-Web,ETS,0.4243826259793997,0.5449399343905824,4,1,1,1.0
-Web,Moving-Average,0.4370340438348723,0.54802384805273,5,1,1,1.0
-Web,Chronos-2,0.4498860462918392,0.5739884284864399,6,1,1,1.0
-Web,TiRex,0.46006192226374054,0.5949360110321206,7,1,1,1.0
-Web,TimesFM-2.5,0.32382897462809757,0.6278956823267848,8,1,1,1.0
-Web,Chronos-Bolt,0.3947163650820193,0.6425428458659833,9,1,1,1.0
-Web,Toto-1.0,1.1828445386062165,0.7008146109363338,10,1,1,1.0
-Web,Seasonal-Naive,1.0,1.0,11,1,1,1.0
-Web,TabPFN-TS,0.8629829581417852,1.004036040514957,12,1,1,1.0

results/aggregates/results_by_frequency.csv

@@ -23,30 +23,18 @@ frequency,model,MSE,CRPS,Rank,n_configs,n_group_configs,coverage
 15min,Toto-1.0,108543418950.29202,0.44809729258219655,10,1,1,1.0
 15min,Seasonal-Naive,1.0,1.0,11,1,1,1.0
 15min,ARIMA,10837674801930.557,5.106079303459772,12,1,1,1.0
-1D,ARIMA,0.35081770342317864,0.47944164778388026,1,1,1,1.0
-1D,Moirai-2.0,0.40785680526551005,0.5118677129116019,2,1,1,1.0
-1D,Sundial,0.303769640015456,0.5282879649026628,3,1,1,1.0
-1D,ETS,0.4243826259793997,0.5449399343905824,4,1,1,1.0
-1D,Moving-Average,0.4370340438348723,0.54802384805273,5,1,1,1.0
-1D,Chronos-2,0.4498860462918392,0.5739884284864399,6,1,1,1.0
-1D,TiRex,0.46006192226374054,0.5949360110321206,7,1,1,1.0
-1D,TimesFM-2.5,0.32382897462809757,0.6278956823267848,8,1,1,1.0
-1D,Chronos-Bolt,0.3947163650820193,0.6425428458659833,9,1,1,1.0
-1D,Toto-1.0,1.1828445386062165,0.7008146109363338,10,1,1,1.0
-1D,Seasonal-Naive,1.0,1.0,11,1,1,1.0
-1D,TabPFN-TS,0.8629829581417852,1.004036040514957,12,1,1,1.0
-1h,TiRex,0.7750233887785511,0.8731280991152257,1,7,7,1.0
-1h,Toto-1.0,3.470346837824563,0.9290676306090473,2,7,7,1.0
-1h,Seasonal-Naive,1.0,1.0,3,7,7,1.0
-1h,Chronos-2,1.0557222332934295,0.9420539527408869,4,7,7,1.0
-1h,Moirai-2.0,1.3074632480924078,1.0409721294042618,5,7,7,1.0
-1h,TabPFN-TS,0.7872457845933027,0.9836899995672891,6,7,7,1.0
-1h,Sundial,1.4082380266832522,1.0309415375783566,7,7,7,1.0
-1h,ARIMA,4.487995071217601,2.1913157129268295,8,7,7,1.0
-1h,Chronos-Bolt,1.2952350198445803,1.0117775807505975,9,7,7,1.0
-1h,TimesFM-2.5,0.9561943420818555,1.08048274208185,10,7,7,1.0
-1h,Moving-Average,2.958841847567672,1.851054181454787,11,7,7,1.0
-1h,ETS,15.27761409327684,4.372686909715655,12,7,7,1.0
+1h,TiRex,0.7610996943071011,0.878746331824924,1,6,6,1.0
+1h,Toto-1.0,3.7496443706833267,0.8493092732352021,2,6,6,1.0
+1h,Chronos-2,1.014246865952099,0.9083287947663047,3,6,6,1.0
+1h,Seasonal-Naive,1.0,1.0,4,6,6,1.0
+1h,Moirai-2.0,1.2211495654602278,0.9979878400623481,5,6,6,1.0
+1h,TabPFN-TS,0.7569397375884535,0.926257234094127,6,6,6,1.0
+1h,Sundial,1.3657000431154327,0.9576090002677531,7,6,6,1.0
+1h,Chronos-Bolt,1.159369986330377,0.9368498734192843,8,6,6,1.0
+1h,TimesFM-2.5,0.905722114763785,1.00979852986364,9,6,6,1.0
+1h,ARIMA,5.5976242951061925,2.579896730748346,10,6,6,1.0
+1h,Moving-Average,3.4357064155293853,1.9348925056984916,11,6,6,1.0
+1h,ETS,22.757191857667504,5.205578498615606,12,6,6,1.0
 1mo,Moirai-2.0,0.5701893078499076,0.5711658118752873,1,1,1,1.0
 1mo,Chronos-Bolt,0.6722527938921684,0.6335328970612374,2,1,1,1.0
 1mo,Toto-1.0,0.6317640740160342,0.6385745807295035,3,1,1,1.0

results/aggregates/results_by_prediction_length.csv

@@ -1,16 +1,4 @@
 prediction_length,model,MSE,CRPS,Rank,n_configs,n_group_configs,coverage
-7,ARIMA,0.35081770342317864,0.47944164778388026,1,1,1,1.0
-7,Moirai-2.0,0.40785680526551005,0.5118677129116019,2,1,1,1.0
-7,Sundial,0.303769640015456,0.5282879649026628,3,1,1,1.0
-7,ETS,0.4243826259793997,0.5449399343905824,4,1,1,1.0
-7,Moving-Average,0.4370340438348723,0.54802384805273,5,1,1,1.0
-7,Chronos-2,0.4498860462918392,0.5739884284864399,6,1,1,1.0
-7,TiRex,0.46006192226374054,0.5949360110321206,7,1,1,1.0
-7,TimesFM-2.5,0.32382897462809757,0.6278956823267848,8,1,1,1.0
-7,Chronos-Bolt,0.3947163650820193,0.6425428458659833,9,1,1,1.0
-7,Toto-1.0,1.1828445386062165,0.7008146109363338,10,1,1,1.0
-7,Seasonal-Naive,1.0,1.0,11,1,1,1.0
-7,TabPFN-TS,0.8629829581417852,1.004036040514957,12,1,1,1.0
 12,Chronos-2,0.8294476545156683,38.946376459967496,1,2,2,1.0
 12,TiRex,0.847802672121473,30.22591777828443,2,2,2,1.0
 12,Moirai-2.0,0.7551088053055054,483.94979926515697,3,2,2,1.0
@@ -23,18 +11,18 @@ prediction_length,model,MSE,CRPS,Rank,n_configs,n_group_configs,coverage
 12,ETS,0.8706436180201554,439.02846065451104,10,2,2,1.0
 12,Toto-1.0,296910.5495133771,13406.731654855768,11,2,2,1.0
 12,Moving-Average,0.9617678019724286,471.56119324419365,12,2,2,1.0
-24,TiRex,5.224871811883289,0.7823730949914425,1,8,8,1.0
-24,Chronos-2,6.0695511895599426,0.42944714703078707,2,8,8,1.0
-24,TabPFN-TS,2.548332405401998,0.29130116026903996,3,8,8,1.0
-24,Moirai-2.0,11.76921551477689,0.6135853025712298,4,8,8,1.0
-24,Toto-1.0,71.16668043850568,0.8481310305890166,5,8,8,1.0
-24,Seasonal-Naive,1.0,1.0,6,8,8,1.0
-24,Sundial,22.22111134675419,0.6698504650169589,7,8,8,1.0
-24,Chronos-Bolt,13.420985696012036,0.5948015273581341,8,8,8,1.0
-24,TimesFM-2.5,6.290754398626646,0.494206717769758,9,8,8,1.0
-24,Moving-Average,46.15004583660541,0.05419905815213889,10,8,8,1.0
-24,ARIMA,158.4575144359654,2.4357223332839517,11,8,8,1.0
-24,ETS,274.4663647782223,3.0878543129477825,12,8,8,1.0
+24,TiRex,6.756496163289516,0.7744482076348079,1,7,7,1.0
+24,Chronos-2,7.5292623256725335,0.37204959265128773,2,7,7,1.0
+24,Toto-1.0,117.08635389952524,0.7751671490693114,3,7,7,1.0
+24,TabPFN-TS,2.9142082085464702,0.23251215265324596,4,7,7,1.0
+24,Moirai-2.0,15.193410236369886,0.5487600429428899,5,7,7,1.0
+24,Sundial,32.09989910427459,0.591233639652137,6,7,7,1.0
+24,Seasonal-Naive,1.0,1.0,7,7,7,1.0
+24,Chronos-Bolt,17.044935754093814,0.5161448606395037,8,7,7,1.0
+24,TimesFM-2.5,7.859510485853854,0.41705491671244116,9,7,7,1.0
+24,Moving-Average,77.66611580503375,0.03399539961874934,10,7,7,1.0
+24,ARIMA,318.62496343381406,2.8441818163453627,11,7,7,1.0
+24,ETS,583.4904598160068,3.4117380320294015,12,7,7,1.0
 36,Seasonal-Naive,1.0,1.0,1,1,1,1.0
 36,Moving-Average,1208318634.7525988,1.0,2,1,1,1.0
 36,ARIMA,1083193423.3268,342.79755336616876,3,1,1,1.0

results/aggregates/results_overall.csv

@@ -1,13 +1,13 @@
 overall,model,MSE,CRPS,Rank,n_configs,n_group_configs,coverage
-Overall,TiRex,5.774975006020239,5.646181090823145,1,14,14,1.0
-Overall,Chronos-2,7.145030886581356,4.44572064522782,2,14,14,1.0
-Overall,Moirai-2.0,11.238482524527065,7.976714576170561,3,14,14,1.0
-Overall,Moving-Average,35.35372366234358,0.4217764566963194,4,14,14,1.0
-Overall,Seasonal-Naive,1.0,1.0,5,14,14,1.0
-Overall,ARIMA,73.34125083823173,5.749266231793158,6,14,14,1.0
-Overall,Toto-1.0,439.2371205401317,15.505646734653837,7,14,14,1.0
-Overall,Sundial,24.74388106301975,7.864794822974638,8,14,14,1.0
-Overall,TabPFN-TS,5.019807509828724,2.306438364864622,9,14,14,1.0
-Overall,TimesFM-2.5,6.716812811069544,2.9647404539042026,10,14,14,1.0
-Overall,Chronos-Bolt,15.624470605913055,10.848082782604182,11,14,14,1.0
-Overall,ETS,105.26199831669565,9.475730029675217,12,14,14,1.0
+Overall,TiRex,8.353354354823885,7.982599955499262,1,12,12,1.0
+Overall,Chronos-2,10.341434437439744,5.892070589574432,2,12,12,1.0
+Overall,Toto-1.0,1119.0489777773864,24.26293090185123,3,12,12,1.0
+Overall,Moirai-2.0,17.129730806331033,11.634687735117273,4,12,12,1.0
+Overall,Seasonal-Naive,1.0,1.0,5,12,12,1.0
+Overall,Moving-Average,67.55410612788627,0.37298957745710887,6,12,12,1.0
+Overall,TabPFN-TS,6.651725853133647,2.575242972556257,7,12,12,1.0
+Overall,TimesFM-2.5,9.900938534912244,3.5481404126522627,8,12,12,1.0
+Overall,Sundial,44.64611919685413,11.244418242242466,9,12,12,1.0
+Overall,ARIMA,161.35690780763707,8.315271787203065,10,12,12,1.0
+Overall,Chronos-Bolt,24.71667942750833,16.098621969410704,11,12,12,1.0
+Overall,ETS,238.88965993603242,13.98997749855162,12,12,12,1.0

results/dataset_eval_intervals.json

@@ -1,5 +1,6 @@
 {
     "binance_btcusdt_1h_close": 300,
+    "coingecko_bitcoin_7d_price_usd": 300,
     "data_gov_sg_sgx_equities_turnover_monthly_turnover_value_million_sgd": 2592000,
     "data_gov_sg_singapore_rainfall_5min_rainfall_5min_total": 900,
     "data_gov_sg_singapore_taxi_availability_available_taxi_count": 300,

results/dataset_properties.csv

@@ -1,6 +1,5 @@
 dataset,domain,frequency,num_variates
 binance_btcusdt_1h_close,Finance,1h,1
-coingecko_bitcoin_7d_price_usd,Finance,1h,1
 data_gov_sg_sgx_equities_turnover_monthly_turnover_value_million_sgd,Finance,M,1
 data_gov_sg_singapore_rainfall_5min_rainfall_5min_total,Climate,5min,1
 data_gov_sg_singapore_taxi_availability_available_taxi_count,Transportation,5min,1
@@ -12,4 +11,3 @@ open_meteo_shanghai_air_quality_hourly_pm2_5,Air Quality,1h,1
 open_meteo_shanghai_hourly_temperature_2m,Weather,1h,1
 usgs_earthquake_all_week_hourly_earthquake_count,Hazards,1h,1
 usgs_potomac_iv_usgs_00060,Hydrology,15min,1
-wikimedia_time_series_hourly_pageviews,Web,1D,1

results/online_status.json

@@ -1,9 +1,9 @@
 {
-    "status": "ok",
+    "status": "partial",
     "evaluation_protocol": "issue-now-score-later",
     "metric_version": "livehouse-canonical-v1",
-    "started_at": "2026-07-27T10:39:45.214847+00:00",
-    "finished_at": "2026-07-27T10:39:50.946321+00:00",
+    "started_at": "2026-07-27T10:51:25.986558+00:00",
+    "finished_at": "2026-07-27T10:51:33.126731+00:00",
     "data_source": "ts_bench",
     "data_config": "configs/datasets/ts_bench.yaml",
     "task_count": 3,
@@ -11,7 +11,13 @@
     "issued_forecasts": 0,
     "resolved_forecasts": 0,
     "pending_tasks": 13,
-    "failed_forecasts": [],
+    "failed_forecasts": [
+        {
+            "model": "Seasonal-Naive",
+            "dataset": "nws_ksfo_observations_temperature/1h/short",
+            "error": "Mean forecast length 10 < 24"
+        }
+    ],
     "models": {
         "Chronos-2": {
             "model": "Chronos-2",
@@ -19,7 +25,7 @@
             "admitted_at": "2026-06-01T00:00:00Z",
             "evaluation_protocol": "issue-now-score-later",
             "metric_version": "livehouse-canonical-v1",
-            "evaluated_at": "2026-07-27T10:39:47.133617+00:00",
+            "evaluated_at": "2026-07-27T10:51:29.050595+00:00",
             "resolved_tasks": []
         },
         "TiRex": {
@@ -28,7 +34,7 @@
             "admitted_at": "2026-06-01T00:00:00Z",
             "evaluation_protocol": "issue-now-score-later",
             "metric_version": "livehouse-canonical-v1",
-            "evaluated_at": "2026-07-27T10:39:47.133626+00:00",
+            "evaluated_at": "2026-07-27T10:51:29.050603+00:00",
             "resolved_tasks": []
         },
         "TimesFM-2.5": {
@@ -37,7 +43,7 @@
             "admitted_at": "2026-06-01T00:00:00Z",
             "evaluation_protocol": "issue-now-score-later",
             "metric_version": "livehouse-canonical-v1",
-            "evaluated_at": "2026-07-27T10:39:47.133627+00:00",
+            "evaluated_at": "2026-07-27T10:51:29.050605+00:00",
             "resolved_tasks": []
         },
         "Toto-1.0": {
@@ -46,7 +52,7 @@
             "admitted_at": "2026-06-01T00:00:00Z",
             "evaluation_protocol": "issue-now-score-later",
             "metric_version": "livehouse-canonical-v1",
-            "evaluated_at": "2026-07-27T10:39:47.133628+00:00",
+            "evaluated_at": "2026-07-27T10:51:29.050606+00:00",
             "resolved_tasks": []
         },
         "Moirai-2.0": {
@@ -55,7 +61,7 @@
             "admitted_at": "2026-06-01T00:00:00Z",
             "evaluation_protocol": "issue-now-score-later",
             "metric_version": "livehouse-canonical-v1",
-            "evaluated_at": "2026-07-27T10:39:47.133630+00:00",
+            "evaluated_at": "2026-07-27T10:51:29.050607+00:00",
             "resolved_tasks": []
         },
         "Chronos-Bolt": {
@@ -64,7 +70,7 @@
             "admitted_at": "2026-06-01T00:00:00Z",
             "evaluation_protocol": "issue-now-score-later",
             "metric_version": "livehouse-canonical-v1",
-            "evaluated_at": "2026-07-27T10:39:47.133630+00:00",
+            "evaluated_at": "2026-07-27T10:51:29.050608+00:00",
             "resolved_tasks": []
         },
         "TabPFN-TS": {
@@ -73,7 +79,7 @@
             "admitted_at": "2026-06-01T00:00:00Z",
             "evaluation_protocol": "issue-now-score-later",
             "metric_version": "livehouse-canonical-v1",
-            "evaluated_at": "2026-07-27T10:39:47.133632+00:00",
+            "evaluated_at": "2026-07-27T10:51:29.050609+00:00",
             "resolved_tasks": []
         },
         "Sundial": {
@@ -82,7 +88,7 @@
             "admitted_at": "2026-06-01T00:00:00Z",
             "evaluation_protocol": "issue-now-score-later",
             "metric_version": "livehouse-canonical-v1",
-            "evaluated_at": "2026-07-27T10:39:47.133632+00:00",
+            "evaluated_at": "2026-07-27T10:51:29.050610+00:00",
             "resolved_tasks": []
         },
         "Moving-Average": {
@@ -91,7 +97,7 @@
             "admitted_at": "2026-06-01T00:00:00Z",
             "evaluation_protocol": "issue-now-score-later",
             "metric_version": "livehouse-canonical-v1",
-            "evaluated_at": "2026-07-27T10:39:47.133633+00:00",
+            "evaluated_at": "2026-07-27T10:51:29.050611+00:00",
             "resolved_tasks": []
         },
         "ETS": {
@@ -100,7 +106,7 @@
             "admitted_at": "2026-06-01T00:00:00Z",
             "evaluation_protocol": "issue-now-score-later",
             "metric_version": "livehouse-canonical-v1",
-            "evaluated_at": "2026-07-27T10:39:47.133634+00:00",
+            "evaluated_at": "2026-07-27T10:51:29.050612+00:00",
             "resolved_tasks": []
         },
         "ARIMA": {
@@ -109,7 +115,7 @@
             "admitted_at": "2026-06-01T00:00:00Z",
             "evaluation_protocol": "issue-now-score-later",
             "metric_version": "livehouse-canonical-v1",
-            "evaluated_at": "2026-07-27T10:39:47.133635+00:00",
+            "evaluated_at": "2026-07-27T10:51:29.050613+00:00",
             "resolved_tasks": []
         },
         "Seasonal-Naive": {
@@ -118,7 +124,7 @@
             "admitted_at": "2026-06-01T00:00:00Z",
             "evaluation_protocol": "issue-now-score-later",
             "metric_version": "livehouse-canonical-v1",
-            "evaluated_at": "2026-07-27T10:39:47.133636+00:00",
+            "evaluated_at": "2026-07-27T10:51:29.050614+00:00",
             "resolved_tasks": []
         }
     },
@@ -142,7 +148,5 @@
         "live_overall_30d": "aggregates/live_overall_30d.csv",
         "live_metadata": "aggregates/live_metadata.json"
     },
-    "push_status": "ok",
-    "pushed_at": "2026-07-27T10:40:37.393174+00:00",
-    "push_detail": "HF Hub API"
+    "push_status": "pending"
 }