Update leaderboard results from local eval

results/aggregates/live_metadata.json

@@ -1,5 +1,5 @@
 {
-  "generated_at": "2026-07-27T23:40:55.846725+00:00",
+  "generated_at": "2026-07-28T01:02:51.013205+00:00",
   "scope": "Canonical metrics-only release history when available; otherwise legacy release history plus latest allowlisted snapshots",
   "release_deduplication": "One row per model, dataset, data source timestamp, and prediction length",
   "dataset_aggregation": "Arithmetic mean over releases, then equal-weight mean over datasets",

results/aggregates/metadata.json

@@ -1,5 +1,5 @@
 {
-  "generated_at": "2026-07-27T23:40:53.793207+00:00",
+  "generated_at": "2026-07-28T01:02:49.053656+00:00",
   "baseline_model": "Seasonal-Naive",
   "normalization": "Per-config metric / Seasonal-Naive metric",
   "metric_aggregation": "Geometric mean over dataset configurations",
@@ -10,7 +10,7 @@
     "frequency",
     "prediction_length"
   ],
-  "num_configs": 14,
+  "num_configs": 12,
   "num_models": 12,
   "num_rows_missing_prediction_length": 0
 }

results/aggregates/results_by_domain.csv

@@ -11,18 +11,18 @@ Climate,Moirai-2.0,1.4824060116514644,14.748371632894651,9,5,5,1.0
 Climate,ARIMA,7.768114077941136,34.84406300542313,10,5,5,1.0
 Climate,Moving-Average,4.448829912179885,27.760038786622744,11,5,5,1.0
 Climate,ETS,41.67150566777602,91.51964639360355,12,5,5,1.0
-Finance,Sundial,0.19445980585053715,0.4743818504842995,1,3,3,1.0
-Finance,Toto-1.0,0.9947021774741156,0.5950506929520395,2,3,3,1.0
-Finance,Chronos-Bolt,0.4199809081352074,0.6752307614065016,3,3,3,1.0
-Finance,Moirai-2.0,0.5671540412820155,0.6826931781544368,4,3,3,1.0
-Finance,TiRex,0.4809226366262122,0.659726851800773,5,3,3,1.0
-Finance,TimesFM-2.5,0.47441744573781885,0.6306218519694167,6,3,3,1.0
-Finance,Chronos-2,0.6332301204356406,0.7064421109615494,7,3,3,1.0
-Finance,Moving-Average,0.487799996024218,0.6635190074539776,8,3,3,1.0
-Finance,TabPFN-TS,0.4489120413494622,0.7033657254188453,9,3,3,1.0
-Finance,ETS,1.1616593432036997,1.0465162529631655,10,3,3,1.0
-Finance,ARIMA,1.0573085089666414,1.5807217993353238,11,3,3,1.0
-Finance,Seasonal-Naive,1.0,1.0,12,3,3,1.0
+Finance,Sundial,0.09952901750099762,0.368263120192457,1,2,2,1.0
+Finance,TimesFM-2.5,0.36989863532881,0.5604116791822299,2,2,2,1.0
+Finance,Toto-1.0,1.2481366622293906,0.5744141563175763,3,2,2,1.0
+Finance,TiRex,0.39338522835316736,0.6627981567759068,4,2,2,1.0
+Finance,TabPFN-TS,0.32911000269320434,0.6243785441568039,5,2,2,1.0
+Finance,Chronos-Bolt,0.33195408280557637,0.6970978498460929,6,2,2,1.0
+Finance,Moving-Average,0.35423580404413224,0.5733245263298223,7,2,2,1.0
+Finance,Moirai-2.0,0.5656424724531764,0.7463752724135839,8,2,2,1.0
+Finance,Chronos-2,0.6075099051917358,0.7400319231159583,9,2,2,1.0
+Finance,Seasonal-Naive,1.0,1.0,10,2,2,1.0
+Finance,ETS,1.438062507760955,1.2197892953489888,11,2,2,1.0
+Finance,ARIMA,1.2583538765342204,2.2658873562028767,12,2,2,1.0
 Hazards,ARIMA,0.7462605338771628,0.6602389941338824,1,1,1,1.0
 Hazards,Moirai-2.0,0.7281972824447458,0.7036277604367918,2,1,1,1.0
 Hazards,Moving-Average,0.7963258085146442,0.7048427412162565,3,1,1,1.0
@@ -71,15 +71,3 @@ Transportation,Moving-Average,1.0,250217.138218906,9,1,1,1.0
 Transportation,Moirai-2.0,1.0,410051.5180343537,10,1,1,1.0
 Transportation,Chronos-Bolt,1.094103907762285,3214861.8499168884,11,1,1,1.0
 Transportation,Toto-1.0,50341153.673178256,277943555.25179964,12,1,1,1.0
-Web,ARIMA,0.35081770342317864,0.47944164778388026,1,1,1,1.0
-Web,Moirai-2.0,0.40785680526551005,0.5118677129116019,2,1,1,1.0
-Web,Sundial,0.303769640015456,0.5282879649026628,3,1,1,1.0
-Web,ETS,0.4243826259793997,0.5449399343905824,4,1,1,1.0
-Web,Moving-Average,0.4370340438348723,0.54802384805273,5,1,1,1.0
-Web,Chronos-2,0.4498860462918392,0.5739884284864399,6,1,1,1.0
-Web,TiRex,0.46006192226374054,0.5949360110321206,7,1,1,1.0
-Web,TimesFM-2.5,0.32382897462809757,0.6278956823267848,8,1,1,1.0
-Web,Chronos-Bolt,0.3947163650820193,0.6425428458659833,9,1,1,1.0
-Web,Toto-1.0,1.1828445386062165,0.7008146109363338,10,1,1,1.0
-Web,Seasonal-Naive,1.0,1.0,11,1,1,1.0
-Web,TabPFN-TS,0.8629829581417852,1.004036040514957,12,1,1,1.0

results/aggregates/results_by_frequency.csv

@@ -23,18 +23,6 @@ frequency,model,MSE,CRPS,Rank,n_configs,n_group_configs,coverage
 15min,TabPFN-TS,1.7823948120487276,1.6723544394535688,10,1,1,1.0
 15min,ETS,2.3101830377998778,1.8643116894683234,11,1,1,1.0
 15min,ARIMA,1.7969075767873166,2.166778271475671,12,1,1,1.0
-1D,ARIMA,0.35081770342317864,0.47944164778388026,1,1,1,1.0
-1D,Moirai-2.0,0.40785680526551005,0.5118677129116019,2,1,1,1.0
-1D,Sundial,0.303769640015456,0.5282879649026628,3,1,1,1.0
-1D,ETS,0.4243826259793997,0.5449399343905824,4,1,1,1.0
-1D,Moving-Average,0.4370340438348723,0.54802384805273,5,1,1,1.0
-1D,Chronos-2,0.4498860462918392,0.5739884284864399,6,1,1,1.0
-1D,TiRex,0.46006192226374054,0.5949360110321206,7,1,1,1.0
-1D,TimesFM-2.5,0.32382897462809757,0.6278956823267848,8,1,1,1.0
-1D,Chronos-Bolt,0.3947163650820193,0.6425428458659833,9,1,1,1.0
-1D,Toto-1.0,1.1828445386062165,0.7008146109363338,10,1,1,1.0
-1D,Seasonal-Naive,1.0,1.0,11,1,1,1.0
-1D,TabPFN-TS,0.8629829581417852,1.004036040514957,12,1,1,1.0
 1h,Toto-1.0,3.631265561890252,0.7862624882593825,1,7,7,1.0
 1h,TiRex,0.7550528102997923,0.886617366686293,2,7,7,1.0
 1h,Sundial,0.8504551179293129,0.823347513995428,3,7,7,1.0
@@ -47,18 +35,6 @@ frequency,model,MSE,CRPS,Rank,n_configs,n_group_configs,coverage
 1h,Moving-Average,2.089917990633689,1.4725595797214652,10,7,7,1.0
 1h,ARIMA,4.428904238820278,2.541255453576966,11,7,7,1.0
 1h,ETS,15.453136887947402,4.288675039942574,12,7,7,1.0
-1mo,Moirai-2.0,0.5701893078499076,0.5711658118752873,1,1,1,1.0
-1mo,Chronos-Bolt,0.6722527938921684,0.6335328970612374,2,1,1,1.0
-1mo,Toto-1.0,0.6317640740160342,0.6385745807295035,3,1,1,1.0
-1mo,Chronos-2,0.6879834115815433,0.6437671766189335,4,1,1,1.0
-1mo,TiRex,0.7187693708563099,0.6536268717700132,5,1,1,1.0
-1mo,ARIMA,0.7464480695996939,0.7692893066778397,6,1,1,1.0
-1mo,ETS,0.7580203095992263,0.7703148978390245,7,1,1,1.0
-1mo,Sundial,0.7423182015400501,0.7871685560956725,8,1,1,1.0
-1mo,TimesFM-2.5,0.7803986235065251,0.7985326751147283,9,1,1,1.0
-1mo,Moving-Average,0.9249973049108767,0.8887079460534169,10,1,1,1.0
-1mo,TabPFN-TS,0.8352213750339312,0.8925810519947165,11,1,1,1.0
-1mo,Seasonal-Naive,1.0,1.0,12,1,1,1.0
 5min,Seasonal-Naive,1.0,1.0,1,2,2,1.0
 5min,TabPFN-TS,1.0,1.0,2,2,2,1.0
 5min,TimesFM-2.5,1.0,1.0,3,2,2,1.0

results/aggregates/results_by_prediction_length.csv

@@ -1,28 +1,16 @@
 prediction_length,model,MSE,CRPS,Rank,n_configs,n_group_configs,coverage
-7,ARIMA,0.35081770342317864,0.47944164778388026,1,1,1,1.0
-7,Moirai-2.0,0.40785680526551005,0.5118677129116019,2,1,1,1.0
-7,Sundial,0.303769640015456,0.5282879649026628,3,1,1,1.0
-7,ETS,0.4243826259793997,0.5449399343905824,4,1,1,1.0
-7,Moving-Average,0.4370340438348723,0.54802384805273,5,1,1,1.0
-7,Chronos-2,0.4498860462918392,0.5739884284864399,6,1,1,1.0
-7,TiRex,0.46006192226374054,0.5949360110321206,7,1,1,1.0
-7,TimesFM-2.5,0.32382897462809757,0.6278956823267848,8,1,1,1.0
-7,Chronos-Bolt,0.3947163650820193,0.6425428458659833,9,1,1,1.0
-7,Toto-1.0,1.1828445386062165,0.7008146109363338,10,1,1,1.0
-7,Seasonal-Naive,1.0,1.0,11,1,1,1.0
-7,TabPFN-TS,0.8629829581417852,1.004036040514957,12,1,1,1.0
-12,Chronos-2,0.8294476545156683,38.946376459967496,1,2,2,1.0
-12,TiRex,0.847802672121473,30.22591777828443,2,2,2,1.0
-12,Moirai-2.0,0.7551088053055054,483.94979926515697,3,2,2,1.0
-12,TimesFM-2.5,0.883401733927733,0.8936065549864372,4,2,2,1.0
-12,Chronos-Bolt,0.8576213668056174,1427.1372538860771,5,2,2,1.0
-12,TabPFN-TS,0.9139044671265871,0.9447650776752475,6,2,2,1.0
-12,Sundial,0.8615789003568101,128.99937631337272,7,2,2,1.0
-12,ARIMA,0.8639722620545719,438.7361038019728,8,2,2,1.0
-12,Seasonal-Naive,1.0,1.0,9,2,2,1.0
-12,ETS,0.8706436180201554,439.02846065451104,10,2,2,1.0
-12,Toto-1.0,5639.479792962678,13322.450572675643,11,2,2,1.0
-12,Moving-Average,0.9617678019724286,471.56119324419365,12,2,2,1.0
+12,Seasonal-Naive,1.0,1.0,1,1,1,1.0
+12,TabPFN-TS,1.0,1.0,2,1,1,1.0
+12,TimesFM-2.5,1.0,1.0,3,1,1,1.0
+12,TiRex,1.0,1397.7486927145103,4,1,1,1.0
+12,Chronos-2,1.0,2356.162747109683,5,1,1,1.0
+12,Sundial,1.0,21140.1217190091,6,1,1,1.0
+12,ARIMA,1.0,250217.138218906,7,1,1,1.0
+12,ETS,1.0,250217.138218906,8,1,1,1.0
+12,Moving-Average,1.0,250217.138218906,9,1,1,1.0
+12,Moirai-2.0,1.0,410051.5180343537,10,1,1,1.0
+12,Chronos-Bolt,1.094103907762285,3214861.8499168884,11,1,1,1.0
+12,Toto-1.0,50341153.673178256,277943555.25179964,12,1,1,1.0
 24,TiRex,0.8393124614051559,0.9225291259963043,1,8,8,1.0
 24,Toto-1.0,3.221488937183407,0.8424427873415189,2,8,8,1.0
 24,Moirai-2.0,1.0996765234197958,0.9902855253418892,3,8,8,1.0

results/aggregates/results_overall.csv

@@ -1,13 +1,13 @@
 overall,model,MSE,CRPS,Rank,n_configs,n_group_configs,coverage
-Overall,TiRex,0.703393241439027,2.487920264306921,1,14,14,1.0
-Overall,Sundial,0.7317203685052369,3.4971862653133186,2,14,14,1.0
-Overall,Chronos-2,0.7609184058549149,2.383744211554301,3,14,14,1.0
-Overall,TimesFM-2.5,0.7658651758591832,0.8853652606290402,4,14,14,1.0
-Overall,Moirai-2.0,1.034855488960128,5.861708922220037,5,14,14,1.0
-Overall,Toto-1.0,48.273768089052126,13.15836737780422,6,14,14,1.0
-Overall,Seasonal-Naive,1.0,1.0,7,14,14,1.0
-Overall,TabPFN-TS,0.8204778598140972,0.9550955125482756,8,14,14,1.0
-Overall,Chronos-Bolt,0.983158788785599,5.880150827877197,9,14,14,1.0
-Overall,Moving-Average,1.2654153157880021,6.624155488526724,10,14,14,1.0
-Overall,ARIMA,2.2049738892736177,9.601367184262196,11,14,14,1.0
-Overall,ETS,4.400531481318116,12.73349124682922,12,14,14,1.0
+Overall,TiRex,0.7274124589463101,3.133231764528986,1,12,12,1.0
+Overall,Sundial,0.786396301241797,4.635469062321733,2,12,12,1.0
+Overall,Chronos-2,0.8016861795629415,2.993414992250487,3,12,12,1.0
+Overall,TimesFM-2.5,0.8215330246701777,0.9189556267630763,4,12,12,1.0
+Overall,Seasonal-Naive,1.0,1.0,5,12,12,1.0
+Overall,TabPFN-TS,0.8158200209107423,0.9565070855775092,6,12,12,1.0
+Overall,Toto-1.0,94.37845208105712,21.618942395242758,7,12,12,1.0
+Overall,Moving-Average,1.419222942029952,9.638787245937582,8,12,12,1.0
+Overall,Chronos-Bolt,1.0949882640042732,8.514267128835796,9,12,12,1.0
+Overall,Moirai-2.0,1.1753009673612107,8.720370995044656,10,12,12,1.0
+Overall,ARIMA,2.8127465532292644,15.210837874073626,11,12,12,1.0
+Overall,ETS,6.191598473811765,20.918024408446165,12,12,12,1.0

results/dataset_properties.csv

@@ -1,7 +1,6 @@
 dataset,domain,frequency,num_variates
 binance_btcusdt_1h_close,Finance,1h,1
 coingecko_bitcoin_7d_price_usd,Finance,1h,1
-data_gov_sg_sgx_equities_turnover_monthly_turnover_value_million_sgd,Finance,M,1
 data_gov_sg_singapore_rainfall_5min_rainfall_5min_total,Climate,5min,1
 data_gov_sg_singapore_taxi_availability_available_taxi_count,Transportation,5min,1
 nasa_power_shanghai_hourly_t2m,Weather,1h,1
@@ -12,4 +11,3 @@ open_meteo_shanghai_air_quality_hourly_pm2_5,Air Quality,1h,1
 open_meteo_shanghai_hourly_temperature_2m,Weather,1h,1
 usgs_earthquake_all_week_hourly_earthquake_count,Hazards,1h,1
 usgs_potomac_iv_usgs_00060,Hydrology,15min,1
-wikimedia_time_series_hourly_pageviews,Web,1D,1

results/online_status.json

@@ -1,9 +1,9 @@
 {
-    "status": "ok",
+    "status": "partial",
     "evaluation_protocol": "issue-now-score-later",
     "metric_version": "livehouse-canonical-v1",
-    "started_at": "2026-07-27T23:40:47.118767+00:00",
-    "finished_at": "2026-07-27T23:40:55.875758+00:00",
+    "started_at": "2026-07-28T01:02:45.223130+00:00",
+    "finished_at": "2026-07-28T01:02:51.024032+00:00",
     "data_source": "ts_bench",
     "data_config": "configs/datasets/ts_bench.yaml",
     "task_count": 4,
@@ -11,7 +11,13 @@
     "issued_forecasts": 0,
     "resolved_forecasts": 0,
     "pending_tasks": 13,
-    "failed_forecasts": [],
+    "failed_forecasts": [
+        {
+            "model": "Seasonal-Naive",
+            "dataset": "nws_ksfo_observations_temperature/1h/short",
+            "error": "Mean forecast length 10 < 24"
+        }
+    ],
     "models": {
         "Chronos-2": {
             "model": "Chronos-2",
@@ -19,7 +25,7 @@
             "admitted_at": "2026-06-01T00:00:00Z",
             "evaluation_protocol": "issue-now-score-later",
             "metric_version": "livehouse-canonical-v1",
-            "evaluated_at": "2026-07-27T23:40:51.541579+00:00",
+            "evaluated_at": "2026-07-28T01:02:47.761732+00:00",
             "resolved_tasks": []
         },
         "TiRex": {
@@ -28,7 +34,7 @@
             "admitted_at": "2026-06-01T00:00:00Z",
             "evaluation_protocol": "issue-now-score-later",
             "metric_version": "livehouse-canonical-v1",
-            "evaluated_at": "2026-07-27T23:40:51.541586+00:00",
+            "evaluated_at": "2026-07-28T01:02:47.761740+00:00",
             "resolved_tasks": []
         },
         "TimesFM-2.5": {
@@ -37,7 +43,7 @@
             "admitted_at": "2026-06-01T00:00:00Z",
             "evaluation_protocol": "issue-now-score-later",
             "metric_version": "livehouse-canonical-v1",
-            "evaluated_at": "2026-07-27T23:40:51.541587+00:00",
+            "evaluated_at": "2026-07-28T01:02:47.761741+00:00",
             "resolved_tasks": []
         },
         "Toto-1.0": {
@@ -46,7 +52,7 @@
             "admitted_at": "2026-06-01T00:00:00Z",
             "evaluation_protocol": "issue-now-score-later",
             "metric_version": "livehouse-canonical-v1",
-            "evaluated_at": "2026-07-27T23:40:51.541588+00:00",
+            "evaluated_at": "2026-07-28T01:02:47.761742+00:00",
             "resolved_tasks": []
         },
         "Moirai-2.0": {
@@ -55,7 +61,7 @@
             "admitted_at": "2026-06-01T00:00:00Z",
             "evaluation_protocol": "issue-now-score-later",
             "metric_version": "livehouse-canonical-v1",
-            "evaluated_at": "2026-07-27T23:40:51.541589+00:00",
+            "evaluated_at": "2026-07-28T01:02:47.761743+00:00",
             "resolved_tasks": []
         },
         "Chronos-Bolt": {
@@ -64,7 +70,7 @@
             "admitted_at": "2026-06-01T00:00:00Z",
             "evaluation_protocol": "issue-now-score-later",
             "metric_version": "livehouse-canonical-v1",
-            "evaluated_at": "2026-07-27T23:40:51.541590+00:00",
+            "evaluated_at": "2026-07-28T01:02:47.761744+00:00",
             "resolved_tasks": []
         },
         "TabPFN-TS": {
@@ -73,7 +79,7 @@
             "admitted_at": "2026-06-01T00:00:00Z",
             "evaluation_protocol": "issue-now-score-later",
             "metric_version": "livehouse-canonical-v1",
-            "evaluated_at": "2026-07-27T23:40:51.541591+00:00",
+            "evaluated_at": "2026-07-28T01:02:47.761745+00:00",
             "resolved_tasks": []
         },
         "Sundial": {
@@ -82,7 +88,7 @@
             "admitted_at": "2026-06-01T00:00:00Z",
             "evaluation_protocol": "issue-now-score-later",
             "metric_version": "livehouse-canonical-v1",
-            "evaluated_at": "2026-07-27T23:40:51.541592+00:00",
+            "evaluated_at": "2026-07-28T01:02:47.761746+00:00",
             "resolved_tasks": []
         },
         "Moving-Average": {
@@ -91,7 +97,7 @@
             "admitted_at": "2026-06-01T00:00:00Z",
             "evaluation_protocol": "issue-now-score-later",
             "metric_version": "livehouse-canonical-v1",
-            "evaluated_at": "2026-07-27T23:40:51.541594+00:00",
+            "evaluated_at": "2026-07-28T01:02:47.761746+00:00",
             "resolved_tasks": []
         },
         "ETS": {
@@ -100,7 +106,7 @@
             "admitted_at": "2026-06-01T00:00:00Z",
             "evaluation_protocol": "issue-now-score-later",
             "metric_version": "livehouse-canonical-v1",
-            "evaluated_at": "2026-07-27T23:40:51.541594+00:00",
+            "evaluated_at": "2026-07-28T01:02:47.761747+00:00",
             "resolved_tasks": []
         },
         "ARIMA": {
@@ -109,7 +115,7 @@
             "admitted_at": "2026-06-01T00:00:00Z",
             "evaluation_protocol": "issue-now-score-later",
             "metric_version": "livehouse-canonical-v1",
-            "evaluated_at": "2026-07-27T23:40:51.541595+00:00",
+            "evaluated_at": "2026-07-28T01:02:47.761748+00:00",
             "resolved_tasks": []
         },
         "Seasonal-Naive": {
@@ -118,7 +124,7 @@
             "admitted_at": "2026-06-01T00:00:00Z",
             "evaluation_protocol": "issue-now-score-later",
             "metric_version": "livehouse-canonical-v1",
-            "evaluated_at": "2026-07-27T23:40:51.541596+00:00",
+            "evaluated_at": "2026-07-28T01:02:47.761749+00:00",
             "resolved_tasks": []
         }
     },
@@ -142,7 +148,5 @@
         "live_overall_30d": "aggregates/live_overall_30d.csv",
         "live_metadata": "aggregates/live_metadata.json"
     },
-    "push_status": "ok",
-    "pushed_at": "2026-07-28T00:51:18.371820+00:00",
-    "push_detail": "HF Hub API"
+    "push_status": "pending"
 }