Update leaderboard results from local eval

results/aggregates/live_metadata.json

@@ -1,5 +1,5 @@
 {
-  "generated_at": "2026-07-29T06:51:26.383616+00:00",
+  "generated_at": "2026-07-29T07:01:43.819196+00:00",
   "scope": "Canonical metrics-only release history when available; otherwise legacy release history plus latest allowlisted snapshots",
   "release_deduplication": "One row per model, dataset, data source timestamp, and prediction length",
   "dataset_aggregation": "Arithmetic mean over releases, then equal-weight mean over datasets",

results/aggregates/metadata.json

@@ -1,5 +1,5 @@
 {
-  "generated_at": "2026-07-29T06:51:24.162285+00:00",
+  "generated_at": "2026-07-29T07:01:41.776971+00:00",
   "baseline_model": "Seasonal-Naive",
   "normalization": "Per-config metric / Seasonal-Naive metric",
   "metric_aggregation": "Geometric mean over dataset configurations",
@@ -10,7 +10,7 @@
     "frequency",
     "prediction_length"
   ],
-  "num_configs": 13,
+  "num_configs": 12,
   "num_models": 12,
   "num_rows_missing_prediction_length": 0
 }

results/aggregates/results_by_domain.csv

@@ -1,16 +1,16 @@
 domain,model,MSE,CRPS,Rank,n_configs,n_group_configs,coverage
-Climate,Toto-1.0,435.5901985713874,53.29156478660997,1,5,5,1.0
-Climate,Seasonal-Naive,1.0,1.0,2,5,5,1.0
-Climate,Moirai-2.0,7.378507435351014,57.141717297913075,3,5,5,1.0
-Climate,Chronos-2,8.207612976283679,58.356072575766355,4,5,5,1.0
-Climate,TiRex,3.6039513362371127,47.1885455613262,5,5,5,1.0
-Climate,TabPFN-TS,10.21051994514313,53.753593411883465,6,5,5,1.0
-Climate,Chronos-Bolt,15.545386249279808,77.74915686334033,7,5,5,1.0
-Climate,Sundial,56.13631154452401,80.26168134874668,8,5,5,1.0
-Climate,Moving-Average,263.19708208074803,1.9356657685209764,9,5,5,1.0
-Climate,TimesFM-2.5,6.024327851477196,70.69945011330122,10,5,5,1.0
-Climate,ARIMA,342.47676221363855,246.06359529884122,11,5,5,1.0
-Climate,ETS,852.1492904678585,44.71268860829721,12,5,5,1.0
+Climate,Toto-1.0,2.10788408676814,0.7444982286752517,1,4,4,1.0
+Climate,Moirai-2.0,0.26139857175176046,0.7987449648560268,2,4,4,1.0
+Climate,Seasonal-Naive,1.0,1.0,3,4,4,1.0
+Climate,Chronos-2,0.3511052230278436,0.8406087624335221,4,4,4,1.0
+Climate,Chronos-Bolt,0.4736642972936194,0.8759108166985273,5,4,4,1.0
+Climate,TabPFN-TS,0.5618684872816783,0.8160147919184494,6,4,4,1.0
+Climate,Sundial,0.8781204086033556,0.8457655435196634,7,4,4,1.0
+Climate,TiRex,0.25645820380075013,0.8748904165726592,8,4,4,1.0
+Climate,TimesFM-2.5,0.48373052394112653,0.9590866974818474,9,4,4,1.0
+Climate,ARIMA,4.800690937631407,2.1894644656096474,10,4,4,1.0
+Climate,Moving-Average,4.164784946934952,2.283168490719553,11,4,4,1.0
+Climate,ETS,18.087801508734604,4.849243130270303,12,4,4,1.0
 Finance,Moirai-2.0,0.22409481207132434,0.487951964417501,1,2,2,1.0
 Finance,Chronos-Bolt,0.2736790800966687,0.5057014143672646,2,2,2,1.0
 Finance,TiRex,0.20014521509059205,0.4804972472127513,3,2,2,1.0

results/aggregates/results_by_frequency.csv

@@ -59,18 +59,18 @@ frequency,model,MSE,CRPS,Rank,n_configs,n_group_configs,coverage
 1mo,Moving-Average,0.9249973049108767,0.8887079460534169,10,1,1,1.0
 1mo,TabPFN-TS,0.8352213750339312,0.8925810519947165,11,1,1,1.0
 1mo,Seasonal-Naive,1.0,1.0,12,1,1,1.0
-5min,Seasonal-Naive,1.0,1.0,1,2,2,1.0
-5min,TabPFN-TS,1055.2376905725948,31814.323472829048,2,2,2,1.0
-5min,TiRex,374.8988453889774,747134.3573618883,3,2,2,1.0
-5min,Moving-Average,64791.422249299285,500.21709109036453,4,2,2,1.0
-5min,TimesFM-2.5,380.68390120897556,45690.33634956956,5,2,2,1.0
-5min,Chronos-2,1565.552346444848,1787022.2357811735,6,2,2,1.0
-5min,ETS,64791.44260399208,284372.48341344163,7,2,2,1.0
-5min,Sundial,30619.743111159518,11429071.166645743,8,2,2,1.0
-5min,Moirai-2.0,2164.2871329316717,24773431.41265387,9,2,2,1.0
-5min,Toto-1.0,273756101415.0104,614429321.3236533,10,2,2,1.0
-5min,ARIMA,94182.51370114152,99106271.41688466,11,2,2,1.0
-5min,Chronos-Bolt,4442.13523689146,124566309.73853382,12,2,2,1.0
+5min,Seasonal-Naive,1.0,1.0,1,1,1,1.0
+5min,TabPFN-TS,1.0,1.0,2,1,1,1.0
+5min,TimesFM-2.5,1.0,1.0,3,1,1,1.0
+5min,TiRex,1.0,1397.7486927145103,4,1,1,1.0
+5min,Chronos-2,1.0,2356.162747109683,5,1,1,1.0
+5min,Sundial,1.0,20066.91263321144,6,1,1,1.0
+5min,ARIMA,1.0,250217.138218906,7,1,1,1.0
+5min,ETS,1.0,250217.138218906,8,1,1,1.0
+5min,Moving-Average,1.0,250217.138218906,9,1,1,1.0
+5min,Moirai-2.0,1.0,410051.5180343537,10,1,1,1.0
+5min,Chronos-Bolt,1.094103907762285,3214861.8499168884,11,1,1,1.0
+5min,Toto-1.0,94346114916.68228,269839906.6073004,12,1,1,1.0
 6min,Chronos-2,0.004065646210503269,0.18017789189149058,1,1,1,1.0
 6min,TimesFM-2.5,0.11897587453818394,0.30185471840281924,2,1,1,1.0
 6min,Moirai-2.0,0.15529767214638196,0.43989214409973665,3,1,1,1.0

results/aggregates/results_by_prediction_length.csv

@@ -35,18 +35,6 @@ prediction_length,model,MSE,CRPS,Rank,n_configs,n_group_configs,coverage
 24,Seasonal-Naive,1.0,1.0,10,7,7,1.0
 24,ARIMA,2.0280756005522766,1.7363039984998985,11,7,7,1.0
 24,ETS,5.524818260678021,2.5285715380243023,12,7,7,1.0
-36,Seasonal-Naive,1.0,1.0,1,1,1,1.0
-36,Moving-Average,4197928397.0869946,1.0,2,1,1,1.0
-36,ETS,4197931034.7064004,323190.1295744973,3,1,1,1.0
-36,TiRex,140549.1442739884,399363455.5769006,4,1,1,1.0
-36,TabPFN-TS,1113526.5836049833,1012151178.0338014,5,1,1,1.0
-36,Chronos-2,2450954.1494589695,1355359885.5146005,6,1,1,1.0
-36,Toto-1.0,794334807831.0154,1399064340.2187998,7,1,1,1.0
-36,Moirai-2.0,4684138.793773595,1496697066.0160985,8,1,1,1.0
-36,TimesFM-2.5,144920.23263968507,2087606835.7367973,9,1,1,1.0
-36,Chronos-Bolt,18035366.95448861,4826573036.809504,10,1,1,1.0
-36,Sundial,937568668.1934007,6509405314.10231,11,1,1,1.0
-36,ARIMA,8870345887.065712,39254117859.681656,12,1,1,1.0
 60,Chronos-2,0.004065646210503269,0.18017789189149058,1,1,1,1.0
 60,TimesFM-2.5,0.11897587453818394,0.30185471840281924,2,1,1,1.0
 60,Moirai-2.0,0.15529767214638196,0.43989214409973665,3,1,1,1.0

results/aggregates/results_overall.csv

@@ -1,13 +1,13 @@
 overall,model,MSE,CRPS,Rank,n_configs,n_group_configs,coverage
-Overall,Chronos-2,1.0798923015805328,6.13850138599413,1,13,13,1.0
-Overall,Moirai-2.0,1.3942772640118302,9.808091977070788,2,13,13,1.0
-Overall,TiRex,1.134168572138379,6.102790819707846,3,13,13,1.0
-Overall,Sundial,3.2550945737330066,8.82210611904731,4,13,13,1.0
-Overall,Chronos-Bolt,2.2766805528745215,14.371919755650465,5,13,13,1.0
-Overall,TimesFM-2.5,1.0710929523348298,3.697931272092179,6,13,13,1.0
-Overall,Moving-Average,6.983561763466352,2.8867005741911806,7,13,13,1.0
-Overall,Toto-1.0,76.93438205584695,18.06722749541169,8,13,13,1.0
-Overall,Seasonal-Naive,1.0,1.0,9,13,13,1.0
-Overall,TabPFN-TS,2.390864374898132,4.570626940041292,10,13,13,1.0
-Overall,ARIMA,7.984903908356004,21.583997422611787,11,13,13,1.0
-Overall,ETS,12.868742631895437,10.652066109852175,12,13,13,1.0
+Overall,Chronos-2,0.31894628649553414,1.2380257253097502,1,12,12,1.0
+Overall,Moirai-2.0,0.39855882338778986,2.0399620196948294,2,12,12,1.0
+Overall,Sundial,0.6421113129877262,1.609069703557794,3,12,12,1.0
+Overall,TiRex,0.4268235684375872,1.362098447165561,4,12,12,1.0
+Overall,Chronos-Bolt,0.6058989372918245,2.799009948857091,5,12,12,1.0
+Overall,TimesFM-2.5,0.4001461782119439,0.6896858974822001,6,12,12,1.0
+Overall,Moving-Average,1.2956781682236393,3.1533236653487573,7,12,12,1.0
+Overall,Toto-1.0,11.262380852733084,3.976301727540666,8,12,12,1.0
+Overall,ARIMA,1.407549863732539,3.651621611113589,9,12,12,1.0
+Overall,Seasonal-Naive,1.0,1.0,10,12,12,1.0
+Overall,TabPFN-TS,0.8057669872813471,0.9215876388113726,11,12,12,1.0
+Overall,ETS,2.512336742921387,4.507459502855504,12,12,12,1.0

results/dataset_properties.csv

@@ -1,7 +1,6 @@
 dataset,domain,frequency,num_variates
 binance_btcusdt_1h_close,Finance,1h,1
 data_gov_sg_sgx_equities_turnover_monthly_turnover_value_million_sgd,Finance,M,1
-data_gov_sg_singapore_rainfall_5min_rainfall_5min_total,Climate,5min,1
 data_gov_sg_singapore_taxi_availability_available_taxi_count,Transportation,5min,1
 nasa_power_shanghai_hourly_t2m,Weather,1h,1
 noaa_coops_sf_water_level_water_level,Ocean,6min,1

results/online_status.json

@@ -2,14 +2,14 @@
     "status": "ok",
     "evaluation_protocol": "issue-now-score-later",
     "metric_version": "livehouse-canonical-v1",
-    "started_at": "2026-07-29T06:46:46.411815+00:00",
-    "finished_at": "2026-07-29T06:51:26.398362+00:00",
+    "started_at": "2026-07-29T07:01:35.096139+00:00",
+    "finished_at": "2026-07-29T07:01:43.830715+00:00",
     "data_source": "ts_bench",
     "data_config": "configs/datasets/ts_bench.yaml",
-    "task_count": 4,
-    "issued_tasks": 1,
-    "issued_forecasts": 12,
-    "resolved_forecasts": 12,
+    "task_count": 2,
+    "issued_tasks": 0,
+    "issued_forecasts": 0,
+    "resolved_forecasts": 0,
     "pending_tasks": 13,
     "failed_forecasts": [],
     "models": {
@@ -19,18 +19,8 @@
             "admitted_at": "2026-06-01T00:00:00Z",
             "evaluation_protocol": "issue-now-score-later",
             "metric_version": "livehouse-canonical-v1",
-            "evaluated_at": "2026-07-29T06:51:21.989130+00:00",
-            "resolved_tasks": [
-                {
-                    "dataset": "data_gov_sg_singapore_taxi_availability_available_taxi_count/5min/short",
-                    "task_id": "future-f62bf1c75151e228c6d4",
-                    "issued_at": "2026-07-29T04:50:10.589491+00:00",
-                    "resolved_at": "2026-07-29T06:46:48.270570+00:00",
-                    "target_end": "2026-07-27T20:50:00+00:00",
-                    "prediction_length": 12,
-                    "metric_version": "livehouse-canonical-v1"
-                }
-            ]
+            "evaluated_at": "2026-07-29T07:01:39.730685+00:00",
+            "resolved_tasks": []
         },
         "TiRex": {
             "model": "TiRex",
@@ -38,18 +28,8 @@
             "admitted_at": "2026-06-01T00:00:00Z",
             "evaluation_protocol": "issue-now-score-later",
             "metric_version": "livehouse-canonical-v1",
-            "evaluated_at": "2026-07-29T06:51:22.000419+00:00",
-            "resolved_tasks": [
-                {
-                    "dataset": "data_gov_sg_singapore_taxi_availability_available_taxi_count/5min/short",
-                    "task_id": "future-f62bf1c75151e228c6d4",
-                    "issued_at": "2026-07-29T04:50:10.589491+00:00",
-                    "resolved_at": "2026-07-29T06:46:48.567951+00:00",
-                    "target_end": "2026-07-27T20:50:00+00:00",
-                    "prediction_length": 12,
-                    "metric_version": "livehouse-canonical-v1"
-                }
-            ]
+            "evaluated_at": "2026-07-29T07:01:39.730708+00:00",
+            "resolved_tasks": []
         },
         "TimesFM-2.5": {
             "model": "TimesFM-2.5",
@@ -57,18 +37,8 @@
             "admitted_at": "2026-06-01T00:00:00Z",
             "evaluation_protocol": "issue-now-score-later",
             "metric_version": "livehouse-canonical-v1",
-            "evaluated_at": "2026-07-29T06:51:22.007817+00:00",
-            "resolved_tasks": [
-                {
-                    "dataset": "data_gov_sg_singapore_taxi_availability_available_taxi_count/5min/short",
-                    "task_id": "future-f62bf1c75151e228c6d4",
-                    "issued_at": "2026-07-29T04:50:10.589491+00:00",
-                    "resolved_at": "2026-07-29T06:46:48.520670+00:00",
-                    "target_end": "2026-07-27T20:50:00+00:00",
-                    "prediction_length": 12,
-                    "metric_version": "livehouse-canonical-v1"
-                }
-            ]
+            "evaluated_at": "2026-07-29T07:01:39.730711+00:00",
+            "resolved_tasks": []
         },
         "Toto-1.0": {
             "model": "Toto-1.0",
@@ -76,18 +46,8 @@
             "admitted_at": "2026-06-01T00:00:00Z",
             "evaluation_protocol": "issue-now-score-later",
             "metric_version": "livehouse-canonical-v1",
-            "evaluated_at": "2026-07-29T06:51:22.012666+00:00",
-            "resolved_tasks": [
-                {
-                    "dataset": "data_gov_sg_singapore_taxi_availability_available_taxi_count/5min/short",
-                    "task_id": "future-f62bf1c75151e228c6d4",
-                    "issued_at": "2026-07-29T04:50:10.589491+00:00",
-                    "resolved_at": "2026-07-29T06:46:48.588538+00:00",
-                    "target_end": "2026-07-27T20:50:00+00:00",
-                    "prediction_length": 12,
-                    "metric_version": "livehouse-canonical-v1"
-                }
-            ]
+            "evaluated_at": "2026-07-29T07:01:39.730714+00:00",
+            "resolved_tasks": []
         },
         "Moirai-2.0": {
             "model": "Moirai-2.0",
@@ -95,18 +55,8 @@
             "admitted_at": "2026-06-01T00:00:00Z",
             "evaluation_protocol": "issue-now-score-later",
             "metric_version": "livehouse-canonical-v1",
-            "evaluated_at": "2026-07-29T06:51:22.052641+00:00",
-            "resolved_tasks": [
-                {
-                    "dataset": "data_gov_sg_singapore_taxi_availability_available_taxi_count/5min/short",
-                    "task_id": "future-f62bf1c75151e228c6d4",
-                    "issued_at": "2026-07-29T04:50:10.589491+00:00",
-                    "resolved_at": "2026-07-29T06:46:48.385759+00:00",
-                    "target_end": "2026-07-27T20:50:00+00:00",
-                    "prediction_length": 12,
-                    "metric_version": "livehouse-canonical-v1"
-                }
-            ]
+            "evaluated_at": "2026-07-29T07:01:39.730716+00:00",
+            "resolved_tasks": []
         },
         "Chronos-Bolt": {
             "model": "Chronos-Bolt",
@@ -114,18 +64,8 @@
             "admitted_at": "2026-06-01T00:00:00Z",
             "evaluation_protocol": "issue-now-score-later",
             "metric_version": "livehouse-canonical-v1",
-            "evaluated_at": "2026-07-29T06:51:22.076509+00:00",
-            "resolved_tasks": [
-                {
-                    "dataset": "data_gov_sg_singapore_taxi_availability_available_taxi_count/5min/short",
-                    "task_id": "future-f62bf1c75151e228c6d4",
-                    "issued_at": "2026-07-29T04:50:10.589491+00:00",
-                    "resolved_at": "2026-07-29T06:46:48.315655+00:00",
-                    "target_end": "2026-07-27T20:50:00+00:00",
-                    "prediction_length": 12,
-                    "metric_version": "livehouse-canonical-v1"
-                }
-            ]
+            "evaluated_at": "2026-07-29T07:01:39.730718+00:00",
+            "resolved_tasks": []
         },
         "TabPFN-TS": {
             "model": "TabPFN-TS",
@@ -133,18 +73,8 @@
             "admitted_at": "2026-06-01T00:00:00Z",
             "evaluation_protocol": "issue-now-score-later",
             "metric_version": "livehouse-canonical-v1",
-            "evaluated_at": "2026-07-29T06:51:22.079264+00:00",
-            "resolved_tasks": [
-                {
-                    "dataset": "data_gov_sg_singapore_taxi_availability_available_taxi_count/5min/short",
-                    "task_id": "future-f62bf1c75151e228c6d4",
-                    "issued_at": "2026-07-29T04:50:10.589491+00:00",
-                    "resolved_at": "2026-07-29T06:46:48.491734+00:00",
-                    "target_end": "2026-07-27T20:50:00+00:00",
-                    "prediction_length": 12,
-                    "metric_version": "livehouse-canonical-v1"
-                }
-            ]
+            "evaluated_at": "2026-07-29T07:01:39.730721+00:00",
+            "resolved_tasks": []
         },
         "Sundial": {
             "model": "Sundial",
@@ -152,18 +82,8 @@
             "admitted_at": "2026-06-01T00:00:00Z",
             "evaluation_protocol": "issue-now-score-later",
             "metric_version": "livehouse-canonical-v1",
-            "evaluated_at": "2026-07-29T06:51:22.096782+00:00",
-            "resolved_tasks": [
-                {
-                    "dataset": "data_gov_sg_singapore_taxi_availability_available_taxi_count/5min/short",
-                    "task_id": "future-f62bf1c75151e228c6d4",
-                    "issued_at": "2026-07-29T04:50:10.589491+00:00",
-                    "resolved_at": "2026-07-29T06:46:48.467874+00:00",
-                    "target_end": "2026-07-27T20:50:00+00:00",
-                    "prediction_length": 12,
-                    "metric_version": "livehouse-canonical-v1"
-                }
-            ]
+            "evaluated_at": "2026-07-29T07:01:39.730723+00:00",
+            "resolved_tasks": []
         },
         "Moving-Average": {
             "model": "Moving-Average",
@@ -171,18 +91,8 @@
             "admitted_at": "2026-06-01T00:00:00Z",
             "evaluation_protocol": "issue-now-score-later",
             "metric_version": "livehouse-canonical-v1",
-            "evaluated_at": "2026-07-29T06:51:22.103962+00:00",
-            "resolved_tasks": [
-                {
-                    "dataset": "data_gov_sg_singapore_taxi_availability_available_taxi_count/5min/short",
-                    "task_id": "future-f62bf1c75151e228c6d4",
-                    "issued_at": "2026-07-29T04:50:10.589491+00:00",
-                    "resolved_at": "2026-07-29T06:46:48.411757+00:00",
-                    "target_end": "2026-07-27T20:50:00+00:00",
-                    "prediction_length": 12,
-                    "metric_version": "livehouse-canonical-v1"
-                }
-            ]
+            "evaluated_at": "2026-07-29T07:01:39.730725+00:00",
+            "resolved_tasks": []
         },
         "ETS": {
             "model": "ETS",
@@ -190,18 +100,8 @@
             "admitted_at": "2026-06-01T00:00:00Z",
             "evaluation_protocol": "issue-now-score-later",
             "metric_version": "livehouse-canonical-v1",
-            "evaluated_at": "2026-07-29T06:51:22.120597+00:00",
-            "resolved_tasks": [
-                {
-                    "dataset": "data_gov_sg_singapore_taxi_availability_available_taxi_count/5min/short",
-                    "task_id": "future-f62bf1c75151e228c6d4",
-                    "issued_at": "2026-07-29T04:50:10.589491+00:00",
-                    "resolved_at": "2026-07-29T06:46:48.355858+00:00",
-                    "target_end": "2026-07-27T20:50:00+00:00",
-                    "prediction_length": 12,
-                    "metric_version": "livehouse-canonical-v1"
-                }
-            ]
+            "evaluated_at": "2026-07-29T07:01:39.730727+00:00",
+            "resolved_tasks": []
         },
         "ARIMA": {
             "model": "ARIMA",
@@ -209,18 +109,8 @@
             "admitted_at": "2026-06-01T00:00:00Z",
             "evaluation_protocol": "issue-now-score-later",
             "metric_version": "livehouse-canonical-v1",
-            "evaluated_at": "2026-07-29T06:51:22.132276+00:00",
-            "resolved_tasks": [
-                {
-                    "dataset": "data_gov_sg_singapore_taxi_availability_available_taxi_count/5min/short",
-                    "task_id": "future-f62bf1c75151e228c6d4",
-                    "issued_at": "2026-07-29T04:50:10.589491+00:00",
-                    "resolved_at": "2026-07-29T06:46:48.233596+00:00",
-                    "target_end": "2026-07-27T20:50:00+00:00",
-                    "prediction_length": 12,
-                    "metric_version": "livehouse-canonical-v1"
-                }
-            ]
+            "evaluated_at": "2026-07-29T07:01:39.730730+00:00",
+            "resolved_tasks": []
         },
         "Seasonal-Naive": {
             "model": "Seasonal-Naive",
@@ -228,18 +118,8 @@
             "admitted_at": "2026-06-01T00:00:00Z",
             "evaluation_protocol": "issue-now-score-later",
             "metric_version": "livehouse-canonical-v1",
-            "evaluated_at": "2026-07-29T06:51:22.135908+00:00",
-            "resolved_tasks": [
-                {
-                    "dataset": "data_gov_sg_singapore_taxi_availability_available_taxi_count/5min/short",
-                    "task_id": "future-f62bf1c75151e228c6d4",
-                    "issued_at": "2026-07-29T04:50:10.589491+00:00",
-                    "resolved_at": "2026-07-29T06:46:48.433558+00:00",
-                    "target_end": "2026-07-27T20:50:00+00:00",
-                    "prediction_length": 12,
-                    "metric_version": "livehouse-canonical-v1"
-                }
-            ]
+            "evaluated_at": "2026-07-29T07:01:39.730732+00:00",
+            "resolved_tasks": []
         }
     },
     "skipped_models": [],
@@ -262,7 +142,5 @@
         "live_overall_30d": "aggregates/live_overall_30d.csv",
         "live_metadata": "aggregates/live_metadata.json"
     },
-    "push_status": "ok",
-    "pushed_at": "2026-07-29T06:52:20.888016+00:00",
-    "push_detail": "HF Hub API"
+    "push_status": "pending"
 }