Update leaderboard results from local eval

results/aggregates/live_metadata.json

@@ -1,5 +1,5 @@
 {
-  "generated_at": "2026-07-28T17:46:14.552914+00:00",
+  "generated_at": "2026-07-28T17:59:02.793511+00:00",
   "scope": "Canonical metrics-only release history when available; otherwise legacy release history plus latest allowlisted snapshots",
   "release_deduplication": "One row per model, dataset, data source timestamp, and prediction length",
   "dataset_aggregation": "Arithmetic mean over releases, then equal-weight mean over datasets",

results/aggregates/metadata.json

@@ -1,5 +1,5 @@
 {
-  "generated_at": "2026-07-28T17:46:12.360995+00:00",
+  "generated_at": "2026-07-28T17:59:00.872037+00:00",
   "baseline_model": "Seasonal-Naive",
   "normalization": "Per-config metric / Seasonal-Naive metric",
   "metric_aggregation": "Geometric mean over dataset configurations",
@@ -10,7 +10,7 @@
     "frequency",
     "prediction_length"
   ],
-  "num_configs": 13,
+  "num_configs": 12,
   "num_models": 12,
   "num_rows_missing_prediction_length": 0
 }

results/aggregates/results_by_domain.csv

@@ -35,18 +35,6 @@ Hazards,Toto-1.0,0.5447067827362504,0.6198857623270299,9,1,1,1.0
 Hazards,TimesFM-2.5,0.5159030493533491,0.6552873663892604,10,1,1,1.0
 Hazards,TabPFN-TS,0.5234081583087049,0.7154825875538315,11,1,1,1.0
 Hazards,Seasonal-Naive,1.0,1.0,12,1,1,1.0
-Hydrology,ARIMA,0.9751306749691749,0.38486182748098285,1,1,1,1.0
-Hydrology,Moirai-2.0,1.0813692464527007,0.9924021595681446,2,1,1,1.0
-Hydrology,Seasonal-Naive,1.0,1.0,3,1,1,1.0
-Hydrology,Chronos-2,1.1276077437591416,1.010443151113958,4,1,1,1.0
-Hydrology,TiRex,1.127237170726061,1.0311739352853535,5,1,1,1.0
-Hydrology,Toto-1.0,1.0790332247777732,1.0470160067587573,6,1,1,1.0
-Hydrology,Chronos-Bolt,1.093508711981219,1.082295632689394,7,1,1,1.0
-Hydrology,Moving-Average,0.9761363565890064,1.1132157631177908,8,1,1,1.0
-Hydrology,ETS,1.0814189920897397,1.17781549355183,9,1,1,1.0
-Hydrology,Sundial,0.9824313277425463,1.1871040987080808,10,1,1,1.0
-Hydrology,TimesFM-2.5,1.13126434578556,1.255403146176928,11,1,1,1.0
-Hydrology,TabPFN-TS,1.130074924317557,1.3225726844593262,12,1,1,1.0
 Ocean,TimesFM-2.5,0.03684048004661691,0.20206259110870364,1,2,2,1.0
 Ocean,Chronos-2,0.0369650586967402,0.21429367126674892,2,2,2,1.0
 Ocean,TiRex,0.06099399090461052,0.2882552025089681,3,2,2,1.0

results/aggregates/results_by_frequency.csv

@@ -11,18 +11,6 @@ frequency,model,MSE,CRPS,Rank,n_configs,n_group_configs,coverage
 10min,Moirai-2.0,0.9931170314987676,0.9506171711084153,10,1,1,1.0
 10min,Seasonal-Naive,1.0,1.0,11,1,1,1.0
 10min,TabPFN-TS,1.0965296750513482,1.1700878991909816,12,1,1,1.0
-15min,ARIMA,0.9751306749691749,0.38486182748098285,1,1,1,1.0
-15min,Moirai-2.0,1.0813692464527007,0.9924021595681446,2,1,1,1.0
-15min,Seasonal-Naive,1.0,1.0,3,1,1,1.0
-15min,Chronos-2,1.1276077437591416,1.010443151113958,4,1,1,1.0
-15min,TiRex,1.127237170726061,1.0311739352853535,5,1,1,1.0
-15min,Toto-1.0,1.0790332247777732,1.0470160067587573,6,1,1,1.0
-15min,Chronos-Bolt,1.093508711981219,1.082295632689394,7,1,1,1.0
-15min,Moving-Average,0.9761363565890064,1.1132157631177908,8,1,1,1.0
-15min,ETS,1.0814189920897397,1.17781549355183,9,1,1,1.0
-15min,Sundial,0.9824313277425463,1.1871040987080808,10,1,1,1.0
-15min,TimesFM-2.5,1.13126434578556,1.255403146176928,11,1,1,1.0
-15min,TabPFN-TS,1.130074924317557,1.3225726844593262,12,1,1,1.0
 1D,ARIMA,0.35081770342317864,0.47944164778388026,1,1,1,1.0
 1D,Moirai-2.0,0.40785680526551005,0.5118677129116019,2,1,1,1.0
 1D,Sundial,0.303769640015456,0.5282879649026628,3,1,1,1.0

results/aggregates/results_by_prediction_length.csv

@@ -23,18 +23,18 @@ prediction_length,model,MSE,CRPS,Rank,n_configs,n_group_configs,coverage
 12,ETS,0.8706436180201554,439.02846065451104,10,2,2,1.0
 12,Toto-1.0,7534.1263954366505,13038.851881781453,11,2,2,1.0
 12,Moving-Average,0.9617678019724286,471.56119324419365,12,2,2,1.0
-24,Chronos-2,0.44339020476480734,0.7317235055481944,1,7,7,1.0
-24,TiRex,0.4145108564513675,0.7448889973096526,2,7,7,1.0
-24,Chronos-Bolt,0.5681562833990131,0.7490469070437524,3,7,7,1.0
-24,Moirai-2.0,0.4799992195385256,0.775419565634457,4,7,7,1.0
-24,Toto-1.0,1.520374525220479,0.7287828451205441,5,7,7,1.0
-24,Sundial,0.7255009526147621,0.7651853252476193,6,7,7,1.0
-24,ARIMA,1.7038724348964605,1.1269151847404395,7,7,7,1.0
-24,Seasonal-Naive,1.0,1.0,8,7,7,1.0
-24,TabPFN-TS,0.594705587085834,0.803937793274539,9,7,7,1.0
-24,Moving-Average,1.4444563689905403,1.2919521417914248,10,7,7,1.0
-24,TimesFM-2.5,0.5331643601238124,0.8304626910315389,11,7,7,1.0
-24,ETS,3.6175474085585955,2.013352278105239,12,7,7,1.0
+24,TiRex,0.3508509668853035,0.7055885069620663,1,6,6,1.0
+24,Chronos-2,0.37951076735916145,0.6934037615919085,2,6,6,1.0
+24,Chronos-Bolt,0.5094193160936651,0.704481403367288,3,6,6,1.0
+24,Sundial,0.6897535606282527,0.7111801131149522,4,6,6,1.0
+24,Toto-1.0,1.6097924218006927,0.6860760798646774,5,6,6,1.0
+24,Moirai-2.0,0.419229289802811,0.7441804627400953,6,6,6,1.0
+24,TabPFN-TS,0.534360665395711,0.7399282263361974,7,6,6,1.0
+24,TimesFM-2.5,0.4703385898665995,0.7751925886506386,8,6,6,1.0
+24,ARIMA,1.8699620239083985,1.3478936574541842,9,6,6,1.0
+24,Moving-Average,1.5419492666405061,1.3244155547545262,10,6,6,1.0
+24,Seasonal-Naive,1.0,1.0,11,6,6,1.0
+24,ETS,4.424025325844782,2.20154139110127,12,6,6,1.0
 36,ETS,0.0003160064870084056,1.000000000000001e-12,1,1,1,1.0
 36,TabPFN-TS,1.0553883777131202e-07,0.0008433365473335877,2,1,1,1.0
 36,Toto-1.0,0.05469192573721511,0.0020086340389211823,3,1,1,1.0

results/aggregates/results_overall.csv

@@ -1,13 +1,13 @@
 overall,model,MSE,CRPS,Rank,n_configs,n_group_configs,coverage
-Overall,Chronos-2,0.13551095513746933,0.9913608467642023,1,13,13,1.0
-Overall,TiRex,0.1224806415900693,0.9807742198495171,2,13,13,1.0
-Overall,Moirai-2.0,0.16287258118758158,1.1658420026831902,3,13,13,1.0
-Overall,Sundial,0.35149707747707265,1.1190372381053564,4,13,13,1.0
-Overall,Toto-1.0,3.989385957790437,1.9736567361922646,5,13,13,1.0
-Overall,ARIMA,0.6341595777487467,1.9472211116426137,6,13,13,1.0
-Overall,Chronos-Bolt,0.21501646886290463,1.817335776045877,7,13,13,1.0
-Overall,TimesFM-2.5,0.1597427274112271,0.4790935805746809,8,13,13,1.0
-Overall,ETS,0.7923273256040497,0.37679264853892164,9,13,13,1.0
-Overall,TabPFN-TS,0.18690240444673736,0.47855028852547243,10,13,13,1.0
-Overall,Moving-Average,0.9256421422364086,2.735750287222228,11,13,13,1.0
-Overall,Seasonal-Naive,1.0,1.0,12,13,13,1.0
+Overall,TiRex,0.10179781709203017,0.9766871365860337,1,12,12,1.0
+Overall,Chronos-2,0.11357753428061074,0.9897870162036663,2,12,12,1.0
+Overall,Moirai-2.0,0.1391033138051579,1.1815960514755193,3,12,12,1.0
+Overall,Sundial,0.32264381538941844,1.1135443446285107,4,12,12,1.0
+Overall,Toto-1.0,4.448654292431631,2.080725611373261,5,12,12,1.0
+Overall,Chronos-Bolt,0.187762639510678,1.8975473220496613,6,12,12,1.0
+Overall,TimesFM-2.5,0.13569874928075795,0.4421369184156991,7,12,12,1.0
+Overall,ARIMA,0.6118240353757025,2.2289028463405396,8,12,12,1.0
+Overall,ETS,0.7720531291181013,0.34265298329472405,9,12,12,1.0
+Overall,TabPFN-TS,0.16087572829886923,0.4396798726030351,10,12,12,1.0
+Overall,Moving-Average,0.9215541049043304,2.948613699720417,11,12,12,1.0
+Overall,Seasonal-Naive,1.0,1.0,12,12,12,1.0

results/dataset_properties.csv

@@ -10,5 +10,4 @@ nws_ksfo_observations_temperature,Weather,1h,1
 open_meteo_shanghai_air_quality_hourly_pm2_5,Air Quality,1h,1
 open_meteo_shanghai_hourly_temperature_2m,Weather,1h,1
 usgs_earthquake_all_week_hourly_earthquake_count,Hazards,1h,1
-usgs_potomac_iv_usgs_00060,Hydrology,15min,1
 wikimedia_time_series_hourly_pageviews,Web,1D,1

results/online_status.json

@@ -2,14 +2,14 @@
     "status": "ok",
     "evaluation_protocol": "issue-now-score-later",
     "metric_version": "livehouse-canonical-v1",
-    "started_at": "2026-07-28T17:42:34.701264+00:00",
-    "finished_at": "2026-07-28T17:46:14.563659+00:00",
+    "started_at": "2026-07-28T17:58:57.354453+00:00",
+    "finished_at": "2026-07-28T17:59:02.804053+00:00",
     "data_source": "ts_bench",
     "data_config": "configs/datasets/ts_bench.yaml",
-    "task_count": 4,
-    "issued_tasks": 1,
-    "issued_forecasts": 12,
-    "resolved_forecasts": 12,
+    "task_count": 1,
+    "issued_tasks": 0,
+    "issued_forecasts": 0,
+    "resolved_forecasts": 0,
     "pending_tasks": 13,
     "failed_forecasts": [],
     "models": {
@@ -19,18 +19,8 @@
             "admitted_at": "2026-06-01T00:00:00Z",
             "evaluation_protocol": "issue-now-score-later",
             "metric_version": "livehouse-canonical-v1",
-            "evaluated_at": "2026-07-28T17:46:10.072503+00:00",
-            "resolved_tasks": [
-                {
-                    "dataset": "data_gov_sg_singapore_taxi_availability_available_taxi_count/5min/short",
-                    "task_id": "future-11a4cfc38ecb272f5f2d",
-                    "issued_at": "2026-07-28T14:22:27.338912+00:00",
-                    "resolved_at": "2026-07-28T17:42:39.369270+00:00",
-                    "target_end": "2026-07-27T14:30:00+00:00",
-                    "prediction_length": 12,
-                    "metric_version": "livehouse-canonical-v1"
-                }
-            ]
+            "evaluated_at": "2026-07-28T17:58:59.675069+00:00",
+            "resolved_tasks": []
         },
         "TiRex": {
             "model": "TiRex",
@@ -38,18 +28,8 @@
             "admitted_at": "2026-06-01T00:00:00Z",
             "evaluation_protocol": "issue-now-score-later",
             "metric_version": "livehouse-canonical-v1",
-            "evaluated_at": "2026-07-28T17:46:10.122596+00:00",
-            "resolved_tasks": [
-                {
-                    "dataset": "data_gov_sg_singapore_taxi_availability_available_taxi_count/5min/short",
-                    "task_id": "future-11a4cfc38ecb272f5f2d",
-                    "issued_at": "2026-07-28T14:22:27.338912+00:00",
-                    "resolved_at": "2026-07-28T17:42:39.760321+00:00",
-                    "target_end": "2026-07-27T14:30:00+00:00",
-                    "prediction_length": 12,
-                    "metric_version": "livehouse-canonical-v1"
-                }
-            ]
+            "evaluated_at": "2026-07-28T17:58:59.675091+00:00",
+            "resolved_tasks": []
         },
         "TimesFM-2.5": {
             "model": "TimesFM-2.5",
@@ -57,18 +37,8 @@
             "admitted_at": "2026-06-01T00:00:00Z",
             "evaluation_protocol": "issue-now-score-later",
             "metric_version": "livehouse-canonical-v1",
-            "evaluated_at": "2026-07-28T17:46:10.142955+00:00",
-            "resolved_tasks": [
-                {
-                    "dataset": "data_gov_sg_singapore_taxi_availability_available_taxi_count/5min/short",
-                    "task_id": "future-11a4cfc38ecb272f5f2d",
-                    "issued_at": "2026-07-28T14:22:27.338912+00:00",
-                    "resolved_at": "2026-07-28T17:42:39.734065+00:00",
-                    "target_end": "2026-07-27T14:30:00+00:00",
-                    "prediction_length": 12,
-                    "metric_version": "livehouse-canonical-v1"
-                }
-            ]
+            "evaluated_at": "2026-07-28T17:58:59.675095+00:00",
+            "resolved_tasks": []
         },
         "Toto-1.0": {
             "model": "Toto-1.0",
@@ -76,18 +46,8 @@
             "admitted_at": "2026-06-01T00:00:00Z",
             "evaluation_protocol": "issue-now-score-later",
             "metric_version": "livehouse-canonical-v1",
-            "evaluated_at": "2026-07-28T17:46:10.152520+00:00",
-            "resolved_tasks": [
-                {
-                    "dataset": "data_gov_sg_singapore_taxi_availability_available_taxi_count/5min/short",
-                    "task_id": "future-11a4cfc38ecb272f5f2d",
-                    "issued_at": "2026-07-28T14:22:27.338912+00:00",
-                    "resolved_at": "2026-07-28T17:42:39.829301+00:00",
-                    "target_end": "2026-07-27T14:30:00+00:00",
-                    "prediction_length": 12,
-                    "metric_version": "livehouse-canonical-v1"
-                }
-            ]
+            "evaluated_at": "2026-07-28T17:58:59.675098+00:00",
+            "resolved_tasks": []
         },
         "Moirai-2.0": {
             "model": "Moirai-2.0",
@@ -95,18 +55,8 @@
             "admitted_at": "2026-06-01T00:00:00Z",
             "evaluation_protocol": "issue-now-score-later",
             "metric_version": "livehouse-canonical-v1",
-            "evaluated_at": "2026-07-28T17:46:10.162006+00:00",
-            "resolved_tasks": [
-                {
-                    "dataset": "data_gov_sg_singapore_taxi_availability_available_taxi_count/5min/short",
-                    "task_id": "future-11a4cfc38ecb272f5f2d",
-                    "issued_at": "2026-07-28T14:22:27.338912+00:00",
-                    "resolved_at": "2026-07-28T17:42:39.511668+00:00",
-                    "target_end": "2026-07-27T14:30:00+00:00",
-                    "prediction_length": 12,
-                    "metric_version": "livehouse-canonical-v1"
-                }
-            ]
+            "evaluated_at": "2026-07-28T17:58:59.675101+00:00",
+            "resolved_tasks": []
         },
         "Chronos-Bolt": {
             "model": "Chronos-Bolt",
@@ -114,18 +64,8 @@
             "admitted_at": "2026-06-01T00:00:00Z",
             "evaluation_protocol": "issue-now-score-later",
             "metric_version": "livehouse-canonical-v1",
-            "evaluated_at": "2026-07-28T17:46:10.168982+00:00",
-            "resolved_tasks": [
-                {
-                    "dataset": "data_gov_sg_singapore_taxi_availability_available_taxi_count/5min/short",
-                    "task_id": "future-11a4cfc38ecb272f5f2d",
-                    "issued_at": "2026-07-28T14:22:27.338912+00:00",
-                    "resolved_at": "2026-07-28T17:42:39.409356+00:00",
-                    "target_end": "2026-07-27T14:30:00+00:00",
-                    "prediction_length": 12,
-                    "metric_version": "livehouse-canonical-v1"
-                }
-            ]
+            "evaluated_at": "2026-07-28T17:58:59.675104+00:00",
+            "resolved_tasks": []
         },
         "TabPFN-TS": {
             "model": "TabPFN-TS",
@@ -133,18 +73,8 @@
             "admitted_at": "2026-06-01T00:00:00Z",
             "evaluation_protocol": "issue-now-score-later",
             "metric_version": "livehouse-canonical-v1",
-            "evaluated_at": "2026-07-28T17:46:10.216276+00:00",
-            "resolved_tasks": [
-                {
-                    "dataset": "data_gov_sg_singapore_taxi_availability_available_taxi_count/5min/short",
-                    "task_id": "future-11a4cfc38ecb272f5f2d",
-                    "issued_at": "2026-07-28T14:22:27.338912+00:00",
-                    "resolved_at": "2026-07-28T17:42:39.695463+00:00",
-                    "target_end": "2026-07-27T14:30:00+00:00",
-                    "prediction_length": 12,
-                    "metric_version": "livehouse-canonical-v1"
-                }
-            ]
+            "evaluated_at": "2026-07-28T17:58:59.675106+00:00",
+            "resolved_tasks": []
         },
         "Sundial": {
             "model": "Sundial",
@@ -152,18 +82,8 @@
             "admitted_at": "2026-06-01T00:00:00Z",
             "evaluation_protocol": "issue-now-score-later",
             "metric_version": "livehouse-canonical-v1",
-            "evaluated_at": "2026-07-28T17:46:10.239806+00:00",
-            "resolved_tasks": [
-                {
-                    "dataset": "data_gov_sg_singapore_taxi_availability_available_taxi_count/5min/short",
-                    "task_id": "future-11a4cfc38ecb272f5f2d",
-                    "issued_at": "2026-07-28T14:22:27.338912+00:00",
-                    "resolved_at": "2026-07-28T17:42:39.642247+00:00",
-                    "target_end": "2026-07-27T14:30:00+00:00",
-                    "prediction_length": 12,
-                    "metric_version": "livehouse-canonical-v1"
-                }
-            ]
+            "evaluated_at": "2026-07-28T17:58:59.675108+00:00",
+            "resolved_tasks": []
         },
         "Moving-Average": {
             "model": "Moving-Average",
@@ -171,18 +91,8 @@
             "admitted_at": "2026-06-01T00:00:00Z",
             "evaluation_protocol": "issue-now-score-later",
             "metric_version": "livehouse-canonical-v1",
-            "evaluated_at": "2026-07-28T17:46:10.253942+00:00",
-            "resolved_tasks": [
-                {
-                    "dataset": "data_gov_sg_singapore_taxi_availability_available_taxi_count/5min/short",
-                    "task_id": "future-11a4cfc38ecb272f5f2d",
-                    "issued_at": "2026-07-28T14:22:27.338912+00:00",
-                    "resolved_at": "2026-07-28T17:42:39.536254+00:00",
-                    "target_end": "2026-07-27T14:30:00+00:00",
-                    "prediction_length": 12,
-                    "metric_version": "livehouse-canonical-v1"
-                }
-            ]
+            "evaluated_at": "2026-07-28T17:58:59.675111+00:00",
+            "resolved_tasks": []
         },
         "ETS": {
             "model": "ETS",
@@ -190,18 +100,8 @@
             "admitted_at": "2026-06-01T00:00:00Z",
             "evaluation_protocol": "issue-now-score-later",
             "metric_version": "livehouse-canonical-v1",
-            "evaluated_at": "2026-07-28T17:46:10.281851+00:00",
-            "resolved_tasks": [
-                {
-                    "dataset": "data_gov_sg_singapore_taxi_availability_available_taxi_count/5min/short",
-                    "task_id": "future-11a4cfc38ecb272f5f2d",
-                    "issued_at": "2026-07-28T14:22:27.338912+00:00",
-                    "resolved_at": "2026-07-28T17:42:39.455925+00:00",
-                    "target_end": "2026-07-27T14:30:00+00:00",
-                    "prediction_length": 12,
-                    "metric_version": "livehouse-canonical-v1"
-                }
-            ]
+            "evaluated_at": "2026-07-28T17:58:59.675113+00:00",
+            "resolved_tasks": []
         },
         "ARIMA": {
             "model": "ARIMA",
@@ -209,18 +109,8 @@
             "admitted_at": "2026-06-01T00:00:00Z",
             "evaluation_protocol": "issue-now-score-later",
             "metric_version": "livehouse-canonical-v1",
-            "evaluated_at": "2026-07-28T17:46:10.290619+00:00",
-            "resolved_tasks": [
-                {
-                    "dataset": "data_gov_sg_singapore_taxi_availability_available_taxi_count/5min/short",
-                    "task_id": "future-11a4cfc38ecb272f5f2d",
-                    "issued_at": "2026-07-28T14:22:27.338912+00:00",
-                    "resolved_at": "2026-07-28T17:42:39.297471+00:00",
-                    "target_end": "2026-07-27T14:30:00+00:00",
-                    "prediction_length": 12,
-                    "metric_version": "livehouse-canonical-v1"
-                }
-            ]
+            "evaluated_at": "2026-07-28T17:58:59.675115+00:00",
+            "resolved_tasks": []
         },
         "Seasonal-Naive": {
             "model": "Seasonal-Naive",
@@ -228,18 +118,8 @@
             "admitted_at": "2026-06-01T00:00:00Z",
             "evaluation_protocol": "issue-now-score-later",
             "metric_version": "livehouse-canonical-v1",
-            "evaluated_at": "2026-07-28T17:46:10.323503+00:00",
-            "resolved_tasks": [
-                {
-                    "dataset": "data_gov_sg_singapore_taxi_availability_available_taxi_count/5min/short",
-                    "task_id": "future-11a4cfc38ecb272f5f2d",
-                    "issued_at": "2026-07-28T14:22:27.338912+00:00",
-                    "resolved_at": "2026-07-28T17:42:39.603257+00:00",
-                    "target_end": "2026-07-27T14:30:00+00:00",
-                    "prediction_length": 12,
-                    "metric_version": "livehouse-canonical-v1"
-                }
-            ]
+            "evaluated_at": "2026-07-28T17:58:59.675118+00:00",
+            "resolved_tasks": []
         }
     },
     "skipped_models": [],
@@ -262,7 +142,5 @@
         "live_overall_30d": "aggregates/live_overall_30d.csv",
         "live_metadata": "aggregates/live_metadata.json"
     },
-    "push_status": "ok",
-    "pushed_at": "2026-07-28T17:47:24.036162+00:00",
-    "push_detail": "HF Hub API"
+    "push_status": "pending"
 }