Update leaderboard results from local eval

results/aggregates/live_metadata.json

@@ -1,5 +1,5 @@
 {
-  "generated_at": "2026-07-28T12:26:01.466941+00:00",
+  "generated_at": "2026-07-28T12:40:11.489228+00:00",
   "scope": "Canonical metrics-only release history when available; otherwise legacy release history plus latest allowlisted snapshots",
   "release_deduplication": "One row per model, dataset, data source timestamp, and prediction length",
   "dataset_aggregation": "Arithmetic mean over releases, then equal-weight mean over datasets",

results/aggregates/metadata.json

@@ -1,5 +1,5 @@
 {
-  "generated_at": "2026-07-28T12:25:37.451335+00:00",
+  "generated_at": "2026-07-28T12:40:09.513780+00:00",
   "baseline_model": "Seasonal-Naive",
   "normalization": "Per-config metric / Seasonal-Naive metric",
   "metric_aggregation": "Geometric mean over dataset configurations",

results/online_status.json

@@ -1,17 +1,23 @@
 {
-    "status": "ok",
+    "status": "partial",
     "evaluation_protocol": "issue-now-score-later",
     "metric_version": "livehouse-canonical-v1",
-    "started_at": "2026-07-28T12:22:46.850740+00:00",
-    "finished_at": "2026-07-28T12:26:01.477346+00:00",
+    "started_at": "2026-07-28T12:40:06.026240+00:00",
+    "finished_at": "2026-07-28T12:40:11.500648+00:00",
     "data_source": "ts_bench",
     "data_config": "configs/datasets/ts_bench.yaml",
-    "task_count": 2,
-    "issued_tasks": 1,
-    "issued_forecasts": 12,
-    "resolved_forecasts": 12,
+    "task_count": 4,
+    "issued_tasks": 0,
+    "issued_forecasts": 0,
+    "resolved_forecasts": 0,
     "pending_tasks": 13,
-    "failed_forecasts": [],
+    "failed_forecasts": [
+        {
+            "model": "Seasonal-Naive",
+            "dataset": "nws_ksfo_observations_temperature/1h/short",
+            "error": "Mean forecast length 10 < 24"
+        }
+    ],
     "models": {
         "Chronos-2": {
             "model": "Chronos-2",
@@ -19,18 +25,8 @@
             "admitted_at": "2026-06-01T00:00:00Z",
             "evaluation_protocol": "issue-now-score-later",
             "metric_version": "livehouse-canonical-v1",
-            "evaluated_at": "2026-07-28T12:25:35.163834+00:00",
-            "resolved_tasks": [
-                {
-                    "dataset": "data_gov_sg_singapore_taxi_availability_available_taxi_count/5min/short",
-                    "task_id": "future-915c3cd2da3f0a7f5bda",
-                    "issued_at": "2026-07-28T11:48:19.276268+00:00",
-                    "resolved_at": "2026-07-28T12:22:51.537982+00:00",
-                    "target_end": "2026-07-27T09:05:00+00:00",
-                    "prediction_length": 12,
-                    "metric_version": "livehouse-canonical-v1"
-                }
-            ]
+            "evaluated_at": "2026-07-28T12:40:08.537782+00:00",
+            "resolved_tasks": []
         },
         "TiRex": {
             "model": "TiRex",
@@ -38,18 +34,8 @@
             "admitted_at": "2026-06-01T00:00:00Z",
             "evaluation_protocol": "issue-now-score-later",
             "metric_version": "livehouse-canonical-v1",
-            "evaluated_at": "2026-07-28T12:25:35.189225+00:00",
-            "resolved_tasks": [
-                {
-                    "dataset": "data_gov_sg_singapore_taxi_availability_available_taxi_count/5min/short",
-                    "task_id": "future-915c3cd2da3f0a7f5bda",
-                    "issued_at": "2026-07-28T11:48:19.276268+00:00",
-                    "resolved_at": "2026-07-28T12:22:51.966847+00:00",
-                    "target_end": "2026-07-27T09:05:00+00:00",
-                    "prediction_length": 12,
-                    "metric_version": "livehouse-canonical-v1"
-                }
-            ]
+            "evaluated_at": "2026-07-28T12:40:08.537789+00:00",
+            "resolved_tasks": []
         },
         "TimesFM-2.5": {
             "model": "TimesFM-2.5",
@@ -57,18 +43,8 @@
             "admitted_at": "2026-06-01T00:00:00Z",
             "evaluation_protocol": "issue-now-score-later",
             "metric_version": "livehouse-canonical-v1",
-            "evaluated_at": "2026-07-28T12:25:35.206125+00:00",
-            "resolved_tasks": [
-                {
-                    "dataset": "data_gov_sg_singapore_taxi_availability_available_taxi_count/5min/short",
-                    "task_id": "future-915c3cd2da3f0a7f5bda",
-                    "issued_at": "2026-07-28T11:48:19.276268+00:00",
-                    "resolved_at": "2026-07-28T12:22:51.872515+00:00",
-                    "target_end": "2026-07-27T09:05:00+00:00",
-                    "prediction_length": 12,
-                    "metric_version": "livehouse-canonical-v1"
-                }
-            ]
+            "evaluated_at": "2026-07-28T12:40:08.537791+00:00",
+            "resolved_tasks": []
         },
         "Toto-1.0": {
             "model": "Toto-1.0",
@@ -76,18 +52,8 @@
             "admitted_at": "2026-06-01T00:00:00Z",
             "evaluation_protocol": "issue-now-score-later",
             "metric_version": "livehouse-canonical-v1",
-            "evaluated_at": "2026-07-28T12:25:35.208881+00:00",
-            "resolved_tasks": [
-                {
-                    "dataset": "data_gov_sg_singapore_taxi_availability_available_taxi_count/5min/short",
-                    "task_id": "future-915c3cd2da3f0a7f5bda",
-                    "issued_at": "2026-07-28T11:48:19.276268+00:00",
-                    "resolved_at": "2026-07-28T12:22:52.008060+00:00",
-                    "target_end": "2026-07-27T09:05:00+00:00",
-                    "prediction_length": 12,
-                    "metric_version": "livehouse-canonical-v1"
-                }
-            ]
+            "evaluated_at": "2026-07-28T12:40:08.537792+00:00",
+            "resolved_tasks": []
         },
         "Moirai-2.0": {
             "model": "Moirai-2.0",
@@ -95,18 +61,8 @@
             "admitted_at": "2026-06-01T00:00:00Z",
             "evaluation_protocol": "issue-now-score-later",
             "metric_version": "livehouse-canonical-v1",
-            "evaluated_at": "2026-07-28T12:25:35.215484+00:00",
-            "resolved_tasks": [
-                {
-                    "dataset": "data_gov_sg_singapore_taxi_availability_available_taxi_count/5min/short",
-                    "task_id": "future-915c3cd2da3f0a7f5bda",
-                    "issued_at": "2026-07-28T11:48:19.276268+00:00",
-                    "resolved_at": "2026-07-28T12:22:51.647830+00:00",
-                    "target_end": "2026-07-27T09:05:00+00:00",
-                    "prediction_length": 12,
-                    "metric_version": "livehouse-canonical-v1"
-                }
-            ]
+            "evaluated_at": "2026-07-28T12:40:08.537793+00:00",
+            "resolved_tasks": []
         },
         "Chronos-Bolt": {
             "model": "Chronos-Bolt",
@@ -114,18 +70,8 @@
             "admitted_at": "2026-06-01T00:00:00Z",
             "evaluation_protocol": "issue-now-score-later",
             "metric_version": "livehouse-canonical-v1",
-            "evaluated_at": "2026-07-28T12:25:35.224737+00:00",
-            "resolved_tasks": [
-                {
-                    "dataset": "data_gov_sg_singapore_taxi_availability_available_taxi_count/5min/short",
-                    "task_id": "future-915c3cd2da3f0a7f5bda",
-                    "issued_at": "2026-07-28T11:48:19.276268+00:00",
-                    "resolved_at": "2026-07-28T12:22:51.562834+00:00",
-                    "target_end": "2026-07-27T09:05:00+00:00",
-                    "prediction_length": 12,
-                    "metric_version": "livehouse-canonical-v1"
-                }
-            ]
+            "evaluated_at": "2026-07-28T12:40:08.537794+00:00",
+            "resolved_tasks": []
         },
         "TabPFN-TS": {
             "model": "TabPFN-TS",
@@ -133,18 +79,8 @@
             "admitted_at": "2026-06-01T00:00:00Z",
             "evaluation_protocol": "issue-now-score-later",
             "metric_version": "livehouse-canonical-v1",
-            "evaluated_at": "2026-07-28T12:25:35.232581+00:00",
-            "resolved_tasks": [
-                {
-                    "dataset": "data_gov_sg_singapore_taxi_availability_available_taxi_count/5min/short",
-                    "task_id": "future-915c3cd2da3f0a7f5bda",
-                    "issued_at": "2026-07-28T11:48:19.276268+00:00",
-                    "resolved_at": "2026-07-28T12:22:51.841833+00:00",
-                    "target_end": "2026-07-27T09:05:00+00:00",
-                    "prediction_length": 12,
-                    "metric_version": "livehouse-canonical-v1"
-                }
-            ]
+            "evaluated_at": "2026-07-28T12:40:08.537795+00:00",
+            "resolved_tasks": []
         },
         "Sundial": {
             "model": "Sundial",
@@ -152,18 +88,8 @@
             "admitted_at": "2026-06-01T00:00:00Z",
             "evaluation_protocol": "issue-now-score-later",
             "metric_version": "livehouse-canonical-v1",
-            "evaluated_at": "2026-07-28T12:25:35.252370+00:00",
-            "resolved_tasks": [
-                {
-                    "dataset": "data_gov_sg_singapore_taxi_availability_available_taxi_count/5min/short",
-                    "task_id": "future-915c3cd2da3f0a7f5bda",
-                    "issued_at": "2026-07-28T11:48:19.276268+00:00",
-                    "resolved_at": "2026-07-28T12:22:51.809922+00:00",
-                    "target_end": "2026-07-27T09:05:00+00:00",
-                    "prediction_length": 12,
-                    "metric_version": "livehouse-canonical-v1"
-                }
-            ]
+            "evaluated_at": "2026-07-28T12:40:08.537796+00:00",
+            "resolved_tasks": []
         },
         "Moving-Average": {
             "model": "Moving-Average",
@@ -171,18 +97,8 @@
             "admitted_at": "2026-06-01T00:00:00Z",
             "evaluation_protocol": "issue-now-score-later",
             "metric_version": "livehouse-canonical-v1",
-            "evaluated_at": "2026-07-28T12:25:35.263355+00:00",
-            "resolved_tasks": [
-                {
-                    "dataset": "data_gov_sg_singapore_taxi_availability_available_taxi_count/5min/short",
-                    "task_id": "future-915c3cd2da3f0a7f5bda",
-                    "issued_at": "2026-07-28T11:48:19.276268+00:00",
-                    "resolved_at": "2026-07-28T12:22:51.690818+00:00",
-                    "target_end": "2026-07-27T09:05:00+00:00",
-                    "prediction_length": 12,
-                    "metric_version": "livehouse-canonical-v1"
-                }
-            ]
+            "evaluated_at": "2026-07-28T12:40:08.537797+00:00",
+            "resolved_tasks": []
         },
         "ETS": {
             "model": "ETS",
@@ -190,18 +106,8 @@
             "admitted_at": "2026-06-01T00:00:00Z",
             "evaluation_protocol": "issue-now-score-later",
             "metric_version": "livehouse-canonical-v1",
-            "evaluated_at": "2026-07-28T12:25:35.265949+00:00",
-            "resolved_tasks": [
-                {
-                    "dataset": "data_gov_sg_singapore_taxi_availability_available_taxi_count/5min/short",
-                    "task_id": "future-915c3cd2da3f0a7f5bda",
-                    "issued_at": "2026-07-28T11:48:19.276268+00:00",
-                    "resolved_at": "2026-07-28T12:22:51.618875+00:00",
-                    "target_end": "2026-07-27T09:05:00+00:00",
-                    "prediction_length": 12,
-                    "metric_version": "livehouse-canonical-v1"
-                }
-            ]
+            "evaluated_at": "2026-07-28T12:40:08.537798+00:00",
+            "resolved_tasks": []
         },
         "ARIMA": {
             "model": "ARIMA",
@@ -209,18 +115,8 @@
             "admitted_at": "2026-06-01T00:00:00Z",
             "evaluation_protocol": "issue-now-score-later",
             "metric_version": "livehouse-canonical-v1",
-            "evaluated_at": "2026-07-28T12:25:35.267990+00:00",
-            "resolved_tasks": [
-                {
-                    "dataset": "data_gov_sg_singapore_taxi_availability_available_taxi_count/5min/short",
-                    "task_id": "future-915c3cd2da3f0a7f5bda",
-                    "issued_at": "2026-07-28T11:48:19.276268+00:00",
-                    "resolved_at": "2026-07-28T12:22:51.455146+00:00",
-                    "target_end": "2026-07-27T09:05:00+00:00",
-                    "prediction_length": 12,
-                    "metric_version": "livehouse-canonical-v1"
-                }
-            ]
+            "evaluated_at": "2026-07-28T12:40:08.537799+00:00",
+            "resolved_tasks": []
         },
         "Seasonal-Naive": {
             "model": "Seasonal-Naive",
@@ -228,18 +124,8 @@
             "admitted_at": "2026-06-01T00:00:00Z",
             "evaluation_protocol": "issue-now-score-later",
             "metric_version": "livehouse-canonical-v1",
-            "evaluated_at": "2026-07-28T12:25:35.269866+00:00",
-            "resolved_tasks": [
-                {
-                    "dataset": "data_gov_sg_singapore_taxi_availability_available_taxi_count/5min/short",
-                    "task_id": "future-915c3cd2da3f0a7f5bda",
-                    "issued_at": "2026-07-28T11:48:19.276268+00:00",
-                    "resolved_at": "2026-07-28T12:22:51.743842+00:00",
-                    "target_end": "2026-07-27T09:05:00+00:00",
-                    "prediction_length": 12,
-                    "metric_version": "livehouse-canonical-v1"
-                }
-            ]
+            "evaluated_at": "2026-07-28T12:40:08.537800+00:00",
+            "resolved_tasks": []
         }
     },
     "skipped_models": [],
@@ -262,7 +148,5 @@
         "live_overall_30d": "aggregates/live_overall_30d.csv",
         "live_metadata": "aggregates/live_metadata.json"
     },
-    "push_status": "ok",
-    "pushed_at": "2026-07-28T12:27:04.899450+00:00",
-    "push_detail": "HF Hub API"
+    "push_status": "pending"
 }