ziyuzhou02 commited on
Commit
75654af
·
verified ·
1 Parent(s): fd1b227

Update leaderboard results from local eval

Browse files
results/aggregates/live_metadata.json CHANGED
@@ -1,5 +1,5 @@
1
  {
2
- "generated_at": "2026-07-26T05:37:28.887181+00:00",
3
  "scope": "Canonical metrics-only release history when available; otherwise legacy release history plus latest allowlisted snapshots",
4
  "release_deduplication": "One row per model, dataset, data source timestamp, and prediction length",
5
  "dataset_aggregation": "Arithmetic mean over releases, then equal-weight mean over datasets",
 
1
  {
2
+ "generated_at": "2026-07-26T06:01:39.224193+00:00",
3
  "scope": "Canonical metrics-only release history when available; otherwise legacy release history plus latest allowlisted snapshots",
4
  "release_deduplication": "One row per model, dataset, data source timestamp, and prediction length",
5
  "dataset_aggregation": "Arithmetic mean over releases, then equal-weight mean over datasets",
results/aggregates/metadata.json CHANGED
@@ -1,5 +1,5 @@
1
  {
2
- "generated_at": "2026-07-26T05:37:18.397974+00:00",
3
  "baseline_model": "Seasonal-Naive",
4
  "normalization": "Per-config metric / Seasonal-Naive metric",
5
  "metric_aggregation": "Geometric mean over dataset configurations",
@@ -10,7 +10,7 @@
10
  "frequency",
11
  "prediction_length"
12
  ],
13
- "num_configs": 13,
14
  "num_models": 12,
15
  "num_rows_missing_prediction_length": 0
16
  }
 
1
  {
2
+ "generated_at": "2026-07-26T06:01:04.419744+00:00",
3
  "baseline_model": "Seasonal-Naive",
4
  "normalization": "Per-config metric / Seasonal-Naive metric",
5
  "metric_aggregation": "Geometric mean over dataset configurations",
 
10
  "frequency",
11
  "prediction_length"
12
  ],
13
+ "num_configs": 12,
14
  "num_models": 12,
15
  "num_rows_missing_prediction_length": 0
16
  }
results/aggregates/results_by_domain.csv CHANGED
@@ -35,18 +35,6 @@ Hazards,TimesFM-2.5,0.46613899613358084,0.7161023290919594,9,1,1,1.0
35
  Hazards,Sundial,1.026729195920962,0.818406495628751,10,1,1,1.0
36
  Hazards,TabPFN-TS,0.6517855412261532,0.8184241183297211,11,1,1,1.0
37
  Hazards,Seasonal-Naive,1.0,1.0,12,1,1,1.0
38
- Hydrology,ARIMA,1.043495770859604,0.3313520193062611,1,1,1,1.0
39
- Hydrology,Toto-1.0,0.48908837219598983,0.5998947164656739,2,1,1,1.0
40
- Hydrology,Moirai-2.0,0.7740156701650377,0.6592439438333478,3,1,1,1.0
41
- Hydrology,Chronos-Bolt,0.8170891657303544,0.836848265918259,4,1,1,1.0
42
- Hydrology,Chronos-2,1.1667569571522571,0.9722131697347309,5,1,1,1.0
43
- Hydrology,Sundial,0.7275725022407682,0.9749230872623076,6,1,1,1.0
44
- Hydrology,ETS,0.7792620768998486,0.9891801797380332,7,1,1,1.0
45
- Hydrology,Seasonal-Naive,1.0,1.0,8,1,1,1.0
46
- Hydrology,TiRex,1.14579172632572,1.0011797241572697,9,1,1,1.0
47
- Hydrology,Moving-Average,0.9463988601048352,1.1032977120563572,10,1,1,1.0
48
- Hydrology,TimesFM-2.5,1.1756474339628833,1.2462786151784258,11,1,1,1.0
49
- Hydrology,TabPFN-TS,1.1541534361133212,1.3656905168289035,12,1,1,1.0
50
  Ocean,Chronos-2,0.2011648667909037,0.5071594517485618,1,2,2,1.0
51
  Ocean,TimesFM-2.5,0.4620918841076899,0.6093658587089561,2,2,2,1.0
52
  Ocean,Sundial,1.1470585646086184,0.9268316639791424,3,2,2,1.0
 
35
  Hazards,Sundial,1.026729195920962,0.818406495628751,10,1,1,1.0
36
  Hazards,TabPFN-TS,0.6517855412261532,0.8184241183297211,11,1,1,1.0
37
  Hazards,Seasonal-Naive,1.0,1.0,12,1,1,1.0
 
 
 
 
 
 
 
 
 
 
 
 
38
  Ocean,Chronos-2,0.2011648667909037,0.5071594517485618,1,2,2,1.0
39
  Ocean,TimesFM-2.5,0.4620918841076899,0.6093658587089561,2,2,2,1.0
40
  Ocean,Sundial,1.1470585646086184,0.9268316639791424,3,2,2,1.0
results/aggregates/results_by_frequency.csv CHANGED
@@ -11,18 +11,6 @@ frequency,model,MSE,CRPS,Rank,n_configs,n_group_configs,coverage
11
  10min,Moving-Average,0.7233618924674304,0.9043628928455715,10,1,1,1.0
12
  10min,Seasonal-Naive,1.0,1.0,11,1,1,1.0
13
  10min,TabPFN-TS,1.0066822711546748,1.0942122219799577,12,1,1,1.0
14
- 15min,ARIMA,1.043495770859604,0.3313520193062611,1,1,1,1.0
15
- 15min,Toto-1.0,0.48908837219598983,0.5998947164656739,2,1,1,1.0
16
- 15min,Moirai-2.0,0.7740156701650377,0.6592439438333478,3,1,1,1.0
17
- 15min,Chronos-Bolt,0.8170891657303544,0.836848265918259,4,1,1,1.0
18
- 15min,Chronos-2,1.1667569571522571,0.9722131697347309,5,1,1,1.0
19
- 15min,Sundial,0.7275725022407682,0.9749230872623076,6,1,1,1.0
20
- 15min,ETS,0.7792620768998486,0.9891801797380332,7,1,1,1.0
21
- 15min,Seasonal-Naive,1.0,1.0,8,1,1,1.0
22
- 15min,TiRex,1.14579172632572,1.0011797241572697,9,1,1,1.0
23
- 15min,Moving-Average,0.9463988601048352,1.1032977120563572,10,1,1,1.0
24
- 15min,TimesFM-2.5,1.1756474339628833,1.2462786151784258,11,1,1,1.0
25
- 15min,TabPFN-TS,1.1541534361133212,1.3656905168289035,12,1,1,1.0
26
  1D,ARIMA,0.35081770342317864,0.47944164778388026,1,1,1,1.0
27
  1D,Moirai-2.0,0.40785680526551005,0.5118677129116019,2,1,1,1.0
28
  1D,Sundial,0.303769640015456,0.5282879649026628,3,1,1,1.0
 
11
  10min,Moving-Average,0.7233618924674304,0.9043628928455715,10,1,1,1.0
12
  10min,Seasonal-Naive,1.0,1.0,11,1,1,1.0
13
  10min,TabPFN-TS,1.0066822711546748,1.0942122219799577,12,1,1,1.0
 
 
 
 
 
 
 
 
 
 
 
 
14
  1D,ARIMA,0.35081770342317864,0.47944164778388026,1,1,1,1.0
15
  1D,Moirai-2.0,0.40785680526551005,0.5118677129116019,2,1,1,1.0
16
  1D,Sundial,0.303769640015456,0.5282879649026628,3,1,1,1.0
results/aggregates/results_by_prediction_length.csv CHANGED
@@ -23,18 +23,18 @@ prediction_length,model,MSE,CRPS,Rank,n_configs,n_group_configs,coverage
23
  12,ETS,0.8706436180201554,439.02846065451104,10,2,2,1.0
24
  12,Toto-1.0,301465.8674852202,13479.58838265291,11,2,2,1.0
25
  12,Moving-Average,0.9617678019724286,471.56119324419365,12,2,2,1.0
26
- 24,Moirai-2.0,0.28874301316337336,0.6017244069194752,1,7,7,1.0
27
- 24,Toto-1.0,0.9733289598511201,0.594647608316137,2,7,7,1.0
28
- 24,Chronos-Bolt,0.29396850244281414,0.6205802828157266,3,7,7,1.0
29
- 24,TiRex,0.350246773442756,0.6801829007896318,4,7,7,1.0
30
- 24,Chronos-2,0.4401118651297656,0.6729284167630043,5,7,7,1.0
31
- 24,TimesFM-2.5,0.3412276623337479,0.695571040535807,6,7,7,1.0
32
- 24,ARIMA,1.2674379448136914,1.051072875195878,7,7,7,1.0
33
- 24,TabPFN-TS,0.3938315241089019,0.7335687325621745,8,7,7,1.0
34
- 24,Sundial,0.7999160604190338,0.7917527062021471,9,7,7,1.0
35
- 24,Moving-Average,1.1508379842929615,1.1378968029072782,10,7,7,1.0
36
- 24,Seasonal-Naive,1.0,1.0,11,7,7,1.0
37
- 24,ETS,4.8594115645361615,2.4161473494589476,12,7,7,1.0
38
  36,Moirai-2.0,0.0007130233539612103,0.02446029720098052,1,1,1,1.0
39
  36,ETS,0.001122399082795723,0.028178285506856342,2,1,1,1.0
40
  36,Chronos-Bolt,0.0007087563775442164,0.02872983747954875,3,1,1,1.0
 
23
  12,ETS,0.8706436180201554,439.02846065451104,10,2,2,1.0
24
  12,Toto-1.0,301465.8674852202,13479.58838265291,11,2,2,1.0
25
  12,Moving-Average,0.9617678019724286,471.56119324419365,12,2,2,1.0
26
+ 24,Moirai-2.0,0.24498440504588392,0.5926380622546753,1,6,6,1.0
27
+ 24,Chronos-Bolt,0.24791683827967076,0.5904138206285896,2,6,6,1.0
28
+ 24,Toto-1.0,1.0916205343851877,0.5937775631859699,3,6,6,1.0
29
+ 24,TiRex,0.2874653827064607,0.637741463383205,4,6,6,1.0
30
+ 24,TimesFM-2.5,0.2776549493115521,0.6311451033045645,5,6,6,1.0
31
+ 24,Chronos-2,0.3741050091657133,0.6329024298794399,6,6,6,1.0
32
+ 24,TabPFN-TS,0.3292190550408538,0.6613868637005982,7,6,6,1.0
33
+ 24,Sundial,0.8126541790626612,0.7647616297287181,8,6,6,1.0
34
+ 24,ARIMA,1.3091800347104579,1.2740604362147936,9,6,6,1.0
35
+ 24,Moving-Average,1.1889698746064494,1.1437678999109642,10,6,6,1.0
36
+ 24,Seasonal-Naive,1.0,1.0,11,6,6,1.0
37
+ 24,ETS,6.592756563291063,2.803914402547216,12,6,6,1.0
38
  36,Moirai-2.0,0.0007130233539612103,0.02446029720098052,1,1,1,1.0
39
  36,ETS,0.001122399082795723,0.028178285506856342,2,1,1,1.0
40
  36,Chronos-Bolt,0.0007087563775442164,0.02872983747954875,3,1,1,1.0
results/aggregates/results_overall.csv CHANGED
@@ -1,13 +1,13 @@
1
  overall,model,MSE,CRPS,Rank,n_configs,n_group_configs,coverage
2
- Overall,Moirai-2.0,0.26903914558625297,1.4015262389717487,1,13,13,1.0
3
- Overall,Toto-1.0,7.985793465971716,2.3979597024852457,2,13,13,1.0
4
- Overall,Chronos-2,0.26284127556810805,1.0844392172684765,3,13,13,1.0
5
- Overall,Chronos-Bolt,0.327075478539634,1.8985761279460167,4,13,13,1.0
6
- Overall,TiRex,0.2957126481025725,1.1454185864481672,5,13,13,1.0
7
- Overall,TimesFM-2.5,0.2563716272285658,0.5733442420051051,6,13,13,1.0
8
- Overall,ARIMA,0.6968972506615401,2.1475076696871875,7,13,13,1.0
9
- Overall,Sundial,0.5189676811848826,1.4019307697122283,8,13,13,1.0
10
- Overall,ETS,1.3904140825603852,3.0892861254436528,9,13,13,1.0
11
- Overall,TabPFN-TS,0.5173373691535904,0.8112786762408226,10,13,13,1.0
12
- Overall,Moving-Average,0.8639347438648337,2.690779420320406,11,13,13,1.0
13
- Overall,Seasonal-Naive,1.0,1.0,12,13,13,1.0
 
1
  overall,model,MSE,CRPS,Rank,n_configs,n_group_configs,coverage
2
+ Overall,Moirai-2.0,0.24636042912699188,1.4924420946686294,1,12,12,1.0
3
+ Overall,Toto-1.0,10.078493854529805,2.6914673606627693,2,12,12,1.0
4
+ Overall,Chronos-2,0.23214161246094414,1.0943565907249047,3,12,12,1.0
5
+ Overall,TimesFM-2.5,0.22581495383495592,0.5374220501550242,4,12,12,1.0
6
+ Overall,TiRex,0.26414977915807664,1.1583378324501594,5,12,12,1.0
7
+ Overall,Chronos-Bolt,0.30304900558452547,2.0327148371824446,6,12,12,1.0
8
+ Overall,Sundial,0.5045594166872331,1.4450168750099175,7,12,12,1.0
9
+ Overall,ARIMA,0.6738427954117607,2.5094111040179787,8,12,12,1.0
10
+ Overall,ETS,1.4591475462306003,3.3968264712794736,9,12,12,1.0
11
+ Overall,TabPFN-TS,0.48387484140702197,0.7768220450141894,10,12,12,1.0
12
+ Overall,Moving-Average,0.8573960942697303,2.898301394353555,11,12,12,1.0
13
+ Overall,Seasonal-Naive,1.0,1.0,12,12,12,1.0
results/dataset_eval_intervals.json CHANGED
@@ -10,5 +10,6 @@
10
  "open_meteo_shanghai_air_quality_hourly_pm2_5": 3600,
11
  "open_meteo_shanghai_hourly_temperature_2m": 3600,
12
  "usgs_earthquake_all_week_hourly_earthquake_count": 3600,
13
- "usgs_potomac_iv_usgs_00060": 900
 
14
  }
 
10
  "open_meteo_shanghai_air_quality_hourly_pm2_5": 3600,
11
  "open_meteo_shanghai_hourly_temperature_2m": 3600,
12
  "usgs_earthquake_all_week_hourly_earthquake_count": 3600,
13
+ "usgs_potomac_iv_usgs_00060": 900,
14
+ "wikimedia_time_series_hourly_pageviews": 86400
15
  }
results/dataset_properties.csv CHANGED
@@ -10,5 +10,4 @@ nws_ksfo_observations_temperature,Weather,1h,1
10
  open_meteo_shanghai_air_quality_hourly_pm2_5,Air Quality,1h,1
11
  open_meteo_shanghai_hourly_temperature_2m,Weather,1h,1
12
  usgs_earthquake_all_week_hourly_earthquake_count,Hazards,1h,1
13
- usgs_potomac_iv_usgs_00060,Hydrology,15min,1
14
  wikimedia_time_series_hourly_pageviews,Web,1D,1
 
10
  open_meteo_shanghai_air_quality_hourly_pm2_5,Air Quality,1h,1
11
  open_meteo_shanghai_hourly_temperature_2m,Weather,1h,1
12
  usgs_earthquake_all_week_hourly_earthquake_count,Hazards,1h,1
 
13
  wikimedia_time_series_hourly_pageviews,Web,1D,1
results/online_status.json CHANGED
@@ -1,23 +1,17 @@
1
  {
2
- "status": "partial",
3
  "evaluation_protocol": "issue-now-score-later",
4
  "metric_version": "livehouse-canonical-v1",
5
- "started_at": "2026-07-26T05:26:48.139617+00:00",
6
- "finished_at": "2026-07-26T05:37:28.970567+00:00",
7
  "data_source": "ts_bench",
8
  "data_config": "configs/datasets/ts_bench.yaml",
9
- "task_count": 4,
10
- "issued_tasks": 2,
11
- "issued_forecasts": 24,
12
- "resolved_forecasts": 24,
13
  "pending_tasks": 13,
14
- "failed_forecasts": [
15
- {
16
- "model": "Seasonal-Naive",
17
- "dataset": "nws_ksfo_observations_temperature/1h/short",
18
- "error": "Mean forecast length 10 < 24"
19
- }
20
- ],
21
  "models": {
22
  "Chronos-2": {
23
  "model": "Chronos-2",
@@ -25,27 +19,8 @@
25
  "admitted_at": "2026-06-01T00:00:00Z",
26
  "evaluation_protocol": "issue-now-score-later",
27
  "metric_version": "livehouse-canonical-v1",
28
- "evaluated_at": "2026-07-26T05:37:16.208165+00:00",
29
- "resolved_tasks": [
30
- {
31
- "dataset": "open_meteo_shanghai_air_quality_hourly_pm2_5/1h/short",
32
- "task_id": "future-7cf15642ebedce11035a",
33
- "issued_at": "2026-07-25T05:57:25.569188+00:00",
34
- "resolved_at": "2026-07-26T05:26:51.503110+00:00",
35
- "target_end": "2026-07-26T04:00:00+00:00",
36
- "prediction_length": 24,
37
- "metric_version": "livehouse-canonical-v1"
38
- },
39
- {
40
- "dataset": "data_gov_sg_singapore_taxi_availability_available_taxi_count/5min/short",
41
- "task_id": "future-c88625e66c964a9a0896",
42
- "issued_at": "2026-07-26T03:14:21.211924+00:00",
43
- "resolved_at": "2026-07-26T05:26:51.996225+00:00",
44
- "target_end": "2026-07-24T18:25:00+00:00",
45
- "prediction_length": 12,
46
- "metric_version": "livehouse-canonical-v1"
47
- }
48
- ]
49
  },
50
  "TiRex": {
51
  "model": "TiRex",
@@ -53,27 +28,8 @@
53
  "admitted_at": "2026-06-01T00:00:00Z",
54
  "evaluation_protocol": "issue-now-score-later",
55
  "metric_version": "livehouse-canonical-v1",
56
- "evaluated_at": "2026-07-26T05:37:16.228469+00:00",
57
- "resolved_tasks": [
58
- {
59
- "dataset": "open_meteo_shanghai_air_quality_hourly_pm2_5/1h/short",
60
- "task_id": "future-7cf15642ebedce11035a",
61
- "issued_at": "2026-07-25T05:57:25.569188+00:00",
62
- "resolved_at": "2026-07-26T05:26:51.871127+00:00",
63
- "target_end": "2026-07-26T04:00:00+00:00",
64
- "prediction_length": 24,
65
- "metric_version": "livehouse-canonical-v1"
66
- },
67
- {
68
- "dataset": "data_gov_sg_singapore_taxi_availability_available_taxi_count/5min/short",
69
- "task_id": "future-c88625e66c964a9a0896",
70
- "issued_at": "2026-07-26T03:14:21.211924+00:00",
71
- "resolved_at": "2026-07-26T05:26:52.278146+00:00",
72
- "target_end": "2026-07-24T18:25:00+00:00",
73
- "prediction_length": 12,
74
- "metric_version": "livehouse-canonical-v1"
75
- }
76
- ]
77
  },
78
  "TimesFM-2.5": {
79
  "model": "TimesFM-2.5",
@@ -81,27 +37,8 @@
81
  "admitted_at": "2026-06-01T00:00:00Z",
82
  "evaluation_protocol": "issue-now-score-later",
83
  "metric_version": "livehouse-canonical-v1",
84
- "evaluated_at": "2026-07-26T05:37:16.252299+00:00",
85
- "resolved_tasks": [
86
- {
87
- "dataset": "open_meteo_shanghai_air_quality_hourly_pm2_5/1h/short",
88
- "task_id": "future-7cf15642ebedce11035a",
89
- "issued_at": "2026-07-25T05:57:25.569188+00:00",
90
- "resolved_at": "2026-07-26T05:26:51.842105+00:00",
91
- "target_end": "2026-07-26T04:00:00+00:00",
92
- "prediction_length": 24,
93
- "metric_version": "livehouse-canonical-v1"
94
- },
95
- {
96
- "dataset": "data_gov_sg_singapore_taxi_availability_available_taxi_count/5min/short",
97
- "task_id": "future-c88625e66c964a9a0896",
98
- "issued_at": "2026-07-26T03:14:21.211924+00:00",
99
- "resolved_at": "2026-07-26T05:26:52.242276+00:00",
100
- "target_end": "2026-07-24T18:25:00+00:00",
101
- "prediction_length": 12,
102
- "metric_version": "livehouse-canonical-v1"
103
- }
104
- ]
105
  },
106
  "Toto-1.0": {
107
  "model": "Toto-1.0",
@@ -109,27 +46,8 @@
109
  "admitted_at": "2026-06-01T00:00:00Z",
110
  "evaluation_protocol": "issue-now-score-later",
111
  "metric_version": "livehouse-canonical-v1",
112
- "evaluated_at": "2026-07-26T05:37:16.292094+00:00",
113
- "resolved_tasks": [
114
- {
115
- "dataset": "open_meteo_shanghai_air_quality_hourly_pm2_5/1h/short",
116
- "task_id": "future-7cf15642ebedce11035a",
117
- "issued_at": "2026-07-25T05:57:25.569188+00:00",
118
- "resolved_at": "2026-07-26T05:26:51.923241+00:00",
119
- "target_end": "2026-07-26T04:00:00+00:00",
120
- "prediction_length": 24,
121
- "metric_version": "livehouse-canonical-v1"
122
- },
123
- {
124
- "dataset": "data_gov_sg_singapore_taxi_availability_available_taxi_count/5min/short",
125
- "task_id": "future-c88625e66c964a9a0896",
126
- "issued_at": "2026-07-26T03:14:21.211924+00:00",
127
- "resolved_at": "2026-07-26T05:26:52.305193+00:00",
128
- "target_end": "2026-07-24T18:25:00+00:00",
129
- "prediction_length": 12,
130
- "metric_version": "livehouse-canonical-v1"
131
- }
132
- ]
133
  },
134
  "Moirai-2.0": {
135
  "model": "Moirai-2.0",
@@ -137,27 +55,8 @@
137
  "admitted_at": "2026-06-01T00:00:00Z",
138
  "evaluation_protocol": "issue-now-score-later",
139
  "metric_version": "livehouse-canonical-v1",
140
- "evaluated_at": "2026-07-26T05:37:16.308608+00:00",
141
- "resolved_tasks": [
142
- {
143
- "dataset": "open_meteo_shanghai_air_quality_hourly_pm2_5/1h/short",
144
- "task_id": "future-7cf15642ebedce11035a",
145
- "issued_at": "2026-07-25T05:57:25.569188+00:00",
146
- "resolved_at": "2026-07-26T05:26:51.632049+00:00",
147
- "target_end": "2026-07-26T04:00:00+00:00",
148
- "prediction_length": 24,
149
- "metric_version": "livehouse-canonical-v1"
150
- },
151
- {
152
- "dataset": "data_gov_sg_singapore_taxi_availability_available_taxi_count/5min/short",
153
- "task_id": "future-c88625e66c964a9a0896",
154
- "issued_at": "2026-07-26T03:14:21.211924+00:00",
155
- "resolved_at": "2026-07-26T05:26:52.108276+00:00",
156
- "target_end": "2026-07-24T18:25:00+00:00",
157
- "prediction_length": 12,
158
- "metric_version": "livehouse-canonical-v1"
159
- }
160
- ]
161
  },
162
  "Chronos-Bolt": {
163
  "model": "Chronos-Bolt",
@@ -165,27 +64,8 @@
165
  "admitted_at": "2026-06-01T00:00:00Z",
166
  "evaluation_protocol": "issue-now-score-later",
167
  "metric_version": "livehouse-canonical-v1",
168
- "evaluated_at": "2026-07-26T05:37:16.326993+00:00",
169
- "resolved_tasks": [
170
- {
171
- "dataset": "open_meteo_shanghai_air_quality_hourly_pm2_5/1h/short",
172
- "task_id": "future-7cf15642ebedce11035a",
173
- "issued_at": "2026-07-25T05:57:25.569188+00:00",
174
- "resolved_at": "2026-07-26T05:26:51.538236+00:00",
175
- "target_end": "2026-07-26T04:00:00+00:00",
176
- "prediction_length": 24,
177
- "metric_version": "livehouse-canonical-v1"
178
- },
179
- {
180
- "dataset": "data_gov_sg_singapore_taxi_availability_available_taxi_count/5min/short",
181
- "task_id": "future-c88625e66c964a9a0896",
182
- "issued_at": "2026-07-26T03:14:21.211924+00:00",
183
- "resolved_at": "2026-07-26T05:26:52.031159+00:00",
184
- "target_end": "2026-07-24T18:25:00+00:00",
185
- "prediction_length": 12,
186
- "metric_version": "livehouse-canonical-v1"
187
- }
188
- ]
189
  },
190
  "TabPFN-TS": {
191
  "model": "TabPFN-TS",
@@ -193,27 +73,8 @@
193
  "admitted_at": "2026-06-01T00:00:00Z",
194
  "evaluation_protocol": "issue-now-score-later",
195
  "metric_version": "livehouse-canonical-v1",
196
- "evaluated_at": "2026-07-26T05:37:16.338122+00:00",
197
- "resolved_tasks": [
198
- {
199
- "dataset": "open_meteo_shanghai_air_quality_hourly_pm2_5/1h/short",
200
- "task_id": "future-7cf15642ebedce11035a",
201
- "issued_at": "2026-07-25T05:57:25.569188+00:00",
202
- "resolved_at": "2026-07-26T05:26:51.810367+00:00",
203
- "target_end": "2026-07-26T04:00:00+00:00",
204
- "prediction_length": 24,
205
- "metric_version": "livehouse-canonical-v1"
206
- },
207
- {
208
- "dataset": "data_gov_sg_singapore_taxi_availability_available_taxi_count/5min/short",
209
- "task_id": "future-c88625e66c964a9a0896",
210
- "issued_at": "2026-07-26T03:14:21.211924+00:00",
211
- "resolved_at": "2026-07-26T05:26:52.214370+00:00",
212
- "target_end": "2026-07-24T18:25:00+00:00",
213
- "prediction_length": 12,
214
- "metric_version": "livehouse-canonical-v1"
215
- }
216
- ]
217
  },
218
  "Sundial": {
219
  "model": "Sundial",
@@ -221,27 +82,8 @@
221
  "admitted_at": "2026-06-01T00:00:00Z",
222
  "evaluation_protocol": "issue-now-score-later",
223
  "metric_version": "livehouse-canonical-v1",
224
- "evaluated_at": "2026-07-26T05:37:16.357766+00:00",
225
- "resolved_tasks": [
226
- {
227
- "dataset": "open_meteo_shanghai_air_quality_hourly_pm2_5/1h/short",
228
- "task_id": "future-7cf15642ebedce11035a",
229
- "issued_at": "2026-07-25T05:57:25.569188+00:00",
230
- "resolved_at": "2026-07-26T05:26:51.762198+00:00",
231
- "target_end": "2026-07-26T04:00:00+00:00",
232
- "prediction_length": 24,
233
- "metric_version": "livehouse-canonical-v1"
234
- },
235
- {
236
- "dataset": "data_gov_sg_singapore_taxi_availability_available_taxi_count/5min/short",
237
- "task_id": "future-c88625e66c964a9a0896",
238
- "issued_at": "2026-07-26T03:14:21.211924+00:00",
239
- "resolved_at": "2026-07-26T05:26:52.190151+00:00",
240
- "target_end": "2026-07-24T18:25:00+00:00",
241
- "prediction_length": 12,
242
- "metric_version": "livehouse-canonical-v1"
243
- }
244
- ]
245
  },
246
  "Moving-Average": {
247
  "model": "Moving-Average",
@@ -249,27 +91,8 @@
249
  "admitted_at": "2026-06-01T00:00:00Z",
250
  "evaluation_protocol": "issue-now-score-later",
251
  "metric_version": "livehouse-canonical-v1",
252
- "evaluated_at": "2026-07-26T05:37:16.367105+00:00",
253
- "resolved_tasks": [
254
- {
255
- "dataset": "open_meteo_shanghai_air_quality_hourly_pm2_5/1h/short",
256
- "task_id": "future-7cf15642ebedce11035a",
257
- "issued_at": "2026-07-25T05:57:25.569188+00:00",
258
- "resolved_at": "2026-07-26T05:26:51.666111+00:00",
259
- "target_end": "2026-07-26T04:00:00+00:00",
260
- "prediction_length": 24,
261
- "metric_version": "livehouse-canonical-v1"
262
- },
263
- {
264
- "dataset": "data_gov_sg_singapore_taxi_availability_available_taxi_count/5min/short",
265
- "task_id": "future-c88625e66c964a9a0896",
266
- "issued_at": "2026-07-26T03:14:21.211924+00:00",
267
- "resolved_at": "2026-07-26T05:26:52.147093+00:00",
268
- "target_end": "2026-07-24T18:25:00+00:00",
269
- "prediction_length": 12,
270
- "metric_version": "livehouse-canonical-v1"
271
- }
272
- ]
273
  },
274
  "ETS": {
275
  "model": "ETS",
@@ -277,27 +100,8 @@
277
  "admitted_at": "2026-06-01T00:00:00Z",
278
  "evaluation_protocol": "issue-now-score-later",
279
  "metric_version": "livehouse-canonical-v1",
280
- "evaluated_at": "2026-07-26T05:37:16.372277+00:00",
281
- "resolved_tasks": [
282
- {
283
- "dataset": "open_meteo_shanghai_air_quality_hourly_pm2_5/1h/short",
284
- "task_id": "future-7cf15642ebedce11035a",
285
- "issued_at": "2026-07-25T05:57:25.569188+00:00",
286
- "resolved_at": "2026-07-26T05:26:51.579097+00:00",
287
- "target_end": "2026-07-26T04:00:00+00:00",
288
- "prediction_length": 24,
289
- "metric_version": "livehouse-canonical-v1"
290
- },
291
- {
292
- "dataset": "data_gov_sg_singapore_taxi_availability_available_taxi_count/5min/short",
293
- "task_id": "future-c88625e66c964a9a0896",
294
- "issued_at": "2026-07-26T03:14:21.211924+00:00",
295
- "resolved_at": "2026-07-26T05:26:52.071035+00:00",
296
- "target_end": "2026-07-24T18:25:00+00:00",
297
- "prediction_length": 12,
298
- "metric_version": "livehouse-canonical-v1"
299
- }
300
- ]
301
  },
302
  "ARIMA": {
303
  "model": "ARIMA",
@@ -305,27 +109,8 @@
305
  "admitted_at": "2026-06-01T00:00:00Z",
306
  "evaluation_protocol": "issue-now-score-later",
307
  "metric_version": "livehouse-canonical-v1",
308
- "evaluated_at": "2026-07-26T05:37:16.376682+00:00",
309
- "resolved_tasks": [
310
- {
311
- "dataset": "open_meteo_shanghai_air_quality_hourly_pm2_5/1h/short",
312
- "task_id": "future-7cf15642ebedce11035a",
313
- "issued_at": "2026-07-25T05:57:25.569188+00:00",
314
- "resolved_at": "2026-07-26T05:26:51.449393+00:00",
315
- "target_end": "2026-07-26T04:00:00+00:00",
316
- "prediction_length": 24,
317
- "metric_version": "livehouse-canonical-v1"
318
- },
319
- {
320
- "dataset": "data_gov_sg_singapore_taxi_availability_available_taxi_count/5min/short",
321
- "task_id": "future-c88625e66c964a9a0896",
322
- "issued_at": "2026-07-26T03:14:21.211924+00:00",
323
- "resolved_at": "2026-07-26T05:26:51.953095+00:00",
324
- "target_end": "2026-07-24T18:25:00+00:00",
325
- "prediction_length": 12,
326
- "metric_version": "livehouse-canonical-v1"
327
- }
328
- ]
329
  },
330
  "Seasonal-Naive": {
331
  "model": "Seasonal-Naive",
@@ -333,27 +118,8 @@
333
  "admitted_at": "2026-06-01T00:00:00Z",
334
  "evaluation_protocol": "issue-now-score-later",
335
  "metric_version": "livehouse-canonical-v1",
336
- "evaluated_at": "2026-07-26T05:37:16.386810+00:00",
337
- "resolved_tasks": [
338
- {
339
- "dataset": "open_meteo_shanghai_air_quality_hourly_pm2_5/1h/short",
340
- "task_id": "future-7cf15642ebedce11035a",
341
- "issued_at": "2026-07-25T05:57:25.569188+00:00",
342
- "resolved_at": "2026-07-26T05:26:51.715425+00:00",
343
- "target_end": "2026-07-26T04:00:00+00:00",
344
- "prediction_length": 24,
345
- "metric_version": "livehouse-canonical-v1"
346
- },
347
- {
348
- "dataset": "data_gov_sg_singapore_taxi_availability_available_taxi_count/5min/short",
349
- "task_id": "future-c88625e66c964a9a0896",
350
- "issued_at": "2026-07-26T03:14:21.211924+00:00",
351
- "resolved_at": "2026-07-26T05:26:52.163038+00:00",
352
- "target_end": "2026-07-24T18:25:00+00:00",
353
- "prediction_length": 12,
354
- "metric_version": "livehouse-canonical-v1"
355
- }
356
- ]
357
  }
358
  },
359
  "skipped_models": [],
@@ -376,7 +142,5 @@
376
  "live_overall_30d": "aggregates/live_overall_30d.csv",
377
  "live_metadata": "aggregates/live_metadata.json"
378
  },
379
- "push_status": "ok",
380
- "pushed_at": "2026-07-26T05:41:37.148182+00:00",
381
- "push_detail": "HF Hub API"
382
  }
 
1
  {
2
+ "status": "ok",
3
  "evaluation_protocol": "issue-now-score-later",
4
  "metric_version": "livehouse-canonical-v1",
5
+ "started_at": "2026-07-26T06:00:58.272024+00:00",
6
+ "finished_at": "2026-07-26T06:01:39.234957+00:00",
7
  "data_source": "ts_bench",
8
  "data_config": "configs/datasets/ts_bench.yaml",
9
+ "task_count": 3,
10
+ "issued_tasks": 0,
11
+ "issued_forecasts": 0,
12
+ "resolved_forecasts": 0,
13
  "pending_tasks": 13,
14
+ "failed_forecasts": [],
 
 
 
 
 
 
15
  "models": {
16
  "Chronos-2": {
17
  "model": "Chronos-2",
 
19
  "admitted_at": "2026-06-01T00:00:00Z",
20
  "evaluation_protocol": "issue-now-score-later",
21
  "metric_version": "livehouse-canonical-v1",
22
+ "evaluated_at": "2026-07-26T06:01:01.688937+00:00",
23
+ "resolved_tasks": []
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
24
  },
25
  "TiRex": {
26
  "model": "TiRex",
 
28
  "admitted_at": "2026-06-01T00:00:00Z",
29
  "evaluation_protocol": "issue-now-score-later",
30
  "metric_version": "livehouse-canonical-v1",
31
+ "evaluated_at": "2026-07-26T06:01:01.688947+00:00",
32
+ "resolved_tasks": []
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
33
  },
34
  "TimesFM-2.5": {
35
  "model": "TimesFM-2.5",
 
37
  "admitted_at": "2026-06-01T00:00:00Z",
38
  "evaluation_protocol": "issue-now-score-later",
39
  "metric_version": "livehouse-canonical-v1",
40
+ "evaluated_at": "2026-07-26T06:01:01.688948+00:00",
41
+ "resolved_tasks": []
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
42
  },
43
  "Toto-1.0": {
44
  "model": "Toto-1.0",
 
46
  "admitted_at": "2026-06-01T00:00:00Z",
47
  "evaluation_protocol": "issue-now-score-later",
48
  "metric_version": "livehouse-canonical-v1",
49
+ "evaluated_at": "2026-07-26T06:01:01.688949+00:00",
50
+ "resolved_tasks": []
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
51
  },
52
  "Moirai-2.0": {
53
  "model": "Moirai-2.0",
 
55
  "admitted_at": "2026-06-01T00:00:00Z",
56
  "evaluation_protocol": "issue-now-score-later",
57
  "metric_version": "livehouse-canonical-v1",
58
+ "evaluated_at": "2026-07-26T06:01:01.688950+00:00",
59
+ "resolved_tasks": []
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
60
  },
61
  "Chronos-Bolt": {
62
  "model": "Chronos-Bolt",
 
64
  "admitted_at": "2026-06-01T00:00:00Z",
65
  "evaluation_protocol": "issue-now-score-later",
66
  "metric_version": "livehouse-canonical-v1",
67
+ "evaluated_at": "2026-07-26T06:01:01.688951+00:00",
68
+ "resolved_tasks": []
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
69
  },
70
  "TabPFN-TS": {
71
  "model": "TabPFN-TS",
 
73
  "admitted_at": "2026-06-01T00:00:00Z",
74
  "evaluation_protocol": "issue-now-score-later",
75
  "metric_version": "livehouse-canonical-v1",
76
+ "evaluated_at": "2026-07-26T06:01:01.688953+00:00",
77
+ "resolved_tasks": []
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
78
  },
79
  "Sundial": {
80
  "model": "Sundial",
 
82
  "admitted_at": "2026-06-01T00:00:00Z",
83
  "evaluation_protocol": "issue-now-score-later",
84
  "metric_version": "livehouse-canonical-v1",
85
+ "evaluated_at": "2026-07-26T06:01:01.688953+00:00",
86
+ "resolved_tasks": []
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
87
  },
88
  "Moving-Average": {
89
  "model": "Moving-Average",
 
91
  "admitted_at": "2026-06-01T00:00:00Z",
92
  "evaluation_protocol": "issue-now-score-later",
93
  "metric_version": "livehouse-canonical-v1",
94
+ "evaluated_at": "2026-07-26T06:01:01.688954+00:00",
95
+ "resolved_tasks": []
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
96
  },
97
  "ETS": {
98
  "model": "ETS",
 
100
  "admitted_at": "2026-06-01T00:00:00Z",
101
  "evaluation_protocol": "issue-now-score-later",
102
  "metric_version": "livehouse-canonical-v1",
103
+ "evaluated_at": "2026-07-26T06:01:01.688955+00:00",
104
+ "resolved_tasks": []
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
105
  },
106
  "ARIMA": {
107
  "model": "ARIMA",
 
109
  "admitted_at": "2026-06-01T00:00:00Z",
110
  "evaluation_protocol": "issue-now-score-later",
111
  "metric_version": "livehouse-canonical-v1",
112
+ "evaluated_at": "2026-07-26T06:01:01.688956+00:00",
113
+ "resolved_tasks": []
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
114
  },
115
  "Seasonal-Naive": {
116
  "model": "Seasonal-Naive",
 
118
  "admitted_at": "2026-06-01T00:00:00Z",
119
  "evaluation_protocol": "issue-now-score-later",
120
  "metric_version": "livehouse-canonical-v1",
121
+ "evaluated_at": "2026-07-26T06:01:01.688957+00:00",
122
+ "resolved_tasks": []
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
123
  }
124
  },
125
  "skipped_models": [],
 
142
  "live_overall_30d": "aggregates/live_overall_30d.csv",
143
  "live_metadata": "aggregates/live_metadata.json"
144
  },
145
+ "push_status": "pending"
 
 
146
  }