ziyuzhou02 commited on
Commit
f2a4fb6
·
verified ·
1 Parent(s): cae7ca2

Update leaderboard results from local eval

Browse files
results/aggregates/live_metadata.json CHANGED
@@ -1,5 +1,5 @@
1
  {
2
- "generated_at": "2026-07-24T00:18:19.651509+00:00",
3
  "scope": "Canonical metrics-only release history when available; otherwise legacy release history plus latest allowlisted snapshots",
4
  "release_deduplication": "One row per model, dataset, data source timestamp, and prediction length",
5
  "dataset_aggregation": "Arithmetic mean over releases, then equal-weight mean over datasets",
 
1
  {
2
+ "generated_at": "2026-07-24T00:24:57.294139+00:00",
3
  "scope": "Canonical metrics-only release history when available; otherwise legacy release history plus latest allowlisted snapshots",
4
  "release_deduplication": "One row per model, dataset, data source timestamp, and prediction length",
5
  "dataset_aggregation": "Arithmetic mean over releases, then equal-weight mean over datasets",
results/aggregates/metadata.json CHANGED
@@ -1,5 +1,5 @@
1
  {
2
- "generated_at": "2026-07-24T00:18:09.422025+00:00",
3
  "baseline_model": "Seasonal-Naive",
4
  "normalization": "Per-config metric / Seasonal-Naive metric",
5
  "metric_aggregation": "Geometric mean over dataset configurations",
 
1
  {
2
+ "generated_at": "2026-07-24T00:24:47.153996+00:00",
3
  "baseline_model": "Seasonal-Naive",
4
  "normalization": "Per-config metric / Seasonal-Naive metric",
5
  "metric_aggregation": "Geometric mean over dataset configurations",
results/dataset_eval_intervals.json CHANGED
@@ -1,7 +1,6 @@
1
  {
2
  "binance_btcusdt_1h_close": 300,
3
  "coingecko_bitcoin_7d_price_usd": 300,
4
- "data_gov_sg_sgx_equities_turnover_monthly_turnover_value_million_sgd": 2592000,
5
  "data_gov_sg_singapore_rainfall_5min_rainfall_5min_total": 900,
6
  "data_gov_sg_singapore_taxi_availability_available_taxi_count": 300,
7
  "nasa_power_shanghai_hourly_T2M": 3600,
 
1
  {
2
  "binance_btcusdt_1h_close": 300,
3
  "coingecko_bitcoin_7d_price_usd": 300,
 
4
  "data_gov_sg_singapore_rainfall_5min_rainfall_5min_total": 900,
5
  "data_gov_sg_singapore_taxi_availability_available_taxi_count": 300,
6
  "nasa_power_shanghai_hourly_T2M": 3600,
results/online_status.json CHANGED
@@ -1,23 +1,17 @@
1
  {
2
- "status": "partial",
3
  "evaluation_protocol": "issue-now-score-later",
4
  "metric_version": "livehouse-canonical-v1",
5
- "started_at": "2026-07-24T00:14:37.001862+00:00",
6
- "finished_at": "2026-07-24T00:18:19.705831+00:00",
7
  "data_source": "ts_bench",
8
  "data_config": "configs/datasets/ts_bench.yaml",
9
  "task_count": 4,
10
- "issued_tasks": 1,
11
- "issued_forecasts": 12,
12
- "resolved_forecasts": 12,
13
  "pending_tasks": 13,
14
- "failed_forecasts": [
15
- {
16
- "model": "Seasonal-Naive",
17
- "dataset": "nws_ksfo_observations_temperature/1h/short",
18
- "error": "Mean forecast length 10 < 24"
19
- }
20
- ],
21
  "models": {
22
  "Chronos-2": {
23
  "model": "Chronos-2",
@@ -25,18 +19,8 @@
25
  "admitted_at": "2026-06-01T00:00:00Z",
26
  "evaluation_protocol": "issue-now-score-later",
27
  "metric_version": "livehouse-canonical-v1",
28
- "evaluated_at": "2026-07-24T00:18:07.227220+00:00",
29
- "resolved_tasks": [
30
- {
31
- "dataset": "data_gov_sg_singapore_rainfall_5min_rainfall_5min_total/5min/short",
32
- "task_id": "future-0175285e7e20e806f971",
33
- "issued_at": "2026-07-23T15:52:33.394311+00:00",
34
- "resolved_at": "2026-07-24T00:14:39.066736+00:00",
35
- "target_end": "2026-07-23T18:45:00+00:00",
36
- "prediction_length": 36,
37
- "metric_version": "livehouse-canonical-v1"
38
- }
39
- ]
40
  },
41
  "TiRex": {
42
  "model": "TiRex",
@@ -44,18 +28,8 @@
44
  "admitted_at": "2026-06-01T00:00:00Z",
45
  "evaluation_protocol": "issue-now-score-later",
46
  "metric_version": "livehouse-canonical-v1",
47
- "evaluated_at": "2026-07-24T00:18:07.245356+00:00",
48
- "resolved_tasks": [
49
- {
50
- "dataset": "data_gov_sg_singapore_rainfall_5min_rainfall_5min_total/5min/short",
51
- "task_id": "future-0175285e7e20e806f971",
52
- "issued_at": "2026-07-23T15:52:33.394311+00:00",
53
- "resolved_at": "2026-07-24T00:14:39.312530+00:00",
54
- "target_end": "2026-07-23T18:45:00+00:00",
55
- "prediction_length": 36,
56
- "metric_version": "livehouse-canonical-v1"
57
- }
58
- ]
59
  },
60
  "TimesFM-2.5": {
61
  "model": "TimesFM-2.5",
@@ -63,18 +37,8 @@
63
  "admitted_at": "2026-06-01T00:00:00Z",
64
  "evaluation_protocol": "issue-now-score-later",
65
  "metric_version": "livehouse-canonical-v1",
66
- "evaluated_at": "2026-07-24T00:18:07.253812+00:00",
67
- "resolved_tasks": [
68
- {
69
- "dataset": "data_gov_sg_singapore_rainfall_5min_rainfall_5min_total/5min/short",
70
- "task_id": "future-0175285e7e20e806f971",
71
- "issued_at": "2026-07-23T15:52:33.394311+00:00",
72
- "resolved_at": "2026-07-24T00:14:39.291528+00:00",
73
- "target_end": "2026-07-23T18:45:00+00:00",
74
- "prediction_length": 36,
75
- "metric_version": "livehouse-canonical-v1"
76
- }
77
- ]
78
  },
79
  "Toto-1.0": {
80
  "model": "Toto-1.0",
@@ -82,18 +46,8 @@
82
  "admitted_at": "2026-06-01T00:00:00Z",
83
  "evaluation_protocol": "issue-now-score-later",
84
  "metric_version": "livehouse-canonical-v1",
85
- "evaluated_at": "2026-07-24T00:18:07.256134+00:00",
86
- "resolved_tasks": [
87
- {
88
- "dataset": "data_gov_sg_singapore_rainfall_5min_rainfall_5min_total/5min/short",
89
- "task_id": "future-0175285e7e20e806f971",
90
- "issued_at": "2026-07-23T15:52:33.394311+00:00",
91
- "resolved_at": "2026-07-24T00:14:39.350525+00:00",
92
- "target_end": "2026-07-23T18:45:00+00:00",
93
- "prediction_length": 36,
94
- "metric_version": "livehouse-canonical-v1"
95
- }
96
- ]
97
  },
98
  "Moirai-2.0": {
99
  "model": "Moirai-2.0",
@@ -101,18 +55,8 @@
101
  "admitted_at": "2026-06-01T00:00:00Z",
102
  "evaluation_protocol": "issue-now-score-later",
103
  "metric_version": "livehouse-canonical-v1",
104
- "evaluated_at": "2026-07-24T00:18:07.267301+00:00",
105
- "resolved_tasks": [
106
- {
107
- "dataset": "data_gov_sg_singapore_rainfall_5min_rainfall_5min_total/5min/short",
108
- "task_id": "future-0175285e7e20e806f971",
109
- "issued_at": "2026-07-23T15:52:33.394311+00:00",
110
- "resolved_at": "2026-07-24T00:14:39.179572+00:00",
111
- "target_end": "2026-07-23T18:45:00+00:00",
112
- "prediction_length": 36,
113
- "metric_version": "livehouse-canonical-v1"
114
- }
115
- ]
116
  },
117
  "Chronos-Bolt": {
118
  "model": "Chronos-Bolt",
@@ -120,18 +64,8 @@
120
  "admitted_at": "2026-06-01T00:00:00Z",
121
  "evaluation_protocol": "issue-now-score-later",
122
  "metric_version": "livehouse-canonical-v1",
123
- "evaluated_at": "2026-07-24T00:18:07.276576+00:00",
124
- "resolved_tasks": [
125
- {
126
- "dataset": "data_gov_sg_singapore_rainfall_5min_rainfall_5min_total/5min/short",
127
- "task_id": "future-0175285e7e20e806f971",
128
- "issued_at": "2026-07-23T15:52:33.394311+00:00",
129
- "resolved_at": "2026-07-24T00:14:39.093362+00:00",
130
- "target_end": "2026-07-23T18:45:00+00:00",
131
- "prediction_length": 36,
132
- "metric_version": "livehouse-canonical-v1"
133
- }
134
- ]
135
  },
136
  "TabPFN-TS": {
137
  "model": "TabPFN-TS",
@@ -139,18 +73,8 @@
139
  "admitted_at": "2026-06-01T00:00:00Z",
140
  "evaluation_protocol": "issue-now-score-later",
141
  "metric_version": "livehouse-canonical-v1",
142
- "evaluated_at": "2026-07-24T00:18:07.279518+00:00",
143
- "resolved_tasks": [
144
- {
145
- "dataset": "data_gov_sg_singapore_rainfall_5min_rainfall_5min_total/5min/short",
146
- "task_id": "future-0175285e7e20e806f971",
147
- "issued_at": "2026-07-23T15:52:33.394311+00:00",
148
- "resolved_at": "2026-07-24T00:14:39.262415+00:00",
149
- "target_end": "2026-07-23T18:45:00+00:00",
150
- "prediction_length": 36,
151
- "metric_version": "livehouse-canonical-v1"
152
- }
153
- ]
154
  },
155
  "Sundial": {
156
  "model": "Sundial",
@@ -158,18 +82,8 @@
158
  "admitted_at": "2026-06-01T00:00:00Z",
159
  "evaluation_protocol": "issue-now-score-later",
160
  "metric_version": "livehouse-canonical-v1",
161
- "evaluated_at": "2026-07-24T00:18:07.303304+00:00",
162
- "resolved_tasks": [
163
- {
164
- "dataset": "data_gov_sg_singapore_rainfall_5min_rainfall_5min_total/5min/short",
165
- "task_id": "future-0175285e7e20e806f971",
166
- "issued_at": "2026-07-23T15:52:33.394311+00:00",
167
- "resolved_at": "2026-07-24T00:14:39.236569+00:00",
168
- "target_end": "2026-07-23T18:45:00+00:00",
169
- "prediction_length": 36,
170
- "metric_version": "livehouse-canonical-v1"
171
- }
172
- ]
173
  },
174
  "Moving-Average": {
175
  "model": "Moving-Average",
@@ -177,18 +91,8 @@
177
  "admitted_at": "2026-06-01T00:00:00Z",
178
  "evaluation_protocol": "issue-now-score-later",
179
  "metric_version": "livehouse-canonical-v1",
180
- "evaluated_at": "2026-07-24T00:18:07.308530+00:00",
181
- "resolved_tasks": [
182
- {
183
- "dataset": "data_gov_sg_singapore_rainfall_5min_rainfall_5min_total/5min/short",
184
- "task_id": "future-0175285e7e20e806f971",
185
- "issued_at": "2026-07-23T15:52:33.394311+00:00",
186
- "resolved_at": "2026-07-24T00:14:39.205192+00:00",
187
- "target_end": "2026-07-23T18:45:00+00:00",
188
- "prediction_length": 36,
189
- "metric_version": "livehouse-canonical-v1"
190
- }
191
- ]
192
  },
193
  "ETS": {
194
  "model": "ETS",
@@ -196,18 +100,8 @@
196
  "admitted_at": "2026-06-01T00:00:00Z",
197
  "evaluation_protocol": "issue-now-score-later",
198
  "metric_version": "livehouse-canonical-v1",
199
- "evaluated_at": "2026-07-24T00:18:07.312690+00:00",
200
- "resolved_tasks": [
201
- {
202
- "dataset": "data_gov_sg_singapore_rainfall_5min_rainfall_5min_total/5min/short",
203
- "task_id": "future-0175285e7e20e806f971",
204
- "issued_at": "2026-07-23T15:52:33.394311+00:00",
205
- "resolved_at": "2026-07-24T00:14:39.154182+00:00",
206
- "target_end": "2026-07-23T18:45:00+00:00",
207
- "prediction_length": 36,
208
- "metric_version": "livehouse-canonical-v1"
209
- }
210
- ]
211
  },
212
  "ARIMA": {
213
  "model": "ARIMA",
@@ -215,18 +109,8 @@
215
  "admitted_at": "2026-06-01T00:00:00Z",
216
  "evaluation_protocol": "issue-now-score-later",
217
  "metric_version": "livehouse-canonical-v1",
218
- "evaluated_at": "2026-07-24T00:18:07.348964+00:00",
219
- "resolved_tasks": [
220
- {
221
- "dataset": "data_gov_sg_singapore_rainfall_5min_rainfall_5min_total/5min/short",
222
- "task_id": "future-0175285e7e20e806f971",
223
- "issued_at": "2026-07-23T15:52:33.394311+00:00",
224
- "resolved_at": "2026-07-24T00:14:38.958955+00:00",
225
- "target_end": "2026-07-23T18:45:00+00:00",
226
- "prediction_length": 36,
227
- "metric_version": "livehouse-canonical-v1"
228
- }
229
- ]
230
  },
231
  "Seasonal-Naive": {
232
  "model": "Seasonal-Naive",
@@ -234,18 +118,8 @@
234
  "admitted_at": "2026-06-01T00:00:00Z",
235
  "evaluation_protocol": "issue-now-score-later",
236
  "metric_version": "livehouse-canonical-v1",
237
- "evaluated_at": "2026-07-24T00:18:07.366747+00:00",
238
- "resolved_tasks": [
239
- {
240
- "dataset": "data_gov_sg_singapore_rainfall_5min_rainfall_5min_total/5min/short",
241
- "task_id": "future-0175285e7e20e806f971",
242
- "issued_at": "2026-07-23T15:52:33.394311+00:00",
243
- "resolved_at": "2026-07-24T00:14:39.221451+00:00",
244
- "target_end": "2026-07-23T18:45:00+00:00",
245
- "prediction_length": 36,
246
- "metric_version": "livehouse-canonical-v1"
247
- }
248
- ]
249
  }
250
  },
251
  "skipped_models": [],
@@ -268,7 +142,5 @@
268
  "live_overall_30d": "aggregates/live_overall_30d.csv",
269
  "live_metadata": "aggregates/live_metadata.json"
270
  },
271
- "push_status": "ok",
272
- "pushed_at": "2026-07-24T00:18:49.265814+00:00",
273
- "push_detail": "HF Hub API"
274
  }
 
1
  {
2
+ "status": "ok",
3
  "evaluation_protocol": "issue-now-score-later",
4
  "metric_version": "livehouse-canonical-v1",
5
+ "started_at": "2026-07-24T00:24:44.409741+00:00",
6
+ "finished_at": "2026-07-24T00:24:57.366173+00:00",
7
  "data_source": "ts_bench",
8
  "data_config": "configs/datasets/ts_bench.yaml",
9
  "task_count": 4,
10
+ "issued_tasks": 0,
11
+ "issued_forecasts": 0,
12
+ "resolved_forecasts": 0,
13
  "pending_tasks": 13,
14
+ "failed_forecasts": [],
 
 
 
 
 
 
15
  "models": {
16
  "Chronos-2": {
17
  "model": "Chronos-2",
 
19
  "admitted_at": "2026-06-01T00:00:00Z",
20
  "evaluation_protocol": "issue-now-score-later",
21
  "metric_version": "livehouse-canonical-v1",
22
+ "evaluated_at": "2026-07-24T00:24:45.776046+00:00",
23
+ "resolved_tasks": []
 
 
 
 
 
 
 
 
 
 
24
  },
25
  "TiRex": {
26
  "model": "TiRex",
 
28
  "admitted_at": "2026-06-01T00:00:00Z",
29
  "evaluation_protocol": "issue-now-score-later",
30
  "metric_version": "livehouse-canonical-v1",
31
+ "evaluated_at": "2026-07-24T00:24:45.776080+00:00",
32
+ "resolved_tasks": []
 
 
 
 
 
 
 
 
 
 
33
  },
34
  "TimesFM-2.5": {
35
  "model": "TimesFM-2.5",
 
37
  "admitted_at": "2026-06-01T00:00:00Z",
38
  "evaluation_protocol": "issue-now-score-later",
39
  "metric_version": "livehouse-canonical-v1",
40
+ "evaluated_at": "2026-07-24T00:24:45.776085+00:00",
41
+ "resolved_tasks": []
 
 
 
 
 
 
 
 
 
 
42
  },
43
  "Toto-1.0": {
44
  "model": "Toto-1.0",
 
46
  "admitted_at": "2026-06-01T00:00:00Z",
47
  "evaluation_protocol": "issue-now-score-later",
48
  "metric_version": "livehouse-canonical-v1",
49
+ "evaluated_at": "2026-07-24T00:24:45.776089+00:00",
50
+ "resolved_tasks": []
 
 
 
 
 
 
 
 
 
 
51
  },
52
  "Moirai-2.0": {
53
  "model": "Moirai-2.0",
 
55
  "admitted_at": "2026-06-01T00:00:00Z",
56
  "evaluation_protocol": "issue-now-score-later",
57
  "metric_version": "livehouse-canonical-v1",
58
+ "evaluated_at": "2026-07-24T00:24:45.776093+00:00",
59
+ "resolved_tasks": []
 
 
 
 
 
 
 
 
 
 
60
  },
61
  "Chronos-Bolt": {
62
  "model": "Chronos-Bolt",
 
64
  "admitted_at": "2026-06-01T00:00:00Z",
65
  "evaluation_protocol": "issue-now-score-later",
66
  "metric_version": "livehouse-canonical-v1",
67
+ "evaluated_at": "2026-07-24T00:24:45.776097+00:00",
68
+ "resolved_tasks": []
 
 
 
 
 
 
 
 
 
 
69
  },
70
  "TabPFN-TS": {
71
  "model": "TabPFN-TS",
 
73
  "admitted_at": "2026-06-01T00:00:00Z",
74
  "evaluation_protocol": "issue-now-score-later",
75
  "metric_version": "livehouse-canonical-v1",
76
+ "evaluated_at": "2026-07-24T00:24:45.776101+00:00",
77
+ "resolved_tasks": []
 
 
 
 
 
 
 
 
 
 
78
  },
79
  "Sundial": {
80
  "model": "Sundial",
 
82
  "admitted_at": "2026-06-01T00:00:00Z",
83
  "evaluation_protocol": "issue-now-score-later",
84
  "metric_version": "livehouse-canonical-v1",
85
+ "evaluated_at": "2026-07-24T00:24:45.776106+00:00",
86
+ "resolved_tasks": []
 
 
 
 
 
 
 
 
 
 
87
  },
88
  "Moving-Average": {
89
  "model": "Moving-Average",
 
91
  "admitted_at": "2026-06-01T00:00:00Z",
92
  "evaluation_protocol": "issue-now-score-later",
93
  "metric_version": "livehouse-canonical-v1",
94
+ "evaluated_at": "2026-07-24T00:24:45.776110+00:00",
95
+ "resolved_tasks": []
 
 
 
 
 
 
 
 
 
 
96
  },
97
  "ETS": {
98
  "model": "ETS",
 
100
  "admitted_at": "2026-06-01T00:00:00Z",
101
  "evaluation_protocol": "issue-now-score-later",
102
  "metric_version": "livehouse-canonical-v1",
103
+ "evaluated_at": "2026-07-24T00:24:45.776114+00:00",
104
+ "resolved_tasks": []
 
 
 
 
 
 
 
 
 
 
105
  },
106
  "ARIMA": {
107
  "model": "ARIMA",
 
109
  "admitted_at": "2026-06-01T00:00:00Z",
110
  "evaluation_protocol": "issue-now-score-later",
111
  "metric_version": "livehouse-canonical-v1",
112
+ "evaluated_at": "2026-07-24T00:24:45.776119+00:00",
113
+ "resolved_tasks": []
 
 
 
 
 
 
 
 
 
 
114
  },
115
  "Seasonal-Naive": {
116
  "model": "Seasonal-Naive",
 
118
  "admitted_at": "2026-06-01T00:00:00Z",
119
  "evaluation_protocol": "issue-now-score-later",
120
  "metric_version": "livehouse-canonical-v1",
121
+ "evaluated_at": "2026-07-24T00:24:45.776123+00:00",
122
+ "resolved_tasks": []
 
 
 
 
 
 
 
 
 
 
123
  }
124
  },
125
  "skipped_models": [],
 
142
  "live_overall_30d": "aggregates/live_overall_30d.csv",
143
  "live_metadata": "aggregates/live_metadata.json"
144
  },
145
+ "push_status": "pending"
 
 
146
  }