ziyuzhou02 commited on
Commit
d8a5162
·
verified ·
1 Parent(s): cf0e833

Update leaderboard results from local eval

Browse files
results/aggregates/live_metadata.json CHANGED
@@ -1,5 +1,5 @@
1
  {
2
- "generated_at": "2026-07-30T03:45:16.469296+00:00",
3
  "scope": "Canonical metrics-only release history when available; otherwise legacy release history plus latest allowlisted snapshots",
4
  "release_deduplication": "One row per model, dataset, data source timestamp, and prediction length",
5
  "dataset_aggregation": "Arithmetic mean over releases, then equal-weight mean over datasets",
 
1
  {
2
+ "generated_at": "2026-07-30T03:56:59.089485+00:00",
3
  "scope": "Canonical metrics-only release history when available; otherwise legacy release history plus latest allowlisted snapshots",
4
  "release_deduplication": "One row per model, dataset, data source timestamp, and prediction length",
5
  "dataset_aggregation": "Arithmetic mean over releases, then equal-weight mean over datasets",
results/aggregates/metadata.json CHANGED
@@ -1,5 +1,5 @@
1
  {
2
- "generated_at": "2026-07-30T03:45:14.236121+00:00",
3
  "baseline_model": "Seasonal-Naive",
4
  "normalization": "Per-config metric / Seasonal-Naive metric",
5
  "metric_aggregation": "Geometric mean over dataset configurations",
 
1
  {
2
+ "generated_at": "2026-07-30T03:56:56.864221+00:00",
3
  "baseline_model": "Seasonal-Naive",
4
  "normalization": "Per-config metric / Seasonal-Naive metric",
5
  "metric_aggregation": "Geometric mean over dataset configurations",
results/online_status.json CHANGED
@@ -1,9 +1,9 @@
1
  {
2
- "status": "partial",
3
  "evaluation_protocol": "issue-now-score-later",
4
  "metric_version": "livehouse-canonical-v1",
5
- "started_at": "2026-07-30T03:45:08.165039+00:00",
6
- "finished_at": "2026-07-30T03:45:16.484348+00:00",
7
  "data_source": "ts_bench",
8
  "data_config": "configs/datasets/ts_bench.yaml",
9
  "task_count": 4,
@@ -11,13 +11,7 @@
11
  "issued_forecasts": 0,
12
  "resolved_forecasts": 0,
13
  "pending_tasks": 13,
14
- "failed_forecasts": [
15
- {
16
- "model": "Seasonal-Naive",
17
- "dataset": "nws_ksfo_observations_temperature/1h/short",
18
- "error": "Mean forecast length 10 < 24"
19
- }
20
- ],
21
  "models": {
22
  "Chronos-2": {
23
  "model": "Chronos-2",
@@ -25,7 +19,7 @@
25
  "admitted_at": "2026-06-01T00:00:00Z",
26
  "evaluation_protocol": "issue-now-score-later",
27
  "metric_version": "livehouse-canonical-v1",
28
- "evaluated_at": "2026-07-30T03:45:12.609348+00:00",
29
  "resolved_tasks": []
30
  },
31
  "TiRex": {
@@ -34,7 +28,7 @@
34
  "admitted_at": "2026-06-01T00:00:00Z",
35
  "evaluation_protocol": "issue-now-score-later",
36
  "metric_version": "livehouse-canonical-v1",
37
- "evaluated_at": "2026-07-30T03:45:12.609365+00:00",
38
  "resolved_tasks": []
39
  },
40
  "TimesFM-2.5": {
@@ -43,7 +37,7 @@
43
  "admitted_at": "2026-06-01T00:00:00Z",
44
  "evaluation_protocol": "issue-now-score-later",
45
  "metric_version": "livehouse-canonical-v1",
46
- "evaluated_at": "2026-07-30T03:45:12.609369+00:00",
47
  "resolved_tasks": []
48
  },
49
  "Toto-1.0": {
@@ -52,7 +46,7 @@
52
  "admitted_at": "2026-06-01T00:00:00Z",
53
  "evaluation_protocol": "issue-now-score-later",
54
  "metric_version": "livehouse-canonical-v1",
55
- "evaluated_at": "2026-07-30T03:45:12.609371+00:00",
56
  "resolved_tasks": []
57
  },
58
  "Moirai-2.0": {
@@ -61,7 +55,7 @@
61
  "admitted_at": "2026-06-01T00:00:00Z",
62
  "evaluation_protocol": "issue-now-score-later",
63
  "metric_version": "livehouse-canonical-v1",
64
- "evaluated_at": "2026-07-30T03:45:12.609373+00:00",
65
  "resolved_tasks": []
66
  },
67
  "Chronos-Bolt": {
@@ -70,7 +64,7 @@
70
  "admitted_at": "2026-06-01T00:00:00Z",
71
  "evaluation_protocol": "issue-now-score-later",
72
  "metric_version": "livehouse-canonical-v1",
73
- "evaluated_at": "2026-07-30T03:45:12.609375+00:00",
74
  "resolved_tasks": []
75
  },
76
  "TabPFN-TS": {
@@ -79,7 +73,7 @@
79
  "admitted_at": "2026-06-01T00:00:00Z",
80
  "evaluation_protocol": "issue-now-score-later",
81
  "metric_version": "livehouse-canonical-v1",
82
- "evaluated_at": "2026-07-30T03:45:12.609377+00:00",
83
  "resolved_tasks": []
84
  },
85
  "Sundial": {
@@ -88,7 +82,7 @@
88
  "admitted_at": "2026-06-01T00:00:00Z",
89
  "evaluation_protocol": "issue-now-score-later",
90
  "metric_version": "livehouse-canonical-v1",
91
- "evaluated_at": "2026-07-30T03:45:12.609379+00:00",
92
  "resolved_tasks": []
93
  },
94
  "Moving-Average": {
@@ -97,7 +91,7 @@
97
  "admitted_at": "2026-06-01T00:00:00Z",
98
  "evaluation_protocol": "issue-now-score-later",
99
  "metric_version": "livehouse-canonical-v1",
100
- "evaluated_at": "2026-07-30T03:45:12.609381+00:00",
101
  "resolved_tasks": []
102
  },
103
  "ETS": {
@@ -106,7 +100,7 @@
106
  "admitted_at": "2026-06-01T00:00:00Z",
107
  "evaluation_protocol": "issue-now-score-later",
108
  "metric_version": "livehouse-canonical-v1",
109
- "evaluated_at": "2026-07-30T03:45:12.609383+00:00",
110
  "resolved_tasks": []
111
  },
112
  "ARIMA": {
@@ -115,7 +109,7 @@
115
  "admitted_at": "2026-06-01T00:00:00Z",
116
  "evaluation_protocol": "issue-now-score-later",
117
  "metric_version": "livehouse-canonical-v1",
118
- "evaluated_at": "2026-07-30T03:45:12.609384+00:00",
119
  "resolved_tasks": []
120
  },
121
  "Seasonal-Naive": {
@@ -124,7 +118,7 @@
124
  "admitted_at": "2026-06-01T00:00:00Z",
125
  "evaluation_protocol": "issue-now-score-later",
126
  "metric_version": "livehouse-canonical-v1",
127
- "evaluated_at": "2026-07-30T03:45:12.609387+00:00",
128
  "resolved_tasks": []
129
  }
130
  },
@@ -148,7 +142,5 @@
148
  "live_overall_30d": "aggregates/live_overall_30d.csv",
149
  "live_metadata": "aggregates/live_metadata.json"
150
  },
151
- "push_status": "ok",
152
- "pushed_at": "2026-07-30T03:46:14.137666+00:00",
153
- "push_detail": "HF Hub API"
154
  }
 
1
  {
2
+ "status": "ok",
3
  "evaluation_protocol": "issue-now-score-later",
4
  "metric_version": "livehouse-canonical-v1",
5
+ "started_at": "2026-07-30T03:56:50.141633+00:00",
6
+ "finished_at": "2026-07-30T03:56:59.101215+00:00",
7
  "data_source": "ts_bench",
8
  "data_config": "configs/datasets/ts_bench.yaml",
9
  "task_count": 4,
 
11
  "issued_forecasts": 0,
12
  "resolved_forecasts": 0,
13
  "pending_tasks": 13,
14
+ "failed_forecasts": [],
 
 
 
 
 
 
15
  "models": {
16
  "Chronos-2": {
17
  "model": "Chronos-2",
 
19
  "admitted_at": "2026-06-01T00:00:00Z",
20
  "evaluation_protocol": "issue-now-score-later",
21
  "metric_version": "livehouse-canonical-v1",
22
+ "evaluated_at": "2026-07-30T03:56:54.779306+00:00",
23
  "resolved_tasks": []
24
  },
25
  "TiRex": {
 
28
  "admitted_at": "2026-06-01T00:00:00Z",
29
  "evaluation_protocol": "issue-now-score-later",
30
  "metric_version": "livehouse-canonical-v1",
31
+ "evaluated_at": "2026-07-30T03:56:54.779315+00:00",
32
  "resolved_tasks": []
33
  },
34
  "TimesFM-2.5": {
 
37
  "admitted_at": "2026-06-01T00:00:00Z",
38
  "evaluation_protocol": "issue-now-score-later",
39
  "metric_version": "livehouse-canonical-v1",
40
+ "evaluated_at": "2026-07-30T03:56:54.779317+00:00",
41
  "resolved_tasks": []
42
  },
43
  "Toto-1.0": {
 
46
  "admitted_at": "2026-06-01T00:00:00Z",
47
  "evaluation_protocol": "issue-now-score-later",
48
  "metric_version": "livehouse-canonical-v1",
49
+ "evaluated_at": "2026-07-30T03:56:54.779318+00:00",
50
  "resolved_tasks": []
51
  },
52
  "Moirai-2.0": {
 
55
  "admitted_at": "2026-06-01T00:00:00Z",
56
  "evaluation_protocol": "issue-now-score-later",
57
  "metric_version": "livehouse-canonical-v1",
58
+ "evaluated_at": "2026-07-30T03:56:54.779319+00:00",
59
  "resolved_tasks": []
60
  },
61
  "Chronos-Bolt": {
 
64
  "admitted_at": "2026-06-01T00:00:00Z",
65
  "evaluation_protocol": "issue-now-score-later",
66
  "metric_version": "livehouse-canonical-v1",
67
+ "evaluated_at": "2026-07-30T03:56:54.779320+00:00",
68
  "resolved_tasks": []
69
  },
70
  "TabPFN-TS": {
 
73
  "admitted_at": "2026-06-01T00:00:00Z",
74
  "evaluation_protocol": "issue-now-score-later",
75
  "metric_version": "livehouse-canonical-v1",
76
+ "evaluated_at": "2026-07-30T03:56:54.779321+00:00",
77
  "resolved_tasks": []
78
  },
79
  "Sundial": {
 
82
  "admitted_at": "2026-06-01T00:00:00Z",
83
  "evaluation_protocol": "issue-now-score-later",
84
  "metric_version": "livehouse-canonical-v1",
85
+ "evaluated_at": "2026-07-30T03:56:54.779322+00:00",
86
  "resolved_tasks": []
87
  },
88
  "Moving-Average": {
 
91
  "admitted_at": "2026-06-01T00:00:00Z",
92
  "evaluation_protocol": "issue-now-score-later",
93
  "metric_version": "livehouse-canonical-v1",
94
+ "evaluated_at": "2026-07-30T03:56:54.779323+00:00",
95
  "resolved_tasks": []
96
  },
97
  "ETS": {
 
100
  "admitted_at": "2026-06-01T00:00:00Z",
101
  "evaluation_protocol": "issue-now-score-later",
102
  "metric_version": "livehouse-canonical-v1",
103
+ "evaluated_at": "2026-07-30T03:56:54.779323+00:00",
104
  "resolved_tasks": []
105
  },
106
  "ARIMA": {
 
109
  "admitted_at": "2026-06-01T00:00:00Z",
110
  "evaluation_protocol": "issue-now-score-later",
111
  "metric_version": "livehouse-canonical-v1",
112
+ "evaluated_at": "2026-07-30T03:56:54.779324+00:00",
113
  "resolved_tasks": []
114
  },
115
  "Seasonal-Naive": {
 
118
  "admitted_at": "2026-06-01T00:00:00Z",
119
  "evaluation_protocol": "issue-now-score-later",
120
  "metric_version": "livehouse-canonical-v1",
121
+ "evaluated_at": "2026-07-30T03:56:54.779325+00:00",
122
  "resolved_tasks": []
123
  }
124
  },
 
142
  "live_overall_30d": "aggregates/live_overall_30d.csv",
143
  "live_metadata": "aggregates/live_metadata.json"
144
  },
145
+ "push_status": "pending"
 
 
146
  }