ziyuzhou02 commited on
Commit
167517f
·
verified ·
1 Parent(s): 183d744

Update leaderboard results from local eval

Browse files
results/aggregates/live_metadata.json CHANGED
@@ -1,5 +1,5 @@
1
  {
2
- "generated_at": "2026-07-18T03:51:21.745466+00:00",
3
  "scope": "Canonical metrics-only release history when available; otherwise legacy release history plus latest allowlisted snapshots",
4
  "release_deduplication": "One row per model, dataset, data source timestamp, and prediction length",
5
  "dataset_aggregation": "Arithmetic mean over releases, then equal-weight mean over datasets",
 
1
  {
2
+ "generated_at": "2026-07-18T08:07:52.767614+00:00",
3
  "scope": "Canonical metrics-only release history when available; otherwise legacy release history plus latest allowlisted snapshots",
4
  "release_deduplication": "One row per model, dataset, data source timestamp, and prediction length",
5
  "dataset_aggregation": "Arithmetic mean over releases, then equal-weight mean over datasets",
results/aggregates/metadata.json CHANGED
@@ -1,5 +1,5 @@
1
  {
2
- "generated_at": "2026-07-18T03:51:19.429029+00:00",
3
  "baseline_model": "Seasonal-Naive",
4
  "normalization": "Per-config metric / Seasonal-Naive metric",
5
  "metric_aggregation": "Geometric mean over dataset configurations",
@@ -10,7 +10,7 @@
10
  "frequency",
11
  "prediction_length"
12
  ],
13
- "num_configs": 13,
14
  "num_models": 12,
15
  "num_rows_missing_prediction_length": 0
16
  }
 
1
  {
2
+ "generated_at": "2026-07-18T08:07:50.925022+00:00",
3
  "baseline_model": "Seasonal-Naive",
4
  "normalization": "Per-config metric / Seasonal-Naive metric",
5
  "metric_aggregation": "Geometric mean over dataset configurations",
 
10
  "frequency",
11
  "prediction_length"
12
  ],
13
+ "num_configs": 6,
14
  "num_models": 12,
15
  "num_rows_missing_prediction_length": 0
16
  }
results/aggregates/results_by_domain.csv CHANGED
@@ -1,16 +1,16 @@
1
  domain,model,MSE,CRPS,Rank,n_configs,n_group_configs,coverage
2
- Climate,Toto-1.0,807.2036579767959,54.36478495700872,1,5,5,1.0
3
- Climate,Moirai-2.0,11.452173716954187,52.361799916692284,2,5,5,1.0
4
- Climate,TiRex,9.47678640615057,44.445503872023814,3,5,5,1.0
5
- Climate,Chronos-2,17.57186770487565,62.11189935516065,4,5,5,1.0
6
- Climate,Seasonal-Naive,1.0,1.0,5,5,5,1.0
7
- Climate,Chronos-Bolt,18.328360911858056,66.24043663187904,6,5,5,1.0
8
- Climate,TabPFN-TS,7.510514269685618,43.71947356987246,7,5,5,1.0
9
- Climate,TimesFM-2.5,13.559730001772747,87.6491807357361,8,5,5,1.0
10
- Climate,Sundial,103.24773784731899,106.309759176328,9,5,5,1.0
11
- Climate,Moving-Average,177.87609966682683,1.5657188035250973,10,5,5,1.0
12
- Climate,ARIMA,203.49884651977817,177.45589008102328,11,5,5,1.0
13
- Climate,ETS,643.2430525036865,42.70933743266084,12,5,5,1.0
14
  Finance,Chronos-Bolt,0.19298771011845486,0.48437065483374686,1,2,2,1.0
15
  Finance,Moirai-2.0,0.2994100671855118,0.49358388757082117,2,2,2,1.0
16
  Finance,Sundial,0.2577663168744378,0.49342482292609496,3,2,2,1.0
@@ -35,30 +35,6 @@ Hazards,Moirai-2.0,0.6152416390305132,0.6641907413416972,9,1,1,1.0
35
  Hazards,TimesFM-2.5,0.6081084151826099,0.6706133785407319,10,1,1,1.0
36
  Hazards,Seasonal-Naive,1.0,1.0,11,1,1,1.0
37
  Hazards,Toto-1.0,3.019745444328402,1.1961880424879558,12,1,1,1.0
38
- Hydrology,ARIMA,1.0822972356882135,0.32310253910911185,1,1,1,1.0
39
- Hydrology,ETS,0.47591835122996634,0.8489234892846661,2,1,1,1.0
40
- Hydrology,Seasonal-Naive,1.0,1.0,3,1,1,1.0
41
- Hydrology,Toto-1.0,1.141911340572928,1.0811352940913546,4,1,1,1.0
42
- Hydrology,TabPFN-TS,0.8791481146033557,1.162889438787299,5,1,1,1.0
43
- Hydrology,TimesFM-2.5,0.8798555794555492,1.174107737396118,6,1,1,1.0
44
- Hydrology,TiRex,0.8814352809130944,1.178397901985136,7,1,1,1.0
45
- Hydrology,Chronos-Bolt,0.87251738236926,1.1786025093498782,8,1,1,1.0
46
- Hydrology,Sundial,0.9997852855837929,1.1947861396031385,9,1,1,1.0
47
- Hydrology,Chronos-2,0.9390817024400493,1.212577711258179,10,1,1,1.0
48
- Hydrology,Moirai-2.0,1.1021814117726327,1.235944432467708,11,1,1,1.0
49
- Hydrology,Moving-Average,0.9835742550209795,1.2360581940724533,12,1,1,1.0
50
- Ocean,Sundial,0.14730047044931643,0.30022093358297336,1,2,2,1.0
51
- Ocean,TimesFM-2.5,0.16648276642467685,0.352010306534625,2,2,2,1.0
52
- Ocean,TiRex,0.25463447930670996,0.41804557068086096,3,2,2,1.0
53
- Ocean,TabPFN-TS,0.3289203365619865,0.5736227248892207,4,2,2,1.0
54
- Ocean,ARIMA,0.3321250816527086,0.5298855308493969,5,2,2,1.0
55
- Ocean,Chronos-2,0.1319621797323891,0.3676798996511592,6,2,2,1.0
56
- Ocean,Moirai-2.0,0.37486865908100847,0.5981565133465596,7,2,2,1.0
57
- Ocean,Moving-Average,0.7625742931438745,0.8373048028382828,8,2,2,1.0
58
- Ocean,ETS,0.46601202940437536,0.6369423351053193,9,2,2,1.0
59
- Ocean,Chronos-Bolt,0.5592709050659052,0.6434931919926125,10,2,2,1.0
60
- Ocean,Toto-1.0,1.1411410804469087,0.6600828391566468,11,2,2,1.0
61
- Ocean,Seasonal-Naive,1.0,1.0,12,2,2,1.0
62
  Transportation,Seasonal-Naive,1.0,1.0,1,1,1,1.0
63
  Transportation,TabPFN-TS,1.0,1.0,2,1,1,1.0
64
  Transportation,TimesFM-2.5,1.0,1.0,3,1,1,1.0
 
1
  domain,model,MSE,CRPS,Rank,n_configs,n_group_configs,coverage
2
+ Climate,Toto-1.0,2.254579608553505,0.24437397891735274,1,1,1,1.0
3
+ Climate,TiRex,0.09355367378402578,0.3066903335793479,2,1,1,1.0
4
+ Climate,ARIMA,0.23366136013662836,0.31955206633207156,3,1,1,1.0
5
+ Climate,Moirai-2.0,0.10106927673354417,0.3353261386571622,4,1,1,1.0
6
+ Climate,TimesFM-2.5,0.15205093135244388,0.34904613835122883,5,1,1,1.0
7
+ Climate,Chronos-Bolt,0.2429913807934176,0.3510211648504888,6,1,1,1.0
8
+ Climate,Chronos-2,0.1600748903303435,0.37661088916030827,7,1,1,1.0
9
+ Climate,TabPFN-TS,0.2675510042683412,0.40509292646312733,8,1,1,1.0
10
+ Climate,Sundial,0.5926489229544686,0.455575861200003,9,1,1,1.0
11
+ Climate,Seasonal-Naive,1.0,1.0,10,1,1,1.0
12
+ Climate,Moving-Average,1.050311963095287,1.0855336597040712,11,1,1,1.0
13
+ Climate,ETS,5.263660955277566,2.3520525011376643,12,1,1,1.0
14
  Finance,Chronos-Bolt,0.19298771011845486,0.48437065483374686,1,2,2,1.0
15
  Finance,Moirai-2.0,0.2994100671855118,0.49358388757082117,2,2,2,1.0
16
  Finance,Sundial,0.2577663168744378,0.49342482292609496,3,2,2,1.0
 
35
  Hazards,TimesFM-2.5,0.6081084151826099,0.6706133785407319,10,1,1,1.0
36
  Hazards,Seasonal-Naive,1.0,1.0,11,1,1,1.0
37
  Hazards,Toto-1.0,3.019745444328402,1.1961880424879558,12,1,1,1.0
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
38
  Transportation,Seasonal-Naive,1.0,1.0,1,1,1,1.0
39
  Transportation,TabPFN-TS,1.0,1.0,2,1,1,1.0
40
  Transportation,TimesFM-2.5,1.0,1.0,3,1,1,1.0
results/aggregates/results_by_frequency.csv CHANGED
@@ -1,28 +1,4 @@
1
  frequency,model,MSE,CRPS,Rank,n_configs,n_group_configs,coverage
2
- 10min,ARIMA,0.3999025881023859,0.5582836232999162,1,1,1,1.0
3
- 10min,Moving-Average,0.5824978820079671,0.7101225153735382,2,1,1,1.0
4
- 10min,Sundial,0.6557649216589705,0.7214286117868186,3,1,1,1.0
5
- 10min,TabPFN-TS,0.520290822706649,0.7497734893148866,4,1,1,1.0
6
- 10min,TiRex,0.811373083056152,0.7501124646461534,5,1,1,1.0
7
- 10min,Moirai-2.0,0.3937216131513764,0.7516908499435677,6,1,1,1.0
8
- 10min,TimesFM-2.5,0.6937196205408115,0.8176063799390647,7,1,1,1.0
9
- 10min,Toto-1.0,2.512628461419206,0.8218858571642068,8,1,1,1.0
10
- 10min,Chronos-Bolt,0.8654655606708073,0.8319712196183959,9,1,1,1.0
11
- 10min,Chronos-2,0.4449822080646386,0.88029165470227,10,1,1,1.0
12
- 10min,Seasonal-Naive,1.0,1.0,11,1,1,1.0
13
- 10min,ETS,1.5161334578715508,1.1227047976128204,12,1,1,1.0
14
- 15min,ARIMA,1.0822972356882135,0.32310253910911185,1,1,1,1.0
15
- 15min,ETS,0.47591835122996634,0.8489234892846661,2,1,1,1.0
16
- 15min,Seasonal-Naive,1.0,1.0,3,1,1,1.0
17
- 15min,Toto-1.0,1.141911340572928,1.0811352940913546,4,1,1,1.0
18
- 15min,TabPFN-TS,0.8791481146033557,1.162889438787299,5,1,1,1.0
19
- 15min,TimesFM-2.5,0.8798555794555492,1.174107737396118,6,1,1,1.0
20
- 15min,TiRex,0.8814352809130944,1.178397901985136,7,1,1,1.0
21
- 15min,Chronos-Bolt,0.87251738236926,1.1786025093498782,8,1,1,1.0
22
- 15min,Sundial,0.9997852855837929,1.1947861396031385,9,1,1,1.0
23
- 15min,Chronos-2,0.9390817024400493,1.212577711258179,10,1,1,1.0
24
- 15min,Moirai-2.0,1.1021814117726327,1.235944432467708,11,1,1,1.0
25
- 15min,Moving-Average,0.9835742550209795,1.2360581940724533,12,1,1,1.0
26
  1D,Sundial,0.20244880832897819,0.31071330341508385,1,1,1,1.0
27
  1D,TabPFN-TS,0.17052198003432278,0.3202668885632687,2,1,1,1.0
28
  1D,Chronos-2,0.26094607467981124,0.3476579918970984,3,1,1,1.0
@@ -35,18 +11,18 @@ frequency,model,MSE,CRPS,Rank,n_configs,n_group_configs,coverage
35
  1D,ARIMA,0.5322502000225762,0.5679944207315325,10,1,1,1.0
36
  1D,ETS,0.6173397898668767,0.5782539888826587,11,1,1,1.0
37
  1D,Seasonal-Naive,1.0,1.0,12,1,1,1.0
38
- 1h,Moirai-2.0,0.42706170997622483,0.6694618916846242,1,6,6,1.0
39
- 1h,Chronos-2,0.5848055540054211,0.7137591138496754,2,6,6,1.0
40
- 1h,Sundial,0.6658170224476011,0.6984444640115564,3,6,6,1.0
41
- 1h,Toto-1.0,3.526873374076283,0.7107403438510198,4,6,6,1.0
42
- 1h,Chronos-Bolt,0.4832998950136812,0.6574042208929408,5,6,6,1.0
43
- 1h,TiRex,0.6230286673571245,0.7318235123024881,6,6,6,1.0
44
- 1h,ARIMA,2.037937992457479,1.1709365965730085,7,6,6,1.0
45
- 1h,TimesFM-2.5,0.7119865543368777,0.8981156417687401,8,6,6,1.0
46
- 1h,Seasonal-Naive,1.0,1.0,9,6,6,1.0
47
- 1h,TabPFN-TS,0.835828125394398,0.95826976716502,10,6,6,1.0
48
- 1h,Moving-Average,1.7697147947224021,1.395902584467836,11,6,6,1.0
49
- 1h,ETS,5.113933654395343,2.378074752343176,12,6,6,1.0
50
  1mo,Moirai-2.0,0.5701893078499076,0.5711658118752873,1,1,1,1.0
51
  1mo,Chronos-Bolt,0.6722527938921684,0.6335328970612374,2,1,1,1.0
52
  1mo,Toto-1.0,0.6317640740160342,0.6385745807295035,3,1,1,1.0
@@ -59,27 +35,15 @@ frequency,model,MSE,CRPS,Rank,n_configs,n_group_configs,coverage
59
  1mo,Moving-Average,0.9249973049108767,0.8887079460534169,10,1,1,1.0
60
  1mo,TabPFN-TS,0.8352213750339312,0.8925810519947165,11,1,1,1.0
61
  1mo,Seasonal-Naive,1.0,1.0,12,1,1,1.0
62
- 5min,Seasonal-Naive,1.0,1.0,1,2,2,1.0
63
- 5min,TabPFN-TS,211.88124381182288,13927.890449264021,2,2,2,1.0
64
- 5min,TiRex,726.1597103015184,844456.8028719203,3,2,2,1.0
65
- 5min,Moving-Average,58637.79047858731,500.21709109036453,4,2,2,1.0
66
- 5min,ETS,58637.70093556705,365259.43770170194,5,2,2,1.0
67
- 5min,TimesFM-2.5,1322.2154037605794,88738.17215266943,6,2,2,1.0
68
- 5min,Chronos-2,3743.6140859211073,2588930.810804653,7,2,2,1.0
69
- 5min,Moirai-2.0,1772.260827362009,22537426.786567364,8,2,2,1.0
70
- 5min,Sundial,88613.44471869215,21415959.409888003,9,2,2,1.0
71
- 5min,Chronos-Bolt,2449.2600992990842,109229487.28537717,10,2,2,1.0
72
- 5min,ARIMA,59842.15474163591,73361911.47380109,11,2,2,1.0
73
- 5min,Toto-1.0,713714701360.155,1031266709.5242426,12,2,2,1.0
74
- 6min,Sundial,0.033087205304759594,0.12493628266030873,1,1,1,1.0
75
- 6min,TimesFM-2.5,0.03995347788319179,0.15155368028786126,2,1,1,1.0
76
- 6min,Chronos-2,0.039134186859879565,0.15357240737810884,3,1,1,1.0
77
- 6min,TiRex,0.07991233552828132,0.23298119602415393,4,1,1,1.0
78
- 6min,ETS,0.14323752992989033,0.36135548642175336,5,1,1,1.0
79
- 6min,TabPFN-TS,0.20793868175731695,0.4388565816190716,6,1,1,1.0
80
- 6min,Moirai-2.0,0.3569184593053172,0.4759818673937215,7,1,1,1.0
81
- 6min,Chronos-Bolt,0.3614054209283652,0.4977137169850308,8,1,1,1.0
82
- 6min,ARIMA,0.2758348486471329,0.5029319580322164,9,1,1,1.0
83
- 6min,Toto-1.0,0.518263239264597,0.5301336563357459,10,1,1,1.0
84
- 6min,Moving-Average,0.9983204583667928,0.9872653206711439,11,1,1,1.0
85
- 6min,Seasonal-Naive,1.0,1.0,12,1,1,1.0
 
1
  frequency,model,MSE,CRPS,Rank,n_configs,n_group_configs,coverage
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2
  1D,Sundial,0.20244880832897819,0.31071330341508385,1,1,1,1.0
3
  1D,TabPFN-TS,0.17052198003432278,0.3202668885632687,2,1,1,1.0
4
  1D,Chronos-2,0.26094607467981124,0.3476579918970984,3,1,1,1.0
 
11
  1D,ARIMA,0.5322502000225762,0.5679944207315325,10,1,1,1.0
12
  1D,ETS,0.6173397898668767,0.5782539888826587,11,1,1,1.0
13
  1D,Seasonal-Naive,1.0,1.0,12,1,1,1.0
14
+ 1h,ARIMA,0.5558775401967115,0.46519818612525576,1,3,3,1.0
15
+ 1h,Chronos-Bolt,0.20172577513083545,0.4352544883543161,2,3,3,1.0
16
+ 1h,Sundial,0.3257175250604903,0.43972492932202084,3,3,3,1.0
17
+ 1h,Moirai-2.0,0.2138255193374551,0.45628899031637443,4,3,3,1.0
18
+ 1h,TiRex,0.3354460031718085,0.5174975330072453,5,3,3,1.0
19
+ 1h,Chronos-2,0.37695966984166646,0.5351244675546674,6,3,3,1.0
20
+ 1h,Toto-1.0,2.2145419802222572,0.6419067079584152,7,3,3,1.0
21
+ 1h,TimesFM-2.5,0.42272346010265477,0.6533100670024531,8,3,3,1.0
22
+ 1h,ETS,1.4325141827653676,1.1686664176050945,9,3,3,1.0
23
+ 1h,Moving-Average,0.854086260368711,0.9485677941114191,10,3,3,1.0
24
+ 1h,TabPFN-TS,0.5554543516234263,0.7249240054757624,11,3,3,1.0
25
+ 1h,Seasonal-Naive,1.0,1.0,12,3,3,1.0
26
  1mo,Moirai-2.0,0.5701893078499076,0.5711658118752873,1,1,1,1.0
27
  1mo,Chronos-Bolt,0.6722527938921684,0.6335328970612374,2,1,1,1.0
28
  1mo,Toto-1.0,0.6317640740160342,0.6385745807295035,3,1,1,1.0
 
35
  1mo,Moving-Average,0.9249973049108767,0.8887079460534169,10,1,1,1.0
36
  1mo,TabPFN-TS,0.8352213750339312,0.8925810519947165,11,1,1,1.0
37
  1mo,Seasonal-Naive,1.0,1.0,12,1,1,1.0
38
+ 5min,Seasonal-Naive,1.0,1.0,1,1,1,1.0
39
+ 5min,TabPFN-TS,1.0,1.0,2,1,1,1.0
40
+ 5min,TimesFM-2.5,1.0,1.0,3,1,1,1.0
41
+ 5min,TiRex,1.0,1397.7486927145103,4,1,1,1.0
42
+ 5min,Chronos-2,1.0,2356.162747109683,5,1,1,1.0
43
+ 5min,Sundial,1.0,21009.616078147144,6,1,1,1.0
44
+ 5min,ARIMA,1.0,250217.138218906,7,1,1,1.0
45
+ 5min,ETS,1.0,250217.138218906,8,1,1,1.0
46
+ 5min,Moving-Average,1.0,250217.138218906,9,1,1,1.0
47
+ 5min,Moirai-2.0,1.0,410051.5180343537,10,1,1,1.0
48
+ 5min,Chronos-Bolt,1.094103907762285,3214861.8499168884,11,1,1,1.0
49
+ 5min,Toto-1.0,593863598336.2706,266721884.25299966,12,1,1,1.0
 
 
 
 
 
 
 
 
 
 
 
 
results/aggregates/results_by_prediction_length.csv CHANGED
@@ -23,51 +23,15 @@ prediction_length,model,MSE,CRPS,Rank,n_configs,n_group_configs,coverage
23
  12,ETS,0.8706436180201554,439.02846065451104,10,2,2,1.0
24
  12,Toto-1.0,612520.7639702874,13050.740032972935,11,2,2,1.0
25
  12,Moving-Average,0.9617678019724286,471.56119324419365,12,2,2,1.0
26
- 24,Toto-1.0,3.0020955650038634,0.7546318341191678,1,7,7,1.0
27
- 24,Chronos-Bolt,0.5258571170027158,0.7145814791494673,2,7,7,1.0
28
- 24,Chronos-2,0.6257430638031326,0.7698947688397185,3,7,7,1.0
29
- 24,Sundial,0.7056292687940565,0.7541195634858877,4,7,7,1.0
30
- 24,Moirai-2.0,0.4890056099901845,0.7307433511677697,5,7,7,1.0
31
- 24,TiRex,0.6546875578470471,0.7833600951808131,6,7,7,1.0
32
- 24,ARIMA,1.861776214920616,0.9742012089071099,7,7,7,1.0
33
- 24,TimesFM-2.5,0.7338478408697692,0.9331626736621227,8,7,7,1.0
34
- 24,Seasonal-Naive,1.0,1.0,9,7,7,1.0
35
- 24,TabPFN-TS,0.8418834859929512,0.9851333483285372,10,7,7,1.0
36
- 24,ETS,3.642818803797478,2.052661639172763,11,7,7,1.0
37
- 24,Moving-Average,1.627275093911534,1.3718604659576152,12,7,7,1.0
38
- 36,Seasonal-Naive,1.0,1.0,1,1,1,1.0
39
- 36,Moving-Average,3438390472.2107043,1.0,2,1,1,1.0
40
- 36,ETS,3438379971.009001,533194.7195137535,3,1,1,1.0
41
- 36,TabPFN-TS,44893.66147924513,193986132.36669993,4,1,1,1.0
42
- 36,TiRex,527307.924865185,510182764.3506999,5,1,1,1.0
43
- 36,Moirai-2.0,3140908.4402018725,1238711683.3386016,6,1,1,1.0
44
- 36,Chronos-2,14014646.42430693,2844694302.7835035,7,1,1,1.0
45
- 36,Chronos-Bolt,5482911.624260396,3711226624.8500977,8,1,1,1.0
46
- 36,Toto-1.0,857753659871.8685,3987340705.6624064,9,1,1,1.0
47
- 36,TimesFM-2.5,1748253.5739417518,7874463196.996796,10,1,1,1.0
48
- 36,ARIMA,3581083484.121897,21509198344.284977,11,1,1,1.0
49
- 36,Sundial,7852342584.912709,21830161757.359337,12,1,1,1.0
50
- 60,Sundial,0.033087205304759594,0.12493628266030873,1,1,1,1.0
51
- 60,TimesFM-2.5,0.03995347788319179,0.15155368028786126,2,1,1,1.0
52
- 60,Chronos-2,0.039134186859879565,0.15357240737810884,3,1,1,1.0
53
- 60,TiRex,0.07991233552828132,0.23298119602415393,4,1,1,1.0
54
- 60,ETS,0.14323752992989033,0.36135548642175336,5,1,1,1.0
55
- 60,TabPFN-TS,0.20793868175731695,0.4388565816190716,6,1,1,1.0
56
- 60,Moirai-2.0,0.3569184593053172,0.4759818673937215,7,1,1,1.0
57
- 60,Chronos-Bolt,0.3614054209283652,0.4977137169850308,8,1,1,1.0
58
- 60,ARIMA,0.2758348486471329,0.5029319580322164,9,1,1,1.0
59
- 60,Toto-1.0,0.518263239264597,0.5301336563357459,10,1,1,1.0
60
- 60,Moving-Average,0.9983204583667928,0.9872653206711439,11,1,1,1.0
61
- 60,Seasonal-Naive,1.0,1.0,12,1,1,1.0
62
- 72,ARIMA,0.3999025881023859,0.5582836232999162,1,1,1,1.0
63
- 72,Moving-Average,0.5824978820079671,0.7101225153735382,2,1,1,1.0
64
- 72,Sundial,0.6557649216589705,0.7214286117868186,3,1,1,1.0
65
- 72,TabPFN-TS,0.520290822706649,0.7497734893148866,4,1,1,1.0
66
- 72,TiRex,0.811373083056152,0.7501124646461534,5,1,1,1.0
67
- 72,Moirai-2.0,0.3937216131513764,0.7516908499435677,6,1,1,1.0
68
- 72,TimesFM-2.5,0.6937196205408115,0.8176063799390647,7,1,1,1.0
69
- 72,Toto-1.0,2.512628461419206,0.8218858571642068,8,1,1,1.0
70
- 72,Chronos-Bolt,0.8654655606708073,0.8319712196183959,9,1,1,1.0
71
- 72,Chronos-2,0.4449822080646386,0.88029165470227,10,1,1,1.0
72
- 72,Seasonal-Naive,1.0,1.0,11,1,1,1.0
73
- 72,ETS,1.5161334578715508,1.1227047976128204,12,1,1,1.0
 
23
  12,ETS,0.8706436180201554,439.02846065451104,10,2,2,1.0
24
  12,Toto-1.0,612520.7639702874,13050.740032972935,11,2,2,1.0
25
  12,Moving-Average,0.9617678019724286,471.56119324419365,12,2,2,1.0
26
+ 24,ARIMA,0.5558775401967115,0.46519818612525576,1,3,3,1.0
27
+ 24,Chronos-Bolt,0.20172577513083545,0.4352544883543161,2,3,3,1.0
28
+ 24,Sundial,0.3257175250604903,0.43972492932202084,3,3,3,1.0
29
+ 24,Moirai-2.0,0.2138255193374551,0.45628899031637443,4,3,3,1.0
30
+ 24,TiRex,0.3354460031718085,0.5174975330072453,5,3,3,1.0
31
+ 24,Chronos-2,0.37695966984166646,0.5351244675546674,6,3,3,1.0
32
+ 24,Toto-1.0,2.2145419802222572,0.6419067079584152,7,3,3,1.0
33
+ 24,TimesFM-2.5,0.42272346010265477,0.6533100670024531,8,3,3,1.0
34
+ 24,ETS,1.4325141827653676,1.1686664176050945,9,3,3,1.0
35
+ 24,Moving-Average,0.854086260368711,0.9485677941114191,10,3,3,1.0
36
+ 24,TabPFN-TS,0.5554543516234263,0.7249240054757624,11,3,3,1.0
37
+ 24,Seasonal-Naive,1.0,1.0,12,3,3,1.0
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
results/aggregates/results_overall.csv CHANGED
@@ -1,13 +1,13 @@
1
  overall,model,MSE,CRPS,Rank,n_configs,n_group_configs,coverage
2
- Overall,TiRex,1.5577704491145108,5.593178711132306,1,13,13,1.0
3
- Overall,Sundial,3.078467226223764,8.597542108923774,2,13,13,1.0
4
- Overall,Chronos-2,1.7677221820677038,6.436083026995778,3,13,13,1.0
5
- Overall,Moirai-2.0,1.6184303236224802,9.383904410274479,4,13,13,1.0
6
- Overall,TabPFN-TS,1.5073310475752202,3.5896314694438605,5,13,13,1.0
7
- Overall,TimesFM-2.5,1.7132890440912785,4.318465472960177,6,13,13,1.0
8
- Overall,Chronos-Bolt,1.9663081211624194,12.359885771340469,7,13,13,1.0
9
- Overall,Toto-1.0,117.69708657663212,18.13233062156029,8,13,13,1.0
10
- Overall,ARIMA,5.967419681569124,13.610299368467002,9,13,13,1.0
11
- Overall,Seasonal-Naive,1.0,1.0,10,13,13,1.0
12
- Overall,Moving-Average,6.364136586323452,2.8304357683470944,11,13,13,1.0
13
- Overall,ETS,9.109479347989522,9.264050076456916,12,13,13,1.0
 
1
  overall,model,MSE,CRPS,Rank,n_configs,n_group_configs,coverage
2
+ Overall,Sundial,0.4161387121497312,2.7546398606440126,1,6,6,1.0
3
+ Overall,TiRex,0.43539080682635134,1.8860974386048113,2,6,6,1.0
4
+ Overall,Chronos-2,0.4611429428103631,2.079258376408379,3,6,6,1.0
5
+ Overall,ARIMA,0.6392569655789688,4.716401940292018,4,6,6,1.0
6
+ Overall,Moirai-2.0,0.3465227168859271,4.511199902395067,5,6,6,1.0
7
+ Overall,Chronos-Bolt,0.37604293977591025,6.676902067763457,6,6,6,1.0
8
+ Overall,TimesFM-2.5,0.49609368733506126,0.662116812817791,7,6,6,1.0
9
+ Overall,TabPFN-TS,0.5385853234421009,0.6910447809453072,8,6,6,1.0
10
+ Overall,Toto-1.0,124.25070484488315,17.102035614829397,9,6,6,1.0
11
+ Overall,Moving-Average,0.8134459607935641,6.80863763093453,10,6,6,1.0
12
+ Overall,ETS,1.0545900039036709,7.4994476883848575,11,6,6,1.0
13
+ Overall,Seasonal-Naive,1.0,1.0,12,6,6,1.0
results/dataset_eval_intervals.json CHANGED
@@ -1,16 +1,8 @@
1
  {
2
  "binance_btcusdt_1h_close": 300,
3
- "coingecko_bitcoin_7d_price_usd": 300,
4
  "data_gov_sg_sgx_equities_turnover_monthly_turnover_value_million_sgd": 2592000,
5
- "data_gov_sg_singapore_rainfall_5min_rainfall_5min_total": 900,
6
  "data_gov_sg_singapore_taxi_availability_available_taxi_count": 300,
7
- "nasa_power_shanghai_hourly_T2M": 3600,
8
- "noaa_coops_sf_water_level_water_level": 360,
9
- "noaa_ndbc_46013_realtime_wave_height": 600,
10
  "nws_ksfo_observations_temperature": 3600,
11
- "open_meteo_shanghai_air_quality_hourly_pm2_5": 3600,
12
- "open_meteo_shanghai_hourly_temperature_2m": 3600,
13
  "usgs_earthquake_all_week_hourly_earthquake_count": 3600,
14
- "usgs_potomac_iv_usgs_00060": 900,
15
  "wikimedia_time_series_hourly_pageviews": 86400
16
  }
 
1
  {
2
  "binance_btcusdt_1h_close": 300,
 
3
  "data_gov_sg_sgx_equities_turnover_monthly_turnover_value_million_sgd": 2592000,
 
4
  "data_gov_sg_singapore_taxi_availability_available_taxi_count": 300,
 
 
 
5
  "nws_ksfo_observations_temperature": 3600,
 
 
6
  "usgs_earthquake_all_week_hourly_earthquake_count": 3600,
 
7
  "wikimedia_time_series_hourly_pageviews": 86400
8
  }
results/dataset_properties.csv CHANGED
@@ -1,14 +1,7 @@
1
  dataset,domain,frequency,num_variates
2
  binance_btcusdt_1h_close,Finance,1h,1
3
  data_gov_sg_sgx_equities_turnover_monthly_turnover_value_million_sgd,Finance,M,1
4
- data_gov_sg_singapore_rainfall_5min_rainfall_5min_total,Climate,5min,1
5
  data_gov_sg_singapore_taxi_availability_available_taxi_count,Transportation,5min,1
6
- nasa_power_shanghai_hourly_t2m,Weather,1h,1
7
- noaa_coops_sf_water_level_water_level,Ocean,6min,1
8
- noaa_ndbc_46013_realtime_wave_height,Ocean,10min,1
9
  nws_ksfo_observations_temperature,Weather,1h,1
10
- open_meteo_shanghai_air_quality_hourly_pm2_5,Air Quality,1h,1
11
- open_meteo_shanghai_hourly_temperature_2m,Weather,1h,1
12
  usgs_earthquake_all_week_hourly_earthquake_count,Hazards,1h,1
13
- usgs_potomac_iv_usgs_00060,Hydrology,15min,1
14
  wikimedia_time_series_hourly_pageviews,Web,1D,1
 
1
  dataset,domain,frequency,num_variates
2
  binance_btcusdt_1h_close,Finance,1h,1
3
  data_gov_sg_sgx_equities_turnover_monthly_turnover_value_million_sgd,Finance,M,1
 
4
  data_gov_sg_singapore_taxi_availability_available_taxi_count,Transportation,5min,1
 
 
 
5
  nws_ksfo_observations_temperature,Weather,1h,1
 
 
6
  usgs_earthquake_all_week_hourly_earthquake_count,Hazards,1h,1
 
7
  wikimedia_time_series_hourly_pageviews,Web,1D,1
results/online_status.json CHANGED
@@ -1,17 +1,23 @@
1
  {
2
- "status": "ok",
3
  "evaluation_protocol": "issue-now-score-later",
4
  "metric_version": "livehouse-canonical-v1",
5
- "started_at": "2026-07-18T03:51:19.015149+00:00",
6
- "finished_at": "2026-07-18T03:51:21.756816+00:00",
7
  "data_source": "ts_bench",
8
  "data_config": "configs/datasets/ts_bench.yaml",
9
- "task_count": 1,
10
  "issued_tasks": 0,
11
  "issued_forecasts": 0,
12
  "resolved_forecasts": 0,
13
  "pending_tasks": 13,
14
- "failed_forecasts": [],
 
 
 
 
 
 
15
  "models": {
16
  "Chronos-2": {
17
  "model": "Chronos-2",
@@ -19,7 +25,7 @@
19
  "admitted_at": "2026-06-01T00:00:00Z",
20
  "evaluation_protocol": "issue-now-score-later",
21
  "metric_version": "livehouse-canonical-v1",
22
- "evaluated_at": "2026-07-18T03:51:19.114816+00:00",
23
  "resolved_tasks": []
24
  },
25
  "TiRex": {
@@ -28,7 +34,7 @@
28
  "admitted_at": "2026-06-01T00:00:00Z",
29
  "evaluation_protocol": "issue-now-score-later",
30
  "metric_version": "livehouse-canonical-v1",
31
- "evaluated_at": "2026-07-18T03:51:19.114824+00:00",
32
  "resolved_tasks": []
33
  },
34
  "TimesFM-2.5": {
@@ -37,7 +43,7 @@
37
  "admitted_at": "2026-06-01T00:00:00Z",
38
  "evaluation_protocol": "issue-now-score-later",
39
  "metric_version": "livehouse-canonical-v1",
40
- "evaluated_at": "2026-07-18T03:51:19.114825+00:00",
41
  "resolved_tasks": []
42
  },
43
  "Toto-1.0": {
@@ -46,7 +52,7 @@
46
  "admitted_at": "2026-06-01T00:00:00Z",
47
  "evaluation_protocol": "issue-now-score-later",
48
  "metric_version": "livehouse-canonical-v1",
49
- "evaluated_at": "2026-07-18T03:51:19.114827+00:00",
50
  "resolved_tasks": []
51
  },
52
  "Moirai-2.0": {
@@ -55,7 +61,7 @@
55
  "admitted_at": "2026-06-01T00:00:00Z",
56
  "evaluation_protocol": "issue-now-score-later",
57
  "metric_version": "livehouse-canonical-v1",
58
- "evaluated_at": "2026-07-18T03:51:19.114828+00:00",
59
  "resolved_tasks": []
60
  },
61
  "Chronos-Bolt": {
@@ -64,7 +70,7 @@
64
  "admitted_at": "2026-06-01T00:00:00Z",
65
  "evaluation_protocol": "issue-now-score-later",
66
  "metric_version": "livehouse-canonical-v1",
67
- "evaluated_at": "2026-07-18T03:51:19.114829+00:00",
68
  "resolved_tasks": []
69
  },
70
  "TabPFN-TS": {
@@ -73,7 +79,7 @@
73
  "admitted_at": "2026-06-01T00:00:00Z",
74
  "evaluation_protocol": "issue-now-score-later",
75
  "metric_version": "livehouse-canonical-v1",
76
- "evaluated_at": "2026-07-18T03:51:19.114830+00:00",
77
  "resolved_tasks": []
78
  },
79
  "Sundial": {
@@ -82,7 +88,7 @@
82
  "admitted_at": "2026-06-01T00:00:00Z",
83
  "evaluation_protocol": "issue-now-score-later",
84
  "metric_version": "livehouse-canonical-v1",
85
- "evaluated_at": "2026-07-18T03:51:19.114831+00:00",
86
  "resolved_tasks": []
87
  },
88
  "Moving-Average": {
@@ -91,7 +97,7 @@
91
  "admitted_at": "2026-06-01T00:00:00Z",
92
  "evaluation_protocol": "issue-now-score-later",
93
  "metric_version": "livehouse-canonical-v1",
94
- "evaluated_at": "2026-07-18T03:51:19.114832+00:00",
95
  "resolved_tasks": []
96
  },
97
  "ETS": {
@@ -100,7 +106,7 @@
100
  "admitted_at": "2026-06-01T00:00:00Z",
101
  "evaluation_protocol": "issue-now-score-later",
102
  "metric_version": "livehouse-canonical-v1",
103
- "evaluated_at": "2026-07-18T03:51:19.114833+00:00",
104
  "resolved_tasks": []
105
  },
106
  "ARIMA": {
@@ -109,7 +115,7 @@
109
  "admitted_at": "2026-06-01T00:00:00Z",
110
  "evaluation_protocol": "issue-now-score-later",
111
  "metric_version": "livehouse-canonical-v1",
112
- "evaluated_at": "2026-07-18T03:51:19.114834+00:00",
113
  "resolved_tasks": []
114
  },
115
  "Seasonal-Naive": {
@@ -118,7 +124,7 @@
118
  "admitted_at": "2026-06-01T00:00:00Z",
119
  "evaluation_protocol": "issue-now-score-later",
120
  "metric_version": "livehouse-canonical-v1",
121
- "evaluated_at": "2026-07-18T03:51:19.114835+00:00",
122
  "resolved_tasks": []
123
  }
124
  },
@@ -142,7 +148,5 @@
142
  "live_overall_30d": "aggregates/live_overall_30d.csv",
143
  "live_metadata": "aggregates/live_metadata.json"
144
  },
145
- "push_status": "ok",
146
- "pushed_at": "2026-07-18T03:51:40.115258+00:00",
147
- "push_detail": "HF Hub API"
148
  }
 
1
  {
2
+ "status": "partial",
3
  "evaluation_protocol": "issue-now-score-later",
4
  "metric_version": "livehouse-canonical-v1",
5
+ "started_at": "2026-07-18T08:07:50.652374+00:00",
6
+ "finished_at": "2026-07-18T08:07:52.776646+00:00",
7
  "data_source": "ts_bench",
8
  "data_config": "configs/datasets/ts_bench.yaml",
9
+ "task_count": 3,
10
  "issued_tasks": 0,
11
  "issued_forecasts": 0,
12
  "resolved_forecasts": 0,
13
  "pending_tasks": 13,
14
+ "failed_forecasts": [
15
+ {
16
+ "model": "Seasonal-Naive",
17
+ "dataset": "nws_ksfo_observations_temperature/1h/short",
18
+ "error": "Mean forecast length 10 < 24"
19
+ }
20
+ ],
21
  "models": {
22
  "Chronos-2": {
23
  "model": "Chronos-2",
 
25
  "admitted_at": "2026-06-01T00:00:00Z",
26
  "evaluation_protocol": "issue-now-score-later",
27
  "metric_version": "livehouse-canonical-v1",
28
+ "evaluated_at": "2026-07-18T08:07:50.742287+00:00",
29
  "resolved_tasks": []
30
  },
31
  "TiRex": {
 
34
  "admitted_at": "2026-06-01T00:00:00Z",
35
  "evaluation_protocol": "issue-now-score-later",
36
  "metric_version": "livehouse-canonical-v1",
37
+ "evaluated_at": "2026-07-18T08:07:50.742291+00:00",
38
  "resolved_tasks": []
39
  },
40
  "TimesFM-2.5": {
 
43
  "admitted_at": "2026-06-01T00:00:00Z",
44
  "evaluation_protocol": "issue-now-score-later",
45
  "metric_version": "livehouse-canonical-v1",
46
+ "evaluated_at": "2026-07-18T08:07:50.742292+00:00",
47
  "resolved_tasks": []
48
  },
49
  "Toto-1.0": {
 
52
  "admitted_at": "2026-06-01T00:00:00Z",
53
  "evaluation_protocol": "issue-now-score-later",
54
  "metric_version": "livehouse-canonical-v1",
55
+ "evaluated_at": "2026-07-18T08:07:50.742293+00:00",
56
  "resolved_tasks": []
57
  },
58
  "Moirai-2.0": {
 
61
  "admitted_at": "2026-06-01T00:00:00Z",
62
  "evaluation_protocol": "issue-now-score-later",
63
  "metric_version": "livehouse-canonical-v1",
64
+ "evaluated_at": "2026-07-18T08:07:50.742294+00:00",
65
  "resolved_tasks": []
66
  },
67
  "Chronos-Bolt": {
 
70
  "admitted_at": "2026-06-01T00:00:00Z",
71
  "evaluation_protocol": "issue-now-score-later",
72
  "metric_version": "livehouse-canonical-v1",
73
+ "evaluated_at": "2026-07-18T08:07:50.742295+00:00",
74
  "resolved_tasks": []
75
  },
76
  "TabPFN-TS": {
 
79
  "admitted_at": "2026-06-01T00:00:00Z",
80
  "evaluation_protocol": "issue-now-score-later",
81
  "metric_version": "livehouse-canonical-v1",
82
+ "evaluated_at": "2026-07-18T08:07:50.742296+00:00",
83
  "resolved_tasks": []
84
  },
85
  "Sundial": {
 
88
  "admitted_at": "2026-06-01T00:00:00Z",
89
  "evaluation_protocol": "issue-now-score-later",
90
  "metric_version": "livehouse-canonical-v1",
91
+ "evaluated_at": "2026-07-18T08:07:50.742297+00:00",
92
  "resolved_tasks": []
93
  },
94
  "Moving-Average": {
 
97
  "admitted_at": "2026-06-01T00:00:00Z",
98
  "evaluation_protocol": "issue-now-score-later",
99
  "metric_version": "livehouse-canonical-v1",
100
+ "evaluated_at": "2026-07-18T08:07:50.742298+00:00",
101
  "resolved_tasks": []
102
  },
103
  "ETS": {
 
106
  "admitted_at": "2026-06-01T00:00:00Z",
107
  "evaluation_protocol": "issue-now-score-later",
108
  "metric_version": "livehouse-canonical-v1",
109
+ "evaluated_at": "2026-07-18T08:07:50.742298+00:00",
110
  "resolved_tasks": []
111
  },
112
  "ARIMA": {
 
115
  "admitted_at": "2026-06-01T00:00:00Z",
116
  "evaluation_protocol": "issue-now-score-later",
117
  "metric_version": "livehouse-canonical-v1",
118
+ "evaluated_at": "2026-07-18T08:07:50.742299+00:00",
119
  "resolved_tasks": []
120
  },
121
  "Seasonal-Naive": {
 
124
  "admitted_at": "2026-06-01T00:00:00Z",
125
  "evaluation_protocol": "issue-now-score-later",
126
  "metric_version": "livehouse-canonical-v1",
127
+ "evaluated_at": "2026-07-18T08:07:50.742300+00:00",
128
  "resolved_tasks": []
129
  }
130
  },
 
148
  "live_overall_30d": "aggregates/live_overall_30d.csv",
149
  "live_metadata": "aggregates/live_metadata.json"
150
  },
151
+ "push_status": "pending"
 
 
152
  }