Spaces:
Runtime error
Runtime error
Update leaderboard results from local eval
Browse files- results/aggregates/live_metadata.json +1 -1
- results/aggregates/metadata.json +1 -1
- results/chronos_2/all_results.csv +15 -3
- results/chronos_2/config.json +5 -2
- results/chronos_2/online_meta.json +12 -12
- results/dataset_properties.csv +7 -0
- results/online_status.json +142 -17
- results/timesfm_2_5/all_results.csv +15 -11
- results/timesfm_2_5/config.json +5 -2
- results/timesfm_2_5/online_meta.json +13 -61
results/aggregates/live_metadata.json
CHANGED
|
@@ -1,5 +1,5 @@
|
|
| 1 |
{
|
| 2 |
-
"generated_at": "2026-07-26T02:
|
| 3 |
"scope": "Canonical metrics-only release history when available; otherwise legacy release history plus latest allowlisted snapshots",
|
| 4 |
"release_deduplication": "One row per model, dataset, data source timestamp, and prediction length",
|
| 5 |
"dataset_aggregation": "Arithmetic mean over releases, then equal-weight mean over datasets",
|
|
|
|
| 1 |
{
|
| 2 |
+
"generated_at": "2026-07-26T02:42:44.183682+00:00",
|
| 3 |
"scope": "Canonical metrics-only release history when available; otherwise legacy release history plus latest allowlisted snapshots",
|
| 4 |
"release_deduplication": "One row per model, dataset, data source timestamp, and prediction length",
|
| 5 |
"dataset_aggregation": "Arithmetic mean over releases, then equal-weight mean over datasets",
|
results/aggregates/metadata.json
CHANGED
|
@@ -1,5 +1,5 @@
|
|
| 1 |
{
|
| 2 |
-
"generated_at": "2026-07-26T02:
|
| 3 |
"baseline_model": "Seasonal-Naive",
|
| 4 |
"normalization": "Per-config metric / Seasonal-Naive metric",
|
| 5 |
"metric_aggregation": "Geometric mean over dataset configurations",
|
|
|
|
| 1 |
{
|
| 2 |
+
"generated_at": "2026-07-26T02:42:41.549389+00:00",
|
| 3 |
"baseline_model": "Seasonal-Naive",
|
| 4 |
"normalization": "Per-config metric / Seasonal-Naive metric",
|
| 5 |
"metric_aggregation": "Geometric mean over dataset configurations",
|
results/chronos_2/all_results.csv
CHANGED
|
@@ -1,3 +1,15 @@
|
|
| 1 |
-
dataset,model,eval_metrics/MSE[mean],eval_metrics/MSE[0.5],eval_metrics/MAE[0.5],eval_metrics/MASE[0.5],eval_metrics/MAPE[0.5],eval_metrics/sMAPE[0.5],eval_metrics/MSIS,eval_metrics/RMSE[mean],eval_metrics/NRMSE[mean],eval_metrics/ND[0.5],eval_metrics/mean_weighted_sum_quantile_loss,domain,num_variates
|
| 2 |
-
binance_btcusdt_1h_close/1h/short,Chronos-2,
|
| 3 |
-
coingecko_bitcoin_7d_price_usd/1h/short,Chronos-2,
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
dataset,model,eval_metrics/MSE[mean],eval_metrics/MSE[0.5],eval_metrics/MAE[0.5],eval_metrics/MASE[0.5],eval_metrics/MAPE[0.5],eval_metrics/sMAPE[0.5],eval_metrics/MSIS,eval_metrics/RMSE[mean],eval_metrics/NRMSE[mean],eval_metrics/ND[0.5],eval_metrics/mean_weighted_sum_quantile_loss,domain,num_variates,prediction_length
|
| 2 |
+
binance_btcusdt_1h_close/1h/short,Chronos-2,0.08840222683743608,0.08840222683743608,nan,nan,0.002894309814063975,nan,nan,0.29732511975518666,nan,nan,0.23711727370139674,Finance,1,24
|
| 3 |
+
coingecko_bitcoin_7d_price_usd/1h/short,Chronos-2,2.2560049558764144,2.2560049558764144,nan,nan,0.013873897978694727,nan,nan,1.5020003182011694,nan,nan,0.9690457209453692,Finance,1,24
|
| 4 |
+
data_gov_sg_sgx_equities_turnover_monthly_turnover_value_million_sgd/1mo/short,Chronos-2,25067.461899718543,25067.461899718543,121.96686029434204,1.2859549577545721,0.6608019070822674,0.9393767274480805,18.969076544647905,158.32707254199624,0.9747204171363386,0.7508734105557137,0.605299940786381,Finance,1,12
|
| 5 |
+
data_gov_sg_singapore_rainfall_5min_rainfall_5min_total/5min/short,Chronos-2,3.4449184919363044e-06,3.4449184919363044e-06,nan,nan,nan,nan,nan,0.0018560491620472515,nan,nan,0.0016391544022115049,Climate,1,36
|
| 6 |
+
data_gov_sg_singapore_taxi_availability_available_taxi_count/5min/short,Chronos-2,1.7296862743653321e-18,1.7296862743653321e-18,nan,nan,nan,nan,nan,1.3151753777977035e-09,nan,nan,2.356162747109682e-09,Transportation,1,12
|
| 7 |
+
nasa_power_shanghai_hourly_T2M/1h/short,Chronos-2,0.12225909892650333,0.12225909892650333,nan,nan,0.01991489605396631,nan,nan,0.34965568624934923,nan,nan,0.22049499664734704,Weather,1,24
|
| 8 |
+
noaa_coops_sf_water_level_water_level/6min/short,Chronos-2,0.9233962613456854,0.9233962613456854,nan,nan,0.30187810637977674,nan,nan,0.9609350973638571,nan,nan,0.5827423701157508,Ocean,1,60
|
| 9 |
+
noaa_ndbc_46013_realtime_wave_height/10min/short,Chronos-2,1.9981401627691777,1.9981401627691777,nan,nan,0.12916795308136578,nan,nan,1.4135558576756624,nan,nan,0.9825547706029509,Ocean,1,72
|
| 10 |
+
nws_ksfo_observations_temperature/1h/short,Chronos-2,0.22763978935275495,0.22763978935275495,0.4551292260487874,0.3293397414624391,0.02734639793034648,0.02740698654518156,1.2449814398684094,0.4771161172636646,0.02853423165374086,0.027219291695583462,0.02293888289302154,Weather,1,24
|
| 11 |
+
open_meteo_shanghai_air_quality_hourly_pm2_5/1h/short,Chronos-2,0.1757138539928659,0.1757138539928659,nan,nan,0.730236178632242,nan,nan,0.4191823636472149,nan,nan,0.2986659566799537,Air Quality,1,24
|
| 12 |
+
open_meteo_shanghai_hourly_temperature_2m/1h/short,Chronos-2,0.058734737754144305,0.058734737754144305,nan,nan,0.02082097310795616,nan,nan,0.24235250721654253,nan,nan,0.19196935991416927,Weather,1,24
|
| 13 |
+
usgs_earthquake_all_week_hourly_earthquake_count/1h/short,Chronos-2,0.37843725573137094,0.37843725573137094,nan,nan,0.2922572074594943,nan,nan,0.6151725414315653,nan,nan,0.4434051867383657,Hazards,1,24
|
| 14 |
+
usgs_potomac_iv_usgs_00060/15min/short,Chronos-2,3.474611558180053,3.474611558180053,nan,nan,0.015378280340456496,nan,nan,1.8640309971081632,nan,nan,1.8566738730698593,Hydrology,1,24
|
| 15 |
+
wikimedia_time_series_hourly_pageviews/1D/short,Chronos-2,0.5379146607901764,0.5379146607901764,nan,nan,0.17100514688638668,nan,nan,0.7334266567218404,nan,nan,0.48842848142456025,Web,1,7
|
results/chronos_2/config.json
CHANGED
|
@@ -3,9 +3,12 @@
|
|
| 3 |
"model_type": "zero-shot",
|
| 4 |
"model_dtype": "float32",
|
| 5 |
"model_link": "https://tsfm.ai/models/amazon/chronos-2",
|
| 6 |
-
"code_link": "https://github.com/zhouziyu02/
|
| 7 |
"org": "Amazon / TSFM.ai",
|
| 8 |
"testdata_leakage": "No",
|
| 9 |
"replication_code_available": "Yes",
|
| 10 |
-
"api_model_id": "amazon/chronos-2"
|
|
|
|
|
|
|
|
|
|
| 11 |
}
|
|
|
|
| 3 |
"model_type": "zero-shot",
|
| 4 |
"model_dtype": "float32",
|
| 5 |
"model_link": "https://tsfm.ai/models/amazon/chronos-2",
|
| 6 |
+
"code_link": "https://github.com/zhouziyu02/TS-Live/blob/main/scripts/run_online_eval.py",
|
| 7 |
"org": "Amazon / TSFM.ai",
|
| 8 |
"testdata_leakage": "No",
|
| 9 |
"replication_code_available": "Yes",
|
| 10 |
+
"api_model_id": "amazon/chronos-2",
|
| 11 |
+
"admitted_at": "2026-06-01T00:00:00Z",
|
| 12 |
+
"evaluation_protocol": "issue-now-score-later",
|
| 13 |
+
"metric_version": "livehouse-canonical-v1"
|
| 14 |
}
|
results/chronos_2/online_meta.json
CHANGED
|
@@ -1,19 +1,19 @@
|
|
| 1 |
{
|
| 2 |
"model": "Chronos-2",
|
| 3 |
"api_model_id": "amazon/chronos-2",
|
| 4 |
-
"
|
| 5 |
-
"
|
|
|
|
|
|
|
|
|
|
| 6 |
{
|
| 7 |
-
"dataset": "
|
| 8 |
-
"
|
| 9 |
-
"
|
| 10 |
-
"
|
| 11 |
-
|
| 12 |
-
|
| 13 |
-
"
|
| 14 |
-
"data_fetched_at": "2026-06-30T10:58:37.207440Z",
|
| 15 |
-
"context_length": 96,
|
| 16 |
-
"prediction_length": 24
|
| 17 |
}
|
| 18 |
]
|
| 19 |
}
|
|
|
|
| 1 |
{
|
| 2 |
"model": "Chronos-2",
|
| 3 |
"api_model_id": "amazon/chronos-2",
|
| 4 |
+
"admitted_at": "2026-06-01T00:00:00Z",
|
| 5 |
+
"evaluation_protocol": "issue-now-score-later",
|
| 6 |
+
"metric_version": "livehouse-canonical-v1",
|
| 7 |
+
"evaluated_at": "2026-07-26T02:22:36.349267+00:00",
|
| 8 |
+
"resolved_tasks": [
|
| 9 |
{
|
| 10 |
+
"dataset": "binance_btcusdt_1h_close/1h/short",
|
| 11 |
+
"task_id": "future-162c5d588c33ce9246a2",
|
| 12 |
+
"issued_at": "2026-07-25T02:15:07.111204+00:00",
|
| 13 |
+
"resolved_at": "2026-07-26T02:20:01.309926+00:00",
|
| 14 |
+
"target_end": "2026-07-26T02:00:00+00:00",
|
| 15 |
+
"prediction_length": 24,
|
| 16 |
+
"metric_version": "livehouse-canonical-v1"
|
|
|
|
|
|
|
|
|
|
| 17 |
}
|
| 18 |
]
|
| 19 |
}
|
results/dataset_properties.csv
CHANGED
|
@@ -1,7 +1,14 @@
|
|
| 1 |
dataset,domain,frequency,num_variates
|
| 2 |
binance_btcusdt_1h_close,Finance,1h,1
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3 |
noaa_coops_sf_water_level_water_level,Ocean,6min,1
|
| 4 |
noaa_ndbc_46013_realtime_wave_height,Ocean,10min,1
|
|
|
|
| 5 |
open_meteo_shanghai_air_quality_hourly_pm2_5,Air Quality,1h,1
|
| 6 |
open_meteo_shanghai_hourly_temperature_2m,Weather,1h,1
|
|
|
|
| 7 |
usgs_potomac_iv_usgs_00060,Hydrology,15min,1
|
|
|
|
|
|
| 1 |
dataset,domain,frequency,num_variates
|
| 2 |
binance_btcusdt_1h_close,Finance,1h,1
|
| 3 |
+
data_gov_sg_sgx_equities_turnover_monthly_turnover_value_million_sgd,Finance,M,1
|
| 4 |
+
data_gov_sg_singapore_rainfall_5min_rainfall_5min_total,Climate,5min,1
|
| 5 |
+
data_gov_sg_singapore_taxi_availability_available_taxi_count,Transportation,5min,1
|
| 6 |
+
nasa_power_shanghai_hourly_t2m,Weather,1h,1
|
| 7 |
noaa_coops_sf_water_level_water_level,Ocean,6min,1
|
| 8 |
noaa_ndbc_46013_realtime_wave_height,Ocean,10min,1
|
| 9 |
+
nws_ksfo_observations_temperature,Weather,1h,1
|
| 10 |
open_meteo_shanghai_air_quality_hourly_pm2_5,Air Quality,1h,1
|
| 11 |
open_meteo_shanghai_hourly_temperature_2m,Weather,1h,1
|
| 12 |
+
usgs_earthquake_all_week_hourly_earthquake_count,Hazards,1h,1
|
| 13 |
usgs_potomac_iv_usgs_00060,Hydrology,15min,1
|
| 14 |
+
wikimedia_time_series_hourly_pageviews,Web,1D,1
|
results/online_status.json
CHANGED
|
@@ -1,21 +1,146 @@
|
|
| 1 |
{
|
| 2 |
-
"status": "
|
| 3 |
-
"
|
| 4 |
-
"
|
| 5 |
-
"
|
|
|
|
| 6 |
"data_source": "ts_bench",
|
| 7 |
"data_config": "configs/datasets/ts_bench.yaml",
|
| 8 |
-
"
|
| 9 |
-
"
|
| 10 |
-
"
|
| 11 |
-
|
| 12 |
-
|
| 13 |
-
|
| 14 |
-
|
| 15 |
-
"
|
| 16 |
-
|
| 17 |
-
|
| 18 |
-
|
| 19 |
-
|
| 20 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 21 |
}
|
|
|
|
| 1 |
{
|
| 2 |
+
"status": "ok",
|
| 3 |
+
"evaluation_protocol": "issue-now-score-later",
|
| 4 |
+
"metric_version": "livehouse-canonical-v1",
|
| 5 |
+
"started_at": "2026-07-26T02:42:39.371334+00:00",
|
| 6 |
+
"finished_at": "2026-07-26T02:42:44.199560+00:00",
|
| 7 |
"data_source": "ts_bench",
|
| 8 |
"data_config": "configs/datasets/ts_bench.yaml",
|
| 9 |
+
"task_count": 4,
|
| 10 |
+
"issued_tasks": 0,
|
| 11 |
+
"issued_forecasts": 0,
|
| 12 |
+
"resolved_forecasts": 0,
|
| 13 |
+
"pending_tasks": 13,
|
| 14 |
+
"failed_forecasts": [],
|
| 15 |
+
"models": {
|
| 16 |
+
"Chronos-2": {
|
| 17 |
+
"model": "Chronos-2",
|
| 18 |
+
"api_model_id": "amazon/chronos-2",
|
| 19 |
+
"admitted_at": "2026-06-01T00:00:00Z",
|
| 20 |
+
"evaluation_protocol": "issue-now-score-later",
|
| 21 |
+
"metric_version": "livehouse-canonical-v1",
|
| 22 |
+
"evaluated_at": "2026-07-26T02:42:39.623056+00:00",
|
| 23 |
+
"resolved_tasks": []
|
| 24 |
+
},
|
| 25 |
+
"TiRex": {
|
| 26 |
+
"model": "TiRex",
|
| 27 |
+
"api_model_id": "NX-AI/TiRex-1.1-gifteval",
|
| 28 |
+
"admitted_at": "2026-06-01T00:00:00Z",
|
| 29 |
+
"evaluation_protocol": "issue-now-score-later",
|
| 30 |
+
"metric_version": "livehouse-canonical-v1",
|
| 31 |
+
"evaluated_at": "2026-07-26T02:42:39.623065+00:00",
|
| 32 |
+
"resolved_tasks": []
|
| 33 |
+
},
|
| 34 |
+
"TimesFM-2.5": {
|
| 35 |
+
"model": "TimesFM-2.5",
|
| 36 |
+
"api_model_id": "google/timesfm-2.5-200m-pytorch",
|
| 37 |
+
"admitted_at": "2026-06-01T00:00:00Z",
|
| 38 |
+
"evaluation_protocol": "issue-now-score-later",
|
| 39 |
+
"metric_version": "livehouse-canonical-v1",
|
| 40 |
+
"evaluated_at": "2026-07-26T02:42:39.623067+00:00",
|
| 41 |
+
"resolved_tasks": []
|
| 42 |
+
},
|
| 43 |
+
"Toto-1.0": {
|
| 44 |
+
"model": "Toto-1.0",
|
| 45 |
+
"api_model_id": "Datadog/Toto-Open-Base-1.0",
|
| 46 |
+
"admitted_at": "2026-06-01T00:00:00Z",
|
| 47 |
+
"evaluation_protocol": "issue-now-score-later",
|
| 48 |
+
"metric_version": "livehouse-canonical-v1",
|
| 49 |
+
"evaluated_at": "2026-07-26T02:42:39.623069+00:00",
|
| 50 |
+
"resolved_tasks": []
|
| 51 |
+
},
|
| 52 |
+
"Moirai-2.0": {
|
| 53 |
+
"model": "Moirai-2.0",
|
| 54 |
+
"api_model_id": "Salesforce/moirai-2.0-R-small",
|
| 55 |
+
"admitted_at": "2026-06-01T00:00:00Z",
|
| 56 |
+
"evaluation_protocol": "issue-now-score-later",
|
| 57 |
+
"metric_version": "livehouse-canonical-v1",
|
| 58 |
+
"evaluated_at": "2026-07-26T02:42:39.623070+00:00",
|
| 59 |
+
"resolved_tasks": []
|
| 60 |
+
},
|
| 61 |
+
"Chronos-Bolt": {
|
| 62 |
+
"model": "Chronos-Bolt",
|
| 63 |
+
"api_model_id": "amazon/chronos-bolt-base",
|
| 64 |
+
"admitted_at": "2026-06-01T00:00:00Z",
|
| 65 |
+
"evaluation_protocol": "issue-now-score-later",
|
| 66 |
+
"metric_version": "livehouse-canonical-v1",
|
| 67 |
+
"evaluated_at": "2026-07-26T02:42:39.623071+00:00",
|
| 68 |
+
"resolved_tasks": []
|
| 69 |
+
},
|
| 70 |
+
"TabPFN-TS": {
|
| 71 |
+
"model": "TabPFN-TS",
|
| 72 |
+
"api_model_id": "priorlabs/tabpfn-ts",
|
| 73 |
+
"admitted_at": "2026-06-01T00:00:00Z",
|
| 74 |
+
"evaluation_protocol": "issue-now-score-later",
|
| 75 |
+
"metric_version": "livehouse-canonical-v1",
|
| 76 |
+
"evaluated_at": "2026-07-26T02:42:39.623072+00:00",
|
| 77 |
+
"resolved_tasks": []
|
| 78 |
+
},
|
| 79 |
+
"Sundial": {
|
| 80 |
+
"model": "Sundial",
|
| 81 |
+
"api_model_id": "thuml/sundial-base-128m",
|
| 82 |
+
"admitted_at": "2026-06-01T00:00:00Z",
|
| 83 |
+
"evaluation_protocol": "issue-now-score-later",
|
| 84 |
+
"metric_version": "livehouse-canonical-v1",
|
| 85 |
+
"evaluated_at": "2026-07-26T02:42:39.623073+00:00",
|
| 86 |
+
"resolved_tasks": []
|
| 87 |
+
},
|
| 88 |
+
"Moving-Average": {
|
| 89 |
+
"model": "Moving-Average",
|
| 90 |
+
"api_model_id": "baseline/moving-average",
|
| 91 |
+
"admitted_at": "2026-06-01T00:00:00Z",
|
| 92 |
+
"evaluation_protocol": "issue-now-score-later",
|
| 93 |
+
"metric_version": "livehouse-canonical-v1",
|
| 94 |
+
"evaluated_at": "2026-07-26T02:42:39.623074+00:00",
|
| 95 |
+
"resolved_tasks": []
|
| 96 |
+
},
|
| 97 |
+
"ETS": {
|
| 98 |
+
"model": "ETS",
|
| 99 |
+
"api_model_id": "baseline/ets",
|
| 100 |
+
"admitted_at": "2026-06-01T00:00:00Z",
|
| 101 |
+
"evaluation_protocol": "issue-now-score-later",
|
| 102 |
+
"metric_version": "livehouse-canonical-v1",
|
| 103 |
+
"evaluated_at": "2026-07-26T02:42:39.623075+00:00",
|
| 104 |
+
"resolved_tasks": []
|
| 105 |
+
},
|
| 106 |
+
"ARIMA": {
|
| 107 |
+
"model": "ARIMA",
|
| 108 |
+
"api_model_id": "baseline/arima",
|
| 109 |
+
"admitted_at": "2026-06-01T00:00:00Z",
|
| 110 |
+
"evaluation_protocol": "issue-now-score-later",
|
| 111 |
+
"metric_version": "livehouse-canonical-v1",
|
| 112 |
+
"evaluated_at": "2026-07-26T02:42:39.623076+00:00",
|
| 113 |
+
"resolved_tasks": []
|
| 114 |
+
},
|
| 115 |
+
"Seasonal-Naive": {
|
| 116 |
+
"model": "Seasonal-Naive",
|
| 117 |
+
"api_model_id": "baseline/seasonal-naive",
|
| 118 |
+
"admitted_at": "2026-06-01T00:00:00Z",
|
| 119 |
+
"evaluation_protocol": "issue-now-score-later",
|
| 120 |
+
"metric_version": "livehouse-canonical-v1",
|
| 121 |
+
"evaluated_at": "2026-07-26T02:42:39.623078+00:00",
|
| 122 |
+
"resolved_tasks": []
|
| 123 |
+
}
|
| 124 |
+
},
|
| 125 |
+
"skipped_models": [],
|
| 126 |
+
"aggregate_status": "ok",
|
| 127 |
+
"aggregate_files": {
|
| 128 |
+
"gift_prediction_length": "aggregates/results_by_prediction_length.csv",
|
| 129 |
+
"gift_domain": "aggregates/results_by_domain.csv",
|
| 130 |
+
"gift_frequency": "aggregates/results_by_frequency.csv",
|
| 131 |
+
"gift_overall": "aggregates/results_overall.csv",
|
| 132 |
+
"gift_metadata": "aggregates/metadata.json",
|
| 133 |
+
"live_releases": "aggregates/live_release_metrics.csv",
|
| 134 |
+
"live_by_dataset": "aggregates/live_results_by_dataset.csv",
|
| 135 |
+
"live_rank": "aggregates/live_rank_up_to_now.csv",
|
| 136 |
+
"live_overall": "aggregates/live_overall_up_to_now.csv",
|
| 137 |
+
"live_rank_1d": "aggregates/live_rank_1d.csv",
|
| 138 |
+
"live_overall_1d": "aggregates/live_overall_1d.csv",
|
| 139 |
+
"live_rank_7d": "aggregates/live_rank_7d.csv",
|
| 140 |
+
"live_overall_7d": "aggregates/live_overall_7d.csv",
|
| 141 |
+
"live_rank_30d": "aggregates/live_rank_30d.csv",
|
| 142 |
+
"live_overall_30d": "aggregates/live_overall_30d.csv",
|
| 143 |
+
"live_metadata": "aggregates/live_metadata.json"
|
| 144 |
+
},
|
| 145 |
+
"push_status": "pending"
|
| 146 |
}
|
results/timesfm_2_5/all_results.csv
CHANGED
|
@@ -1,11 +1,15 @@
|
|
| 1 |
-
dataset,model,eval_metrics/MSE[mean],eval_metrics/MSE[0.5],eval_metrics/MAE[0.5],eval_metrics/MASE[0.5],eval_metrics/MAPE[0.5],eval_metrics/sMAPE[0.5],eval_metrics/MSIS,eval_metrics/RMSE[mean],eval_metrics/NRMSE[mean],eval_metrics/ND[0.5],eval_metrics/mean_weighted_sum_quantile_loss,domain,num_variates
|
| 2 |
-
binance_btcusdt_1h_close/1h/short,TimesFM-2.5,
|
| 3 |
-
coingecko_bitcoin_7d_price_usd/1h/short,TimesFM-2.5,
|
| 4 |
-
|
| 5 |
-
|
| 6 |
-
|
| 7 |
-
|
| 8 |
-
|
| 9 |
-
|
| 10 |
-
|
| 11 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
dataset,model,eval_metrics/MSE[mean],eval_metrics/MSE[0.5],eval_metrics/MAE[0.5],eval_metrics/MASE[0.5],eval_metrics/MAPE[0.5],eval_metrics/sMAPE[0.5],eval_metrics/MSIS,eval_metrics/RMSE[mean],eval_metrics/NRMSE[mean],eval_metrics/ND[0.5],eval_metrics/mean_weighted_sum_quantile_loss,domain,num_variates,prediction_length
|
| 2 |
+
binance_btcusdt_1h_close/1h/short,TimesFM-2.5,0.05812463634081555,0.10802260526101197,nan,nan,0.003137198117540541,nan,nan,0.24109051482962898,nan,nan,0.20632353111651106,Finance,1,24
|
| 3 |
+
coingecko_bitcoin_7d_price_usd/1h/short,TimesFM-2.5,2.224373829174787,1.7006943045362455,nan,nan,0.012061613174095603,nan,nan,1.4914334813107781,nan,nan,1.340259097269519,Finance,1,24
|
| 4 |
+
data_gov_sg_sgx_equities_turnover_monthly_turnover_value_million_sgd/1mo/short,TimesFM-2.5,28434.715767887308,30940.314541637516,136.81071631113687,1.4424608331286375,0.6894371389934627,1.1431172573156296,43.032391046456816,168.6259640977252,1.0381242287022936,0.8422577158188848,0.7508176845882389,Finance,1,12
|
| 5 |
+
data_gov_sg_singapore_rainfall_5min_rainfall_5min_total/5min/short,TimesFM-2.5,6.612880078732847e-07,1.54467580239774e-07,nan,nan,nan,nan,nan,0.0008131961681373595,nan,nan,0.0023361815645300717,Climate,1,36
|
| 6 |
+
data_gov_sg_singapore_taxi_availability_available_taxi_count/5min/short,TimesFM-2.5,0.0,0.0,nan,nan,nan,nan,nan,0.0,nan,nan,0.0,Transportation,1,12
|
| 7 |
+
nasa_power_shanghai_hourly_T2M/1h/short,TimesFM-2.5,0.07745835228564939,0.09483093411445913,nan,nan,0.01912431619826608,nan,nan,0.2783134065862609,nan,nan,0.2170178035434303,Weather,1,24
|
| 8 |
+
noaa_coops_sf_water_level_water_level/6min/short,TimesFM-2.5,0.06167793384500841,0.1195747314766337,nan,nan,0.1275039546356878,nan,nan,0.24835042549794112,nan,nan,0.18449296205337073,Ocean,1,60
|
| 9 |
+
noaa_ndbc_46013_realtime_wave_height/10min/short,TimesFM-2.5,1.7475519811316367,1.8494084864429456,nan,nan,0.1105426369996072,nan,nan,1.321950067563687,nan,nan,0.9879443474987509,Ocean,1,72
|
| 10 |
+
nws_ksfo_observations_temperature/1h/short,TimesFM-2.5,0.21622905324208355,0.1617179170793861,0.26413559913635254,0.1911332978675114,0.01630353750267156,0.01601217538840988,10.994616695384883,0.4650043583043965,0.027809880236186378,0.015796796840529412,0.021259949519123158,Weather,1,24
|
| 11 |
+
open_meteo_shanghai_air_quality_hourly_pm2_5/1h/short,TimesFM-2.5,0.2732677218958713,0.18238185017633832,nan,nan,0.7368445501174264,nan,nan,0.5227501524589652,nan,nan,0.5026766357193823,Air Quality,1,24
|
| 12 |
+
open_meteo_shanghai_hourly_temperature_2m/1h/short,TimesFM-2.5,0.08025561586661849,0.15277379446484235,nan,nan,0.03696867790982377,nan,nan,0.2832942213787964,nan,nan,0.2171996720098492,Weather,1,24
|
| 13 |
+
usgs_earthquake_all_week_hourly_earthquake_count/1h/short,TimesFM-2.5,0.33661738301844374,0.3792209101512718,nan,nan,0.22727704298135887,nan,nan,0.580187368889089,nan,nan,0.4741699619381801,Hazards,1,24
|
| 14 |
+
usgs_potomac_iv_usgs_00060/15min/short,TimesFM-2.5,3.3821215253553216,3.556957252873515,nan,nan,0.015576871907429337,nan,nan,1.8390545194080903,nan,nan,1.6776971747265543,Hydrology,1,24
|
| 15 |
+
wikimedia_time_series_hourly_pageviews/1D/short,TimesFM-2.5,0.38719216672060525,0.341005570724447,nan,nan,0.12667853599052806,nan,nan,0.6222476731339421,nan,nan,0.5343002043100503,Web,1,7
|
results/timesfm_2_5/config.json
CHANGED
|
@@ -3,9 +3,12 @@
|
|
| 3 |
"model_type": "zero-shot",
|
| 4 |
"model_dtype": "float32",
|
| 5 |
"model_link": "https://tsfm.ai/models/google/timesfm-2.5-200m-pytorch",
|
| 6 |
-
"code_link": "https://github.com/zhouziyu02/
|
| 7 |
"org": "Google / TSFM.ai",
|
| 8 |
"testdata_leakage": "No",
|
| 9 |
"replication_code_available": "Yes",
|
| 10 |
-
"api_model_id": "google/timesfm-2.5-200m-pytorch"
|
|
|
|
|
|
|
|
|
|
| 11 |
}
|
|
|
|
| 3 |
"model_type": "zero-shot",
|
| 4 |
"model_dtype": "float32",
|
| 5 |
"model_link": "https://tsfm.ai/models/google/timesfm-2.5-200m-pytorch",
|
| 6 |
+
"code_link": "https://github.com/zhouziyu02/TS-Live/blob/main/scripts/run_online_eval.py",
|
| 7 |
"org": "Google / TSFM.ai",
|
| 8 |
"testdata_leakage": "No",
|
| 9 |
"replication_code_available": "Yes",
|
| 10 |
+
"api_model_id": "google/timesfm-2.5-200m-pytorch",
|
| 11 |
+
"admitted_at": "2026-06-01T00:00:00Z",
|
| 12 |
+
"evaluation_protocol": "issue-now-score-later",
|
| 13 |
+
"metric_version": "livehouse-canonical-v1"
|
| 14 |
}
|
results/timesfm_2_5/online_meta.json
CHANGED
|
@@ -1,67 +1,19 @@
|
|
| 1 |
{
|
| 2 |
"model": "TimesFM-2.5",
|
| 3 |
"api_model_id": "google/timesfm-2.5-200m-pytorch",
|
| 4 |
-
"
|
| 5 |
-
"
|
| 6 |
-
|
| 7 |
-
|
| 8 |
-
|
| 9 |
-
|
| 10 |
-
"
|
| 11 |
-
|
| 12 |
-
|
| 13 |
-
"
|
| 14 |
-
"
|
| 15 |
-
"
|
| 16 |
-
"
|
| 17 |
-
},
|
| 18 |
-
{
|
| 19 |
-
"dataset": "task_nasa_power_shanghai_hourly_point_shanghai_31.2304_121.4737_t2m_20260625t230000z",
|
| 20 |
-
"data_fetched_at": "2026-06-30T02:16:24.198018Z",
|
| 21 |
-
"context_length": 69,
|
| 22 |
-
"prediction_length": 24
|
| 23 |
-
},
|
| 24 |
-
{
|
| 25 |
-
"dataset": "task_noaa_coops_sf_water_level_coops_station_9414290_water_level_20260629t234200z",
|
| 26 |
-
"data_fetched_at": "2026-06-30T02:16:24.198018Z",
|
| 27 |
-
"context_length": 96,
|
| 28 |
-
"prediction_length": 24
|
| 29 |
-
},
|
| 30 |
-
{
|
| 31 |
-
"dataset": "task_noaa_ndbc_46013_realtime_ndbc_station_46013_wave_height_20260629t171000z",
|
| 32 |
-
"data_fetched_at": "2026-06-30T02:16:24.198018Z",
|
| 33 |
-
"context_length": 96,
|
| 34 |
-
"prediction_length": 24
|
| 35 |
-
},
|
| 36 |
-
{
|
| 37 |
-
"dataset": "task_nws_ksfo_observations_nws_station_ksfo_temperature_20260630t001000z",
|
| 38 |
-
"data_fetched_at": "2026-06-30T02:16:24.198018Z",
|
| 39 |
-
"context_length": 96,
|
| 40 |
-
"prediction_length": 24
|
| 41 |
-
},
|
| 42 |
-
{
|
| 43 |
-
"dataset": "task_open_meteo_shanghai_air_quality_hourly_grid_shanghai_31.2304_121.4737_pm2_5_20260629t020000z",
|
| 44 |
-
"data_fetched_at": "2026-06-30T02:16:24.198018Z",
|
| 45 |
-
"context_length": 96,
|
| 46 |
-
"prediction_length": 24
|
| 47 |
-
},
|
| 48 |
-
{
|
| 49 |
-
"dataset": "task_open_meteo_shanghai_hourly_grid_shanghai_31.2304_121.4737_temperature_2m_20260629t020000z",
|
| 50 |
-
"data_fetched_at": "2026-06-30T02:16:24.198018Z",
|
| 51 |
-
"context_length": 96,
|
| 52 |
-
"prediction_length": 24
|
| 53 |
-
},
|
| 54 |
-
{
|
| 55 |
-
"dataset": "task_usgs_earthquake_all_week_hourly_earth_global_earthquake_count_20260629t010000z",
|
| 56 |
-
"data_fetched_at": "2026-06-30T02:16:24.198018Z",
|
| 57 |
-
"context_length": 96,
|
| 58 |
-
"prediction_length": 24
|
| 59 |
-
},
|
| 60 |
-
{
|
| 61 |
-
"dataset": "task_usgs_potomac_iv_usgs_site_01646500_usgs_00060_20260629t235000z",
|
| 62 |
-
"data_fetched_at": "2026-06-30T02:16:24.198018Z",
|
| 63 |
-
"context_length": 96,
|
| 64 |
-
"prediction_length": 24
|
| 65 |
}
|
| 66 |
]
|
| 67 |
}
|
|
|
|
| 1 |
{
|
| 2 |
"model": "TimesFM-2.5",
|
| 3 |
"api_model_id": "google/timesfm-2.5-200m-pytorch",
|
| 4 |
+
"admitted_at": "2026-06-01T00:00:00Z",
|
| 5 |
+
"evaluation_protocol": "issue-now-score-later",
|
| 6 |
+
"metric_version": "livehouse-canonical-v1",
|
| 7 |
+
"evaluated_at": "2026-07-26T02:22:36.372552+00:00",
|
| 8 |
+
"resolved_tasks": [
|
| 9 |
+
{
|
| 10 |
+
"dataset": "binance_btcusdt_1h_close/1h/short",
|
| 11 |
+
"task_id": "future-162c5d588c33ce9246a2",
|
| 12 |
+
"issued_at": "2026-07-25T02:15:07.111204+00:00",
|
| 13 |
+
"resolved_at": "2026-07-26T02:20:01.550129+00:00",
|
| 14 |
+
"target_end": "2026-07-26T02:00:00+00:00",
|
| 15 |
+
"prediction_length": 24,
|
| 16 |
+
"metric_version": "livehouse-canonical-v1"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 17 |
}
|
| 18 |
]
|
| 19 |
}
|