Spaces:
Runtime error
Runtime error
Update leaderboard results from local eval
Browse files- results/aggregates/live_metadata.json +1 -1
- results/aggregates/metadata.json +1 -1
- results/chronos_2/all_results.csv +15 -3
- results/chronos_2/config.json +5 -2
- results/chronos_2/online_meta.json +12 -12
- results/dataset_properties.csv +6 -1
- results/online_status.json +31 -153
- results/timesfm_2_5/all_results.csv +15 -11
- results/timesfm_2_5/config.json +5 -2
- results/timesfm_2_5/online_meta.json +13 -61
results/aggregates/live_metadata.json
CHANGED
|
@@ -1,5 +1,5 @@
|
|
| 1 |
{
|
| 2 |
-
"generated_at": "2026-07-21T08:
|
| 3 |
"scope": "Canonical metrics-only release history when available; otherwise legacy release history plus latest allowlisted snapshots",
|
| 4 |
"release_deduplication": "One row per model, dataset, data source timestamp, and prediction length",
|
| 5 |
"dataset_aggregation": "Arithmetic mean over releases, then equal-weight mean over datasets",
|
|
|
|
| 1 |
{
|
| 2 |
+
"generated_at": "2026-07-21T08:18:22.363377+00:00",
|
| 3 |
"scope": "Canonical metrics-only release history when available; otherwise legacy release history plus latest allowlisted snapshots",
|
| 4 |
"release_deduplication": "One row per model, dataset, data source timestamp, and prediction length",
|
| 5 |
"dataset_aggregation": "Arithmetic mean over releases, then equal-weight mean over datasets",
|
results/aggregates/metadata.json
CHANGED
|
@@ -1,5 +1,5 @@
|
|
| 1 |
{
|
| 2 |
-
"generated_at": "2026-07-21T08:
|
| 3 |
"baseline_model": "Seasonal-Naive",
|
| 4 |
"normalization": "Per-config metric / Seasonal-Naive metric",
|
| 5 |
"metric_aggregation": "Geometric mean over dataset configurations",
|
|
|
|
| 1 |
{
|
| 2 |
+
"generated_at": "2026-07-21T08:18:19.749472+00:00",
|
| 3 |
"baseline_model": "Seasonal-Naive",
|
| 4 |
"normalization": "Per-config metric / Seasonal-Naive metric",
|
| 5 |
"metric_aggregation": "Geometric mean over dataset configurations",
|
results/chronos_2/all_results.csv
CHANGED
|
@@ -1,3 +1,15 @@
|
|
| 1 |
-
dataset,model,eval_metrics/MSE[mean],eval_metrics/MSE[0.5],eval_metrics/MAE[0.5],eval_metrics/MASE[0.5],eval_metrics/MAPE[0.5],eval_metrics/sMAPE[0.5],eval_metrics/MSIS,eval_metrics/RMSE[mean],eval_metrics/NRMSE[mean],eval_metrics/ND[0.5],eval_metrics/mean_weighted_sum_quantile_loss,domain,num_variates
|
| 2 |
-
binance_btcusdt_1h_close/1h/short,Chronos-2,
|
| 3 |
-
coingecko_bitcoin_7d_price_usd/1h/short,Chronos-2,
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
dataset,model,eval_metrics/MSE[mean],eval_metrics/MSE[0.5],eval_metrics/MAE[0.5],eval_metrics/MASE[0.5],eval_metrics/MAPE[0.5],eval_metrics/sMAPE[0.5],eval_metrics/MSIS,eval_metrics/RMSE[mean],eval_metrics/NRMSE[mean],eval_metrics/ND[0.5],eval_metrics/mean_weighted_sum_quantile_loss,domain,num_variates,prediction_length
|
| 2 |
+
binance_btcusdt_1h_close/1h/short,Chronos-2,1.0954658710884135,1.0954658710884135,nan,nan,0.006112263091766741,nan,nan,1.0466450549677353,nan,nan,0.722600276563339,Finance,1,24
|
| 3 |
+
coingecko_bitcoin_7d_price_usd/1h/short,Chronos-2,8.113285903904208,8.113285903904208,nan,nan,0.02025038877595049,nan,nan,2.8483830332144953,nan,nan,2.2157723378543412,Finance,1,24
|
| 4 |
+
data_gov_sg_sgx_equities_turnover_monthly_turnover_value_million_sgd/1mo/short,Chronos-2,25067.461899718543,25067.461899718543,121.96686029434204,1.2859549577545721,0.6608019070822674,0.9393767274480805,18.969076544647905,158.32707254199624,0.9747204171363386,0.7508734105557137,0.605299940786381,Finance,1,12
|
| 5 |
+
data_gov_sg_singapore_rainfall_5min_rainfall_5min_total/5min/short,Chronos-2,1.6284716271939715e-18,1.6284716271939715e-18,nan,nan,nan,nan,nan,1.2761158361191085e-09,nan,nan,1.8284365387902989e-09,Climate,1,36
|
| 6 |
+
data_gov_sg_singapore_taxi_availability_available_taxi_count/5min/short,Chronos-2,1.7296862743653321e-18,1.7296862743653321e-18,nan,nan,nan,nan,nan,1.3151753777977035e-09,nan,nan,2.356162747109682e-09,Transportation,1,12
|
| 7 |
+
nasa_power_shanghai_hourly_T2M/1h/short,Chronos-2,0.12252786453930815,0.12252786453930815,nan,nan,0.02591956633833244,nan,nan,0.3500398042213316,nan,nan,0.1913345897260998,Weather,1,24
|
| 8 |
+
noaa_coops_sf_water_level_water_level/6min/short,Chronos-2,0.09964296912706637,0.09964296912706637,nan,nan,0.15265952769534913,nan,nan,0.3156627458650551,nan,nan,0.17831068805766545,Ocean,1,60
|
| 9 |
+
noaa_ndbc_46013_realtime_wave_height/10min/short,Chronos-2,0.6510470864707055,0.6510470864707055,nan,nan,0.04780668123999115,nan,nan,0.806874888982614,nan,nan,0.6237732424271116,Ocean,1,72
|
| 10 |
+
nws_ksfo_observations_temperature/1h/short,Chronos-2,0.22763978935275495,0.22763978935275495,0.4551292260487874,0.3293397414624391,0.02734639793034648,0.02740698654518156,1.2449814398684094,0.4771161172636646,0.02853423165374086,0.027219291695583462,0.02293888289302154,Weather,1,24
|
| 11 |
+
open_meteo_shanghai_air_quality_hourly_pm2_5/1h/short,Chronos-2,0.6084556830997082,0.6084556830997082,nan,nan,0.35668324586229727,nan,nan,0.7800356934780025,nan,nan,0.5313169904102858,Air Quality,1,24
|
| 12 |
+
open_meteo_shanghai_hourly_temperature_2m/1h/short,Chronos-2,0.44307234654846844,0.44307234654846844,nan,nan,0.06204079587436979,nan,nan,0.6656367977722298,nan,nan,0.3525119272495888,Weather,1,24
|
| 13 |
+
usgs_earthquake_all_week_hourly_earthquake_count/1h/short,Chronos-2,0.8018000685909197,0.8018000685909197,nan,nan,0.3751168635754351,nan,nan,0.8954328945213704,nan,nan,0.6004368640045449,Hazards,1,24
|
| 14 |
+
usgs_potomac_iv_usgs_00060/15min/short,Chronos-2,8.98468642325276,8.98468642325276,nan,nan,0.0739047920146499,nan,nan,2.997446650609942,nan,nan,2.7077797851984093,Hydrology,1,24
|
| 15 |
+
wikimedia_time_series_hourly_pageviews/1D/short,Chronos-2,3124.158240098706,3124.158240098706,40.642015729631694,0.933189593158606,0.1502772757369302,0.16165680139439306,14.255586850024258,55.894170000982264,0.22563967128424212,0.16406811424880155,0.13862344481983843,Web,1,7
|
results/chronos_2/config.json
CHANGED
|
@@ -3,9 +3,12 @@
|
|
| 3 |
"model_type": "zero-shot",
|
| 4 |
"model_dtype": "float32",
|
| 5 |
"model_link": "https://tsfm.ai/models/amazon/chronos-2",
|
| 6 |
-
"code_link": "https://github.com/zhouziyu02/
|
| 7 |
"org": "Amazon / TSFM.ai",
|
| 8 |
"testdata_leakage": "No",
|
| 9 |
"replication_code_available": "Yes",
|
| 10 |
-
"api_model_id": "amazon/chronos-2"
|
|
|
|
|
|
|
|
|
|
| 11 |
}
|
|
|
|
| 3 |
"model_type": "zero-shot",
|
| 4 |
"model_dtype": "float32",
|
| 5 |
"model_link": "https://tsfm.ai/models/amazon/chronos-2",
|
| 6 |
+
"code_link": "https://github.com/zhouziyu02/TS-Live/blob/main/scripts/run_online_eval.py",
|
| 7 |
"org": "Amazon / TSFM.ai",
|
| 8 |
"testdata_leakage": "No",
|
| 9 |
"replication_code_available": "Yes",
|
| 10 |
+
"api_model_id": "amazon/chronos-2",
|
| 11 |
+
"admitted_at": "2026-06-01T00:00:00Z",
|
| 12 |
+
"evaluation_protocol": "issue-now-score-later",
|
| 13 |
+
"metric_version": "livehouse-canonical-v1"
|
| 14 |
}
|
results/chronos_2/online_meta.json
CHANGED
|
@@ -1,19 +1,19 @@
|
|
| 1 |
{
|
| 2 |
"model": "Chronos-2",
|
| 3 |
"api_model_id": "amazon/chronos-2",
|
| 4 |
-
"
|
| 5 |
-
"
|
|
|
|
|
|
|
|
|
|
| 6 |
{
|
| 7 |
-
"dataset": "
|
| 8 |
-
"
|
| 9 |
-
"
|
| 10 |
-
"
|
| 11 |
-
|
| 12 |
-
|
| 13 |
-
"
|
| 14 |
-
"data_fetched_at": "2026-06-30T10:58:37.207440Z",
|
| 15 |
-
"context_length": 96,
|
| 16 |
-
"prediction_length": 24
|
| 17 |
}
|
| 18 |
]
|
| 19 |
}
|
|
|
|
| 1 |
{
|
| 2 |
"model": "Chronos-2",
|
| 3 |
"api_model_id": "amazon/chronos-2",
|
| 4 |
+
"admitted_at": "2026-06-01T00:00:00Z",
|
| 5 |
+
"evaluation_protocol": "issue-now-score-later",
|
| 6 |
+
"metric_version": "livehouse-canonical-v1",
|
| 7 |
+
"evaluated_at": "2026-07-21T08:07:46.638690+00:00",
|
| 8 |
+
"resolved_tasks": [
|
| 9 |
{
|
| 10 |
+
"dataset": "usgs_potomac_iv_usgs_00060/15min/short",
|
| 11 |
+
"task_id": "future-9e27750023baa2a61fb2",
|
| 12 |
+
"issued_at": "2026-07-21T02:08:31.469837+00:00",
|
| 13 |
+
"resolved_at": "2026-07-21T08:05:18.720769+00:00",
|
| 14 |
+
"target_end": "2026-07-21T07:55:00+00:00",
|
| 15 |
+
"prediction_length": 24,
|
| 16 |
+
"metric_version": "livehouse-canonical-v1"
|
|
|
|
|
|
|
|
|
|
| 17 |
}
|
| 18 |
]
|
| 19 |
}
|
results/dataset_properties.csv
CHANGED
|
@@ -1,6 +1,11 @@
|
|
| 1 |
dataset,domain,frequency,num_variates
|
| 2 |
-
|
|
|
|
|
|
|
|
|
|
| 3 |
nasa_power_shanghai_hourly_t2m,Weather,1h,1
|
|
|
|
|
|
|
| 4 |
nws_ksfo_observations_temperature,Weather,1h,1
|
| 5 |
open_meteo_shanghai_air_quality_hourly_pm2_5,Air Quality,1h,1
|
| 6 |
open_meteo_shanghai_hourly_temperature_2m,Weather,1h,1
|
|
|
|
| 1 |
dataset,domain,frequency,num_variates
|
| 2 |
+
binance_btcusdt_1h_close,Finance,1h,1
|
| 3 |
+
data_gov_sg_sgx_equities_turnover_monthly_turnover_value_million_sgd,Finance,M,1
|
| 4 |
+
data_gov_sg_singapore_rainfall_5min_rainfall_5min_total,Climate,5min,1
|
| 5 |
+
data_gov_sg_singapore_taxi_availability_available_taxi_count,Transportation,5min,1
|
| 6 |
nasa_power_shanghai_hourly_t2m,Weather,1h,1
|
| 7 |
+
noaa_coops_sf_water_level_water_level,Ocean,6min,1
|
| 8 |
+
noaa_ndbc_46013_realtime_wave_height,Ocean,10min,1
|
| 9 |
nws_ksfo_observations_temperature,Weather,1h,1
|
| 10 |
open_meteo_shanghai_air_quality_hourly_pm2_5,Air Quality,1h,1
|
| 11 |
open_meteo_shanghai_hourly_temperature_2m,Weather,1h,1
|
results/online_status.json
CHANGED
|
@@ -2,14 +2,14 @@
|
|
| 2 |
"status": "ok",
|
| 3 |
"evaluation_protocol": "issue-now-score-later",
|
| 4 |
"metric_version": "livehouse-canonical-v1",
|
| 5 |
-
"started_at": "2026-07-21T08:
|
| 6 |
-
"finished_at": "2026-07-21T08:
|
| 7 |
"data_source": "ts_bench",
|
| 8 |
"data_config": "configs/datasets/ts_bench.yaml",
|
| 9 |
-
"task_count":
|
| 10 |
-
"issued_tasks":
|
| 11 |
-
"issued_forecasts":
|
| 12 |
-
"resolved_forecasts":
|
| 13 |
"pending_tasks": 13,
|
| 14 |
"failed_forecasts": [],
|
| 15 |
"models": {
|
|
@@ -19,18 +19,8 @@
|
|
| 19 |
"admitted_at": "2026-06-01T00:00:00Z",
|
| 20 |
"evaluation_protocol": "issue-now-score-later",
|
| 21 |
"metric_version": "livehouse-canonical-v1",
|
| 22 |
-
"evaluated_at": "2026-07-21T08:
|
| 23 |
-
"resolved_tasks": [
|
| 24 |
-
{
|
| 25 |
-
"dataset": "usgs_potomac_iv_usgs_00060/15min/short",
|
| 26 |
-
"task_id": "future-9e27750023baa2a61fb2",
|
| 27 |
-
"issued_at": "2026-07-21T02:08:31.469837+00:00",
|
| 28 |
-
"resolved_at": "2026-07-21T08:05:18.720769+00:00",
|
| 29 |
-
"target_end": "2026-07-21T07:55:00+00:00",
|
| 30 |
-
"prediction_length": 24,
|
| 31 |
-
"metric_version": "livehouse-canonical-v1"
|
| 32 |
-
}
|
| 33 |
-
]
|
| 34 |
},
|
| 35 |
"TiRex": {
|
| 36 |
"model": "TiRex",
|
|
@@ -38,18 +28,8 @@
|
|
| 38 |
"admitted_at": "2026-06-01T00:00:00Z",
|
| 39 |
"evaluation_protocol": "issue-now-score-later",
|
| 40 |
"metric_version": "livehouse-canonical-v1",
|
| 41 |
-
"evaluated_at": "2026-07-21T08:
|
| 42 |
-
"resolved_tasks": [
|
| 43 |
-
{
|
| 44 |
-
"dataset": "usgs_potomac_iv_usgs_00060/15min/short",
|
| 45 |
-
"task_id": "future-9e27750023baa2a61fb2",
|
| 46 |
-
"issued_at": "2026-07-21T02:08:31.469837+00:00",
|
| 47 |
-
"resolved_at": "2026-07-21T08:05:18.772115+00:00",
|
| 48 |
-
"target_end": "2026-07-21T07:55:00+00:00",
|
| 49 |
-
"prediction_length": 24,
|
| 50 |
-
"metric_version": "livehouse-canonical-v1"
|
| 51 |
-
}
|
| 52 |
-
]
|
| 53 |
},
|
| 54 |
"TimesFM-2.5": {
|
| 55 |
"model": "TimesFM-2.5",
|
|
@@ -57,18 +37,8 @@
|
|
| 57 |
"admitted_at": "2026-06-01T00:00:00Z",
|
| 58 |
"evaluation_protocol": "issue-now-score-later",
|
| 59 |
"metric_version": "livehouse-canonical-v1",
|
| 60 |
-
"evaluated_at": "2026-07-21T08:
|
| 61 |
-
"resolved_tasks": [
|
| 62 |
-
{
|
| 63 |
-
"dataset": "usgs_potomac_iv_usgs_00060/15min/short",
|
| 64 |
-
"task_id": "future-9e27750023baa2a61fb2",
|
| 65 |
-
"issued_at": "2026-07-21T02:08:31.469837+00:00",
|
| 66 |
-
"resolved_at": "2026-07-21T08:05:18.766480+00:00",
|
| 67 |
-
"target_end": "2026-07-21T07:55:00+00:00",
|
| 68 |
-
"prediction_length": 24,
|
| 69 |
-
"metric_version": "livehouse-canonical-v1"
|
| 70 |
-
}
|
| 71 |
-
]
|
| 72 |
},
|
| 73 |
"Toto-1.0": {
|
| 74 |
"model": "Toto-1.0",
|
|
@@ -76,18 +46,8 @@
|
|
| 76 |
"admitted_at": "2026-06-01T00:00:00Z",
|
| 77 |
"evaluation_protocol": "issue-now-score-later",
|
| 78 |
"metric_version": "livehouse-canonical-v1",
|
| 79 |
-
"evaluated_at": "2026-07-21T08:
|
| 80 |
-
"resolved_tasks": [
|
| 81 |
-
{
|
| 82 |
-
"dataset": "usgs_potomac_iv_usgs_00060/15min/short",
|
| 83 |
-
"task_id": "future-9e27750023baa2a61fb2",
|
| 84 |
-
"issued_at": "2026-07-21T02:08:31.469837+00:00",
|
| 85 |
-
"resolved_at": "2026-07-21T08:05:18.783033+00:00",
|
| 86 |
-
"target_end": "2026-07-21T07:55:00+00:00",
|
| 87 |
-
"prediction_length": 24,
|
| 88 |
-
"metric_version": "livehouse-canonical-v1"
|
| 89 |
-
}
|
| 90 |
-
]
|
| 91 |
},
|
| 92 |
"Moirai-2.0": {
|
| 93 |
"model": "Moirai-2.0",
|
|
@@ -95,18 +55,8 @@
|
|
| 95 |
"admitted_at": "2026-06-01T00:00:00Z",
|
| 96 |
"evaluation_protocol": "issue-now-score-later",
|
| 97 |
"metric_version": "livehouse-canonical-v1",
|
| 98 |
-
"evaluated_at": "2026-07-21T08:
|
| 99 |
-
"resolved_tasks": [
|
| 100 |
-
{
|
| 101 |
-
"dataset": "usgs_potomac_iv_usgs_00060/15min/short",
|
| 102 |
-
"task_id": "future-9e27750023baa2a61fb2",
|
| 103 |
-
"issued_at": "2026-07-21T02:08:31.469837+00:00",
|
| 104 |
-
"resolved_at": "2026-07-21T08:05:18.738047+00:00",
|
| 105 |
-
"target_end": "2026-07-21T07:55:00+00:00",
|
| 106 |
-
"prediction_length": 24,
|
| 107 |
-
"metric_version": "livehouse-canonical-v1"
|
| 108 |
-
}
|
| 109 |
-
]
|
| 110 |
},
|
| 111 |
"Chronos-Bolt": {
|
| 112 |
"model": "Chronos-Bolt",
|
|
@@ -114,18 +64,8 @@
|
|
| 114 |
"admitted_at": "2026-06-01T00:00:00Z",
|
| 115 |
"evaluation_protocol": "issue-now-score-later",
|
| 116 |
"metric_version": "livehouse-canonical-v1",
|
| 117 |
-
"evaluated_at": "2026-07-21T08:
|
| 118 |
-
"resolved_tasks": [
|
| 119 |
-
{
|
| 120 |
-
"dataset": "usgs_potomac_iv_usgs_00060/15min/short",
|
| 121 |
-
"task_id": "future-9e27750023baa2a61fb2",
|
| 122 |
-
"issued_at": "2026-07-21T02:08:31.469837+00:00",
|
| 123 |
-
"resolved_at": "2026-07-21T08:05:18.727008+00:00",
|
| 124 |
-
"target_end": "2026-07-21T07:55:00+00:00",
|
| 125 |
-
"prediction_length": 24,
|
| 126 |
-
"metric_version": "livehouse-canonical-v1"
|
| 127 |
-
}
|
| 128 |
-
]
|
| 129 |
},
|
| 130 |
"TabPFN-TS": {
|
| 131 |
"model": "TabPFN-TS",
|
|
@@ -133,18 +73,8 @@
|
|
| 133 |
"admitted_at": "2026-06-01T00:00:00Z",
|
| 134 |
"evaluation_protocol": "issue-now-score-later",
|
| 135 |
"metric_version": "livehouse-canonical-v1",
|
| 136 |
-
"evaluated_at": "2026-07-21T08:
|
| 137 |
-
"resolved_tasks": [
|
| 138 |
-
{
|
| 139 |
-
"dataset": "usgs_potomac_iv_usgs_00060/15min/short",
|
| 140 |
-
"task_id": "future-9e27750023baa2a61fb2",
|
| 141 |
-
"issued_at": "2026-07-21T02:08:31.469837+00:00",
|
| 142 |
-
"resolved_at": "2026-07-21T08:05:18.761258+00:00",
|
| 143 |
-
"target_end": "2026-07-21T07:55:00+00:00",
|
| 144 |
-
"prediction_length": 24,
|
| 145 |
-
"metric_version": "livehouse-canonical-v1"
|
| 146 |
-
}
|
| 147 |
-
]
|
| 148 |
},
|
| 149 |
"Sundial": {
|
| 150 |
"model": "Sundial",
|
|
@@ -152,18 +82,8 @@
|
|
| 152 |
"admitted_at": "2026-06-01T00:00:00Z",
|
| 153 |
"evaluation_protocol": "issue-now-score-later",
|
| 154 |
"metric_version": "livehouse-canonical-v1",
|
| 155 |
-
"evaluated_at": "2026-07-21T08:
|
| 156 |
-
"resolved_tasks": [
|
| 157 |
-
{
|
| 158 |
-
"dataset": "usgs_potomac_iv_usgs_00060/15min/short",
|
| 159 |
-
"task_id": "future-9e27750023baa2a61fb2",
|
| 160 |
-
"issued_at": "2026-07-21T02:08:31.469837+00:00",
|
| 161 |
-
"resolved_at": "2026-07-21T08:05:18.756331+00:00",
|
| 162 |
-
"target_end": "2026-07-21T07:55:00+00:00",
|
| 163 |
-
"prediction_length": 24,
|
| 164 |
-
"metric_version": "livehouse-canonical-v1"
|
| 165 |
-
}
|
| 166 |
-
]
|
| 167 |
},
|
| 168 |
"Moving-Average": {
|
| 169 |
"model": "Moving-Average",
|
|
@@ -171,18 +91,8 @@
|
|
| 171 |
"admitted_at": "2026-06-01T00:00:00Z",
|
| 172 |
"evaluation_protocol": "issue-now-score-later",
|
| 173 |
"metric_version": "livehouse-canonical-v1",
|
| 174 |
-
"evaluated_at": "2026-07-21T08:
|
| 175 |
-
"resolved_tasks": [
|
| 176 |
-
{
|
| 177 |
-
"dataset": "usgs_potomac_iv_usgs_00060/15min/short",
|
| 178 |
-
"task_id": "future-9e27750023baa2a61fb2",
|
| 179 |
-
"issued_at": "2026-07-21T02:08:31.469837+00:00",
|
| 180 |
-
"resolved_at": "2026-07-21T08:05:18.745339+00:00",
|
| 181 |
-
"target_end": "2026-07-21T07:55:00+00:00",
|
| 182 |
-
"prediction_length": 24,
|
| 183 |
-
"metric_version": "livehouse-canonical-v1"
|
| 184 |
-
}
|
| 185 |
-
]
|
| 186 |
},
|
| 187 |
"ETS": {
|
| 188 |
"model": "ETS",
|
|
@@ -190,18 +100,8 @@
|
|
| 190 |
"admitted_at": "2026-06-01T00:00:00Z",
|
| 191 |
"evaluation_protocol": "issue-now-score-later",
|
| 192 |
"metric_version": "livehouse-canonical-v1",
|
| 193 |
-
"evaluated_at": "2026-07-21T08:
|
| 194 |
-
"resolved_tasks": [
|
| 195 |
-
{
|
| 196 |
-
"dataset": "usgs_potomac_iv_usgs_00060/15min/short",
|
| 197 |
-
"task_id": "future-9e27750023baa2a61fb2",
|
| 198 |
-
"issued_at": "2026-07-21T02:08:31.469837+00:00",
|
| 199 |
-
"resolved_at": "2026-07-21T08:05:18.732604+00:00",
|
| 200 |
-
"target_end": "2026-07-21T07:55:00+00:00",
|
| 201 |
-
"prediction_length": 24,
|
| 202 |
-
"metric_version": "livehouse-canonical-v1"
|
| 203 |
-
}
|
| 204 |
-
]
|
| 205 |
},
|
| 206 |
"ARIMA": {
|
| 207 |
"model": "ARIMA",
|
|
@@ -209,18 +109,8 @@
|
|
| 209 |
"admitted_at": "2026-06-01T00:00:00Z",
|
| 210 |
"evaluation_protocol": "issue-now-score-later",
|
| 211 |
"metric_version": "livehouse-canonical-v1",
|
| 212 |
-
"evaluated_at": "2026-07-21T08:
|
| 213 |
-
"resolved_tasks": [
|
| 214 |
-
{
|
| 215 |
-
"dataset": "usgs_potomac_iv_usgs_00060/15min/short",
|
| 216 |
-
"task_id": "future-9e27750023baa2a61fb2",
|
| 217 |
-
"issued_at": "2026-07-21T02:08:31.469837+00:00",
|
| 218 |
-
"resolved_at": "2026-07-21T08:05:18.716529+00:00",
|
| 219 |
-
"target_end": "2026-07-21T07:55:00+00:00",
|
| 220 |
-
"prediction_length": 24,
|
| 221 |
-
"metric_version": "livehouse-canonical-v1"
|
| 222 |
-
}
|
| 223 |
-
]
|
| 224 |
},
|
| 225 |
"Seasonal-Naive": {
|
| 226 |
"model": "Seasonal-Naive",
|
|
@@ -228,18 +118,8 @@
|
|
| 228 |
"admitted_at": "2026-06-01T00:00:00Z",
|
| 229 |
"evaluation_protocol": "issue-now-score-later",
|
| 230 |
"metric_version": "livehouse-canonical-v1",
|
| 231 |
-
"evaluated_at": "2026-07-21T08:
|
| 232 |
-
"resolved_tasks": [
|
| 233 |
-
{
|
| 234 |
-
"dataset": "usgs_potomac_iv_usgs_00060/15min/short",
|
| 235 |
-
"task_id": "future-9e27750023baa2a61fb2",
|
| 236 |
-
"issued_at": "2026-07-21T02:08:31.469837+00:00",
|
| 237 |
-
"resolved_at": "2026-07-21T08:05:18.751544+00:00",
|
| 238 |
-
"target_end": "2026-07-21T07:55:00+00:00",
|
| 239 |
-
"prediction_length": 24,
|
| 240 |
-
"metric_version": "livehouse-canonical-v1"
|
| 241 |
-
}
|
| 242 |
-
]
|
| 243 |
}
|
| 244 |
},
|
| 245 |
"skipped_models": [],
|
|
@@ -262,7 +142,5 @@
|
|
| 262 |
"live_overall_30d": "aggregates/live_overall_30d.csv",
|
| 263 |
"live_metadata": "aggregates/live_metadata.json"
|
| 264 |
},
|
| 265 |
-
"push_status": "
|
| 266 |
-
"pushed_at": "2026-07-21T08:08:13.469607+00:00",
|
| 267 |
-
"push_detail": "HF Hub API"
|
| 268 |
}
|
|
|
|
| 2 |
"status": "ok",
|
| 3 |
"evaluation_protocol": "issue-now-score-later",
|
| 4 |
"metric_version": "livehouse-canonical-v1",
|
| 5 |
+
"started_at": "2026-07-21T08:18:19.408402+00:00",
|
| 6 |
+
"finished_at": "2026-07-21T08:18:22.378148+00:00",
|
| 7 |
"data_source": "ts_bench",
|
| 8 |
"data_config": "configs/datasets/ts_bench.yaml",
|
| 9 |
+
"task_count": 1,
|
| 10 |
+
"issued_tasks": 0,
|
| 11 |
+
"issued_forecasts": 0,
|
| 12 |
+
"resolved_forecasts": 0,
|
| 13 |
"pending_tasks": 13,
|
| 14 |
"failed_forecasts": [],
|
| 15 |
"models": {
|
|
|
|
| 19 |
"admitted_at": "2026-06-01T00:00:00Z",
|
| 20 |
"evaluation_protocol": "issue-now-score-later",
|
| 21 |
"metric_version": "livehouse-canonical-v1",
|
| 22 |
+
"evaluated_at": "2026-07-21T08:18:19.531521+00:00",
|
| 23 |
+
"resolved_tasks": []
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 24 |
},
|
| 25 |
"TiRex": {
|
| 26 |
"model": "TiRex",
|
|
|
|
| 28 |
"admitted_at": "2026-06-01T00:00:00Z",
|
| 29 |
"evaluation_protocol": "issue-now-score-later",
|
| 30 |
"metric_version": "livehouse-canonical-v1",
|
| 31 |
+
"evaluated_at": "2026-07-21T08:18:19.531527+00:00",
|
| 32 |
+
"resolved_tasks": []
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 33 |
},
|
| 34 |
"TimesFM-2.5": {
|
| 35 |
"model": "TimesFM-2.5",
|
|
|
|
| 37 |
"admitted_at": "2026-06-01T00:00:00Z",
|
| 38 |
"evaluation_protocol": "issue-now-score-later",
|
| 39 |
"metric_version": "livehouse-canonical-v1",
|
| 40 |
+
"evaluated_at": "2026-07-21T08:18:19.531528+00:00",
|
| 41 |
+
"resolved_tasks": []
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 42 |
},
|
| 43 |
"Toto-1.0": {
|
| 44 |
"model": "Toto-1.0",
|
|
|
|
| 46 |
"admitted_at": "2026-06-01T00:00:00Z",
|
| 47 |
"evaluation_protocol": "issue-now-score-later",
|
| 48 |
"metric_version": "livehouse-canonical-v1",
|
| 49 |
+
"evaluated_at": "2026-07-21T08:18:19.531529+00:00",
|
| 50 |
+
"resolved_tasks": []
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 51 |
},
|
| 52 |
"Moirai-2.0": {
|
| 53 |
"model": "Moirai-2.0",
|
|
|
|
| 55 |
"admitted_at": "2026-06-01T00:00:00Z",
|
| 56 |
"evaluation_protocol": "issue-now-score-later",
|
| 57 |
"metric_version": "livehouse-canonical-v1",
|
| 58 |
+
"evaluated_at": "2026-07-21T08:18:19.531530+00:00",
|
| 59 |
+
"resolved_tasks": []
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 60 |
},
|
| 61 |
"Chronos-Bolt": {
|
| 62 |
"model": "Chronos-Bolt",
|
|
|
|
| 64 |
"admitted_at": "2026-06-01T00:00:00Z",
|
| 65 |
"evaluation_protocol": "issue-now-score-later",
|
| 66 |
"metric_version": "livehouse-canonical-v1",
|
| 67 |
+
"evaluated_at": "2026-07-21T08:18:19.531531+00:00",
|
| 68 |
+
"resolved_tasks": []
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 69 |
},
|
| 70 |
"TabPFN-TS": {
|
| 71 |
"model": "TabPFN-TS",
|
|
|
|
| 73 |
"admitted_at": "2026-06-01T00:00:00Z",
|
| 74 |
"evaluation_protocol": "issue-now-score-later",
|
| 75 |
"metric_version": "livehouse-canonical-v1",
|
| 76 |
+
"evaluated_at": "2026-07-21T08:18:19.531532+00:00",
|
| 77 |
+
"resolved_tasks": []
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 78 |
},
|
| 79 |
"Sundial": {
|
| 80 |
"model": "Sundial",
|
|
|
|
| 82 |
"admitted_at": "2026-06-01T00:00:00Z",
|
| 83 |
"evaluation_protocol": "issue-now-score-later",
|
| 84 |
"metric_version": "livehouse-canonical-v1",
|
| 85 |
+
"evaluated_at": "2026-07-21T08:18:19.531533+00:00",
|
| 86 |
+
"resolved_tasks": []
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 87 |
},
|
| 88 |
"Moving-Average": {
|
| 89 |
"model": "Moving-Average",
|
|
|
|
| 91 |
"admitted_at": "2026-06-01T00:00:00Z",
|
| 92 |
"evaluation_protocol": "issue-now-score-later",
|
| 93 |
"metric_version": "livehouse-canonical-v1",
|
| 94 |
+
"evaluated_at": "2026-07-21T08:18:19.531534+00:00",
|
| 95 |
+
"resolved_tasks": []
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 96 |
},
|
| 97 |
"ETS": {
|
| 98 |
"model": "ETS",
|
|
|
|
| 100 |
"admitted_at": "2026-06-01T00:00:00Z",
|
| 101 |
"evaluation_protocol": "issue-now-score-later",
|
| 102 |
"metric_version": "livehouse-canonical-v1",
|
| 103 |
+
"evaluated_at": "2026-07-21T08:18:19.531534+00:00",
|
| 104 |
+
"resolved_tasks": []
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 105 |
},
|
| 106 |
"ARIMA": {
|
| 107 |
"model": "ARIMA",
|
|
|
|
| 109 |
"admitted_at": "2026-06-01T00:00:00Z",
|
| 110 |
"evaluation_protocol": "issue-now-score-later",
|
| 111 |
"metric_version": "livehouse-canonical-v1",
|
| 112 |
+
"evaluated_at": "2026-07-21T08:18:19.531535+00:00",
|
| 113 |
+
"resolved_tasks": []
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 114 |
},
|
| 115 |
"Seasonal-Naive": {
|
| 116 |
"model": "Seasonal-Naive",
|
|
|
|
| 118 |
"admitted_at": "2026-06-01T00:00:00Z",
|
| 119 |
"evaluation_protocol": "issue-now-score-later",
|
| 120 |
"metric_version": "livehouse-canonical-v1",
|
| 121 |
+
"evaluated_at": "2026-07-21T08:18:19.531536+00:00",
|
| 122 |
+
"resolved_tasks": []
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 123 |
}
|
| 124 |
},
|
| 125 |
"skipped_models": [],
|
|
|
|
| 142 |
"live_overall_30d": "aggregates/live_overall_30d.csv",
|
| 143 |
"live_metadata": "aggregates/live_metadata.json"
|
| 144 |
},
|
| 145 |
+
"push_status": "pending"
|
|
|
|
|
|
|
| 146 |
}
|
results/timesfm_2_5/all_results.csv
CHANGED
|
@@ -1,11 +1,15 @@
|
|
| 1 |
-
dataset,model,eval_metrics/MSE[mean],eval_metrics/MSE[0.5],eval_metrics/MAE[0.5],eval_metrics/MASE[0.5],eval_metrics/MAPE[0.5],eval_metrics/sMAPE[0.5],eval_metrics/MSIS,eval_metrics/RMSE[mean],eval_metrics/NRMSE[mean],eval_metrics/ND[0.5],eval_metrics/mean_weighted_sum_quantile_loss,domain,num_variates
|
| 2 |
-
binance_btcusdt_1h_close/1h/short,TimesFM-2.5,
|
| 3 |
-
coingecko_bitcoin_7d_price_usd/1h/short,TimesFM-2.5,
|
| 4 |
-
|
| 5 |
-
|
| 6 |
-
|
| 7 |
-
|
| 8 |
-
|
| 9 |
-
|
| 10 |
-
|
| 11 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
dataset,model,eval_metrics/MSE[mean],eval_metrics/MSE[0.5],eval_metrics/MAE[0.5],eval_metrics/MASE[0.5],eval_metrics/MAPE[0.5],eval_metrics/sMAPE[0.5],eval_metrics/MSIS,eval_metrics/RMSE[mean],eval_metrics/NRMSE[mean],eval_metrics/ND[0.5],eval_metrics/mean_weighted_sum_quantile_loss,domain,num_variates,prediction_length
|
| 2 |
+
binance_btcusdt_1h_close/1h/short,TimesFM-2.5,0.7703098987450588,0.9093325421510349,nan,nan,0.005658163673677518,nan,nan,0.8776730021739639,nan,nan,0.5957596964832665,Finance,1,24
|
| 3 |
+
coingecko_bitcoin_7d_price_usd/1h/short,TimesFM-2.5,2.2915529265767343,2.878913041885473,nan,nan,0.011582480558357755,nan,nan,1.5137876094673037,nan,nan,1.224248705324749,Finance,1,24
|
| 4 |
+
data_gov_sg_sgx_equities_turnover_monthly_turnover_value_million_sgd/1mo/short,TimesFM-2.5,28434.715767887308,30940.314541637516,136.81071631113687,1.4424608331286375,0.6894371389934627,1.1431172573156296,43.032391046456816,168.6259640977252,1.0381242287022936,0.8422577158188848,0.7508176845882389,Finance,1,12
|
| 5 |
+
data_gov_sg_singapore_rainfall_5min_rainfall_5min_total/5min/short,TimesFM-2.5,0.0,0.0,nan,nan,nan,nan,nan,0.0,nan,nan,0.0,Climate,1,36
|
| 6 |
+
data_gov_sg_singapore_taxi_availability_available_taxi_count/5min/short,TimesFM-2.5,0.0,0.0,nan,nan,nan,nan,nan,0.0,nan,nan,0.0,Transportation,1,12
|
| 7 |
+
nasa_power_shanghai_hourly_T2M/1h/short,TimesFM-2.5,0.09315097364023417,0.03152586577332402,nan,nan,0.01182277602777041,nan,nan,0.30520644429669924,nan,nan,0.19657836406005544,Weather,1,24
|
| 8 |
+
noaa_coops_sf_water_level_water_level/6min/short,TimesFM-2.5,0.0017283360726366261,0.008407491788168448,nan,nan,0.03954588883668247,nan,nan,0.0415732615106949,nan,nan,0.07057594160718611,Ocean,1,60
|
| 9 |
+
noaa_ndbc_46013_realtime_wave_height/10min/short,TimesFM-2.5,0.8725495709279553,0.7291968803467479,nan,nan,0.05559633929648969,nan,nan,0.9341036189459686,nan,nan,0.7852210439131277,Ocean,1,72
|
| 10 |
+
nws_ksfo_observations_temperature/1h/short,TimesFM-2.5,0.21622905324208355,0.1617179170793861,0.26413559913635254,0.1911332978675114,0.01630353750267156,0.01601217538840988,10.994616695384883,0.4650043583043965,0.027809880236186378,0.015796796840529412,0.021259949519123158,Weather,1,24
|
| 11 |
+
open_meteo_shanghai_air_quality_hourly_pm2_5/1h/short,TimesFM-2.5,0.6577112567858043,0.5181570518033817,nan,nan,0.3284370667916474,nan,nan,0.8109939930639464,nan,nan,0.70629918281875,Air Quality,1,24
|
| 12 |
+
open_meteo_shanghai_hourly_temperature_2m/1h/short,TimesFM-2.5,0.3086313952502291,0.21861691523319612,nan,nan,0.0442540357821819,nan,nan,0.55554603342138,nan,nan,0.4279171921606936,Weather,1,24
|
| 13 |
+
usgs_earthquake_all_week_hourly_earthquake_count/1h/short,TimesFM-2.5,0.614839892078722,0.7277212159226245,nan,nan,0.36404848689119734,nan,nan,0.784117269340959,nan,nan,0.6131553587651006,Hazards,1,24
|
| 14 |
+
usgs_potomac_iv_usgs_00060/15min/short,TimesFM-2.5,13.194295051320376,11.709681064438575,nan,nan,0.08459454021962103,nan,nan,3.63239522234577,nan,nan,3.625870191777938,Hydrology,1,24
|
| 15 |
+
wikimedia_time_series_hourly_pageviews/1D/short,TimesFM-2.5,3027.463475894449,3382.4058134857833,43.392713274274556,0.9963489192034538,0.161770023788917,0.17298843679453008,24.862357017155514,55.02239067774545,0.2221203775918213,0.17517242959626406,0.15088338611157417,Web,1,7
|
results/timesfm_2_5/config.json
CHANGED
|
@@ -3,9 +3,12 @@
|
|
| 3 |
"model_type": "zero-shot",
|
| 4 |
"model_dtype": "float32",
|
| 5 |
"model_link": "https://tsfm.ai/models/google/timesfm-2.5-200m-pytorch",
|
| 6 |
-
"code_link": "https://github.com/zhouziyu02/
|
| 7 |
"org": "Google / TSFM.ai",
|
| 8 |
"testdata_leakage": "No",
|
| 9 |
"replication_code_available": "Yes",
|
| 10 |
-
"api_model_id": "google/timesfm-2.5-200m-pytorch"
|
|
|
|
|
|
|
|
|
|
| 11 |
}
|
|
|
|
| 3 |
"model_type": "zero-shot",
|
| 4 |
"model_dtype": "float32",
|
| 5 |
"model_link": "https://tsfm.ai/models/google/timesfm-2.5-200m-pytorch",
|
| 6 |
+
"code_link": "https://github.com/zhouziyu02/TS-Live/blob/main/scripts/run_online_eval.py",
|
| 7 |
"org": "Google / TSFM.ai",
|
| 8 |
"testdata_leakage": "No",
|
| 9 |
"replication_code_available": "Yes",
|
| 10 |
+
"api_model_id": "google/timesfm-2.5-200m-pytorch",
|
| 11 |
+
"admitted_at": "2026-06-01T00:00:00Z",
|
| 12 |
+
"evaluation_protocol": "issue-now-score-later",
|
| 13 |
+
"metric_version": "livehouse-canonical-v1"
|
| 14 |
}
|
results/timesfm_2_5/online_meta.json
CHANGED
|
@@ -1,67 +1,19 @@
|
|
| 1 |
{
|
| 2 |
"model": "TimesFM-2.5",
|
| 3 |
"api_model_id": "google/timesfm-2.5-200m-pytorch",
|
| 4 |
-
"
|
| 5 |
-
"
|
| 6 |
-
|
| 7 |
-
|
| 8 |
-
|
| 9 |
-
|
| 10 |
-
"
|
| 11 |
-
|
| 12 |
-
|
| 13 |
-
"
|
| 14 |
-
"
|
| 15 |
-
"
|
| 16 |
-
"
|
| 17 |
-
},
|
| 18 |
-
{
|
| 19 |
-
"dataset": "task_nasa_power_shanghai_hourly_point_shanghai_31.2304_121.4737_t2m_20260625t230000z",
|
| 20 |
-
"data_fetched_at": "2026-06-30T02:16:24.198018Z",
|
| 21 |
-
"context_length": 69,
|
| 22 |
-
"prediction_length": 24
|
| 23 |
-
},
|
| 24 |
-
{
|
| 25 |
-
"dataset": "task_noaa_coops_sf_water_level_coops_station_9414290_water_level_20260629t234200z",
|
| 26 |
-
"data_fetched_at": "2026-06-30T02:16:24.198018Z",
|
| 27 |
-
"context_length": 96,
|
| 28 |
-
"prediction_length": 24
|
| 29 |
-
},
|
| 30 |
-
{
|
| 31 |
-
"dataset": "task_noaa_ndbc_46013_realtime_ndbc_station_46013_wave_height_20260629t171000z",
|
| 32 |
-
"data_fetched_at": "2026-06-30T02:16:24.198018Z",
|
| 33 |
-
"context_length": 96,
|
| 34 |
-
"prediction_length": 24
|
| 35 |
-
},
|
| 36 |
-
{
|
| 37 |
-
"dataset": "task_nws_ksfo_observations_nws_station_ksfo_temperature_20260630t001000z",
|
| 38 |
-
"data_fetched_at": "2026-06-30T02:16:24.198018Z",
|
| 39 |
-
"context_length": 96,
|
| 40 |
-
"prediction_length": 24
|
| 41 |
-
},
|
| 42 |
-
{
|
| 43 |
-
"dataset": "task_open_meteo_shanghai_air_quality_hourly_grid_shanghai_31.2304_121.4737_pm2_5_20260629t020000z",
|
| 44 |
-
"data_fetched_at": "2026-06-30T02:16:24.198018Z",
|
| 45 |
-
"context_length": 96,
|
| 46 |
-
"prediction_length": 24
|
| 47 |
-
},
|
| 48 |
-
{
|
| 49 |
-
"dataset": "task_open_meteo_shanghai_hourly_grid_shanghai_31.2304_121.4737_temperature_2m_20260629t020000z",
|
| 50 |
-
"data_fetched_at": "2026-06-30T02:16:24.198018Z",
|
| 51 |
-
"context_length": 96,
|
| 52 |
-
"prediction_length": 24
|
| 53 |
-
},
|
| 54 |
-
{
|
| 55 |
-
"dataset": "task_usgs_earthquake_all_week_hourly_earth_global_earthquake_count_20260629t010000z",
|
| 56 |
-
"data_fetched_at": "2026-06-30T02:16:24.198018Z",
|
| 57 |
-
"context_length": 96,
|
| 58 |
-
"prediction_length": 24
|
| 59 |
-
},
|
| 60 |
-
{
|
| 61 |
-
"dataset": "task_usgs_potomac_iv_usgs_site_01646500_usgs_00060_20260629t235000z",
|
| 62 |
-
"data_fetched_at": "2026-06-30T02:16:24.198018Z",
|
| 63 |
-
"context_length": 96,
|
| 64 |
-
"prediction_length": 24
|
| 65 |
}
|
| 66 |
]
|
| 67 |
}
|
|
|
|
| 1 |
{
|
| 2 |
"model": "TimesFM-2.5",
|
| 3 |
"api_model_id": "google/timesfm-2.5-200m-pytorch",
|
| 4 |
+
"admitted_at": "2026-06-01T00:00:00Z",
|
| 5 |
+
"evaluation_protocol": "issue-now-score-later",
|
| 6 |
+
"metric_version": "livehouse-canonical-v1",
|
| 7 |
+
"evaluated_at": "2026-07-21T08:07:46.642682+00:00",
|
| 8 |
+
"resolved_tasks": [
|
| 9 |
+
{
|
| 10 |
+
"dataset": "usgs_potomac_iv_usgs_00060/15min/short",
|
| 11 |
+
"task_id": "future-9e27750023baa2a61fb2",
|
| 12 |
+
"issued_at": "2026-07-21T02:08:31.469837+00:00",
|
| 13 |
+
"resolved_at": "2026-07-21T08:05:18.766480+00:00",
|
| 14 |
+
"target_end": "2026-07-21T07:55:00+00:00",
|
| 15 |
+
"prediction_length": 24,
|
| 16 |
+
"metric_version": "livehouse-canonical-v1"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 17 |
}
|
| 18 |
]
|
| 19 |
}
|