Spaces:
Running
Running
zhouziyu02 Cursor commited on
Commit ·
5063745
1
Parent(s): 1d5b407
Deploy live 1h benchmark: scheduler, tsfm_bench vendor, latest results.
Browse filesSpace auto-collects TS-Bench data, runs TSFM.ai eval hourly, and reloads leaderboard.
Co-authored-by: Cursor <cursoragent@cursor.com>
This view is limited to 50 files because it contains too many changes. See raw diff
- README.md +18 -2
- app.py +29 -12
- configs/datasets/ts_bench.yaml +29 -0
- configs/models/online_tsfm.yaml +26 -0
- requirements.txt +8 -3
- results/chronos_2/all_results.csv +7 -10
- results/chronos_2/config.json +1 -1
- results/chronos_2/online_meta.json +15 -33
- results/chronos_bolt_base/all_results.csv +7 -10
- results/chronos_bolt_base/config.json +1 -1
- results/chronos_bolt_base/online_meta.json +15 -33
- results/chronos_bolt_mini/all_results.csv +7 -10
- results/chronos_bolt_mini/config.json +1 -1
- results/chronos_bolt_mini/online_meta.json +15 -33
- results/chronos_bolt_tiny/all_results.csv +7 -10
- results/chronos_bolt_tiny/config.json +1 -1
- results/chronos_bolt_tiny/online_meta.json +15 -33
- results/dataset_properties.csv +0 -3
- results/eval_state.json +10 -0
- results/moirai_1_1_small/all_results.csv +7 -10
- results/moirai_1_1_small/config.json +1 -1
- results/moirai_1_1_small/online_meta.json +15 -33
- results/online_status.json +94 -203
- results/timesfm_2_5/all_results.csv +7 -10
- results/timesfm_2_5/config.json +1 -1
- results/timesfm_2_5/online_meta.json +15 -33
- scripts/run_online_daemon.py +63 -0
- scripts/run_space_eval.py +281 -0
- src/about.py +33 -21
- src/benchmark_config.py +19 -0
- src/display/css_html_js.py +2 -2
- src/eval_scheduler.py +123 -0
- tsfm_bench/__about__.py +1 -0
- tsfm_bench/__init__.py +1 -0
- tsfm_bench/data/__init__.py +18 -0
- tsfm_bench/data/dataset.py +197 -0
- tsfm_bench/data/interface.py +46 -0
- tsfm_bench/data/open_meteo.py +194 -0
- tsfm_bench/data/registry.py +76 -0
- tsfm_bench/data/synthetic.py +151 -0
- tsfm_bench/data/ts_bench.py +304 -0
- tsfm_bench/eval/__init__.py +1 -0
- tsfm_bench/eval/api_predictor.py +166 -0
- tsfm_bench/eval/metrics.py +50 -0
- tsfm_bench/eval/online_eval.py +154 -0
- tsfm_bench/eval/predictors.py +85 -0
- vendor/ts_bench/collect_weekly_data.py +1457 -0
- vendor/ts_bench/data/sample_week/model_inputs_latest/manifest.csv +8 -0
- vendor/ts_bench/data/sample_week/model_inputs_latest/manifest.json +72 -0
- vendor/ts_bench/data/sample_week/model_inputs_latest/task_binance_btcusdt_1h_crypto_btcusdt_close_20260527t230000z/context.csv +97 -0
README.md
CHANGED
|
@@ -13,6 +13,22 @@ python_version: 3.11.9
|
|
| 13 |
|
| 14 |
# TSFM Realworld Bench Leaderboard
|
| 15 |
|
| 16 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 17 |
|
| 18 |
-
Results are submitted via pull request to the [TSFM_Bench](https://github.com/zhouziyu02/TSFM_Bench) repository.
|
|
|
|
| 13 |
|
| 14 |
# TSFM Realworld Bench Leaderboard
|
| 15 |
|
| 16 |
+
Live benchmark: every **1 hour** the Space collects TS-Bench data, runs all TSFM models via TSFM.ai, and updates the leaderboard.
|
| 17 |
+
|
| 18 |
+
## Enable live evaluation (required)
|
| 19 |
+
|
| 20 |
+
In **Settings → Repository secrets**, add:
|
| 21 |
+
|
| 22 |
+
| Secret | Value |
|
| 23 |
+
|---|---|
|
| 24 |
+
| `TSFM_API_KEY` | Your key from [tsfm.ai](https://tsfm.ai/) |
|
| 25 |
+
|
| 26 |
+
Optional variables (Settings → Variables):
|
| 27 |
+
|
| 28 |
+
| Variable | Default | Purpose |
|
| 29 |
+
|---|---|---|
|
| 30 |
+
| `TSFM_BENCH_INTERVAL_SECONDS` | `3600` | Collect + eval cycle (1 hour) |
|
| 31 |
+
| `TSFM_REFRESH_SECONDS` | `60` | UI reload from disk (seconds) |
|
| 32 |
+
|
| 33 |
+
Without `TSFM_API_KEY`, the leaderboard only reloads static CSV files and **does not** call the API.
|
| 34 |
|
|
|
app.py
CHANGED
|
@@ -16,6 +16,8 @@ from src.display.css_html_js import (
|
|
| 16 |
refresh_countdown_head_js,
|
| 17 |
refresh_countdown_markup,
|
| 18 |
)
|
|
|
|
|
|
|
| 19 |
from src.utils import (
|
| 20 |
get_grouped_dfs,
|
| 21 |
prepare_ranks_table,
|
|
@@ -23,25 +25,38 @@ from src.utils import (
|
|
| 23 |
)
|
| 24 |
|
| 25 |
RESULTS_PATH = os.getenv("TSFM_RESULTS_PATH", "results")
|
| 26 |
-
|
|
|
|
| 27 |
|
| 28 |
|
| 29 |
def load_online_status(results_path: str = RESULTS_PATH) -> dict:
|
| 30 |
status_path = Path(results_path) / "online_status.json"
|
| 31 |
if not status_path.exists():
|
| 32 |
-
return {"status": "unknown", "message": "No online_status.json yet.
|
| 33 |
return json.loads(status_path.read_text())
|
| 34 |
|
| 35 |
|
| 36 |
def build_status_line(status: dict) -> str:
|
|
|
|
| 37 |
finished = status.get("finished_at", "n/a")
|
| 38 |
-
|
| 39 |
-
|
| 40 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 41 |
return (
|
| 42 |
-
f"
|
| 43 |
-
f"
|
| 44 |
-
f"Auto-refresh every {REFRESH_SECONDS}s"
|
| 45 |
)
|
| 46 |
|
| 47 |
|
|
@@ -73,14 +88,14 @@ DOMAINS: list[str] = grouped_initial["domains"]
|
|
| 73 |
INITIAL_OUTPUTS = refresh_leaderboard()
|
| 74 |
|
| 75 |
demo = gr.Blocks(
|
| 76 |
-
css=build_custom_css(
|
| 77 |
-
head=refresh_countdown_head_js(
|
| 78 |
)
|
| 79 |
with demo:
|
| 80 |
gr.HTML(TITLE)
|
| 81 |
gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text")
|
| 82 |
status = gr.Markdown(INITIAL_OUTPUTS[2], elem_classes="markdown-text")
|
| 83 |
-
gr.HTML(refresh_countdown_markup(
|
| 84 |
|
| 85 |
domain_tables: list[tuple[gr.Dataframe, gr.Dataframe]] = []
|
| 86 |
with gr.Tabs():
|
|
@@ -130,6 +145,8 @@ with demo:
|
|
| 130 |
refresh_outputs.extend([values_df, ranks_df])
|
| 131 |
|
| 132 |
demo.load(refresh_leaderboard, inputs=[], outputs=refresh_outputs)
|
| 133 |
-
gr.Timer(value=
|
|
|
|
|
|
|
| 134 |
|
| 135 |
demo.queue(default_concurrency_limit=20).launch()
|
|
|
|
| 16 |
refresh_countdown_head_js,
|
| 17 |
refresh_countdown_markup,
|
| 18 |
)
|
| 19 |
+
from src.benchmark_config import BENCHMARK_INTERVAL_SECONDS, DISPLAY_REFRESH_SECONDS
|
| 20 |
+
from src.eval_scheduler import get_scheduler, start_live_eval_scheduler
|
| 21 |
from src.utils import (
|
| 22 |
get_grouped_dfs,
|
| 23 |
prepare_ranks_table,
|
|
|
|
| 25 |
)
|
| 26 |
|
| 27 |
RESULTS_PATH = os.getenv("TSFM_RESULTS_PATH", "results")
|
| 28 |
+
|
| 29 |
+
start_live_eval_scheduler()
|
| 30 |
|
| 31 |
|
| 32 |
def load_online_status(results_path: str = RESULTS_PATH) -> dict:
|
| 33 |
status_path = Path(results_path) / "online_status.json"
|
| 34 |
if not status_path.exists():
|
| 35 |
+
return {"status": "unknown", "message": "No online_status.json yet."}
|
| 36 |
return json.loads(status_path.read_text())
|
| 37 |
|
| 38 |
|
| 39 |
def build_status_line(status: dict) -> str:
|
| 40 |
+
eval_state = get_scheduler().snapshot()
|
| 41 |
finished = status.get("finished_at", "n/a")
|
| 42 |
+
eval_status = eval_state.get("status", "unknown")
|
| 43 |
+
eval_message = eval_state.get("message", "")
|
| 44 |
+
|
| 45 |
+
if eval_status == "running":
|
| 46 |
+
live_hint = "**Evaluating now** — collecting TS-Bench data and calling TSFM.ai…"
|
| 47 |
+
elif eval_status == "disabled":
|
| 48 |
+
live_hint = f"**Live eval off** — {eval_message}"
|
| 49 |
+
else:
|
| 50 |
+
secs = get_scheduler().seconds_until_next_run()
|
| 51 |
+
next_hint = f" · Next eval in ~{secs}s" if secs is not None else ""
|
| 52 |
+
live_hint = (
|
| 53 |
+
f"**Live eval** · Last run `{finished}` · "
|
| 54 |
+
f"collect + eval every **{BENCHMARK_INTERVAL_SECONDS // 60} min**{next_hint}"
|
| 55 |
+
)
|
| 56 |
+
|
| 57 |
return (
|
| 58 |
+
f"{live_hint} · "
|
| 59 |
+
f"Display reloads every {DISPLAY_REFRESH_SECONDS}s"
|
|
|
|
| 60 |
)
|
| 61 |
|
| 62 |
|
|
|
|
| 88 |
INITIAL_OUTPUTS = refresh_leaderboard()
|
| 89 |
|
| 90 |
demo = gr.Blocks(
|
| 91 |
+
css=build_custom_css(DISPLAY_REFRESH_SECONDS),
|
| 92 |
+
head=refresh_countdown_head_js(DISPLAY_REFRESH_SECONDS),
|
| 93 |
)
|
| 94 |
with demo:
|
| 95 |
gr.HTML(TITLE)
|
| 96 |
gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text")
|
| 97 |
status = gr.Markdown(INITIAL_OUTPUTS[2], elem_classes="markdown-text")
|
| 98 |
+
gr.HTML(refresh_countdown_markup(DISPLAY_REFRESH_SECONDS))
|
| 99 |
|
| 100 |
domain_tables: list[tuple[gr.Dataframe, gr.Dataframe]] = []
|
| 101 |
with gr.Tabs():
|
|
|
|
| 145 |
refresh_outputs.extend([values_df, ranks_df])
|
| 146 |
|
| 147 |
demo.load(refresh_leaderboard, inputs=[], outputs=refresh_outputs)
|
| 148 |
+
gr.Timer(value=DISPLAY_REFRESH_SECONDS).tick(
|
| 149 |
+
refresh_leaderboard, inputs=[], outputs=refresh_outputs
|
| 150 |
+
)
|
| 151 |
|
| 152 |
demo.queue(default_concurrency_limit=20).launch()
|
configs/datasets/ts_bench.yaml
ADDED
|
@@ -0,0 +1,29 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
source_type: ts_bench
|
| 2 |
+
|
| 3 |
+
settings:
|
| 4 |
+
ts_bench_root: vendor/ts_bench
|
| 5 |
+
output_subdir: data/sample_week
|
| 6 |
+
collect_days: 7
|
| 7 |
+
collect_timeout: 45
|
| 8 |
+
limit_entities: 10
|
| 9 |
+
task_limit: 20
|
| 10 |
+
max_tasks_per_dataset: 1
|
| 11 |
+
min_future_steps: 2
|
| 12 |
+
skip_frequencies:
|
| 13 |
+
- snapshot
|
| 14 |
+
auto_refresh: true
|
| 15 |
+
|
| 16 |
+
source_domains:
|
| 17 |
+
open_meteo_weather: Weather
|
| 18 |
+
open_meteo_air_quality: Air Quality
|
| 19 |
+
nasa_power: Weather
|
| 20 |
+
usgs_water: Hydrology
|
| 21 |
+
noaa_coops: Ocean
|
| 22 |
+
noaa_ndbc: Ocean
|
| 23 |
+
gbfs_citibike: Transport
|
| 24 |
+
binance: Finance
|
| 25 |
+
coingecko: Finance
|
| 26 |
+
wikimedia_pageviews: Web
|
| 27 |
+
nws_observations: Weather
|
| 28 |
+
usgs_earthquake: Hazards
|
| 29 |
+
noaa_ncei: Weather
|
configs/models/online_tsfm.yaml
ADDED
|
@@ -0,0 +1,26 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
models:
|
| 2 |
+
- model_id: amazon/chronos-bolt-tiny
|
| 3 |
+
display_name: Chronos-Bolt-Tiny
|
| 4 |
+
model_type: zero-shot
|
| 5 |
+
org: Amazon / TSFM.ai
|
| 6 |
+
- model_id: amazon/chronos-bolt-mini
|
| 7 |
+
display_name: Chronos-Bolt-Mini
|
| 8 |
+
model_type: zero-shot
|
| 9 |
+
org: Amazon / TSFM.ai
|
| 10 |
+
- model_id: amazon/chronos-bolt-base
|
| 11 |
+
display_name: Chronos-Bolt-Base
|
| 12 |
+
model_type: zero-shot
|
| 13 |
+
org: Amazon / TSFM.ai
|
| 14 |
+
baseline: true
|
| 15 |
+
- model_id: amazon/chronos-2
|
| 16 |
+
display_name: Chronos-2
|
| 17 |
+
model_type: zero-shot
|
| 18 |
+
org: Amazon / TSFM.ai
|
| 19 |
+
- model_id: google/timesfm-2.5-200m-pytorch
|
| 20 |
+
display_name: TimesFM-2.5
|
| 21 |
+
model_type: zero-shot
|
| 22 |
+
org: Google / TSFM.ai
|
| 23 |
+
- model_id: Salesforce/moirai-1.1-R-small
|
| 24 |
+
display_name: Moirai-1.1-Small
|
| 25 |
+
model_type: zero-shot
|
| 26 |
+
org: Salesforce / TSFM.ai
|
requirements.txt
CHANGED
|
@@ -4,6 +4,11 @@ starlette==0.37.2
|
|
| 4 |
fastapi==0.112.4
|
| 5 |
jinja2==3.1.4
|
| 6 |
uvicorn==0.30.6
|
| 7 |
-
pandas
|
| 8 |
-
numpy
|
| 9 |
-
scipy
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 4 |
fastapi==0.112.4
|
| 5 |
jinja2==3.1.4
|
| 6 |
uvicorn==0.30.6
|
| 7 |
+
pandas>=2.0.0
|
| 8 |
+
numpy~=1.26.0
|
| 9 |
+
scipy~=1.11.3
|
| 10 |
+
gluonts~=0.15.1
|
| 11 |
+
httpx>=0.27.0
|
| 12 |
+
pyyaml>=6.0
|
| 13 |
+
python-dotenv==1.0.0
|
| 14 |
+
requests>=2.31.0
|
results/chronos_2/all_results.csv
CHANGED
|
@@ -1,11 +1,8 @@
|
|
| 1 |
dataset,model,eval_metrics/MSE[mean],eval_metrics/MSE[0.5],eval_metrics/MAE[0.5],eval_metrics/MASE[0.5],eval_metrics/MAPE[0.5],eval_metrics/sMAPE[0.5],eval_metrics/MSIS,eval_metrics/RMSE[mean],eval_metrics/NRMSE[mean],eval_metrics/ND[0.5],eval_metrics/mean_weighted_sum_quantile_loss,domain,num_variates
|
| 2 |
-
binance_btcusdt_1h_close/1h/short,Chronos-2,
|
| 3 |
-
coingecko_bitcoin_7d_price_usd/1h/short,Chronos-2,
|
| 4 |
-
nasa_power_shanghai_hourly_T2M/1h/short,Chronos-2,16.
|
| 5 |
-
noaa_coops_sf_water_level_water_level/6min/short,Chronos-2,0.
|
| 6 |
-
|
| 7 |
-
|
| 8 |
-
|
| 9 |
-
open_meteo_shanghai_hourly_temperature_2m/1h/short,Chronos-2,0.30751845326813054,0.30751845326813054,0.4332726796468099,0.23597305034316118,0.017424389083777924,0.017404320116509994,2.091343357081799,0.5545434638223865,0.0221301007506678,0.0172905618380289,0.022378411607540734,Weather,1
|
| 10 |
-
usgs_potomac_iv_usgs_00060/15min/short,Chronos-2,5269.190091451009,5269.190091451009,65.57242838541667,nan,0.0025808082601376345,0.0025768785670678665,nan,72.58918715243344,0.0028545641351112607,0.002578630642716697,0.001904930492972491,Hydrology,1
|
| 11 |
-
wikimedia_time_series_hourly_pageviews/1D/short,Chronos-2,4063.1116605430143,4063.1116605430143,63.214759826660156,1.3546019962855749,0.2031273434135618,0.22642697598080944,5.089582116263253,63.74254199938228,0.20528999033617482,0.20359020878151418,0.18650881592714202,Web,1
|
|
|
|
| 1 |
dataset,model,eval_metrics/MSE[mean],eval_metrics/MSE[0.5],eval_metrics/MAE[0.5],eval_metrics/MASE[0.5],eval_metrics/MAPE[0.5],eval_metrics/sMAPE[0.5],eval_metrics/MSIS,eval_metrics/RMSE[mean],eval_metrics/NRMSE[mean],eval_metrics/ND[0.5],eval_metrics/mean_weighted_sum_quantile_loss,domain,num_variates
|
| 2 |
+
binance_btcusdt_1h_close/1h/short,Chronos-2,823714.9134292603,823714.9134292603,858.2594401041666,0.9497371860172586,0.011705204247515538,0.011628998753724112,11.651465649030945,907.5874136573624,0.01236475636114196,0.01169272371106779,0.008400224593419353,Finance,1
|
| 3 |
+
coingecko_bitcoin_7d_price_usd/1h/short,Chronos-2,855220.5976460775,855220.5976460775,874.1985677083334,0.9460587958444222,0.011942090149853535,0.011862712920852017,11.805964342783492,924.7813782976372,0.012618568650165292,0.011928370206599736,0.008589137695491324,Finance,1
|
| 4 |
+
nasa_power_shanghai_hourly_T2M/1h/short,Chronos-2,16.932999722573793,16.932999722573793,3.1624552408854165,0.9207458206411238,0.11271485949261918,0.12390656428505213,20.748803742282863,4.114972627196176,0.1603262140917773,0.12321454404213872,0.10622444551676062,Weather,1
|
| 5 |
+
noaa_coops_sf_water_level_water_level/6min/short,Chronos-2,0.0002494680528126736,0.0002494680528126736,0.013614242275555929,0.853698368574301,0.010921521056233332,0.01099381527380565,7.152836038203941,0.015794557696012686,0.012375351058861507,0.010667030429280775,0.014505464409588711,Ocean,1
|
| 6 |
+
nws_ksfo_observations_temperature/1h/short,Chronos-2,6.805250338098176,6.805250338098176,2.0215774377187095,2.4880953079614887,0.13046011501961127,0.11750503806375708,54.233638763427734,2.608687474209622,0.1540184486618227,0.11935512547416734,0.0937472313337025,Weather,1
|
| 7 |
+
open_meteo_shanghai_air_quality_hourly_pm2_5/1h/short,Chronos-2,787.8530339560867,787.8530339560867,22.037151257197063,0.9668951191457578,0.3169416718860694,0.39892875770053077,5.485232905888252,28.06871984890096,0.4783761537649721,0.3755799236696442,0.30724907240289284,Air Quality,1
|
| 8 |
+
usgs_potomac_iv_usgs_00060/15min/short,Chronos-2,12352.058849016825,12352.058849016825,90.12906901041667,nan,0.003257321805455632,0.0032514783083314887,nan,111.13981666809076,0.003992449633339587,0.0032376854606346356,0.0034832326643549696,Hydrology,1
|
|
|
|
|
|
|
|
|
results/chronos_2/config.json
CHANGED
|
@@ -3,7 +3,7 @@
|
|
| 3 |
"model_type": "zero-shot",
|
| 4 |
"model_dtype": "float32",
|
| 5 |
"model_link": "https://tsfm.ai/models/amazon/chronos-2",
|
| 6 |
-
"code_link": "https://github.com/zhouziyu02/TSFM_Bench/blob/main/scripts/
|
| 7 |
"org": "Amazon / TSFM.ai",
|
| 8 |
"testdata_leakage": "No",
|
| 9 |
"replication_code_available": "Yes",
|
|
|
|
| 3 |
"model_type": "zero-shot",
|
| 4 |
"model_dtype": "float32",
|
| 5 |
"model_link": "https://tsfm.ai/models/amazon/chronos-2",
|
| 6 |
+
"code_link": "https://github.com/zhouziyu02/TSFM_Bench/blob/main/space/scripts/run_space_eval.py",
|
| 7 |
"org": "Amazon / TSFM.ai",
|
| 8 |
"testdata_leakage": "No",
|
| 9 |
"replication_code_available": "Yes",
|
results/chronos_2/online_meta.json
CHANGED
|
@@ -1,67 +1,49 @@
|
|
| 1 |
{
|
| 2 |
"model": "Chronos-2",
|
| 3 |
"api_model_id": "amazon/chronos-2",
|
| 4 |
-
"evaluated_at": "2026-05-
|
| 5 |
"datasets": [
|
| 6 |
{
|
| 7 |
-
"dataset": "
|
| 8 |
-
"data_fetched_at": "2026-05-
|
| 9 |
"context_length": 96,
|
| 10 |
"prediction_length": 24
|
| 11 |
},
|
| 12 |
{
|
| 13 |
-
"dataset": "
|
| 14 |
-
"data_fetched_at": "2026-05-
|
| 15 |
"context_length": 96,
|
| 16 |
"prediction_length": 24
|
| 17 |
},
|
| 18 |
{
|
| 19 |
"dataset": "task_nasa_power_shanghai_hourly_point_shanghai_31.2304_121.4737_t2m_20260524t230000z",
|
| 20 |
-
"data_fetched_at": "2026-05-
|
| 21 |
-
"context_length":
|
| 22 |
"prediction_length": 24
|
| 23 |
},
|
| 24 |
{
|
| 25 |
-
"dataset": "
|
| 26 |
-
"data_fetched_at": "2026-05-
|
| 27 |
"context_length": 96,
|
| 28 |
"prediction_length": 24
|
| 29 |
},
|
| 30 |
{
|
| 31 |
-
"dataset": "
|
| 32 |
-
"data_fetched_at": "2026-05-
|
| 33 |
"context_length": 96,
|
| 34 |
"prediction_length": 24
|
| 35 |
},
|
| 36 |
{
|
| 37 |
-
"dataset": "
|
| 38 |
-
"data_fetched_at": "2026-05-
|
| 39 |
"context_length": 96,
|
| 40 |
"prediction_length": 24
|
| 41 |
},
|
| 42 |
{
|
| 43 |
-
"dataset": "
|
| 44 |
-
"data_fetched_at": "2026-05-
|
| 45 |
"context_length": 96,
|
| 46 |
"prediction_length": 24
|
| 47 |
-
},
|
| 48 |
-
{
|
| 49 |
-
"dataset": "task_open_meteo_shanghai_hourly_grid_shanghai_31.2304_121.4737_temperature_2m_20260527t120000z",
|
| 50 |
-
"data_fetched_at": "2026-05-28T12:16:39.187896Z",
|
| 51 |
-
"context_length": 96,
|
| 52 |
-
"prediction_length": 24
|
| 53 |
-
},
|
| 54 |
-
{
|
| 55 |
-
"dataset": "task_usgs_potomac_iv_usgs_site_01646500_usgs_00060_20260528t095000z",
|
| 56 |
-
"data_fetched_at": "2026-05-28T12:16:39.187896Z",
|
| 57 |
-
"context_length": 96,
|
| 58 |
-
"prediction_length": 24
|
| 59 |
-
},
|
| 60 |
-
{
|
| 61 |
-
"dataset": "task_wikimedia_time_series_hourly_enwiki_article_time_series_pageviews_20260525t000000z",
|
| 62 |
-
"data_fetched_at": "2026-05-28T12:16:39.187896Z",
|
| 63 |
-
"context_length": 4,
|
| 64 |
-
"prediction_length": 2
|
| 65 |
}
|
| 66 |
]
|
| 67 |
}
|
|
|
|
| 1 |
{
|
| 2 |
"model": "Chronos-2",
|
| 3 |
"api_model_id": "amazon/chronos-2",
|
| 4 |
+
"evaluated_at": "2026-05-29T02:53:16.309267+00:00",
|
| 5 |
"datasets": [
|
| 6 |
{
|
| 7 |
+
"dataset": "task_binance_btcusdt_1h_crypto_btcusdt_close_20260528t020000z",
|
| 8 |
+
"data_fetched_at": "2026-05-29T02:50:05.083862Z",
|
| 9 |
"context_length": 96,
|
| 10 |
"prediction_length": 24
|
| 11 |
},
|
| 12 |
{
|
| 13 |
+
"dataset": "task_coingecko_bitcoin_7d_coin_bitcoin_price_usd_20260528t030012z",
|
| 14 |
+
"data_fetched_at": "2026-05-29T02:50:05.083862Z",
|
| 15 |
"context_length": 96,
|
| 16 |
"prediction_length": 24
|
| 17 |
},
|
| 18 |
{
|
| 19 |
"dataset": "task_nasa_power_shanghai_hourly_point_shanghai_31.2304_121.4737_t2m_20260524t230000z",
|
| 20 |
+
"data_fetched_at": "2026-05-29T02:50:05.083862Z",
|
| 21 |
+
"context_length": 69,
|
| 22 |
"prediction_length": 24
|
| 23 |
},
|
| 24 |
{
|
| 25 |
+
"dataset": "task_noaa_coops_sf_water_level_coops_station_9414290_water_level_20260529t001800z",
|
| 26 |
+
"data_fetched_at": "2026-05-29T02:50:05.083862Z",
|
| 27 |
"context_length": 96,
|
| 28 |
"prediction_length": 24
|
| 29 |
},
|
| 30 |
{
|
| 31 |
+
"dataset": "task_nws_ksfo_observations_nws_station_ksfo_temperature_20260529t004500z",
|
| 32 |
+
"data_fetched_at": "2026-05-29T02:50:05.083862Z",
|
| 33 |
"context_length": 96,
|
| 34 |
"prediction_length": 24
|
| 35 |
},
|
| 36 |
{
|
| 37 |
+
"dataset": "task_open_meteo_shanghai_air_quality_hourly_grid_shanghai_31.2304_121.4737_pm2_5_20260528t020000z",
|
| 38 |
+
"data_fetched_at": "2026-05-29T02:50:05.083862Z",
|
| 39 |
"context_length": 96,
|
| 40 |
"prediction_length": 24
|
| 41 |
},
|
| 42 |
{
|
| 43 |
+
"dataset": "task_usgs_potomac_iv_usgs_site_01646500_usgs_00060_20260528t235000z",
|
| 44 |
+
"data_fetched_at": "2026-05-29T02:50:05.083862Z",
|
| 45 |
"context_length": 96,
|
| 46 |
"prediction_length": 24
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 47 |
}
|
| 48 |
]
|
| 49 |
}
|
results/chronos_bolt_base/all_results.csv
CHANGED
|
@@ -1,11 +1,8 @@
|
|
| 1 |
dataset,model,eval_metrics/MSE[mean],eval_metrics/MSE[0.5],eval_metrics/MAE[0.5],eval_metrics/MASE[0.5],eval_metrics/MAPE[0.5],eval_metrics/sMAPE[0.5],eval_metrics/MSIS,eval_metrics/RMSE[mean],eval_metrics/NRMSE[mean],eval_metrics/ND[0.5],eval_metrics/mean_weighted_sum_quantile_loss,domain,num_variates
|
| 2 |
-
binance_btcusdt_1h_close/1h/short,Chronos-Bolt-Base,
|
| 3 |
-
coingecko_bitcoin_7d_price_usd/1h/short,Chronos-Bolt-Base,
|
| 4 |
-
nasa_power_shanghai_hourly_T2M/1h/short,Chronos-Bolt-Base,
|
| 5 |
-
noaa_coops_sf_water_level_water_level/6min/short,Chronos-Bolt-Base,0.
|
| 6 |
-
|
| 7 |
-
|
| 8 |
-
|
| 9 |
-
open_meteo_shanghai_hourly_temperature_2m/1h/short,Chronos-Bolt-Base,0.6043374313377777,0.6043374313377777,0.6502586205800375,0.3541499785661307,0.026396943053871125,0.026084632371007877,3.108420925181368,0.7773914273631899,0.03102326820637271,0.02594979424739109,0.027723991507018,Weather,1
|
| 10 |
-
usgs_potomac_iv_usgs_00060/15min/short,Chronos-Bolt-Base,4639.297880808513,4639.297880808513,62.649576822916664,nan,0.0024655696006461968,0.0024621813211091674,nan,68.11239153640483,0.002678514495942513,0.002463689732508602,0.0015585257620341544,Hydrology,1
|
| 11 |
-
wikimedia_time_series_hourly_pageviews/1D/short,Chronos-Bolt-Base,4028.8496973256115,4028.8496973256115,63.05271911621094,1.3511296953473773,0.2026653811521117,0.22577298837325582,4.94127665928432,63.473220316331926,0.20442260971443454,0.20306833853852155,0.181499555051636,Web,1
|
|
|
|
| 1 |
dataset,model,eval_metrics/MSE[mean],eval_metrics/MSE[0.5],eval_metrics/MAE[0.5],eval_metrics/MASE[0.5],eval_metrics/MAPE[0.5],eval_metrics/sMAPE[0.5],eval_metrics/MSIS,eval_metrics/RMSE[mean],eval_metrics/NRMSE[mean],eval_metrics/ND[0.5],eval_metrics/mean_weighted_sum_quantile_loss,domain,num_variates
|
| 2 |
+
binance_btcusdt_1h_close/1h/short,Chronos-Bolt-Base,1020807.1654103597,1020807.1654103597,978.3776041666666,1.082658167477634,0.013340172949584,0.013245844896014864,12.505715915951397,1010.3500212353933,0.013764769832701284,0.013329185181148515,0.009075649577973174,Finance,1
|
| 3 |
+
coingecko_bitcoin_7d_price_usd/1h/short,Chronos-Bolt-Base,1032518.6303456625,1032518.6303456625,981.5572916666666,1.0622425427220556,0.013405463836439183,0.013309736826454506,12.85205207248118,1016.1292389975118,0.01386500297327957,0.013393271490572569,0.009228772376426409,Finance,1
|
| 4 |
+
nasa_power_shanghai_hourly_T2M/1h/short,Chronos-Bolt-Base,14.357317871038353,14.357317871038353,3.029151995976766,0.8819347083000757,0.1091449078122792,0.11893372374924811,7.6038772586826555,3.7891051543917795,0.1476298724769894,0.1180208267276896,0.09493130206210582,Weather,1
|
| 5 |
+
noaa_coops_sf_water_level_water_level/6min/short,Chronos-Bolt-Base,0.007210553512415767,0.007210553512415767,0.07268983622392018,4.558108585133124,0.054106267823969455,0.055993512387166634,21.160505010226164,0.08491497813940581,0.06653257944006775,0.05695393685568374,0.049410255141733854,Ocean,1
|
| 6 |
+
nws_ksfo_observations_temperature/1h/short,Chronos-Bolt-Base,6.471037574445442,6.471037574445442,1.968206803003947,2.4224083729279346,0.1270358446325033,0.11470028206444982,37.253308711907806,2.543823416521957,0.15018883639982034,0.11620409169026993,0.08526762779478787,Weather,1
|
| 7 |
+
open_meteo_shanghai_air_quality_hourly_pm2_5/1h/short,Chronos-Bolt-Base,964.608039366991,964.608039366991,25.43541646003723,1.115996334626862,0.3828018459905362,0.4930053251111303,8.94359384637864,31.058139663653247,0.5293249380582461,0.4334966739154405,0.35288186109406783,Air Quality,1
|
| 8 |
+
usgs_potomac_iv_usgs_00060/15min/short,Chronos-Bolt-Base,31798.887013276417,31798.887013276417,151.68888346354166,nan,0.00542055609772574,0.005430840108326886,nan,178.32242431415185,0.006405834730638593,0.005449084273499476,0.0052516232677867584,Hydrology,1
|
|
|
|
|
|
|
|
|
results/chronos_bolt_base/config.json
CHANGED
|
@@ -3,7 +3,7 @@
|
|
| 3 |
"model_type": "zero-shot",
|
| 4 |
"model_dtype": "float32",
|
| 5 |
"model_link": "https://tsfm.ai/models/amazon/chronos-bolt-base",
|
| 6 |
-
"code_link": "https://github.com/zhouziyu02/TSFM_Bench/blob/main/scripts/
|
| 7 |
"org": "Amazon / TSFM.ai",
|
| 8 |
"testdata_leakage": "No",
|
| 9 |
"replication_code_available": "Yes",
|
|
|
|
| 3 |
"model_type": "zero-shot",
|
| 4 |
"model_dtype": "float32",
|
| 5 |
"model_link": "https://tsfm.ai/models/amazon/chronos-bolt-base",
|
| 6 |
+
"code_link": "https://github.com/zhouziyu02/TSFM_Bench/blob/main/space/scripts/run_space_eval.py",
|
| 7 |
"org": "Amazon / TSFM.ai",
|
| 8 |
"testdata_leakage": "No",
|
| 9 |
"replication_code_available": "Yes",
|
results/chronos_bolt_base/online_meta.json
CHANGED
|
@@ -1,67 +1,49 @@
|
|
| 1 |
{
|
| 2 |
"model": "Chronos-Bolt-Base",
|
| 3 |
"api_model_id": "amazon/chronos-bolt-base",
|
| 4 |
-
"evaluated_at": "2026-05-
|
| 5 |
"datasets": [
|
| 6 |
{
|
| 7 |
-
"dataset": "
|
| 8 |
-
"data_fetched_at": "2026-05-
|
| 9 |
"context_length": 96,
|
| 10 |
"prediction_length": 24
|
| 11 |
},
|
| 12 |
{
|
| 13 |
-
"dataset": "
|
| 14 |
-
"data_fetched_at": "2026-05-
|
| 15 |
"context_length": 96,
|
| 16 |
"prediction_length": 24
|
| 17 |
},
|
| 18 |
{
|
| 19 |
"dataset": "task_nasa_power_shanghai_hourly_point_shanghai_31.2304_121.4737_t2m_20260524t230000z",
|
| 20 |
-
"data_fetched_at": "2026-05-
|
| 21 |
-
"context_length":
|
| 22 |
"prediction_length": 24
|
| 23 |
},
|
| 24 |
{
|
| 25 |
-
"dataset": "
|
| 26 |
-
"data_fetched_at": "2026-05-
|
| 27 |
"context_length": 96,
|
| 28 |
"prediction_length": 24
|
| 29 |
},
|
| 30 |
{
|
| 31 |
-
"dataset": "
|
| 32 |
-
"data_fetched_at": "2026-05-
|
| 33 |
"context_length": 96,
|
| 34 |
"prediction_length": 24
|
| 35 |
},
|
| 36 |
{
|
| 37 |
-
"dataset": "
|
| 38 |
-
"data_fetched_at": "2026-05-
|
| 39 |
"context_length": 96,
|
| 40 |
"prediction_length": 24
|
| 41 |
},
|
| 42 |
{
|
| 43 |
-
"dataset": "
|
| 44 |
-
"data_fetched_at": "2026-05-
|
| 45 |
"context_length": 96,
|
| 46 |
"prediction_length": 24
|
| 47 |
-
},
|
| 48 |
-
{
|
| 49 |
-
"dataset": "task_open_meteo_shanghai_hourly_grid_shanghai_31.2304_121.4737_temperature_2m_20260527t120000z",
|
| 50 |
-
"data_fetched_at": "2026-05-28T12:16:39.187896Z",
|
| 51 |
-
"context_length": 96,
|
| 52 |
-
"prediction_length": 24
|
| 53 |
-
},
|
| 54 |
-
{
|
| 55 |
-
"dataset": "task_usgs_potomac_iv_usgs_site_01646500_usgs_00060_20260528t095000z",
|
| 56 |
-
"data_fetched_at": "2026-05-28T12:16:39.187896Z",
|
| 57 |
-
"context_length": 96,
|
| 58 |
-
"prediction_length": 24
|
| 59 |
-
},
|
| 60 |
-
{
|
| 61 |
-
"dataset": "task_wikimedia_time_series_hourly_enwiki_article_time_series_pageviews_20260525t000000z",
|
| 62 |
-
"data_fetched_at": "2026-05-28T12:16:39.187896Z",
|
| 63 |
-
"context_length": 4,
|
| 64 |
-
"prediction_length": 2
|
| 65 |
}
|
| 66 |
]
|
| 67 |
}
|
|
|
|
| 1 |
{
|
| 2 |
"model": "Chronos-Bolt-Base",
|
| 3 |
"api_model_id": "amazon/chronos-bolt-base",
|
| 4 |
+
"evaluated_at": "2026-05-29T02:51:30.013825+00:00",
|
| 5 |
"datasets": [
|
| 6 |
{
|
| 7 |
+
"dataset": "task_binance_btcusdt_1h_crypto_btcusdt_close_20260528t020000z",
|
| 8 |
+
"data_fetched_at": "2026-05-29T02:50:05.083862Z",
|
| 9 |
"context_length": 96,
|
| 10 |
"prediction_length": 24
|
| 11 |
},
|
| 12 |
{
|
| 13 |
+
"dataset": "task_coingecko_bitcoin_7d_coin_bitcoin_price_usd_20260528t030012z",
|
| 14 |
+
"data_fetched_at": "2026-05-29T02:50:05.083862Z",
|
| 15 |
"context_length": 96,
|
| 16 |
"prediction_length": 24
|
| 17 |
},
|
| 18 |
{
|
| 19 |
"dataset": "task_nasa_power_shanghai_hourly_point_shanghai_31.2304_121.4737_t2m_20260524t230000z",
|
| 20 |
+
"data_fetched_at": "2026-05-29T02:50:05.083862Z",
|
| 21 |
+
"context_length": 69,
|
| 22 |
"prediction_length": 24
|
| 23 |
},
|
| 24 |
{
|
| 25 |
+
"dataset": "task_noaa_coops_sf_water_level_coops_station_9414290_water_level_20260529t001800z",
|
| 26 |
+
"data_fetched_at": "2026-05-29T02:50:05.083862Z",
|
| 27 |
"context_length": 96,
|
| 28 |
"prediction_length": 24
|
| 29 |
},
|
| 30 |
{
|
| 31 |
+
"dataset": "task_nws_ksfo_observations_nws_station_ksfo_temperature_20260529t004500z",
|
| 32 |
+
"data_fetched_at": "2026-05-29T02:50:05.083862Z",
|
| 33 |
"context_length": 96,
|
| 34 |
"prediction_length": 24
|
| 35 |
},
|
| 36 |
{
|
| 37 |
+
"dataset": "task_open_meteo_shanghai_air_quality_hourly_grid_shanghai_31.2304_121.4737_pm2_5_20260528t020000z",
|
| 38 |
+
"data_fetched_at": "2026-05-29T02:50:05.083862Z",
|
| 39 |
"context_length": 96,
|
| 40 |
"prediction_length": 24
|
| 41 |
},
|
| 42 |
{
|
| 43 |
+
"dataset": "task_usgs_potomac_iv_usgs_site_01646500_usgs_00060_20260528t235000z",
|
| 44 |
+
"data_fetched_at": "2026-05-29T02:50:05.083862Z",
|
| 45 |
"context_length": 96,
|
| 46 |
"prediction_length": 24
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 47 |
}
|
| 48 |
]
|
| 49 |
}
|
results/chronos_bolt_mini/all_results.csv
CHANGED
|
@@ -1,11 +1,8 @@
|
|
| 1 |
dataset,model,eval_metrics/MSE[mean],eval_metrics/MSE[0.5],eval_metrics/MAE[0.5],eval_metrics/MASE[0.5],eval_metrics/MAPE[0.5],eval_metrics/sMAPE[0.5],eval_metrics/MSIS,eval_metrics/RMSE[mean],eval_metrics/NRMSE[mean],eval_metrics/ND[0.5],eval_metrics/mean_weighted_sum_quantile_loss,domain,num_variates
|
| 2 |
-
binance_btcusdt_1h_close/1h/short,Chronos-Bolt-Mini,
|
| 3 |
-
coingecko_bitcoin_7d_price_usd/1h/short,Chronos-Bolt-Mini,
|
| 4 |
-
nasa_power_shanghai_hourly_T2M/1h/short,Chronos-Bolt-Mini,9.
|
| 5 |
-
noaa_coops_sf_water_level_water_level/6min/short,Chronos-Bolt-Mini,0.
|
| 6 |
-
|
| 7 |
-
|
| 8 |
-
|
| 9 |
-
open_meteo_shanghai_hourly_temperature_2m/1h/short,Chronos-Bolt-Mini,0.8828099140087033,0.8828099140087033,0.7188208103179932,0.39149096453335225,0.028733217601040043,0.028834013771296452,2.773892978739956,0.9395796475066408,0.03749569442091312,0.02868589748468996,0.02969891826111223,Weather,1
|
| 10 |
-
usgs_potomac_iv_usgs_00060/15min/short,Chronos-Bolt-Mini,4843.679802894592,4843.679802894592,64.029296875,nan,0.002519860464793693,0.0025163275141648713,nan,69.59655022265538,0.002736878920766392,0.0025179471161723742,0.001616091514077776,Hydrology,1
|
| 11 |
-
wikimedia_time_series_hourly_pageviews/1D/short,Chronos-Bolt-Mini,4577.245442453423,4577.245442453423,67.47705841064453,1.445936965942383,0.21708162745704251,0.24360223675984644,8.007688195364816,67.65534304438506,0.21789160400768137,0.21731741839176982,0.23123530744333173,Web,1
|
|
|
|
| 1 |
dataset,model,eval_metrics/MSE[mean],eval_metrics/MSE[0.5],eval_metrics/MAE[0.5],eval_metrics/MASE[0.5],eval_metrics/MAPE[0.5],eval_metrics/sMAPE[0.5],eval_metrics/MSIS,eval_metrics/RMSE[mean],eval_metrics/NRMSE[mean],eval_metrics/ND[0.5],eval_metrics/mean_weighted_sum_quantile_loss,domain,num_variates
|
| 2 |
+
binance_btcusdt_1h_close/1h/short,Chronos-Bolt-Mini,851287.6321029663,851287.6321029663,882.4768880208334,0.976535855233298,0.012034191309566289,0.011955457296705658,10.656671858587634,922.6524980202278,0.012569999508968603,0.012022656496243319,0.008244497111170809,Finance,1
|
| 3 |
+
coingecko_bitcoin_7d_price_usd/1h/short,Chronos-Bolt-Mini,864730.2875595093,864730.2875595093,885.3583984375,0.9581359787768308,0.012093613190106718,0.012013367698897761,11.016269186291742,929.9087522760011,0.01268853126193431,0.012080645212871425,0.0083666342836689,Finance,1
|
| 4 |
+
nasa_power_shanghai_hourly_T2M/1h/short,Chronos-Bolt-Mini,9.548240780425354,9.548240780425354,2.5835858980814614,0.7522085647795073,0.09503784443039846,0.1018251998426517,3.8696742236218,3.090022779920134,0.12039245425054106,0.10066082653447453,0.08411044052817845,Weather,1
|
| 5 |
+
noaa_coops_sf_water_level_water_level/6min/short,Chronos-Bolt-Mini,0.01640126373424522,0.01640126373424522,0.0986495316028595,6.185944284331548,0.07142778184577067,0.07563235656687009,25.340507814660793,0.12806741870688743,0.10034337752300845,0.07729387608529449,0.0749025769287721,Ocean,1
|
| 6 |
+
nws_ksfo_observations_temperature/1h/short,Chronos-Bolt-Mini,6.4003104863786575,6.4003104863786575,1.9677637418111165,2.4218630668444514,0.1268957925752334,0.1146813703822039,35.354552146715996,2.5298834926491494,0.14936581506415642,0.11617793309585928,0.0828891098816994,Weather,1
|
| 7 |
+
open_meteo_shanghai_air_quality_hourly_pm2_5/1h/short,Chronos-Bolt-Mini,1257.403550816446,1257.403550816446,29.262885729471844,1.2839291727739017,0.4375496892711623,0.5885718970399296,15.36094857308285,35.45988650315235,0.6043440602078141,0.4987283637688504,0.40799980188904916,Air Quality,1
|
| 8 |
+
usgs_potomac_iv_usgs_00060/15min/short,Chronos-Bolt-Mini,68632.59780025482,68632.59780025482,211.42757161458334,nan,0.007524633240620554,0.007560872203681779,nan,261.9782391731321,0.00941098299678966,0.007595063192261637,0.006913356011138552,Hydrology,1
|
|
|
|
|
|
|
|
|
results/chronos_bolt_mini/config.json
CHANGED
|
@@ -3,7 +3,7 @@
|
|
| 3 |
"model_type": "zero-shot",
|
| 4 |
"model_dtype": "float32",
|
| 5 |
"model_link": "https://tsfm.ai/models/amazon/chronos-bolt-mini",
|
| 6 |
-
"code_link": "https://github.com/zhouziyu02/TSFM_Bench/blob/main/scripts/
|
| 7 |
"org": "Amazon / TSFM.ai",
|
| 8 |
"testdata_leakage": "No",
|
| 9 |
"replication_code_available": "Yes",
|
|
|
|
| 3 |
"model_type": "zero-shot",
|
| 4 |
"model_dtype": "float32",
|
| 5 |
"model_link": "https://tsfm.ai/models/amazon/chronos-bolt-mini",
|
| 6 |
+
"code_link": "https://github.com/zhouziyu02/TSFM_Bench/blob/main/space/scripts/run_space_eval.py",
|
| 7 |
"org": "Amazon / TSFM.ai",
|
| 8 |
"testdata_leakage": "No",
|
| 9 |
"replication_code_available": "Yes",
|
results/chronos_bolt_mini/online_meta.json
CHANGED
|
@@ -1,67 +1,49 @@
|
|
| 1 |
{
|
| 2 |
"model": "Chronos-Bolt-Mini",
|
| 3 |
"api_model_id": "amazon/chronos-bolt-mini",
|
| 4 |
-
"evaluated_at": "2026-05-
|
| 5 |
"datasets": [
|
| 6 |
{
|
| 7 |
-
"dataset": "
|
| 8 |
-
"data_fetched_at": "2026-05-
|
| 9 |
"context_length": 96,
|
| 10 |
"prediction_length": 24
|
| 11 |
},
|
| 12 |
{
|
| 13 |
-
"dataset": "
|
| 14 |
-
"data_fetched_at": "2026-05-
|
| 15 |
"context_length": 96,
|
| 16 |
"prediction_length": 24
|
| 17 |
},
|
| 18 |
{
|
| 19 |
"dataset": "task_nasa_power_shanghai_hourly_point_shanghai_31.2304_121.4737_t2m_20260524t230000z",
|
| 20 |
-
"data_fetched_at": "2026-05-
|
| 21 |
-
"context_length":
|
| 22 |
"prediction_length": 24
|
| 23 |
},
|
| 24 |
{
|
| 25 |
-
"dataset": "
|
| 26 |
-
"data_fetched_at": "2026-05-
|
| 27 |
"context_length": 96,
|
| 28 |
"prediction_length": 24
|
| 29 |
},
|
| 30 |
{
|
| 31 |
-
"dataset": "
|
| 32 |
-
"data_fetched_at": "2026-05-
|
| 33 |
"context_length": 96,
|
| 34 |
"prediction_length": 24
|
| 35 |
},
|
| 36 |
{
|
| 37 |
-
"dataset": "
|
| 38 |
-
"data_fetched_at": "2026-05-
|
| 39 |
"context_length": 96,
|
| 40 |
"prediction_length": 24
|
| 41 |
},
|
| 42 |
{
|
| 43 |
-
"dataset": "
|
| 44 |
-
"data_fetched_at": "2026-05-
|
| 45 |
"context_length": 96,
|
| 46 |
"prediction_length": 24
|
| 47 |
-
},
|
| 48 |
-
{
|
| 49 |
-
"dataset": "task_open_meteo_shanghai_hourly_grid_shanghai_31.2304_121.4737_temperature_2m_20260527t120000z",
|
| 50 |
-
"data_fetched_at": "2026-05-28T12:16:39.187896Z",
|
| 51 |
-
"context_length": 96,
|
| 52 |
-
"prediction_length": 24
|
| 53 |
-
},
|
| 54 |
-
{
|
| 55 |
-
"dataset": "task_usgs_potomac_iv_usgs_site_01646500_usgs_00060_20260528t095000z",
|
| 56 |
-
"data_fetched_at": "2026-05-28T12:16:39.187896Z",
|
| 57 |
-
"context_length": 96,
|
| 58 |
-
"prediction_length": 24
|
| 59 |
-
},
|
| 60 |
-
{
|
| 61 |
-
"dataset": "task_wikimedia_time_series_hourly_enwiki_article_time_series_pageviews_20260525t000000z",
|
| 62 |
-
"data_fetched_at": "2026-05-28T12:16:39.187896Z",
|
| 63 |
-
"context_length": 4,
|
| 64 |
-
"prediction_length": 2
|
| 65 |
}
|
| 66 |
]
|
| 67 |
}
|
|
|
|
| 1 |
{
|
| 2 |
"model": "Chronos-Bolt-Mini",
|
| 3 |
"api_model_id": "amazon/chronos-bolt-mini",
|
| 4 |
+
"evaluated_at": "2026-05-29T02:51:10.536027+00:00",
|
| 5 |
"datasets": [
|
| 6 |
{
|
| 7 |
+
"dataset": "task_binance_btcusdt_1h_crypto_btcusdt_close_20260528t020000z",
|
| 8 |
+
"data_fetched_at": "2026-05-29T02:50:05.083862Z",
|
| 9 |
"context_length": 96,
|
| 10 |
"prediction_length": 24
|
| 11 |
},
|
| 12 |
{
|
| 13 |
+
"dataset": "task_coingecko_bitcoin_7d_coin_bitcoin_price_usd_20260528t030012z",
|
| 14 |
+
"data_fetched_at": "2026-05-29T02:50:05.083862Z",
|
| 15 |
"context_length": 96,
|
| 16 |
"prediction_length": 24
|
| 17 |
},
|
| 18 |
{
|
| 19 |
"dataset": "task_nasa_power_shanghai_hourly_point_shanghai_31.2304_121.4737_t2m_20260524t230000z",
|
| 20 |
+
"data_fetched_at": "2026-05-29T02:50:05.083862Z",
|
| 21 |
+
"context_length": 69,
|
| 22 |
"prediction_length": 24
|
| 23 |
},
|
| 24 |
{
|
| 25 |
+
"dataset": "task_noaa_coops_sf_water_level_coops_station_9414290_water_level_20260529t001800z",
|
| 26 |
+
"data_fetched_at": "2026-05-29T02:50:05.083862Z",
|
| 27 |
"context_length": 96,
|
| 28 |
"prediction_length": 24
|
| 29 |
},
|
| 30 |
{
|
| 31 |
+
"dataset": "task_nws_ksfo_observations_nws_station_ksfo_temperature_20260529t004500z",
|
| 32 |
+
"data_fetched_at": "2026-05-29T02:50:05.083862Z",
|
| 33 |
"context_length": 96,
|
| 34 |
"prediction_length": 24
|
| 35 |
},
|
| 36 |
{
|
| 37 |
+
"dataset": "task_open_meteo_shanghai_air_quality_hourly_grid_shanghai_31.2304_121.4737_pm2_5_20260528t020000z",
|
| 38 |
+
"data_fetched_at": "2026-05-29T02:50:05.083862Z",
|
| 39 |
"context_length": 96,
|
| 40 |
"prediction_length": 24
|
| 41 |
},
|
| 42 |
{
|
| 43 |
+
"dataset": "task_usgs_potomac_iv_usgs_site_01646500_usgs_00060_20260528t235000z",
|
| 44 |
+
"data_fetched_at": "2026-05-29T02:50:05.083862Z",
|
| 45 |
"context_length": 96,
|
| 46 |
"prediction_length": 24
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 47 |
}
|
| 48 |
]
|
| 49 |
}
|
results/chronos_bolt_tiny/all_results.csv
CHANGED
|
@@ -1,11 +1,8 @@
|
|
| 1 |
dataset,model,eval_metrics/MSE[mean],eval_metrics/MSE[0.5],eval_metrics/MAE[0.5],eval_metrics/MASE[0.5],eval_metrics/MAPE[0.5],eval_metrics/sMAPE[0.5],eval_metrics/MSIS,eval_metrics/RMSE[mean],eval_metrics/NRMSE[mean],eval_metrics/ND[0.5],eval_metrics/mean_weighted_sum_quantile_loss,domain,num_variates
|
| 2 |
-
binance_btcusdt_1h_close/1h/short,Chronos-Bolt-Tiny,
|
| 3 |
-
coingecko_bitcoin_7d_price_usd/1h/short,Chronos-Bolt-Tiny,
|
| 4 |
-
nasa_power_shanghai_hourly_T2M/1h/short,Chronos-Bolt-Tiny,
|
| 5 |
-
noaa_coops_sf_water_level_water_level/6min/short,Chronos-Bolt-Tiny,0.
|
| 6 |
-
|
| 7 |
-
|
| 8 |
-
|
| 9 |
-
open_meteo_shanghai_hourly_temperature_2m/1h/short,Chronos-Bolt-Tiny,0.5948811111434225,0.5948811111434225,0.6484154860178629,0.35314615324951076,0.026197955231929627,0.025978455964565026,2.995656525072318,0.7712853629775573,0.030779594213512538,0.025876240493323176,0.02777387188201828,Weather,1
|
| 10 |
-
usgs_potomac_iv_usgs_00060/15min/short,Chronos-Bolt-Tiny,3149.2314387957254,3149.2314387957254,54.302978515625,nan,0.0021364303572376296,0.0021344827808550478,nan,56.11801349652111,0.002206836513053427,0.002135460403694904,0.0014279900766016713,Hydrology,1
|
| 11 |
-
wikimedia_time_series_hourly_pageviews/1D/short,Chronos-Bolt-Tiny,4641.08992965892,4641.08992965892,67.62164306640625,1.4490352085658482,0.21732069855015004,0.24415806200991483,8.605713272094727,68.125545353112,0.21940594316622222,0.21778306945702497,0.2325466885637525,Web,1
|
|
|
|
| 1 |
dataset,model,eval_metrics/MSE[mean],eval_metrics/MSE[0.5],eval_metrics/MAE[0.5],eval_metrics/MASE[0.5],eval_metrics/MAPE[0.5],eval_metrics/sMAPE[0.5],eval_metrics/MSIS,eval_metrics/RMSE[mean],eval_metrics/NRMSE[mean],eval_metrics/ND[0.5],eval_metrics/mean_weighted_sum_quantile_loss,domain,num_variates
|
| 2 |
+
binance_btcusdt_1h_close/1h/short,Chronos-Bolt-Tiny,1029025.1603978475,1029025.1603978475,984.4554036458334,1.0893837703720461,0.01342267473206749,0.013327594824025202,11.50025719460482,1014.4087738174624,0.013820065318350654,0.013411987684401543,0.008981437172713323,Finance,1
|
| 3 |
+
coingecko_bitcoin_7d_price_usd/1h/short,Chronos-Bolt-Tiny,1049676.785677592,1049676.785677592,991.6748046875,1.073191728111963,0.013543286102919765,0.013445979208987893,11.929663258797348,1024.537352016798,0.0139797310093777,0.013531324154281413,0.009152163608285595,Finance,1
|
| 4 |
+
nasa_power_shanghai_hourly_T2M/1h/short,Chronos-Bolt-Tiny,9.325186076793065,9.325186076793065,2.57205859820048,0.7488524024372003,0.09445699084526753,0.10095493097514746,2.3972798950588126,3.0537167643370373,0.11897791117710119,0.10021170365662015,0.08079254781825497,Weather,1
|
| 5 |
+
noaa_coops_sf_water_level_water_level/6min/short,Chronos-Bolt-Tiny,0.015442472707887234,0.015442472707887234,0.08998485902945201,5.64261497591828,0.06416666330862907,0.06807652745844346,21.565446339911507,0.12426774604814893,0.09736625819078952,0.07050493226238033,0.06671574848406818,Ocean,1
|
| 6 |
+
nws_ksfo_observations_temperature/1h/short,Chronos-Bolt-Tiny,7.537212424866539,7.537212424866539,2.111232042312622,2.5984394366924577,0.13644075438988787,0.12216720632877787,52.66092036320614,2.745398409132368,0.16208994297460477,0.12464838626199982,0.09697055688920332,Weather,1
|
| 7 |
+
open_meteo_shanghai_air_quality_hourly_pm2_5/1h/short,Chronos-Bolt-Tiny,1461.9665035634837,1461.9665035634837,32.67136557896932,1.4334785628798237,0.5069763645738149,0.7007815976613591,18.987324775681852,38.23567056510823,0.6516518430489527,0.5568192025874885,0.44639292146149856,Air Quality,1
|
| 8 |
+
usgs_potomac_iv_usgs_00060/15min/short,Chronos-Bolt-Tiny,78316.96788072586,78316.96788072586,221.53458658854166,nan,0.007875539639408598,0.007920176132274283,nan,279.8516890796371,0.010053046756340802,0.00795813512666517,0.007018508513986596,Hydrology,1
|
|
|
|
|
|
|
|
|
results/chronos_bolt_tiny/config.json
CHANGED
|
@@ -3,7 +3,7 @@
|
|
| 3 |
"model_type": "zero-shot",
|
| 4 |
"model_dtype": "float32",
|
| 5 |
"model_link": "https://tsfm.ai/models/amazon/chronos-bolt-tiny",
|
| 6 |
-
"code_link": "https://github.com/zhouziyu02/TSFM_Bench/blob/main/scripts/
|
| 7 |
"org": "Amazon / TSFM.ai",
|
| 8 |
"testdata_leakage": "No",
|
| 9 |
"replication_code_available": "Yes",
|
|
|
|
| 3 |
"model_type": "zero-shot",
|
| 4 |
"model_dtype": "float32",
|
| 5 |
"model_link": "https://tsfm.ai/models/amazon/chronos-bolt-tiny",
|
| 6 |
+
"code_link": "https://github.com/zhouziyu02/TSFM_Bench/blob/main/space/scripts/run_space_eval.py",
|
| 7 |
"org": "Amazon / TSFM.ai",
|
| 8 |
"testdata_leakage": "No",
|
| 9 |
"replication_code_available": "Yes",
|
results/chronos_bolt_tiny/online_meta.json
CHANGED
|
@@ -1,67 +1,49 @@
|
|
| 1 |
{
|
| 2 |
"model": "Chronos-Bolt-Tiny",
|
| 3 |
"api_model_id": "amazon/chronos-bolt-tiny",
|
| 4 |
-
"evaluated_at": "2026-05-
|
| 5 |
"datasets": [
|
| 6 |
{
|
| 7 |
-
"dataset": "
|
| 8 |
-
"data_fetched_at": "2026-05-
|
| 9 |
"context_length": 96,
|
| 10 |
"prediction_length": 24
|
| 11 |
},
|
| 12 |
{
|
| 13 |
-
"dataset": "
|
| 14 |
-
"data_fetched_at": "2026-05-
|
| 15 |
"context_length": 96,
|
| 16 |
"prediction_length": 24
|
| 17 |
},
|
| 18 |
{
|
| 19 |
"dataset": "task_nasa_power_shanghai_hourly_point_shanghai_31.2304_121.4737_t2m_20260524t230000z",
|
| 20 |
-
"data_fetched_at": "2026-05-
|
| 21 |
-
"context_length":
|
| 22 |
"prediction_length": 24
|
| 23 |
},
|
| 24 |
{
|
| 25 |
-
"dataset": "
|
| 26 |
-
"data_fetched_at": "2026-05-
|
| 27 |
"context_length": 96,
|
| 28 |
"prediction_length": 24
|
| 29 |
},
|
| 30 |
{
|
| 31 |
-
"dataset": "
|
| 32 |
-
"data_fetched_at": "2026-05-
|
| 33 |
"context_length": 96,
|
| 34 |
"prediction_length": 24
|
| 35 |
},
|
| 36 |
{
|
| 37 |
-
"dataset": "
|
| 38 |
-
"data_fetched_at": "2026-05-
|
| 39 |
"context_length": 96,
|
| 40 |
"prediction_length": 24
|
| 41 |
},
|
| 42 |
{
|
| 43 |
-
"dataset": "
|
| 44 |
-
"data_fetched_at": "2026-05-
|
| 45 |
"context_length": 96,
|
| 46 |
"prediction_length": 24
|
| 47 |
-
},
|
| 48 |
-
{
|
| 49 |
-
"dataset": "task_open_meteo_shanghai_hourly_grid_shanghai_31.2304_121.4737_temperature_2m_20260527t120000z",
|
| 50 |
-
"data_fetched_at": "2026-05-28T12:16:39.187896Z",
|
| 51 |
-
"context_length": 96,
|
| 52 |
-
"prediction_length": 24
|
| 53 |
-
},
|
| 54 |
-
{
|
| 55 |
-
"dataset": "task_usgs_potomac_iv_usgs_site_01646500_usgs_00060_20260528t095000z",
|
| 56 |
-
"data_fetched_at": "2026-05-28T12:16:39.187896Z",
|
| 57 |
-
"context_length": 96,
|
| 58 |
-
"prediction_length": 24
|
| 59 |
-
},
|
| 60 |
-
{
|
| 61 |
-
"dataset": "task_wikimedia_time_series_hourly_enwiki_article_time_series_pageviews_20260525t000000z",
|
| 62 |
-
"data_fetched_at": "2026-05-28T12:16:39.187896Z",
|
| 63 |
-
"context_length": 4,
|
| 64 |
-
"prediction_length": 2
|
| 65 |
}
|
| 66 |
]
|
| 67 |
}
|
|
|
|
| 1 |
{
|
| 2 |
"model": "Chronos-Bolt-Tiny",
|
| 3 |
"api_model_id": "amazon/chronos-bolt-tiny",
|
| 4 |
+
"evaluated_at": "2026-05-29T02:50:52.849106+00:00",
|
| 5 |
"datasets": [
|
| 6 |
{
|
| 7 |
+
"dataset": "task_binance_btcusdt_1h_crypto_btcusdt_close_20260528t020000z",
|
| 8 |
+
"data_fetched_at": "2026-05-29T02:50:05.083862Z",
|
| 9 |
"context_length": 96,
|
| 10 |
"prediction_length": 24
|
| 11 |
},
|
| 12 |
{
|
| 13 |
+
"dataset": "task_coingecko_bitcoin_7d_coin_bitcoin_price_usd_20260528t030012z",
|
| 14 |
+
"data_fetched_at": "2026-05-29T02:50:05.083862Z",
|
| 15 |
"context_length": 96,
|
| 16 |
"prediction_length": 24
|
| 17 |
},
|
| 18 |
{
|
| 19 |
"dataset": "task_nasa_power_shanghai_hourly_point_shanghai_31.2304_121.4737_t2m_20260524t230000z",
|
| 20 |
+
"data_fetched_at": "2026-05-29T02:50:05.083862Z",
|
| 21 |
+
"context_length": 69,
|
| 22 |
"prediction_length": 24
|
| 23 |
},
|
| 24 |
{
|
| 25 |
+
"dataset": "task_noaa_coops_sf_water_level_coops_station_9414290_water_level_20260529t001800z",
|
| 26 |
+
"data_fetched_at": "2026-05-29T02:50:05.083862Z",
|
| 27 |
"context_length": 96,
|
| 28 |
"prediction_length": 24
|
| 29 |
},
|
| 30 |
{
|
| 31 |
+
"dataset": "task_nws_ksfo_observations_nws_station_ksfo_temperature_20260529t004500z",
|
| 32 |
+
"data_fetched_at": "2026-05-29T02:50:05.083862Z",
|
| 33 |
"context_length": 96,
|
| 34 |
"prediction_length": 24
|
| 35 |
},
|
| 36 |
{
|
| 37 |
+
"dataset": "task_open_meteo_shanghai_air_quality_hourly_grid_shanghai_31.2304_121.4737_pm2_5_20260528t020000z",
|
| 38 |
+
"data_fetched_at": "2026-05-29T02:50:05.083862Z",
|
| 39 |
"context_length": 96,
|
| 40 |
"prediction_length": 24
|
| 41 |
},
|
| 42 |
{
|
| 43 |
+
"dataset": "task_usgs_potomac_iv_usgs_site_01646500_usgs_00060_20260528t235000z",
|
| 44 |
+
"data_fetched_at": "2026-05-29T02:50:05.083862Z",
|
| 45 |
"context_length": 96,
|
| 46 |
"prediction_length": 24
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 47 |
}
|
| 48 |
]
|
| 49 |
}
|
results/dataset_properties.csv
CHANGED
|
@@ -3,9 +3,6 @@ binance_btcusdt_1h_close,Finance,1h,1
|
|
| 3 |
coingecko_bitcoin_7d_price_usd,Finance,1h,1
|
| 4 |
nasa_power_shanghai_hourly_t2m,Weather,1h,1
|
| 5 |
noaa_coops_sf_water_level_water_level,Ocean,6min,1
|
| 6 |
-
noaa_ndbc_46013_realtime_wave_height,Ocean,10min,1
|
| 7 |
nws_ksfo_observations_temperature,Weather,1h,1
|
| 8 |
open_meteo_shanghai_air_quality_hourly_pm2_5,Air Quality,1h,1
|
| 9 |
-
open_meteo_shanghai_hourly_temperature_2m,Weather,1h,1
|
| 10 |
usgs_potomac_iv_usgs_00060,Hydrology,15min,1
|
| 11 |
-
wikimedia_time_series_hourly_pageviews,Web,1D,1
|
|
|
|
| 3 |
coingecko_bitcoin_7d_price_usd,Finance,1h,1
|
| 4 |
nasa_power_shanghai_hourly_t2m,Weather,1h,1
|
| 5 |
noaa_coops_sf_water_level_water_level,Ocean,6min,1
|
|
|
|
| 6 |
nws_ksfo_observations_temperature,Weather,1h,1
|
| 7 |
open_meteo_shanghai_air_quality_hourly_pm2_5,Air Quality,1h,1
|
|
|
|
| 8 |
usgs_potomac_iv_usgs_00060,Hydrology,15min,1
|
|
|
results/eval_state.json
ADDED
|
@@ -0,0 +1,10 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"status": "ok",
|
| 3 |
+
"started_at": "2026-05-29T02:49:52.974580+00:00",
|
| 4 |
+
"finished_at": "2026-05-29T02:55:13.595519+00:00",
|
| 5 |
+
"failed_models": [],
|
| 6 |
+
"task_count": 7,
|
| 7 |
+
"model_count": 6,
|
| 8 |
+
"message": "Evaluation complete",
|
| 9 |
+
"updated_at": "2026-05-29T02:55:13.595519+00:00"
|
| 10 |
+
}
|
results/moirai_1_1_small/all_results.csv
CHANGED
|
@@ -1,11 +1,8 @@
|
|
| 1 |
dataset,model,eval_metrics/MSE[mean],eval_metrics/MSE[0.5],eval_metrics/MAE[0.5],eval_metrics/MASE[0.5],eval_metrics/MAPE[0.5],eval_metrics/sMAPE[0.5],eval_metrics/MSIS,eval_metrics/RMSE[mean],eval_metrics/NRMSE[mean],eval_metrics/ND[0.5],eval_metrics/mean_weighted_sum_quantile_loss,domain,num_variates
|
| 2 |
-
binance_btcusdt_1h_close/1h/short,Moirai-1.1-Small,
|
| 3 |
-
coingecko_bitcoin_7d_price_usd/1h/short,Moirai-1.1-Small,
|
| 4 |
-
nasa_power_shanghai_hourly_T2M/1h/short,Moirai-1.1-Small,
|
| 5 |
-
noaa_coops_sf_water_level_water_level/6min/short,Moirai-1.1-Small,0.
|
| 6 |
-
|
| 7 |
-
|
| 8 |
-
|
| 9 |
-
open_meteo_shanghai_hourly_temperature_2m/1h/short,Moirai-1.1-Small,0.7451148641512191,0.7336263607690702,0.7014484405517578,0.38202946077832767,0.028982091534334033,0.02845294724981401,3.3017538202957617,0.8632003615332996,0.03444763524408459,0.027992620368853683,0.026208127846003952,Weather,1
|
| 10 |
-
usgs_potomac_iv_usgs_00060/15min/short,Moirai-1.1-Small,10798.155609925589,6040.659053166707,68.469970703125,nan,0.0026951572531489633,0.0026905473764017203,nan,103.91417424935632,0.004086416814655992,0.002692576268843192,0.0026380449775952625,Hydrology,1
|
| 11 |
-
wikimedia_time_series_hourly_pageviews/1D/short,Moirai-1.1-Small,3089.0964928199537,2815.892555135797,52.98386764526367,1.1353685923985073,0.17051746000903956,0.18643314794028967,2.6726661137172156,55.579640992183045,0.17900045408110482,0.17064047550809555,0.13484585527060686,Web,1
|
|
|
|
| 1 |
dataset,model,eval_metrics/MSE[mean],eval_metrics/MSE[0.5],eval_metrics/MAE[0.5],eval_metrics/MASE[0.5],eval_metrics/MAPE[0.5],eval_metrics/sMAPE[0.5],eval_metrics/MSIS,eval_metrics/RMSE[mean],eval_metrics/NRMSE[mean],eval_metrics/ND[0.5],eval_metrics/mean_weighted_sum_quantile_loss,domain,num_variates
|
| 2 |
+
binance_btcusdt_1h_close/1h/short,Moirai-1.1-Small,952374.008026123,920160.1939131418,925.12646484375,1.0237312453261904,0.012614736997089234,0.012529663657250362,10.904908958873357,975.8965150189456,0.013295383409153661,0.012603704248102358,0.008648531284962633,Finance,1
|
| 3 |
+
coingecko_bitcoin_7d_price_usd/1h/short,Moirai-1.1-Small,980882.1796239218,946688.7947781881,936.5074869791666,1.0134895871007623,0.012790573136249405,0.012702768980917836,11.206580901590142,990.3949614289855,0.013513860794401974,0.012778570474239172,0.008733789549838429,Finance,1
|
| 4 |
+
nasa_power_shanghai_hourly_T2M/1h/short,Moirai-1.1-Small,22.11944576046153,23.27638368696455,3.802117625872294,1.106982912627002,0.1363775798751412,0.15233264027581442,7.944486966482941,4.703131484496423,0.18324186661696704,0.14813686012367747,0.13622131038191412,Weather,1
|
| 5 |
+
noaa_coops_sf_water_level_water_level/6min/short,Moirai-1.1-Small,0.07894352857742426,0.08123021915922779,0.24160874634981155,15.1503835775314,0.17799479341783764,0.2016127114400773,229.84756618905703,0.2809689103396037,0.2201447466272617,0.18930527289948496,0.1584442261775357,Ocean,1
|
| 6 |
+
nws_ksfo_observations_temperature/1h/short,Moirai-1.1-Small,7.333502640536911,7.098776238255065,2.0732465585072837,2.551688072008964,0.13368541403511328,0.12020856200648522,67.84784727830153,2.708044061779075,0.1598845202526391,0.12240570087127874,0.1052653917054816,Weather,1
|
| 7 |
+
open_meteo_shanghai_air_quality_hourly_pm2_5/1h/short,Moirai-1.1-Small,973.8622120598106,1023.9657259982699,23.962568004926045,1.051374098936426,0.3256804509855051,0.4334222612374714,12.957339274599958,31.206765485384906,0.5318579730285935,0.4083948672485306,0.3651104991221374,Air Quality,1
|
| 8 |
+
usgs_potomac_iv_usgs_00060/15min/short,Moirai-1.1-Small,96855.45377286275,112976.57937427361,257.6378173828125,nan,0.009145275820772208,0.009214959719790085,nan,311.2160885508054,0.011179742740935982,0.009255063040244723,0.008606347849519367,Hydrology,1
|
|
|
|
|
|
|
|
|
results/moirai_1_1_small/config.json
CHANGED
|
@@ -3,7 +3,7 @@
|
|
| 3 |
"model_type": "zero-shot",
|
| 4 |
"model_dtype": "float32",
|
| 5 |
"model_link": "https://tsfm.ai/models/Salesforce/moirai-1.1-R-small",
|
| 6 |
-
"code_link": "https://github.com/zhouziyu02/TSFM_Bench/blob/main/scripts/
|
| 7 |
"org": "Salesforce / TSFM.ai",
|
| 8 |
"testdata_leakage": "No",
|
| 9 |
"replication_code_available": "Yes",
|
|
|
|
| 3 |
"model_type": "zero-shot",
|
| 4 |
"model_dtype": "float32",
|
| 5 |
"model_link": "https://tsfm.ai/models/Salesforce/moirai-1.1-R-small",
|
| 6 |
+
"code_link": "https://github.com/zhouziyu02/TSFM_Bench/blob/main/space/scripts/run_space_eval.py",
|
| 7 |
"org": "Salesforce / TSFM.ai",
|
| 8 |
"testdata_leakage": "No",
|
| 9 |
"replication_code_available": "Yes",
|
results/moirai_1_1_small/online_meta.json
CHANGED
|
@@ -1,67 +1,49 @@
|
|
| 1 |
{
|
| 2 |
"model": "Moirai-1.1-Small",
|
| 3 |
"api_model_id": "Salesforce/moirai-1.1-R-small",
|
| 4 |
-
"evaluated_at": "2026-05-
|
| 5 |
"datasets": [
|
| 6 |
{
|
| 7 |
-
"dataset": "
|
| 8 |
-
"data_fetched_at": "2026-05-
|
| 9 |
"context_length": 96,
|
| 10 |
"prediction_length": 24
|
| 11 |
},
|
| 12 |
{
|
| 13 |
-
"dataset": "
|
| 14 |
-
"data_fetched_at": "2026-05-
|
| 15 |
"context_length": 96,
|
| 16 |
"prediction_length": 24
|
| 17 |
},
|
| 18 |
{
|
| 19 |
"dataset": "task_nasa_power_shanghai_hourly_point_shanghai_31.2304_121.4737_t2m_20260524t230000z",
|
| 20 |
-
"data_fetched_at": "2026-05-
|
| 21 |
-
"context_length":
|
| 22 |
"prediction_length": 24
|
| 23 |
},
|
| 24 |
{
|
| 25 |
-
"dataset": "
|
| 26 |
-
"data_fetched_at": "2026-05-
|
| 27 |
"context_length": 96,
|
| 28 |
"prediction_length": 24
|
| 29 |
},
|
| 30 |
{
|
| 31 |
-
"dataset": "
|
| 32 |
-
"data_fetched_at": "2026-05-
|
| 33 |
"context_length": 96,
|
| 34 |
"prediction_length": 24
|
| 35 |
},
|
| 36 |
{
|
| 37 |
-
"dataset": "
|
| 38 |
-
"data_fetched_at": "2026-05-
|
| 39 |
"context_length": 96,
|
| 40 |
"prediction_length": 24
|
| 41 |
},
|
| 42 |
{
|
| 43 |
-
"dataset": "
|
| 44 |
-
"data_fetched_at": "2026-05-
|
| 45 |
"context_length": 96,
|
| 46 |
"prediction_length": 24
|
| 47 |
-
},
|
| 48 |
-
{
|
| 49 |
-
"dataset": "task_open_meteo_shanghai_hourly_grid_shanghai_31.2304_121.4737_temperature_2m_20260527t120000z",
|
| 50 |
-
"data_fetched_at": "2026-05-28T12:16:39.187896Z",
|
| 51 |
-
"context_length": 96,
|
| 52 |
-
"prediction_length": 24
|
| 53 |
-
},
|
| 54 |
-
{
|
| 55 |
-
"dataset": "task_usgs_potomac_iv_usgs_site_01646500_usgs_00060_20260528t095000z",
|
| 56 |
-
"data_fetched_at": "2026-05-28T12:16:39.187896Z",
|
| 57 |
-
"context_length": 96,
|
| 58 |
-
"prediction_length": 24
|
| 59 |
-
},
|
| 60 |
-
{
|
| 61 |
-
"dataset": "task_wikimedia_time_series_hourly_enwiki_article_time_series_pageviews_20260525t000000z",
|
| 62 |
-
"data_fetched_at": "2026-05-28T12:16:39.187896Z",
|
| 63 |
-
"context_length": 4,
|
| 64 |
-
"prediction_length": 2
|
| 65 |
}
|
| 66 |
]
|
| 67 |
}
|
|
|
|
| 1 |
{
|
| 2 |
"model": "Moirai-1.1-Small",
|
| 3 |
"api_model_id": "Salesforce/moirai-1.1-R-small",
|
| 4 |
+
"evaluated_at": "2026-05-29T02:55:13.592899+00:00",
|
| 5 |
"datasets": [
|
| 6 |
{
|
| 7 |
+
"dataset": "task_binance_btcusdt_1h_crypto_btcusdt_close_20260528t020000z",
|
| 8 |
+
"data_fetched_at": "2026-05-29T02:50:05.083862Z",
|
| 9 |
"context_length": 96,
|
| 10 |
"prediction_length": 24
|
| 11 |
},
|
| 12 |
{
|
| 13 |
+
"dataset": "task_coingecko_bitcoin_7d_coin_bitcoin_price_usd_20260528t030012z",
|
| 14 |
+
"data_fetched_at": "2026-05-29T02:50:05.083862Z",
|
| 15 |
"context_length": 96,
|
| 16 |
"prediction_length": 24
|
| 17 |
},
|
| 18 |
{
|
| 19 |
"dataset": "task_nasa_power_shanghai_hourly_point_shanghai_31.2304_121.4737_t2m_20260524t230000z",
|
| 20 |
+
"data_fetched_at": "2026-05-29T02:50:05.083862Z",
|
| 21 |
+
"context_length": 69,
|
| 22 |
"prediction_length": 24
|
| 23 |
},
|
| 24 |
{
|
| 25 |
+
"dataset": "task_noaa_coops_sf_water_level_coops_station_9414290_water_level_20260529t001800z",
|
| 26 |
+
"data_fetched_at": "2026-05-29T02:50:05.083862Z",
|
| 27 |
"context_length": 96,
|
| 28 |
"prediction_length": 24
|
| 29 |
},
|
| 30 |
{
|
| 31 |
+
"dataset": "task_nws_ksfo_observations_nws_station_ksfo_temperature_20260529t004500z",
|
| 32 |
+
"data_fetched_at": "2026-05-29T02:50:05.083862Z",
|
| 33 |
"context_length": 96,
|
| 34 |
"prediction_length": 24
|
| 35 |
},
|
| 36 |
{
|
| 37 |
+
"dataset": "task_open_meteo_shanghai_air_quality_hourly_grid_shanghai_31.2304_121.4737_pm2_5_20260528t020000z",
|
| 38 |
+
"data_fetched_at": "2026-05-29T02:50:05.083862Z",
|
| 39 |
"context_length": 96,
|
| 40 |
"prediction_length": 24
|
| 41 |
},
|
| 42 |
{
|
| 43 |
+
"dataset": "task_usgs_potomac_iv_usgs_site_01646500_usgs_00060_20260528t235000z",
|
| 44 |
+
"data_fetched_at": "2026-05-29T02:50:05.083862Z",
|
| 45 |
"context_length": 96,
|
| 46 |
"prediction_length": 24
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 47 |
}
|
| 48 |
]
|
| 49 |
}
|
results/online_status.json
CHANGED
|
@@ -1,411 +1,302 @@
|
|
| 1 |
{
|
| 2 |
"status": "ok",
|
| 3 |
-
"started_at": "2026-05-
|
| 4 |
-
"finished_at": "2026-05-
|
| 5 |
"data_source": "ts_bench",
|
| 6 |
-
"data_config": "configs/datasets/ts_bench.yaml",
|
| 7 |
-
"ts_bench_root": "/Users/zhouziyu/Desktop/
|
| 8 |
-
"baseline_model": "Chronos-Bolt-Base",
|
| 9 |
"models": {
|
| 10 |
"Chronos-Bolt-Tiny": {
|
| 11 |
"model": "Chronos-Bolt-Tiny",
|
| 12 |
"api_model_id": "amazon/chronos-bolt-tiny",
|
| 13 |
-
"evaluated_at": "2026-05-
|
| 14 |
"datasets": [
|
| 15 |
{
|
| 16 |
-
"dataset": "
|
| 17 |
-
"data_fetched_at": "2026-05-
|
| 18 |
"context_length": 96,
|
| 19 |
"prediction_length": 24
|
| 20 |
},
|
| 21 |
{
|
| 22 |
-
"dataset": "
|
| 23 |
-
"data_fetched_at": "2026-05-
|
| 24 |
"context_length": 96,
|
| 25 |
"prediction_length": 24
|
| 26 |
},
|
| 27 |
{
|
| 28 |
"dataset": "task_nasa_power_shanghai_hourly_point_shanghai_31.2304_121.4737_t2m_20260524t230000z",
|
| 29 |
-
"data_fetched_at": "2026-05-
|
| 30 |
-
"context_length":
|
| 31 |
"prediction_length": 24
|
| 32 |
},
|
| 33 |
{
|
| 34 |
-
"dataset": "
|
| 35 |
-
"data_fetched_at": "2026-05-
|
| 36 |
"context_length": 96,
|
| 37 |
"prediction_length": 24
|
| 38 |
},
|
| 39 |
{
|
| 40 |
-
"dataset": "
|
| 41 |
-
"data_fetched_at": "2026-05-
|
| 42 |
"context_length": 96,
|
| 43 |
"prediction_length": 24
|
| 44 |
},
|
| 45 |
{
|
| 46 |
-
"dataset": "
|
| 47 |
-
"data_fetched_at": "2026-05-
|
| 48 |
"context_length": 96,
|
| 49 |
"prediction_length": 24
|
| 50 |
},
|
| 51 |
{
|
| 52 |
-
"dataset": "
|
| 53 |
-
"data_fetched_at": "2026-05-
|
| 54 |
"context_length": 96,
|
| 55 |
"prediction_length": 24
|
| 56 |
-
},
|
| 57 |
-
{
|
| 58 |
-
"dataset": "task_open_meteo_shanghai_hourly_grid_shanghai_31.2304_121.4737_temperature_2m_20260527t120000z",
|
| 59 |
-
"data_fetched_at": "2026-05-28T12:16:39.187896Z",
|
| 60 |
-
"context_length": 96,
|
| 61 |
-
"prediction_length": 24
|
| 62 |
-
},
|
| 63 |
-
{
|
| 64 |
-
"dataset": "task_usgs_potomac_iv_usgs_site_01646500_usgs_00060_20260528t095000z",
|
| 65 |
-
"data_fetched_at": "2026-05-28T12:16:39.187896Z",
|
| 66 |
-
"context_length": 96,
|
| 67 |
-
"prediction_length": 24
|
| 68 |
-
},
|
| 69 |
-
{
|
| 70 |
-
"dataset": "task_wikimedia_time_series_hourly_enwiki_article_time_series_pageviews_20260525t000000z",
|
| 71 |
-
"data_fetched_at": "2026-05-28T12:16:39.187896Z",
|
| 72 |
-
"context_length": 4,
|
| 73 |
-
"prediction_length": 2
|
| 74 |
}
|
| 75 |
]
|
| 76 |
},
|
| 77 |
"Chronos-Bolt-Mini": {
|
| 78 |
"model": "Chronos-Bolt-Mini",
|
| 79 |
"api_model_id": "amazon/chronos-bolt-mini",
|
| 80 |
-
"evaluated_at": "2026-05-
|
| 81 |
"datasets": [
|
| 82 |
{
|
| 83 |
-
"dataset": "
|
| 84 |
-
"data_fetched_at": "2026-05-
|
| 85 |
"context_length": 96,
|
| 86 |
"prediction_length": 24
|
| 87 |
},
|
| 88 |
{
|
| 89 |
-
"dataset": "
|
| 90 |
-
"data_fetched_at": "2026-05-
|
| 91 |
"context_length": 96,
|
| 92 |
"prediction_length": 24
|
| 93 |
},
|
| 94 |
{
|
| 95 |
"dataset": "task_nasa_power_shanghai_hourly_point_shanghai_31.2304_121.4737_t2m_20260524t230000z",
|
| 96 |
-
"data_fetched_at": "2026-05-
|
| 97 |
-
"context_length":
|
| 98 |
-
"prediction_length": 24
|
| 99 |
-
},
|
| 100 |
-
{
|
| 101 |
-
"dataset": "task_noaa_coops_sf_water_level_coops_station_9414290_water_level_20260528t094200z",
|
| 102 |
-
"data_fetched_at": "2026-05-28T12:16:39.187896Z",
|
| 103 |
-
"context_length": 96,
|
| 104 |
-
"prediction_length": 24
|
| 105 |
-
},
|
| 106 |
-
{
|
| 107 |
-
"dataset": "task_noaa_ndbc_46013_realtime_ndbc_station_46013_wave_height_20260528t045000z",
|
| 108 |
-
"data_fetched_at": "2026-05-28T12:16:39.187896Z",
|
| 109 |
-
"context_length": 96,
|
| 110 |
"prediction_length": 24
|
| 111 |
},
|
| 112 |
{
|
| 113 |
-
"dataset": "
|
| 114 |
-
"data_fetched_at": "2026-05-
|
| 115 |
"context_length": 96,
|
| 116 |
"prediction_length": 24
|
| 117 |
},
|
| 118 |
{
|
| 119 |
-
"dataset": "
|
| 120 |
-
"data_fetched_at": "2026-05-
|
| 121 |
"context_length": 96,
|
| 122 |
"prediction_length": 24
|
| 123 |
},
|
| 124 |
{
|
| 125 |
-
"dataset": "
|
| 126 |
-
"data_fetched_at": "2026-05-
|
| 127 |
"context_length": 96,
|
| 128 |
"prediction_length": 24
|
| 129 |
},
|
| 130 |
{
|
| 131 |
-
"dataset": "
|
| 132 |
-
"data_fetched_at": "2026-05-
|
| 133 |
"context_length": 96,
|
| 134 |
"prediction_length": 24
|
| 135 |
-
},
|
| 136 |
-
{
|
| 137 |
-
"dataset": "task_wikimedia_time_series_hourly_enwiki_article_time_series_pageviews_20260525t000000z",
|
| 138 |
-
"data_fetched_at": "2026-05-28T12:16:39.187896Z",
|
| 139 |
-
"context_length": 4,
|
| 140 |
-
"prediction_length": 2
|
| 141 |
}
|
| 142 |
]
|
| 143 |
},
|
| 144 |
"Chronos-Bolt-Base": {
|
| 145 |
"model": "Chronos-Bolt-Base",
|
| 146 |
"api_model_id": "amazon/chronos-bolt-base",
|
| 147 |
-
"evaluated_at": "2026-05-
|
| 148 |
"datasets": [
|
| 149 |
{
|
| 150 |
-
"dataset": "
|
| 151 |
-
"data_fetched_at": "2026-05-
|
| 152 |
"context_length": 96,
|
| 153 |
"prediction_length": 24
|
| 154 |
},
|
| 155 |
{
|
| 156 |
-
"dataset": "
|
| 157 |
-
"data_fetched_at": "2026-05-
|
| 158 |
"context_length": 96,
|
| 159 |
"prediction_length": 24
|
| 160 |
},
|
| 161 |
{
|
| 162 |
"dataset": "task_nasa_power_shanghai_hourly_point_shanghai_31.2304_121.4737_t2m_20260524t230000z",
|
| 163 |
-
"data_fetched_at": "2026-05-
|
| 164 |
-
"context_length":
|
| 165 |
"prediction_length": 24
|
| 166 |
},
|
| 167 |
{
|
| 168 |
-
"dataset": "
|
| 169 |
-
"data_fetched_at": "2026-05-
|
| 170 |
"context_length": 96,
|
| 171 |
"prediction_length": 24
|
| 172 |
},
|
| 173 |
{
|
| 174 |
-
"dataset": "
|
| 175 |
-
"data_fetched_at": "2026-05-
|
| 176 |
"context_length": 96,
|
| 177 |
"prediction_length": 24
|
| 178 |
},
|
| 179 |
{
|
| 180 |
-
"dataset": "
|
| 181 |
-
"data_fetched_at": "2026-05-
|
| 182 |
"context_length": 96,
|
| 183 |
"prediction_length": 24
|
| 184 |
},
|
| 185 |
{
|
| 186 |
-
"dataset": "
|
| 187 |
-
"data_fetched_at": "2026-05-
|
| 188 |
"context_length": 96,
|
| 189 |
"prediction_length": 24
|
| 190 |
-
},
|
| 191 |
-
{
|
| 192 |
-
"dataset": "task_open_meteo_shanghai_hourly_grid_shanghai_31.2304_121.4737_temperature_2m_20260527t120000z",
|
| 193 |
-
"data_fetched_at": "2026-05-28T12:16:39.187896Z",
|
| 194 |
-
"context_length": 96,
|
| 195 |
-
"prediction_length": 24
|
| 196 |
-
},
|
| 197 |
-
{
|
| 198 |
-
"dataset": "task_usgs_potomac_iv_usgs_site_01646500_usgs_00060_20260528t095000z",
|
| 199 |
-
"data_fetched_at": "2026-05-28T12:16:39.187896Z",
|
| 200 |
-
"context_length": 96,
|
| 201 |
-
"prediction_length": 24
|
| 202 |
-
},
|
| 203 |
-
{
|
| 204 |
-
"dataset": "task_wikimedia_time_series_hourly_enwiki_article_time_series_pageviews_20260525t000000z",
|
| 205 |
-
"data_fetched_at": "2026-05-28T12:16:39.187896Z",
|
| 206 |
-
"context_length": 4,
|
| 207 |
-
"prediction_length": 2
|
| 208 |
}
|
| 209 |
]
|
| 210 |
},
|
| 211 |
"Chronos-2": {
|
| 212 |
"model": "Chronos-2",
|
| 213 |
"api_model_id": "amazon/chronos-2",
|
| 214 |
-
"evaluated_at": "2026-05-
|
| 215 |
"datasets": [
|
| 216 |
{
|
| 217 |
-
"dataset": "
|
| 218 |
-
"data_fetched_at": "2026-05-
|
| 219 |
"context_length": 96,
|
| 220 |
"prediction_length": 24
|
| 221 |
},
|
| 222 |
{
|
| 223 |
-
"dataset": "
|
| 224 |
-
"data_fetched_at": "2026-05-
|
| 225 |
"context_length": 96,
|
| 226 |
"prediction_length": 24
|
| 227 |
},
|
| 228 |
{
|
| 229 |
"dataset": "task_nasa_power_shanghai_hourly_point_shanghai_31.2304_121.4737_t2m_20260524t230000z",
|
| 230 |
-
"data_fetched_at": "2026-05-
|
| 231 |
-
"context_length":
|
| 232 |
"prediction_length": 24
|
| 233 |
},
|
| 234 |
{
|
| 235 |
-
"dataset": "
|
| 236 |
-
"data_fetched_at": "2026-05-
|
| 237 |
"context_length": 96,
|
| 238 |
"prediction_length": 24
|
| 239 |
},
|
| 240 |
{
|
| 241 |
-
"dataset": "
|
| 242 |
-
"data_fetched_at": "2026-05-
|
| 243 |
"context_length": 96,
|
| 244 |
"prediction_length": 24
|
| 245 |
},
|
| 246 |
{
|
| 247 |
-
"dataset": "
|
| 248 |
-
"data_fetched_at": "2026-05-
|
| 249 |
"context_length": 96,
|
| 250 |
"prediction_length": 24
|
| 251 |
},
|
| 252 |
{
|
| 253 |
-
"dataset": "
|
| 254 |
-
"data_fetched_at": "2026-05-
|
| 255 |
"context_length": 96,
|
| 256 |
"prediction_length": 24
|
| 257 |
-
},
|
| 258 |
-
{
|
| 259 |
-
"dataset": "task_open_meteo_shanghai_hourly_grid_shanghai_31.2304_121.4737_temperature_2m_20260527t120000z",
|
| 260 |
-
"data_fetched_at": "2026-05-28T12:16:39.187896Z",
|
| 261 |
-
"context_length": 96,
|
| 262 |
-
"prediction_length": 24
|
| 263 |
-
},
|
| 264 |
-
{
|
| 265 |
-
"dataset": "task_usgs_potomac_iv_usgs_site_01646500_usgs_00060_20260528t095000z",
|
| 266 |
-
"data_fetched_at": "2026-05-28T12:16:39.187896Z",
|
| 267 |
-
"context_length": 96,
|
| 268 |
-
"prediction_length": 24
|
| 269 |
-
},
|
| 270 |
-
{
|
| 271 |
-
"dataset": "task_wikimedia_time_series_hourly_enwiki_article_time_series_pageviews_20260525t000000z",
|
| 272 |
-
"data_fetched_at": "2026-05-28T12:16:39.187896Z",
|
| 273 |
-
"context_length": 4,
|
| 274 |
-
"prediction_length": 2
|
| 275 |
}
|
| 276 |
]
|
| 277 |
},
|
| 278 |
"TimesFM-2.5": {
|
| 279 |
"model": "TimesFM-2.5",
|
| 280 |
"api_model_id": "google/timesfm-2.5-200m-pytorch",
|
| 281 |
-
"evaluated_at": "2026-05-
|
| 282 |
"datasets": [
|
| 283 |
{
|
| 284 |
-
"dataset": "
|
| 285 |
-
"data_fetched_at": "2026-05-
|
| 286 |
"context_length": 96,
|
| 287 |
"prediction_length": 24
|
| 288 |
},
|
| 289 |
{
|
| 290 |
-
"dataset": "
|
| 291 |
-
"data_fetched_at": "2026-05-
|
| 292 |
"context_length": 96,
|
| 293 |
"prediction_length": 24
|
| 294 |
},
|
| 295 |
{
|
| 296 |
"dataset": "task_nasa_power_shanghai_hourly_point_shanghai_31.2304_121.4737_t2m_20260524t230000z",
|
| 297 |
-
"data_fetched_at": "2026-05-
|
| 298 |
-
"context_length":
|
| 299 |
-
"prediction_length": 24
|
| 300 |
-
},
|
| 301 |
-
{
|
| 302 |
-
"dataset": "task_noaa_coops_sf_water_level_coops_station_9414290_water_level_20260528t094200z",
|
| 303 |
-
"data_fetched_at": "2026-05-28T12:16:39.187896Z",
|
| 304 |
-
"context_length": 96,
|
| 305 |
-
"prediction_length": 24
|
| 306 |
-
},
|
| 307 |
-
{
|
| 308 |
-
"dataset": "task_noaa_ndbc_46013_realtime_ndbc_station_46013_wave_height_20260528t045000z",
|
| 309 |
-
"data_fetched_at": "2026-05-28T12:16:39.187896Z",
|
| 310 |
-
"context_length": 96,
|
| 311 |
"prediction_length": 24
|
| 312 |
},
|
| 313 |
{
|
| 314 |
-
"dataset": "
|
| 315 |
-
"data_fetched_at": "2026-05-
|
| 316 |
"context_length": 96,
|
| 317 |
"prediction_length": 24
|
| 318 |
},
|
| 319 |
{
|
| 320 |
-
"dataset": "
|
| 321 |
-
"data_fetched_at": "2026-05-
|
| 322 |
"context_length": 96,
|
| 323 |
"prediction_length": 24
|
| 324 |
},
|
| 325 |
{
|
| 326 |
-
"dataset": "
|
| 327 |
-
"data_fetched_at": "2026-05-
|
| 328 |
"context_length": 96,
|
| 329 |
"prediction_length": 24
|
| 330 |
},
|
| 331 |
{
|
| 332 |
-
"dataset": "
|
| 333 |
-
"data_fetched_at": "2026-05-
|
| 334 |
"context_length": 96,
|
| 335 |
"prediction_length": 24
|
| 336 |
-
},
|
| 337 |
-
{
|
| 338 |
-
"dataset": "task_wikimedia_time_series_hourly_enwiki_article_time_series_pageviews_20260525t000000z",
|
| 339 |
-
"data_fetched_at": "2026-05-28T12:16:39.187896Z",
|
| 340 |
-
"context_length": 4,
|
| 341 |
-
"prediction_length": 2
|
| 342 |
}
|
| 343 |
]
|
| 344 |
},
|
| 345 |
"Moirai-1.1-Small": {
|
| 346 |
"model": "Moirai-1.1-Small",
|
| 347 |
"api_model_id": "Salesforce/moirai-1.1-R-small",
|
| 348 |
-
"evaluated_at": "2026-05-
|
| 349 |
"datasets": [
|
| 350 |
{
|
| 351 |
-
"dataset": "
|
| 352 |
-
"data_fetched_at": "2026-05-
|
| 353 |
"context_length": 96,
|
| 354 |
"prediction_length": 24
|
| 355 |
},
|
| 356 |
{
|
| 357 |
-
"dataset": "
|
| 358 |
-
"data_fetched_at": "2026-05-
|
| 359 |
"context_length": 96,
|
| 360 |
"prediction_length": 24
|
| 361 |
},
|
| 362 |
{
|
| 363 |
"dataset": "task_nasa_power_shanghai_hourly_point_shanghai_31.2304_121.4737_t2m_20260524t230000z",
|
| 364 |
-
"data_fetched_at": "2026-05-
|
| 365 |
-
"context_length":
|
| 366 |
"prediction_length": 24
|
| 367 |
},
|
| 368 |
{
|
| 369 |
-
"dataset": "
|
| 370 |
-
"data_fetched_at": "2026-05-
|
| 371 |
"context_length": 96,
|
| 372 |
"prediction_length": 24
|
| 373 |
},
|
| 374 |
{
|
| 375 |
-
"dataset": "
|
| 376 |
-
"data_fetched_at": "2026-05-
|
| 377 |
"context_length": 96,
|
| 378 |
"prediction_length": 24
|
| 379 |
},
|
| 380 |
{
|
| 381 |
-
"dataset": "
|
| 382 |
-
"data_fetched_at": "2026-05-
|
| 383 |
"context_length": 96,
|
| 384 |
"prediction_length": 24
|
| 385 |
},
|
| 386 |
{
|
| 387 |
-
"dataset": "
|
| 388 |
-
"data_fetched_at": "2026-05-
|
| 389 |
"context_length": 96,
|
| 390 |
"prediction_length": 24
|
| 391 |
-
},
|
| 392 |
-
{
|
| 393 |
-
"dataset": "task_open_meteo_shanghai_hourly_grid_shanghai_31.2304_121.4737_temperature_2m_20260527t120000z",
|
| 394 |
-
"data_fetched_at": "2026-05-28T12:16:39.187896Z",
|
| 395 |
-
"context_length": 96,
|
| 396 |
-
"prediction_length": 24
|
| 397 |
-
},
|
| 398 |
-
{
|
| 399 |
-
"dataset": "task_usgs_potomac_iv_usgs_site_01646500_usgs_00060_20260528t095000z",
|
| 400 |
-
"data_fetched_at": "2026-05-28T12:16:39.187896Z",
|
| 401 |
-
"context_length": 96,
|
| 402 |
-
"prediction_length": 24
|
| 403 |
-
},
|
| 404 |
-
{
|
| 405 |
-
"dataset": "task_wikimedia_time_series_hourly_enwiki_article_time_series_pageviews_20260525t000000z",
|
| 406 |
-
"data_fetched_at": "2026-05-28T12:16:39.187896Z",
|
| 407 |
-
"context_length": 4,
|
| 408 |
-
"prediction_length": 2
|
| 409 |
}
|
| 410 |
]
|
| 411 |
}
|
|
|
|
| 1 |
{
|
| 2 |
"status": "ok",
|
| 3 |
+
"started_at": "2026-05-29T02:49:52.974580+00:00",
|
| 4 |
+
"finished_at": "2026-05-29T02:55:13.595519+00:00",
|
| 5 |
"data_source": "ts_bench",
|
| 6 |
+
"data_config": "/Users/zhouziyu/Desktop/TSFM_Bench/space/configs/datasets/ts_bench.yaml",
|
| 7 |
+
"ts_bench_root": "/Users/zhouziyu/Desktop/TSFM_Bench/space/vendor/ts_bench",
|
|
|
|
| 8 |
"models": {
|
| 9 |
"Chronos-Bolt-Tiny": {
|
| 10 |
"model": "Chronos-Bolt-Tiny",
|
| 11 |
"api_model_id": "amazon/chronos-bolt-tiny",
|
| 12 |
+
"evaluated_at": "2026-05-29T02:50:52.849106+00:00",
|
| 13 |
"datasets": [
|
| 14 |
{
|
| 15 |
+
"dataset": "task_binance_btcusdt_1h_crypto_btcusdt_close_20260528t020000z",
|
| 16 |
+
"data_fetched_at": "2026-05-29T02:50:05.083862Z",
|
| 17 |
"context_length": 96,
|
| 18 |
"prediction_length": 24
|
| 19 |
},
|
| 20 |
{
|
| 21 |
+
"dataset": "task_coingecko_bitcoin_7d_coin_bitcoin_price_usd_20260528t030012z",
|
| 22 |
+
"data_fetched_at": "2026-05-29T02:50:05.083862Z",
|
| 23 |
"context_length": 96,
|
| 24 |
"prediction_length": 24
|
| 25 |
},
|
| 26 |
{
|
| 27 |
"dataset": "task_nasa_power_shanghai_hourly_point_shanghai_31.2304_121.4737_t2m_20260524t230000z",
|
| 28 |
+
"data_fetched_at": "2026-05-29T02:50:05.083862Z",
|
| 29 |
+
"context_length": 69,
|
| 30 |
"prediction_length": 24
|
| 31 |
},
|
| 32 |
{
|
| 33 |
+
"dataset": "task_noaa_coops_sf_water_level_coops_station_9414290_water_level_20260529t001800z",
|
| 34 |
+
"data_fetched_at": "2026-05-29T02:50:05.083862Z",
|
| 35 |
"context_length": 96,
|
| 36 |
"prediction_length": 24
|
| 37 |
},
|
| 38 |
{
|
| 39 |
+
"dataset": "task_nws_ksfo_observations_nws_station_ksfo_temperature_20260529t004500z",
|
| 40 |
+
"data_fetched_at": "2026-05-29T02:50:05.083862Z",
|
| 41 |
"context_length": 96,
|
| 42 |
"prediction_length": 24
|
| 43 |
},
|
| 44 |
{
|
| 45 |
+
"dataset": "task_open_meteo_shanghai_air_quality_hourly_grid_shanghai_31.2304_121.4737_pm2_5_20260528t020000z",
|
| 46 |
+
"data_fetched_at": "2026-05-29T02:50:05.083862Z",
|
| 47 |
"context_length": 96,
|
| 48 |
"prediction_length": 24
|
| 49 |
},
|
| 50 |
{
|
| 51 |
+
"dataset": "task_usgs_potomac_iv_usgs_site_01646500_usgs_00060_20260528t235000z",
|
| 52 |
+
"data_fetched_at": "2026-05-29T02:50:05.083862Z",
|
| 53 |
"context_length": 96,
|
| 54 |
"prediction_length": 24
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 55 |
}
|
| 56 |
]
|
| 57 |
},
|
| 58 |
"Chronos-Bolt-Mini": {
|
| 59 |
"model": "Chronos-Bolt-Mini",
|
| 60 |
"api_model_id": "amazon/chronos-bolt-mini",
|
| 61 |
+
"evaluated_at": "2026-05-29T02:51:10.536027+00:00",
|
| 62 |
"datasets": [
|
| 63 |
{
|
| 64 |
+
"dataset": "task_binance_btcusdt_1h_crypto_btcusdt_close_20260528t020000z",
|
| 65 |
+
"data_fetched_at": "2026-05-29T02:50:05.083862Z",
|
| 66 |
"context_length": 96,
|
| 67 |
"prediction_length": 24
|
| 68 |
},
|
| 69 |
{
|
| 70 |
+
"dataset": "task_coingecko_bitcoin_7d_coin_bitcoin_price_usd_20260528t030012z",
|
| 71 |
+
"data_fetched_at": "2026-05-29T02:50:05.083862Z",
|
| 72 |
"context_length": 96,
|
| 73 |
"prediction_length": 24
|
| 74 |
},
|
| 75 |
{
|
| 76 |
"dataset": "task_nasa_power_shanghai_hourly_point_shanghai_31.2304_121.4737_t2m_20260524t230000z",
|
| 77 |
+
"data_fetched_at": "2026-05-29T02:50:05.083862Z",
|
| 78 |
+
"context_length": 69,
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 79 |
"prediction_length": 24
|
| 80 |
},
|
| 81 |
{
|
| 82 |
+
"dataset": "task_noaa_coops_sf_water_level_coops_station_9414290_water_level_20260529t001800z",
|
| 83 |
+
"data_fetched_at": "2026-05-29T02:50:05.083862Z",
|
| 84 |
"context_length": 96,
|
| 85 |
"prediction_length": 24
|
| 86 |
},
|
| 87 |
{
|
| 88 |
+
"dataset": "task_nws_ksfo_observations_nws_station_ksfo_temperature_20260529t004500z",
|
| 89 |
+
"data_fetched_at": "2026-05-29T02:50:05.083862Z",
|
| 90 |
"context_length": 96,
|
| 91 |
"prediction_length": 24
|
| 92 |
},
|
| 93 |
{
|
| 94 |
+
"dataset": "task_open_meteo_shanghai_air_quality_hourly_grid_shanghai_31.2304_121.4737_pm2_5_20260528t020000z",
|
| 95 |
+
"data_fetched_at": "2026-05-29T02:50:05.083862Z",
|
| 96 |
"context_length": 96,
|
| 97 |
"prediction_length": 24
|
| 98 |
},
|
| 99 |
{
|
| 100 |
+
"dataset": "task_usgs_potomac_iv_usgs_site_01646500_usgs_00060_20260528t235000z",
|
| 101 |
+
"data_fetched_at": "2026-05-29T02:50:05.083862Z",
|
| 102 |
"context_length": 96,
|
| 103 |
"prediction_length": 24
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 104 |
}
|
| 105 |
]
|
| 106 |
},
|
| 107 |
"Chronos-Bolt-Base": {
|
| 108 |
"model": "Chronos-Bolt-Base",
|
| 109 |
"api_model_id": "amazon/chronos-bolt-base",
|
| 110 |
+
"evaluated_at": "2026-05-29T02:51:30.013825+00:00",
|
| 111 |
"datasets": [
|
| 112 |
{
|
| 113 |
+
"dataset": "task_binance_btcusdt_1h_crypto_btcusdt_close_20260528t020000z",
|
| 114 |
+
"data_fetched_at": "2026-05-29T02:50:05.083862Z",
|
| 115 |
"context_length": 96,
|
| 116 |
"prediction_length": 24
|
| 117 |
},
|
| 118 |
{
|
| 119 |
+
"dataset": "task_coingecko_bitcoin_7d_coin_bitcoin_price_usd_20260528t030012z",
|
| 120 |
+
"data_fetched_at": "2026-05-29T02:50:05.083862Z",
|
| 121 |
"context_length": 96,
|
| 122 |
"prediction_length": 24
|
| 123 |
},
|
| 124 |
{
|
| 125 |
"dataset": "task_nasa_power_shanghai_hourly_point_shanghai_31.2304_121.4737_t2m_20260524t230000z",
|
| 126 |
+
"data_fetched_at": "2026-05-29T02:50:05.083862Z",
|
| 127 |
+
"context_length": 69,
|
| 128 |
"prediction_length": 24
|
| 129 |
},
|
| 130 |
{
|
| 131 |
+
"dataset": "task_noaa_coops_sf_water_level_coops_station_9414290_water_level_20260529t001800z",
|
| 132 |
+
"data_fetched_at": "2026-05-29T02:50:05.083862Z",
|
| 133 |
"context_length": 96,
|
| 134 |
"prediction_length": 24
|
| 135 |
},
|
| 136 |
{
|
| 137 |
+
"dataset": "task_nws_ksfo_observations_nws_station_ksfo_temperature_20260529t004500z",
|
| 138 |
+
"data_fetched_at": "2026-05-29T02:50:05.083862Z",
|
| 139 |
"context_length": 96,
|
| 140 |
"prediction_length": 24
|
| 141 |
},
|
| 142 |
{
|
| 143 |
+
"dataset": "task_open_meteo_shanghai_air_quality_hourly_grid_shanghai_31.2304_121.4737_pm2_5_20260528t020000z",
|
| 144 |
+
"data_fetched_at": "2026-05-29T02:50:05.083862Z",
|
| 145 |
"context_length": 96,
|
| 146 |
"prediction_length": 24
|
| 147 |
},
|
| 148 |
{
|
| 149 |
+
"dataset": "task_usgs_potomac_iv_usgs_site_01646500_usgs_00060_20260528t235000z",
|
| 150 |
+
"data_fetched_at": "2026-05-29T02:50:05.083862Z",
|
| 151 |
"context_length": 96,
|
| 152 |
"prediction_length": 24
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 153 |
}
|
| 154 |
]
|
| 155 |
},
|
| 156 |
"Chronos-2": {
|
| 157 |
"model": "Chronos-2",
|
| 158 |
"api_model_id": "amazon/chronos-2",
|
| 159 |
+
"evaluated_at": "2026-05-29T02:53:16.309267+00:00",
|
| 160 |
"datasets": [
|
| 161 |
{
|
| 162 |
+
"dataset": "task_binance_btcusdt_1h_crypto_btcusdt_close_20260528t020000z",
|
| 163 |
+
"data_fetched_at": "2026-05-29T02:50:05.083862Z",
|
| 164 |
"context_length": 96,
|
| 165 |
"prediction_length": 24
|
| 166 |
},
|
| 167 |
{
|
| 168 |
+
"dataset": "task_coingecko_bitcoin_7d_coin_bitcoin_price_usd_20260528t030012z",
|
| 169 |
+
"data_fetched_at": "2026-05-29T02:50:05.083862Z",
|
| 170 |
"context_length": 96,
|
| 171 |
"prediction_length": 24
|
| 172 |
},
|
| 173 |
{
|
| 174 |
"dataset": "task_nasa_power_shanghai_hourly_point_shanghai_31.2304_121.4737_t2m_20260524t230000z",
|
| 175 |
+
"data_fetched_at": "2026-05-29T02:50:05.083862Z",
|
| 176 |
+
"context_length": 69,
|
| 177 |
"prediction_length": 24
|
| 178 |
},
|
| 179 |
{
|
| 180 |
+
"dataset": "task_noaa_coops_sf_water_level_coops_station_9414290_water_level_20260529t001800z",
|
| 181 |
+
"data_fetched_at": "2026-05-29T02:50:05.083862Z",
|
| 182 |
"context_length": 96,
|
| 183 |
"prediction_length": 24
|
| 184 |
},
|
| 185 |
{
|
| 186 |
+
"dataset": "task_nws_ksfo_observations_nws_station_ksfo_temperature_20260529t004500z",
|
| 187 |
+
"data_fetched_at": "2026-05-29T02:50:05.083862Z",
|
| 188 |
"context_length": 96,
|
| 189 |
"prediction_length": 24
|
| 190 |
},
|
| 191 |
{
|
| 192 |
+
"dataset": "task_open_meteo_shanghai_air_quality_hourly_grid_shanghai_31.2304_121.4737_pm2_5_20260528t020000z",
|
| 193 |
+
"data_fetched_at": "2026-05-29T02:50:05.083862Z",
|
| 194 |
"context_length": 96,
|
| 195 |
"prediction_length": 24
|
| 196 |
},
|
| 197 |
{
|
| 198 |
+
"dataset": "task_usgs_potomac_iv_usgs_site_01646500_usgs_00060_20260528t235000z",
|
| 199 |
+
"data_fetched_at": "2026-05-29T02:50:05.083862Z",
|
| 200 |
"context_length": 96,
|
| 201 |
"prediction_length": 24
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 202 |
}
|
| 203 |
]
|
| 204 |
},
|
| 205 |
"TimesFM-2.5": {
|
| 206 |
"model": "TimesFM-2.5",
|
| 207 |
"api_model_id": "google/timesfm-2.5-200m-pytorch",
|
| 208 |
+
"evaluated_at": "2026-05-29T02:54:46.508756+00:00",
|
| 209 |
"datasets": [
|
| 210 |
{
|
| 211 |
+
"dataset": "task_binance_btcusdt_1h_crypto_btcusdt_close_20260528t020000z",
|
| 212 |
+
"data_fetched_at": "2026-05-29T02:50:05.083862Z",
|
| 213 |
"context_length": 96,
|
| 214 |
"prediction_length": 24
|
| 215 |
},
|
| 216 |
{
|
| 217 |
+
"dataset": "task_coingecko_bitcoin_7d_coin_bitcoin_price_usd_20260528t030012z",
|
| 218 |
+
"data_fetched_at": "2026-05-29T02:50:05.083862Z",
|
| 219 |
"context_length": 96,
|
| 220 |
"prediction_length": 24
|
| 221 |
},
|
| 222 |
{
|
| 223 |
"dataset": "task_nasa_power_shanghai_hourly_point_shanghai_31.2304_121.4737_t2m_20260524t230000z",
|
| 224 |
+
"data_fetched_at": "2026-05-29T02:50:05.083862Z",
|
| 225 |
+
"context_length": 69,
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 226 |
"prediction_length": 24
|
| 227 |
},
|
| 228 |
{
|
| 229 |
+
"dataset": "task_noaa_coops_sf_water_level_coops_station_9414290_water_level_20260529t001800z",
|
| 230 |
+
"data_fetched_at": "2026-05-29T02:50:05.083862Z",
|
| 231 |
"context_length": 96,
|
| 232 |
"prediction_length": 24
|
| 233 |
},
|
| 234 |
{
|
| 235 |
+
"dataset": "task_nws_ksfo_observations_nws_station_ksfo_temperature_20260529t004500z",
|
| 236 |
+
"data_fetched_at": "2026-05-29T02:50:05.083862Z",
|
| 237 |
"context_length": 96,
|
| 238 |
"prediction_length": 24
|
| 239 |
},
|
| 240 |
{
|
| 241 |
+
"dataset": "task_open_meteo_shanghai_air_quality_hourly_grid_shanghai_31.2304_121.4737_pm2_5_20260528t020000z",
|
| 242 |
+
"data_fetched_at": "2026-05-29T02:50:05.083862Z",
|
| 243 |
"context_length": 96,
|
| 244 |
"prediction_length": 24
|
| 245 |
},
|
| 246 |
{
|
| 247 |
+
"dataset": "task_usgs_potomac_iv_usgs_site_01646500_usgs_00060_20260528t235000z",
|
| 248 |
+
"data_fetched_at": "2026-05-29T02:50:05.083862Z",
|
| 249 |
"context_length": 96,
|
| 250 |
"prediction_length": 24
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 251 |
}
|
| 252 |
]
|
| 253 |
},
|
| 254 |
"Moirai-1.1-Small": {
|
| 255 |
"model": "Moirai-1.1-Small",
|
| 256 |
"api_model_id": "Salesforce/moirai-1.1-R-small",
|
| 257 |
+
"evaluated_at": "2026-05-29T02:55:13.592899+00:00",
|
| 258 |
"datasets": [
|
| 259 |
{
|
| 260 |
+
"dataset": "task_binance_btcusdt_1h_crypto_btcusdt_close_20260528t020000z",
|
| 261 |
+
"data_fetched_at": "2026-05-29T02:50:05.083862Z",
|
| 262 |
"context_length": 96,
|
| 263 |
"prediction_length": 24
|
| 264 |
},
|
| 265 |
{
|
| 266 |
+
"dataset": "task_coingecko_bitcoin_7d_coin_bitcoin_price_usd_20260528t030012z",
|
| 267 |
+
"data_fetched_at": "2026-05-29T02:50:05.083862Z",
|
| 268 |
"context_length": 96,
|
| 269 |
"prediction_length": 24
|
| 270 |
},
|
| 271 |
{
|
| 272 |
"dataset": "task_nasa_power_shanghai_hourly_point_shanghai_31.2304_121.4737_t2m_20260524t230000z",
|
| 273 |
+
"data_fetched_at": "2026-05-29T02:50:05.083862Z",
|
| 274 |
+
"context_length": 69,
|
| 275 |
"prediction_length": 24
|
| 276 |
},
|
| 277 |
{
|
| 278 |
+
"dataset": "task_noaa_coops_sf_water_level_coops_station_9414290_water_level_20260529t001800z",
|
| 279 |
+
"data_fetched_at": "2026-05-29T02:50:05.083862Z",
|
| 280 |
"context_length": 96,
|
| 281 |
"prediction_length": 24
|
| 282 |
},
|
| 283 |
{
|
| 284 |
+
"dataset": "task_nws_ksfo_observations_nws_station_ksfo_temperature_20260529t004500z",
|
| 285 |
+
"data_fetched_at": "2026-05-29T02:50:05.083862Z",
|
| 286 |
"context_length": 96,
|
| 287 |
"prediction_length": 24
|
| 288 |
},
|
| 289 |
{
|
| 290 |
+
"dataset": "task_open_meteo_shanghai_air_quality_hourly_grid_shanghai_31.2304_121.4737_pm2_5_20260528t020000z",
|
| 291 |
+
"data_fetched_at": "2026-05-29T02:50:05.083862Z",
|
| 292 |
"context_length": 96,
|
| 293 |
"prediction_length": 24
|
| 294 |
},
|
| 295 |
{
|
| 296 |
+
"dataset": "task_usgs_potomac_iv_usgs_site_01646500_usgs_00060_20260528t235000z",
|
| 297 |
+
"data_fetched_at": "2026-05-29T02:50:05.083862Z",
|
| 298 |
"context_length": 96,
|
| 299 |
"prediction_length": 24
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 300 |
}
|
| 301 |
]
|
| 302 |
}
|
results/timesfm_2_5/all_results.csv
CHANGED
|
@@ -1,11 +1,8 @@
|
|
| 1 |
dataset,model,eval_metrics/MSE[mean],eval_metrics/MSE[0.5],eval_metrics/MAE[0.5],eval_metrics/MASE[0.5],eval_metrics/MAPE[0.5],eval_metrics/sMAPE[0.5],eval_metrics/MSIS,eval_metrics/RMSE[mean],eval_metrics/NRMSE[mean],eval_metrics/ND[0.5],eval_metrics/mean_weighted_sum_quantile_loss,domain,num_variates
|
| 2 |
-
binance_btcusdt_1h_close/1h/short,TimesFM-2.5,
|
| 3 |
-
coingecko_bitcoin_7d_price_usd/1h/short,TimesFM-2.5,
|
| 4 |
-
nasa_power_shanghai_hourly_T2M/1h/short,TimesFM-2.5,7.
|
| 5 |
-
noaa_coops_sf_water_level_water_level/6min/short,TimesFM-2.5,0.
|
| 6 |
-
|
| 7 |
-
|
| 8 |
-
|
| 9 |
-
open_meteo_shanghai_hourly_temperature_2m/1h/short,TimesFM-2.5,0.4488093384068937,0.42550756879063556,0.5227727890014648,0.28471744342993616,0.021116368406687125,0.020967622620989168,11.867516613268592,0.6699323386782382,0.026734911000282555,0.020862232169443492,0.02480197235359902,Weather,1
|
| 10 |
-
usgs_potomac_iv_usgs_00060/15min/short,TimesFM-2.5,3964.9586008389792,2807.6309466362,52.009847005208336,nan,0.002045911466877187,0.0020444056954015345,nan,62.9679172344058,0.0024762084444137954,0.002045283185523513,0.002015193888024105,Hydrology,1
|
| 11 |
-
wikimedia_time_series_hourly_pageviews/1D/short,TimesFM-2.5,4207.680999754579,5861.192346271244,75.43997955322266,1.6165709904262,0.24218997554725216,0.2765631520841845,22.796233912876673,64.86664011458107,0.20891027412103405,0.2429628971118282,0.177518524153541,Web,1
|
|
|
|
| 1 |
dataset,model,eval_metrics/MSE[mean],eval_metrics/MSE[0.5],eval_metrics/MAE[0.5],eval_metrics/MASE[0.5],eval_metrics/MAPE[0.5],eval_metrics/sMAPE[0.5],eval_metrics/MSIS,eval_metrics/RMSE[mean],eval_metrics/NRMSE[mean],eval_metrics/ND[0.5],eval_metrics/mean_weighted_sum_quantile_loss,domain,num_variates
|
| 2 |
+
binance_btcusdt_1h_close/1h/short,TimesFM-2.5,1219064.9854685466,835565.3181101481,884.322265625,0.97857792276093,0.012057590756120376,0.011980322700327062,53.58648546814461,1104.1127593993951,0.015042171211020878,0.0120477974844572,0.015083192708183289,Finance,1
|
| 3 |
+
coingecko_bitcoin_7d_price_usd/1h/short,TimesFM-2.5,1230621.6017862957,849700.2018839518,888.9869791666666,0.9620628334321281,0.012141146014077656,0.012062308830606919,52.43584047207273,1109.3338549716652,0.015136772575028212,0.012130156909482296,0.015134561186691181,Finance,1
|
| 4 |
+
nasa_power_shanghai_hourly_T2M/1h/short,TimesFM-2.5,7.933177144345336,11.399316419632365,2.7634520530700684,0.8045764238845047,0.10092765997658189,0.10842496844555281,9.535674024816261,2.8165896300926296,0.10973904153296879,0.10766871268224146,0.07858026268608374,Weather,1
|
| 5 |
+
noaa_coops_sf_water_level_water_level/6min/short,TimesFM-2.5,0.0015351705280940802,0.0024778745030360483,0.04266408085823059,2.675305425612182,0.03174085850119802,0.03239086232536759,12.171508226109887,0.03918125225275578,0.030699292812490016,0.03342815850788563,0.021909742273566946,Ocean,1
|
| 6 |
+
nws_ksfo_observations_temperature/1h/short,TimesFM-2.5,7.230681060140189,6.612742760992963,2.0171178181966147,2.4826065454727564,0.12987325798582047,0.11728228771068923,97.8729589413374,2.68899257346319,0.15875970913435808,0.11909182690459717,0.12032840390405451,Weather,1
|
| 7 |
+
open_meteo_shanghai_air_quality_hourly_pm2_5/1h/short,TimesFM-2.5,1370.1968184538339,1568.2607702783378,32.57473762830099,1.429238945300226,0.48036513469849956,0.6789653743383695,33.04442361416721,37.016169689121455,0.6308678478504538,0.5551723691146495,0.45629084448376683,Air Quality,1
|
| 8 |
+
usgs_potomac_iv_usgs_00060/15min/short,TimesFM-2.5,203315.88810078302,267129.3128611247,408.2639973958333,nan,0.014499470105041025,0.014667487665793741,nan,450.90563103689783,0.01619777749571254,0.01466597206630744,0.011492126628270052,Hydrology,1
|
|
|
|
|
|
|
|
|
results/timesfm_2_5/config.json
CHANGED
|
@@ -3,7 +3,7 @@
|
|
| 3 |
"model_type": "zero-shot",
|
| 4 |
"model_dtype": "float32",
|
| 5 |
"model_link": "https://tsfm.ai/models/google/timesfm-2.5-200m-pytorch",
|
| 6 |
-
"code_link": "https://github.com/zhouziyu02/TSFM_Bench/blob/main/scripts/
|
| 7 |
"org": "Google / TSFM.ai",
|
| 8 |
"testdata_leakage": "No",
|
| 9 |
"replication_code_available": "Yes",
|
|
|
|
| 3 |
"model_type": "zero-shot",
|
| 4 |
"model_dtype": "float32",
|
| 5 |
"model_link": "https://tsfm.ai/models/google/timesfm-2.5-200m-pytorch",
|
| 6 |
+
"code_link": "https://github.com/zhouziyu02/TSFM_Bench/blob/main/space/scripts/run_space_eval.py",
|
| 7 |
"org": "Google / TSFM.ai",
|
| 8 |
"testdata_leakage": "No",
|
| 9 |
"replication_code_available": "Yes",
|
results/timesfm_2_5/online_meta.json
CHANGED
|
@@ -1,67 +1,49 @@
|
|
| 1 |
{
|
| 2 |
"model": "TimesFM-2.5",
|
| 3 |
"api_model_id": "google/timesfm-2.5-200m-pytorch",
|
| 4 |
-
"evaluated_at": "2026-05-
|
| 5 |
"datasets": [
|
| 6 |
{
|
| 7 |
-
"dataset": "
|
| 8 |
-
"data_fetched_at": "2026-05-
|
| 9 |
"context_length": 96,
|
| 10 |
"prediction_length": 24
|
| 11 |
},
|
| 12 |
{
|
| 13 |
-
"dataset": "
|
| 14 |
-
"data_fetched_at": "2026-05-
|
| 15 |
"context_length": 96,
|
| 16 |
"prediction_length": 24
|
| 17 |
},
|
| 18 |
{
|
| 19 |
"dataset": "task_nasa_power_shanghai_hourly_point_shanghai_31.2304_121.4737_t2m_20260524t230000z",
|
| 20 |
-
"data_fetched_at": "2026-05-
|
| 21 |
-
"context_length":
|
| 22 |
"prediction_length": 24
|
| 23 |
},
|
| 24 |
{
|
| 25 |
-
"dataset": "
|
| 26 |
-
"data_fetched_at": "2026-05-
|
| 27 |
"context_length": 96,
|
| 28 |
"prediction_length": 24
|
| 29 |
},
|
| 30 |
{
|
| 31 |
-
"dataset": "
|
| 32 |
-
"data_fetched_at": "2026-05-
|
| 33 |
"context_length": 96,
|
| 34 |
"prediction_length": 24
|
| 35 |
},
|
| 36 |
{
|
| 37 |
-
"dataset": "
|
| 38 |
-
"data_fetched_at": "2026-05-
|
| 39 |
"context_length": 96,
|
| 40 |
"prediction_length": 24
|
| 41 |
},
|
| 42 |
{
|
| 43 |
-
"dataset": "
|
| 44 |
-
"data_fetched_at": "2026-05-
|
| 45 |
"context_length": 96,
|
| 46 |
"prediction_length": 24
|
| 47 |
-
},
|
| 48 |
-
{
|
| 49 |
-
"dataset": "task_open_meteo_shanghai_hourly_grid_shanghai_31.2304_121.4737_temperature_2m_20260527t120000z",
|
| 50 |
-
"data_fetched_at": "2026-05-28T12:16:39.187896Z",
|
| 51 |
-
"context_length": 96,
|
| 52 |
-
"prediction_length": 24
|
| 53 |
-
},
|
| 54 |
-
{
|
| 55 |
-
"dataset": "task_usgs_potomac_iv_usgs_site_01646500_usgs_00060_20260528t095000z",
|
| 56 |
-
"data_fetched_at": "2026-05-28T12:16:39.187896Z",
|
| 57 |
-
"context_length": 96,
|
| 58 |
-
"prediction_length": 24
|
| 59 |
-
},
|
| 60 |
-
{
|
| 61 |
-
"dataset": "task_wikimedia_time_series_hourly_enwiki_article_time_series_pageviews_20260525t000000z",
|
| 62 |
-
"data_fetched_at": "2026-05-28T12:16:39.187896Z",
|
| 63 |
-
"context_length": 4,
|
| 64 |
-
"prediction_length": 2
|
| 65 |
}
|
| 66 |
]
|
| 67 |
}
|
|
|
|
| 1 |
{
|
| 2 |
"model": "TimesFM-2.5",
|
| 3 |
"api_model_id": "google/timesfm-2.5-200m-pytorch",
|
| 4 |
+
"evaluated_at": "2026-05-29T02:54:46.508756+00:00",
|
| 5 |
"datasets": [
|
| 6 |
{
|
| 7 |
+
"dataset": "task_binance_btcusdt_1h_crypto_btcusdt_close_20260528t020000z",
|
| 8 |
+
"data_fetched_at": "2026-05-29T02:50:05.083862Z",
|
| 9 |
"context_length": 96,
|
| 10 |
"prediction_length": 24
|
| 11 |
},
|
| 12 |
{
|
| 13 |
+
"dataset": "task_coingecko_bitcoin_7d_coin_bitcoin_price_usd_20260528t030012z",
|
| 14 |
+
"data_fetched_at": "2026-05-29T02:50:05.083862Z",
|
| 15 |
"context_length": 96,
|
| 16 |
"prediction_length": 24
|
| 17 |
},
|
| 18 |
{
|
| 19 |
"dataset": "task_nasa_power_shanghai_hourly_point_shanghai_31.2304_121.4737_t2m_20260524t230000z",
|
| 20 |
+
"data_fetched_at": "2026-05-29T02:50:05.083862Z",
|
| 21 |
+
"context_length": 69,
|
| 22 |
"prediction_length": 24
|
| 23 |
},
|
| 24 |
{
|
| 25 |
+
"dataset": "task_noaa_coops_sf_water_level_coops_station_9414290_water_level_20260529t001800z",
|
| 26 |
+
"data_fetched_at": "2026-05-29T02:50:05.083862Z",
|
| 27 |
"context_length": 96,
|
| 28 |
"prediction_length": 24
|
| 29 |
},
|
| 30 |
{
|
| 31 |
+
"dataset": "task_nws_ksfo_observations_nws_station_ksfo_temperature_20260529t004500z",
|
| 32 |
+
"data_fetched_at": "2026-05-29T02:50:05.083862Z",
|
| 33 |
"context_length": 96,
|
| 34 |
"prediction_length": 24
|
| 35 |
},
|
| 36 |
{
|
| 37 |
+
"dataset": "task_open_meteo_shanghai_air_quality_hourly_grid_shanghai_31.2304_121.4737_pm2_5_20260528t020000z",
|
| 38 |
+
"data_fetched_at": "2026-05-29T02:50:05.083862Z",
|
| 39 |
"context_length": 96,
|
| 40 |
"prediction_length": 24
|
| 41 |
},
|
| 42 |
{
|
| 43 |
+
"dataset": "task_usgs_potomac_iv_usgs_site_01646500_usgs_00060_20260528t235000z",
|
| 44 |
+
"data_fetched_at": "2026-05-29T02:50:05.083862Z",
|
| 45 |
"context_length": 96,
|
| 46 |
"prediction_length": 24
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 47 |
}
|
| 48 |
]
|
| 49 |
}
|
scripts/run_online_daemon.py
ADDED
|
@@ -0,0 +1,63 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python3
|
| 2 |
+
"""Periodically run space/scripts/run_space_eval.py (1-hour live benchmark loop)."""
|
| 3 |
+
|
| 4 |
+
from __future__ import annotations
|
| 5 |
+
|
| 6 |
+
import argparse
|
| 7 |
+
import logging
|
| 8 |
+
import os
|
| 9 |
+
import subprocess
|
| 10 |
+
import sys
|
| 11 |
+
import time
|
| 12 |
+
from pathlib import Path
|
| 13 |
+
|
| 14 |
+
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
|
| 15 |
+
logger = logging.getLogger(__name__)
|
| 16 |
+
|
| 17 |
+
SPACE_ROOT = Path(__file__).resolve().parents[1]
|
| 18 |
+
EVAL_SCRIPT = SPACE_ROOT / "scripts" / "run_space_eval.py"
|
| 19 |
+
DEFAULT_INTERVAL_SECONDS = int(
|
| 20 |
+
os.getenv(
|
| 21 |
+
"TSFM_BENCH_INTERVAL_SECONDS",
|
| 22 |
+
os.getenv("TSFM_EVAL_INTERVAL_SECONDS", "3600"),
|
| 23 |
+
)
|
| 24 |
+
)
|
| 25 |
+
|
| 26 |
+
|
| 27 |
+
def parse_args() -> argparse.Namespace:
|
| 28 |
+
parser = argparse.ArgumentParser(description=__doc__)
|
| 29 |
+
parser.add_argument(
|
| 30 |
+
"--interval-minutes",
|
| 31 |
+
type=int,
|
| 32 |
+
default=max(1, DEFAULT_INTERVAL_SECONDS // 60),
|
| 33 |
+
help="Minutes between collect + eval cycles (default: 60)",
|
| 34 |
+
)
|
| 35 |
+
parser.add_argument(
|
| 36 |
+
"--once",
|
| 37 |
+
action="store_true",
|
| 38 |
+
help="Run a single evaluation and exit",
|
| 39 |
+
)
|
| 40 |
+
return parser.parse_args()
|
| 41 |
+
|
| 42 |
+
|
| 43 |
+
def run_eval() -> int:
|
| 44 |
+
cmd = [sys.executable, str(EVAL_SCRIPT)]
|
| 45 |
+
logger.info("Running: %s", " ".join(cmd))
|
| 46 |
+
return subprocess.run(cmd, cwd=str(SPACE_ROOT), check=False).returncode
|
| 47 |
+
|
| 48 |
+
|
| 49 |
+
def main() -> None:
|
| 50 |
+
args = parse_args()
|
| 51 |
+
if args.once:
|
| 52 |
+
raise SystemExit(run_eval())
|
| 53 |
+
|
| 54 |
+
while True:
|
| 55 |
+
code = run_eval()
|
| 56 |
+
if code != 0:
|
| 57 |
+
logger.error("Evaluation failed with exit code %s", code)
|
| 58 |
+
logger.info("Sleeping %s minutes until next cycle", args.interval_minutes)
|
| 59 |
+
time.sleep(args.interval_minutes * 60)
|
| 60 |
+
|
| 61 |
+
|
| 62 |
+
if __name__ == "__main__":
|
| 63 |
+
main()
|
scripts/run_space_eval.py
ADDED
|
@@ -0,0 +1,281 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python3
|
| 2 |
+
"""Run live TS-Bench + TSFM.ai evaluation inside the HF Space (or locally)."""
|
| 3 |
+
|
| 4 |
+
from __future__ import annotations
|
| 5 |
+
|
| 6 |
+
import csv
|
| 7 |
+
import json
|
| 8 |
+
import logging
|
| 9 |
+
import os
|
| 10 |
+
import sys
|
| 11 |
+
from datetime import datetime, timezone
|
| 12 |
+
from pathlib import Path
|
| 13 |
+
from typing import Any
|
| 14 |
+
|
| 15 |
+
import yaml
|
| 16 |
+
|
| 17 |
+
SPACE_ROOT = Path(__file__).resolve().parents[1]
|
| 18 |
+
if str(SPACE_ROOT) not in sys.path:
|
| 19 |
+
sys.path.insert(0, str(SPACE_ROOT))
|
| 20 |
+
|
| 21 |
+
os.environ.setdefault("TS_BENCH_ROOT", str(SPACE_ROOT / "vendor" / "ts_bench"))
|
| 22 |
+
|
| 23 |
+
from tsfm_bench.data.registry import load_data_source, load_dataset_properties
|
| 24 |
+
from tsfm_bench.data.ts_bench import TsBenchDataSource
|
| 25 |
+
from tsfm_bench.eval.api_predictor import DEFAULT_QUANTILES, TsfmApiConfig, TsfmApiPredictor
|
| 26 |
+
from tsfm_bench.eval.online_eval import run_online_eval_for_dataset
|
| 27 |
+
|
| 28 |
+
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
|
| 29 |
+
logger = logging.getLogger(__name__)
|
| 30 |
+
|
| 31 |
+
DATA_CONFIG = SPACE_ROOT / "configs" / "datasets" / "ts_bench.yaml"
|
| 32 |
+
MODEL_CONFIG = SPACE_ROOT / "configs" / "models" / "online_tsfm.yaml"
|
| 33 |
+
OUTPUT_ROOT = Path(os.getenv("TSFM_RESULTS_PATH", str(SPACE_ROOT / "results")))
|
| 34 |
+
EVAL_STATE_PATH = OUTPUT_ROOT / "eval_state.json"
|
| 35 |
+
|
| 36 |
+
|
| 37 |
+
def model_display_name(model_spec: dict[str, Any]) -> str:
|
| 38 |
+
return model_spec.get("display_name") or model_spec["model_id"]
|
| 39 |
+
|
| 40 |
+
|
| 41 |
+
def model_output_slug(model_spec: dict[str, Any]) -> str:
|
| 42 |
+
import re
|
| 43 |
+
|
| 44 |
+
name = model_spec.get("display_name") or model_spec["model_id"]
|
| 45 |
+
return re.sub(r"[^a-zA-Z0-9]+", "_", name).strip("_").lower()
|
| 46 |
+
|
| 47 |
+
|
| 48 |
+
def load_model_specs(path: Path) -> list[dict[str, Any]]:
|
| 49 |
+
return yaml.safe_load(path.read_text()).get("models", [])
|
| 50 |
+
|
| 51 |
+
|
| 52 |
+
def write_model_results(
|
| 53 |
+
model_spec: dict[str, Any],
|
| 54 |
+
rows: list[list[Any]],
|
| 55 |
+
output_root: Path,
|
| 56 |
+
meta: dict[str, Any],
|
| 57 |
+
) -> None:
|
| 58 |
+
model_name = model_display_name(model_spec)
|
| 59 |
+
out_dir = output_root / model_output_slug(model_spec)
|
| 60 |
+
out_dir.mkdir(parents=True, exist_ok=True)
|
| 61 |
+
|
| 62 |
+
csv_path = out_dir / "all_results.csv"
|
| 63 |
+
with csv_path.open("w", newline="") as handle:
|
| 64 |
+
writer = csv.writer(handle)
|
| 65 |
+
writer.writerow(
|
| 66 |
+
[
|
| 67 |
+
"dataset",
|
| 68 |
+
"model",
|
| 69 |
+
"eval_metrics/MSE[mean]",
|
| 70 |
+
"eval_metrics/MSE[0.5]",
|
| 71 |
+
"eval_metrics/MAE[0.5]",
|
| 72 |
+
"eval_metrics/MASE[0.5]",
|
| 73 |
+
"eval_metrics/MAPE[0.5]",
|
| 74 |
+
"eval_metrics/sMAPE[0.5]",
|
| 75 |
+
"eval_metrics/MSIS",
|
| 76 |
+
"eval_metrics/RMSE[mean]",
|
| 77 |
+
"eval_metrics/NRMSE[mean]",
|
| 78 |
+
"eval_metrics/ND[0.5]",
|
| 79 |
+
"eval_metrics/mean_weighted_sum_quantile_loss",
|
| 80 |
+
"domain",
|
| 81 |
+
"num_variates",
|
| 82 |
+
]
|
| 83 |
+
)
|
| 84 |
+
writer.writerows(rows)
|
| 85 |
+
|
| 86 |
+
config = {
|
| 87 |
+
"model": model_name,
|
| 88 |
+
"model_type": model_spec.get("model_type", "zero-shot"),
|
| 89 |
+
"model_dtype": "float32",
|
| 90 |
+
"model_link": model_spec.get(
|
| 91 |
+
"model_link", f"https://tsfm.ai/models/{model_spec['model_id']}"
|
| 92 |
+
),
|
| 93 |
+
"code_link": "https://github.com/zhouziyu02/TSFM_Bench/blob/main/space/scripts/run_space_eval.py",
|
| 94 |
+
"org": model_spec.get("org", "TSFM.ai"),
|
| 95 |
+
"testdata_leakage": "No",
|
| 96 |
+
"replication_code_available": "Yes",
|
| 97 |
+
"api_model_id": model_spec["model_id"],
|
| 98 |
+
}
|
| 99 |
+
(out_dir / "config.json").write_text(json.dumps(config, indent=4) + "\n")
|
| 100 |
+
(out_dir / "online_meta.json").write_text(json.dumps(meta, indent=4) + "\n")
|
| 101 |
+
|
| 102 |
+
|
| 103 |
+
def write_dataset_properties(properties: dict[str, dict[str, Any]], output_root: Path) -> None:
|
| 104 |
+
rows = [
|
| 105 |
+
{
|
| 106 |
+
"dataset": key,
|
| 107 |
+
"domain": values["domain"],
|
| 108 |
+
"frequency": values["frequency"],
|
| 109 |
+
"num_variates": values["num_variates"],
|
| 110 |
+
}
|
| 111 |
+
for key, values in sorted(properties.items())
|
| 112 |
+
]
|
| 113 |
+
csv_path = output_root / "dataset_properties.csv"
|
| 114 |
+
with csv_path.open("w", newline="") as handle:
|
| 115 |
+
writer = csv.DictWriter(
|
| 116 |
+
handle,
|
| 117 |
+
fieldnames=["dataset", "domain", "frequency", "num_variates"],
|
| 118 |
+
)
|
| 119 |
+
writer.writeheader()
|
| 120 |
+
writer.writerows(rows)
|
| 121 |
+
|
| 122 |
+
|
| 123 |
+
def write_eval_state(payload: dict[str, Any]) -> None:
|
| 124 |
+
OUTPUT_ROOT.mkdir(parents=True, exist_ok=True)
|
| 125 |
+
EVAL_STATE_PATH.write_text(json.dumps(payload, indent=4) + "\n")
|
| 126 |
+
|
| 127 |
+
|
| 128 |
+
def write_run_metadata(output_root: Path, payload: dict[str, Any]) -> None:
|
| 129 |
+
(output_root / "online_status.json").write_text(json.dumps(payload, indent=4) + "\n")
|
| 130 |
+
|
| 131 |
+
|
| 132 |
+
def make_predictor(model_spec: dict[str, Any], pred_len: int) -> TsfmApiPredictor:
|
| 133 |
+
predictor = TsfmApiPredictor(
|
| 134 |
+
config=TsfmApiConfig(model_id=model_spec["model_id"]),
|
| 135 |
+
prediction_length=pred_len,
|
| 136 |
+
quantile_levels=DEFAULT_QUANTILES,
|
| 137 |
+
)
|
| 138 |
+
predictor.leaderboard_name = model_display_name(model_spec)
|
| 139 |
+
return predictor
|
| 140 |
+
|
| 141 |
+
|
| 142 |
+
def run_evaluation() -> int:
|
| 143 |
+
if not os.getenv("TSFM_API_KEY"):
|
| 144 |
+
write_eval_state(
|
| 145 |
+
{
|
| 146 |
+
"status": "disabled",
|
| 147 |
+
"message": "TSFM_API_KEY not configured — set it in HF Space secrets.",
|
| 148 |
+
"updated_at": datetime.now(timezone.utc).isoformat(),
|
| 149 |
+
}
|
| 150 |
+
)
|
| 151 |
+
logger.error("TSFM_API_KEY is missing; skipping evaluation")
|
| 152 |
+
return 1
|
| 153 |
+
|
| 154 |
+
started = datetime.now(timezone.utc).isoformat()
|
| 155 |
+
write_eval_state(
|
| 156 |
+
{
|
| 157 |
+
"status": "running",
|
| 158 |
+
"started_at": started,
|
| 159 |
+
"message": "Collecting TS-Bench data and calling TSFM.ai API…",
|
| 160 |
+
"updated_at": started,
|
| 161 |
+
}
|
| 162 |
+
)
|
| 163 |
+
|
| 164 |
+
try:
|
| 165 |
+
source = load_data_source(DATA_CONFIG)
|
| 166 |
+
if isinstance(source, TsBenchDataSource) and source._settings.auto_refresh:
|
| 167 |
+
source.refresh()
|
| 168 |
+
|
| 169 |
+
datasets = source.list_datasets()
|
| 170 |
+
if not datasets:
|
| 171 |
+
raise RuntimeError("No TS-Bench tasks available after data collection")
|
| 172 |
+
|
| 173 |
+
properties = load_dataset_properties(DATA_CONFIG)
|
| 174 |
+
model_specs = load_model_specs(MODEL_CONFIG)
|
| 175 |
+
OUTPUT_ROOT.mkdir(parents=True, exist_ok=True)
|
| 176 |
+
write_dataset_properties(properties, OUTPUT_ROOT)
|
| 177 |
+
|
| 178 |
+
failed_models: list[str] = []
|
| 179 |
+
all_model_meta: dict[str, Any] = {}
|
| 180 |
+
|
| 181 |
+
for model_spec in model_specs:
|
| 182 |
+
model_name = model_display_name(model_spec)
|
| 183 |
+
rows: list[list[Any]] = []
|
| 184 |
+
dataset_meta: list[dict[str, Any]] = []
|
| 185 |
+
|
| 186 |
+
try:
|
| 187 |
+
for ds_name in datasets:
|
| 188 |
+
pred_len = source.get_prediction_length(ds_name)
|
| 189 |
+
predictor = make_predictor(model_spec, pred_len)
|
| 190 |
+
try:
|
| 191 |
+
logger.info("Evaluating %s on %s", model_name, ds_name)
|
| 192 |
+
result = run_online_eval_for_dataset(source, ds_name, predictor)
|
| 193 |
+
except Exception:
|
| 194 |
+
logger.exception("Skipping %s on %s", model_name, ds_name)
|
| 195 |
+
continue
|
| 196 |
+
rows.append(
|
| 197 |
+
[
|
| 198 |
+
result.dataset,
|
| 199 |
+
model_name,
|
| 200 |
+
result.metrics["MSE[mean]"],
|
| 201 |
+
result.metrics["MSE[0.5]"],
|
| 202 |
+
result.metrics["MAE[0.5]"],
|
| 203 |
+
result.metrics["MASE[0.5]"],
|
| 204 |
+
result.metrics["MAPE[0.5]"],
|
| 205 |
+
result.metrics["sMAPE[0.5]"],
|
| 206 |
+
result.metrics["MSIS"],
|
| 207 |
+
result.metrics["RMSE[mean]"],
|
| 208 |
+
result.metrics["NRMSE[mean]"],
|
| 209 |
+
result.metrics["ND[0.5]"],
|
| 210 |
+
result.metrics["mean_weighted_sum_quantile_loss"],
|
| 211 |
+
result.domain,
|
| 212 |
+
result.num_variates,
|
| 213 |
+
]
|
| 214 |
+
)
|
| 215 |
+
dataset_meta.append(
|
| 216 |
+
{
|
| 217 |
+
"dataset": ds_name,
|
| 218 |
+
"data_fetched_at": result.data_fetched_at,
|
| 219 |
+
"context_length": result.context_length,
|
| 220 |
+
"prediction_length": result.prediction_length,
|
| 221 |
+
}
|
| 222 |
+
)
|
| 223 |
+
if not rows:
|
| 224 |
+
raise RuntimeError(f"No successful datasets for {model_name}")
|
| 225 |
+
model_meta = {
|
| 226 |
+
"model": model_name,
|
| 227 |
+
"api_model_id": model_spec["model_id"],
|
| 228 |
+
"evaluated_at": datetime.now(timezone.utc).isoformat(),
|
| 229 |
+
"datasets": dataset_meta,
|
| 230 |
+
}
|
| 231 |
+
write_model_results(model_spec, rows, OUTPUT_ROOT, model_meta)
|
| 232 |
+
all_model_meta[model_name] = model_meta
|
| 233 |
+
except Exception:
|
| 234 |
+
logger.exception("Failed evaluating %s", model_name)
|
| 235 |
+
failed_models.append(model_name)
|
| 236 |
+
|
| 237 |
+
finished = datetime.now(timezone.utc).isoformat()
|
| 238 |
+
status = "ok" if not failed_models else "partial"
|
| 239 |
+
write_run_metadata(
|
| 240 |
+
OUTPUT_ROOT,
|
| 241 |
+
{
|
| 242 |
+
"status": status,
|
| 243 |
+
"started_at": started,
|
| 244 |
+
"finished_at": finished,
|
| 245 |
+
"data_source": "ts_bench",
|
| 246 |
+
"data_config": str(DATA_CONFIG),
|
| 247 |
+
"ts_bench_root": os.environ.get("TS_BENCH_ROOT", ""),
|
| 248 |
+
"models": all_model_meta,
|
| 249 |
+
"failed_models": failed_models,
|
| 250 |
+
},
|
| 251 |
+
)
|
| 252 |
+
write_eval_state(
|
| 253 |
+
{
|
| 254 |
+
"status": status,
|
| 255 |
+
"started_at": started,
|
| 256 |
+
"finished_at": finished,
|
| 257 |
+
"failed_models": failed_models,
|
| 258 |
+
"task_count": len(datasets),
|
| 259 |
+
"model_count": len(model_specs) - len(failed_models),
|
| 260 |
+
"message": "Evaluation complete",
|
| 261 |
+
"updated_at": finished,
|
| 262 |
+
}
|
| 263 |
+
)
|
| 264 |
+
return 0 if status == "ok" else 2
|
| 265 |
+
except Exception as exc:
|
| 266 |
+
finished = datetime.now(timezone.utc).isoformat()
|
| 267 |
+
logger.exception("Evaluation run failed")
|
| 268 |
+
write_eval_state(
|
| 269 |
+
{
|
| 270 |
+
"status": "error",
|
| 271 |
+
"started_at": started,
|
| 272 |
+
"finished_at": finished,
|
| 273 |
+
"message": str(exc),
|
| 274 |
+
"updated_at": finished,
|
| 275 |
+
}
|
| 276 |
+
)
|
| 277 |
+
return 1
|
| 278 |
+
|
| 279 |
+
|
| 280 |
+
if __name__ == "__main__":
|
| 281 |
+
raise SystemExit(run_evaluation())
|
src/about.py
CHANGED
|
@@ -3,42 +3,54 @@ TITLE = """ TSFM Realworld Bench Leaderboard """
|
|
| 3 |
INTRODUCTION_TEXT = """
|
| 4 |
**TSFM Realworld Bench** evaluates time series foundation models on **live TS-Bench
|
| 5 |
data** with **zero-shot API inference** via [TSFM.ai](https://tsfm.ai/).
|
| 6 |
-
|
| 7 |
-
**
|
| 8 |
-
|
|
|
|
| 9 |
"""
|
| 10 |
|
| 11 |
LLM_BENCHMARKS_TEXT = """
|
| 12 |
-
## How
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 13 |
|
| 14 |
-
|
| 15 |
-
|
| 16 |
-
|
| 17 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 18 |
|
| 19 |
## Metrics
|
| 20 |
|
| 21 |
- **MASE** — Mean Absolute Scaled Error
|
| 22 |
-
- **CRPS** — Mean Weighted Sum Quantile Loss
|
| 23 |
-
-
|
| 24 |
|
| 25 |
## Aggregation (GIFT-Eval style)
|
| 26 |
|
| 27 |
-
1. Compute evaluation metrics on each dataset
|
| 28 |
-
2. Rank models per dataset
|
| 29 |
-
3. Aggregate
|
| 30 |
-
4. Aggregate ranks with an **arithmetic mean** across datasets.
|
| 31 |
-
|
| 32 |
-
## Data interface
|
| 33 |
|
| 34 |
-
|
| 35 |
-
holds out the most recent forecast horizon as ground truth and calls TSFM models through
|
| 36 |
-
the **TSFM.ai** API for zero-shot forecasts.
|
| 37 |
|
| 38 |
```bash
|
| 39 |
export TSFM_API_KEY=your_key
|
| 40 |
-
|
| 41 |
-
python scripts/
|
|
|
|
| 42 |
```
|
| 43 |
"""
|
| 44 |
|
|
|
|
| 3 |
INTRODUCTION_TEXT = """
|
| 4 |
**TSFM Realworld Bench** evaluates time series foundation models on **live TS-Bench
|
| 5 |
data** with **zero-shot API inference** via [TSFM.ai](https://tsfm.ai/).
|
| 6 |
+
|
| 7 |
+
The Space runs a **unified 1-hour loop**: collect TS-Bench data → call TSFM.ai for all models →
|
| 8 |
+
write new `results/` CSVs → leaderboard reloads. Set **`TSFM_API_KEY`** in Space Settings →
|
| 9 |
+
Repository secrets to enable it.
|
| 10 |
"""
|
| 11 |
|
| 12 |
LLM_BENCHMARKS_TEXT = """
|
| 13 |
+
## How it works (live benchmark)
|
| 14 |
+
|
| 15 |
+
1. **Background scheduler** runs every **`TSFM_BENCH_INTERVAL_SECONDS`** (default **3600 = 1 hour**):
|
| 16 |
+
TS-Bench collect → TSFM.ai eval → update `results/`.
|
| 17 |
+
2. Each run calls **TSFM.ai** for all configured models.
|
| 18 |
+
3. The Gradio UI reloads tables every **`TSFM_REFRESH_SECONDS`** (default 60s) from disk.
|
| 19 |
+
|
| 20 |
+
### Required HF Space secret
|
| 21 |
|
| 22 |
+
| Variable | Purpose |
|
| 23 |
+
|---|---|
|
| 24 |
+
| `TSFM_API_KEY` | TSFM.ai API key — **required** for live evaluation |
|
| 25 |
+
|
| 26 |
+
### Optional environment variables
|
| 27 |
+
|
| 28 |
+
| Variable | Default | Purpose |
|
| 29 |
+
|---|---|---|
|
| 30 |
+
| `TSFM_BENCH_INTERVAL_SECONDS` | `3600` | **Main knob**: seconds between collect+eval cycles (1 h) |
|
| 31 |
+
| `TSFM_EVAL_INTERVAL_SECONDS` | (alias) | Same as above if `TSFM_BENCH_INTERVAL_SECONDS` unset |
|
| 32 |
+
| `TSFM_REFRESH_SECONDS` | `60` | UI table reload interval (display only) |
|
| 33 |
+
| `TS_BENCH_ROOT` | `vendor/ts_bench` | Path to TS-Bench collect scripts |
|
| 34 |
|
| 35 |
## Metrics
|
| 36 |
|
| 37 |
- **MASE** — Mean Absolute Scaled Error
|
| 38 |
+
- **CRPS** — Mean Weighted Sum Quantile Loss
|
| 39 |
+
- Ranks are averaged per-dataset on raw metrics (lower is better)
|
| 40 |
|
| 41 |
## Aggregation (GIFT-Eval style)
|
| 42 |
|
| 43 |
+
1. Compute evaluation metrics on each dataset (raw values).
|
| 44 |
+
2. Rank models per dataset.
|
| 45 |
+
3. Aggregate with **geometric mean** (metrics) and **arithmetic mean** (ranks).
|
|
|
|
|
|
|
|
|
|
| 46 |
|
| 47 |
+
## Local development
|
|
|
|
|
|
|
| 48 |
|
| 49 |
```bash
|
| 50 |
export TSFM_API_KEY=your_key
|
| 51 |
+
cd space
|
| 52 |
+
python scripts/run_space_eval.py # single eval run
|
| 53 |
+
python app.py # leaderboard + scheduler
|
| 54 |
```
|
| 55 |
"""
|
| 56 |
|
src/benchmark_config.py
ADDED
|
@@ -0,0 +1,19 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Unified timing for the live benchmark loop."""
|
| 2 |
+
|
| 3 |
+
from __future__ import annotations
|
| 4 |
+
|
| 5 |
+
import os
|
| 6 |
+
|
| 7 |
+
# One full cycle: TS-Bench collect → TSFM.ai eval → write results → leaderboard reload.
|
| 8 |
+
# Override with TSFM_BENCH_INTERVAL_SECONDS (preferred) or TSFM_EVAL_INTERVAL_SECONDS.
|
| 9 |
+
BENCHMARK_INTERVAL_SECONDS = int(
|
| 10 |
+
os.getenv(
|
| 11 |
+
"TSFM_BENCH_INTERVAL_SECONDS",
|
| 12 |
+
os.getenv("TSFM_EVAL_INTERVAL_SECONDS", "3600"),
|
| 13 |
+
)
|
| 14 |
+
)
|
| 15 |
+
|
| 16 |
+
# UI table reload (reads results/ from disk; does not call TSFM.ai).
|
| 17 |
+
DISPLAY_REFRESH_SECONDS = int(os.getenv("TSFM_REFRESH_SECONDS", "60"))
|
| 18 |
+
|
| 19 |
+
BENCHMARK_INTERVAL_MINUTES = BENCHMARK_INTERVAL_SECONDS // 60
|
src/display/css_html_js.py
CHANGED
|
@@ -37,7 +37,7 @@ def refresh_countdown_markup(seconds: int) -> str:
|
|
| 37 |
"""Static HTML only — Gradio on HF Spaces strips <script> inside gr.HTML."""
|
| 38 |
return f"""
|
| 39 |
<div class="auto-refresh-wrap" id="tsfm-refresh-widget">
|
| 40 |
-
<div class="auto-refresh-label" id="tsfm-refresh-label">Next
|
| 41 |
<div class="auto-refresh-track">
|
| 42 |
<div class="auto-refresh-fill" id="tsfm-refresh-fill"></div>
|
| 43 |
</div>
|
|
@@ -55,7 +55,7 @@ def refresh_countdown_head_js(seconds: int) -> str:
|
|
| 55 |
function render(remaining) {{
|
| 56 |
const label = document.getElementById("tsfm-refresh-label");
|
| 57 |
if (label) {{
|
| 58 |
-
label.textContent = "Next
|
| 59 |
}}
|
| 60 |
}}
|
| 61 |
|
|
|
|
| 37 |
"""Static HTML only — Gradio on HF Spaces strips <script> inside gr.HTML."""
|
| 38 |
return f"""
|
| 39 |
<div class="auto-refresh-wrap" id="tsfm-refresh-widget">
|
| 40 |
+
<div class="auto-refresh-label" id="tsfm-refresh-label">Next display reload in {seconds}s</div>
|
| 41 |
<div class="auto-refresh-track">
|
| 42 |
<div class="auto-refresh-fill" id="tsfm-refresh-fill"></div>
|
| 43 |
</div>
|
|
|
|
| 55 |
function render(remaining) {{
|
| 56 |
const label = document.getElementById("tsfm-refresh-label");
|
| 57 |
if (label) {{
|
| 58 |
+
label.textContent = "Next display reload in " + remaining + "s";
|
| 59 |
}}
|
| 60 |
}}
|
| 61 |
|
src/eval_scheduler.py
ADDED
|
@@ -0,0 +1,123 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Background scheduler: periodically runs live TS-Bench + TSFM.ai evaluation."""
|
| 2 |
+
|
| 3 |
+
from __future__ import annotations
|
| 4 |
+
|
| 5 |
+
import json
|
| 6 |
+
import logging
|
| 7 |
+
import os
|
| 8 |
+
import subprocess
|
| 9 |
+
import sys
|
| 10 |
+
import threading
|
| 11 |
+
import time
|
| 12 |
+
from datetime import datetime, timezone
|
| 13 |
+
from pathlib import Path
|
| 14 |
+
|
| 15 |
+
from src.benchmark_config import BENCHMARK_INTERVAL_SECONDS, DISPLAY_REFRESH_SECONDS
|
| 16 |
+
|
| 17 |
+
logger = logging.getLogger(__name__)
|
| 18 |
+
|
| 19 |
+
SPACE_ROOT = Path(__file__).resolve().parents[1]
|
| 20 |
+
EVAL_SCRIPT = SPACE_ROOT / "scripts" / "run_space_eval.py"
|
| 21 |
+
DEFAULT_EVAL_INTERVAL = BENCHMARK_INTERVAL_SECONDS
|
| 22 |
+
|
| 23 |
+
|
| 24 |
+
class LiveEvalScheduler:
|
| 25 |
+
def __init__(
|
| 26 |
+
self,
|
| 27 |
+
interval_seconds: int = DEFAULT_EVAL_INTERVAL,
|
| 28 |
+
run_on_startup: bool = True,
|
| 29 |
+
):
|
| 30 |
+
self.interval_seconds = max(300, interval_seconds)
|
| 31 |
+
self.run_on_startup = run_on_startup
|
| 32 |
+
self._lock = threading.Lock()
|
| 33 |
+
self._stop = threading.Event()
|
| 34 |
+
self._thread: threading.Thread | None = None
|
| 35 |
+
self._next_run_at: float | None = None
|
| 36 |
+
|
| 37 |
+
@property
|
| 38 |
+
def enabled(self) -> bool:
|
| 39 |
+
return bool(os.getenv("TSFM_API_KEY"))
|
| 40 |
+
|
| 41 |
+
def start(self) -> None:
|
| 42 |
+
if self._thread is not None:
|
| 43 |
+
return
|
| 44 |
+
if not self.enabled:
|
| 45 |
+
logger.warning(
|
| 46 |
+
"Live eval disabled: TSFM_API_KEY not set. "
|
| 47 |
+
"Leaderboard will only show static CSV results."
|
| 48 |
+
)
|
| 49 |
+
self._write_idle_state(
|
| 50 |
+
"TSFM_API_KEY not set — configure it in HF Space Settings → Secrets."
|
| 51 |
+
)
|
| 52 |
+
return
|
| 53 |
+
|
| 54 |
+
self._thread = threading.Thread(target=self._loop, name="live-eval-scheduler", daemon=True)
|
| 55 |
+
self._thread.start()
|
| 56 |
+
logger.info(
|
| 57 |
+
"Live eval scheduler started (interval=%ss, script=%s)",
|
| 58 |
+
self.interval_seconds,
|
| 59 |
+
EVAL_SCRIPT,
|
| 60 |
+
)
|
| 61 |
+
|
| 62 |
+
def snapshot(self) -> dict:
|
| 63 |
+
state_path = SPACE_ROOT / "results" / "eval_state.json"
|
| 64 |
+
if state_path.exists():
|
| 65 |
+
try:
|
| 66 |
+
return json.loads(state_path.read_text())
|
| 67 |
+
except json.JSONDecodeError:
|
| 68 |
+
pass
|
| 69 |
+
return {
|
| 70 |
+
"status": "idle" if self.enabled else "disabled",
|
| 71 |
+
"message": "Waiting for first evaluation run…" if self.enabled else "No API key",
|
| 72 |
+
}
|
| 73 |
+
|
| 74 |
+
def seconds_until_next_run(self) -> int | None:
|
| 75 |
+
if self._next_run_at is None:
|
| 76 |
+
return None
|
| 77 |
+
return max(0, int(self._next_run_at - time.monotonic()))
|
| 78 |
+
|
| 79 |
+
def _loop(self) -> None:
|
| 80 |
+
if self.run_on_startup:
|
| 81 |
+
self._run_once()
|
| 82 |
+
while not self._stop.wait(self.interval_seconds):
|
| 83 |
+
self._run_once()
|
| 84 |
+
|
| 85 |
+
def _run_once(self) -> None:
|
| 86 |
+
if not self._lock.acquire(blocking=False):
|
| 87 |
+
logger.info("Evaluation already in progress, skipping tick")
|
| 88 |
+
return
|
| 89 |
+
try:
|
| 90 |
+
logger.info("Starting scheduled live evaluation")
|
| 91 |
+
cmd = [sys.executable, str(EVAL_SCRIPT)]
|
| 92 |
+
completed = subprocess.run(cmd, cwd=str(SPACE_ROOT), check=False)
|
| 93 |
+
if completed.returncode != 0:
|
| 94 |
+
logger.error("Evaluation exited with code %s", completed.returncode)
|
| 95 |
+
finally:
|
| 96 |
+
self._lock.release()
|
| 97 |
+
self._next_run_at = time.monotonic() + self.interval_seconds
|
| 98 |
+
|
| 99 |
+
def _write_idle_state(self, message: str) -> None:
|
| 100 |
+
results = SPACE_ROOT / "results"
|
| 101 |
+
results.mkdir(parents=True, exist_ok=True)
|
| 102 |
+
payload = {
|
| 103 |
+
"status": "disabled",
|
| 104 |
+
"message": message,
|
| 105 |
+
"updated_at": datetime.now(timezone.utc).isoformat(),
|
| 106 |
+
}
|
| 107 |
+
(results / "eval_state.json").write_text(json.dumps(payload, indent=4) + "\n")
|
| 108 |
+
|
| 109 |
+
|
| 110 |
+
_scheduler: LiveEvalScheduler | None = None
|
| 111 |
+
|
| 112 |
+
|
| 113 |
+
def get_scheduler() -> LiveEvalScheduler:
|
| 114 |
+
global _scheduler
|
| 115 |
+
if _scheduler is None:
|
| 116 |
+
_scheduler = LiveEvalScheduler()
|
| 117 |
+
return _scheduler
|
| 118 |
+
|
| 119 |
+
|
| 120 |
+
def start_live_eval_scheduler() -> LiveEvalScheduler:
|
| 121 |
+
scheduler = get_scheduler()
|
| 122 |
+
scheduler.start()
|
| 123 |
+
return scheduler
|
tsfm_bench/__about__.py
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
__version__ = "0.1.0"
|
tsfm_bench/__init__.py
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
from tsfm_bench.__about__ import __version__
|
tsfm_bench/data/__init__.py
ADDED
|
@@ -0,0 +1,18 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from tsfm_bench.data.dataset import Dataset, Term
|
| 2 |
+
from tsfm_bench.data.interface import DatasetMetadata, TimeSeriesDataSource, TimeSeriesRecord
|
| 3 |
+
from tsfm_bench.data.open_meteo import OpenMeteoDataSource
|
| 4 |
+
from tsfm_bench.data.registry import load_data_source
|
| 5 |
+
from tsfm_bench.data.synthetic import SyntheticDataSource
|
| 6 |
+
from tsfm_bench.data.ts_bench import TsBenchDataSource
|
| 7 |
+
|
| 8 |
+
__all__ = [
|
| 9 |
+
"Dataset",
|
| 10 |
+
"Term",
|
| 11 |
+
"DatasetMetadata",
|
| 12 |
+
"TimeSeriesDataSource",
|
| 13 |
+
"TimeSeriesRecord",
|
| 14 |
+
"SyntheticDataSource",
|
| 15 |
+
"OpenMeteoDataSource",
|
| 16 |
+
"TsBenchDataSource",
|
| 17 |
+
"load_data_source",
|
| 18 |
+
]
|
tsfm_bench/data/dataset.py
ADDED
|
@@ -0,0 +1,197 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
import math
|
| 4 |
+
import os
|
| 5 |
+
from enum import Enum
|
| 6 |
+
from functools import cached_property
|
| 7 |
+
from pathlib import Path
|
| 8 |
+
from typing import Iterable, Iterator
|
| 9 |
+
|
| 10 |
+
from dotenv import load_dotenv
|
| 11 |
+
from gluonts.dataset import DataEntry
|
| 12 |
+
from gluonts.dataset.common import ProcessDataEntry
|
| 13 |
+
from gluonts.dataset.split import TestData, TrainingDataset, split
|
| 14 |
+
from gluonts.itertools import Map
|
| 15 |
+
from gluonts.time_feature import norm_freq_str
|
| 16 |
+
from gluonts.transform import Transformation
|
| 17 |
+
from pandas.tseries.frequencies import to_offset
|
| 18 |
+
from toolz import compose
|
| 19 |
+
|
| 20 |
+
from tsfm_bench.data.interface import TimeSeriesDataSource, TimeSeriesRecord
|
| 21 |
+
from tsfm_bench.data.registry import load_data_source
|
| 22 |
+
|
| 23 |
+
TEST_SPLIT = 0.1
|
| 24 |
+
MAX_WINDOW = 20
|
| 25 |
+
|
| 26 |
+
M4_PRED_LENGTH_MAP = {
|
| 27 |
+
"A": 6,
|
| 28 |
+
"Q": 8,
|
| 29 |
+
"M": 18,
|
| 30 |
+
"W": 13,
|
| 31 |
+
"D": 14,
|
| 32 |
+
"H": 48,
|
| 33 |
+
}
|
| 34 |
+
|
| 35 |
+
PRED_LENGTH_MAP = {
|
| 36 |
+
"M": 12,
|
| 37 |
+
"W": 8,
|
| 38 |
+
"D": 30,
|
| 39 |
+
"H": 48,
|
| 40 |
+
"T": 48,
|
| 41 |
+
"S": 60,
|
| 42 |
+
}
|
| 43 |
+
|
| 44 |
+
|
| 45 |
+
class Term(Enum):
|
| 46 |
+
SHORT = "short"
|
| 47 |
+
MEDIUM = "medium"
|
| 48 |
+
LONG = "long"
|
| 49 |
+
|
| 50 |
+
@property
|
| 51 |
+
def multiplier(self) -> int:
|
| 52 |
+
if self == Term.SHORT:
|
| 53 |
+
return 1
|
| 54 |
+
if self == Term.MEDIUM:
|
| 55 |
+
return 10
|
| 56 |
+
return 15
|
| 57 |
+
|
| 58 |
+
|
| 59 |
+
def itemize_start(data_entry: DataEntry) -> DataEntry:
|
| 60 |
+
start = data_entry["start"]
|
| 61 |
+
if hasattr(start, "item"):
|
| 62 |
+
data_entry["start"] = start.item()
|
| 63 |
+
return data_entry
|
| 64 |
+
|
| 65 |
+
|
| 66 |
+
def maybe_reconvert_freq(freq: str) -> str:
|
| 67 |
+
deprecated_map = {
|
| 68 |
+
"Y": "A",
|
| 69 |
+
"YE": "A",
|
| 70 |
+
"QE": "Q",
|
| 71 |
+
"ME": "M",
|
| 72 |
+
"h": "H",
|
| 73 |
+
"min": "T",
|
| 74 |
+
"s": "S",
|
| 75 |
+
"us": "U",
|
| 76 |
+
}
|
| 77 |
+
return deprecated_map.get(freq, freq)
|
| 78 |
+
|
| 79 |
+
|
| 80 |
+
class MultivariateToUnivariate(Transformation):
|
| 81 |
+
def __init__(self, field: str):
|
| 82 |
+
self.field = field
|
| 83 |
+
|
| 84 |
+
def __call__(
|
| 85 |
+
self, data_it: Iterable[DataEntry], is_train: bool = False
|
| 86 |
+
) -> Iterator[DataEntry]:
|
| 87 |
+
for data_entry in data_it:
|
| 88 |
+
item_id = data_entry["item_id"]
|
| 89 |
+
val_ls = list(data_entry[self.field])
|
| 90 |
+
for dim_id, val in enumerate(val_ls):
|
| 91 |
+
univariate_entry = data_entry.copy()
|
| 92 |
+
univariate_entry[self.field] = val
|
| 93 |
+
univariate_entry["item_id"] = f"{item_id}_dim{dim_id}"
|
| 94 |
+
yield univariate_entry
|
| 95 |
+
|
| 96 |
+
|
| 97 |
+
def _record_to_entry(record: TimeSeriesRecord) -> dict:
|
| 98 |
+
import pandas as pd
|
| 99 |
+
|
| 100 |
+
return {
|
| 101 |
+
"item_id": record.item_id,
|
| 102 |
+
"start": pd.Period(record.start),
|
| 103 |
+
"target": record.target,
|
| 104 |
+
"freq": record.freq,
|
| 105 |
+
}
|
| 106 |
+
|
| 107 |
+
|
| 108 |
+
class Dataset:
|
| 109 |
+
"""GluonTS-compatible dataset backed by a streaming data source."""
|
| 110 |
+
|
| 111 |
+
def __init__(
|
| 112 |
+
self,
|
| 113 |
+
name: str,
|
| 114 |
+
term: Term | str = Term.SHORT,
|
| 115 |
+
to_univariate: bool = False,
|
| 116 |
+
source: TimeSeriesDataSource | None = None,
|
| 117 |
+
config_path: str | Path | None = None,
|
| 118 |
+
):
|
| 119 |
+
load_dotenv()
|
| 120 |
+
if source is None:
|
| 121 |
+
env_config = os.getenv("TSFM_BENCH_CONFIG")
|
| 122 |
+
source = load_data_source(config_path or env_config)
|
| 123 |
+
|
| 124 |
+
self.source = source
|
| 125 |
+
self.metadata = source.get_metadata(name)
|
| 126 |
+
self.name = name
|
| 127 |
+
self.term = Term(term)
|
| 128 |
+
|
| 129 |
+
records = list(source.stream(name))
|
| 130 |
+
self._raw_entries = [_record_to_entry(record) for record in records]
|
| 131 |
+
|
| 132 |
+
first_target = self._raw_entries[0]["target"]
|
| 133 |
+
target_dim = first_target.shape[0] if getattr(first_target, "ndim", 1) > 1 else 1
|
| 134 |
+
|
| 135 |
+
process = ProcessDataEntry(
|
| 136 |
+
self.freq,
|
| 137 |
+
one_dim_target=target_dim == 1,
|
| 138 |
+
)
|
| 139 |
+
self.gluonts_dataset = Map(compose(process, itemize_start), self._raw_entries)
|
| 140 |
+
if to_univariate:
|
| 141 |
+
self.gluonts_dataset = MultivariateToUnivariate("target").apply(
|
| 142 |
+
self.gluonts_dataset
|
| 143 |
+
)
|
| 144 |
+
|
| 145 |
+
@cached_property
|
| 146 |
+
def prediction_length(self) -> int:
|
| 147 |
+
freq = norm_freq_str(to_offset(self.freq).name)
|
| 148 |
+
freq = maybe_reconvert_freq(freq)
|
| 149 |
+
pred_len = PRED_LENGTH_MAP.get(freq, 48)
|
| 150 |
+
return self.term.multiplier * pred_len
|
| 151 |
+
|
| 152 |
+
@cached_property
|
| 153 |
+
def freq(self) -> str:
|
| 154 |
+
return self.metadata.frequency
|
| 155 |
+
|
| 156 |
+
@cached_property
|
| 157 |
+
def target_dim(self) -> int:
|
| 158 |
+
target = self._raw_entries[0]["target"]
|
| 159 |
+
return target.shape[0] if getattr(target, "ndim", 1) > 1 else 1
|
| 160 |
+
|
| 161 |
+
@cached_property
|
| 162 |
+
def windows(self) -> int:
|
| 163 |
+
w = math.ceil(TEST_SPLIT * self._min_series_length / self.prediction_length)
|
| 164 |
+
return min(max(1, w), MAX_WINDOW)
|
| 165 |
+
|
| 166 |
+
@cached_property
|
| 167 |
+
def _min_series_length(self) -> int:
|
| 168 |
+
lengths = []
|
| 169 |
+
for entry in self._raw_entries:
|
| 170 |
+
target = entry["target"]
|
| 171 |
+
lengths.append(target.shape[-1] if getattr(target, "ndim", 1) > 1 else len(target))
|
| 172 |
+
return min(lengths)
|
| 173 |
+
|
| 174 |
+
@property
|
| 175 |
+
def training_dataset(self) -> TrainingDataset:
|
| 176 |
+
training_dataset, _ = split(
|
| 177 |
+
self.gluonts_dataset, offset=-self.prediction_length * (self.windows + 1)
|
| 178 |
+
)
|
| 179 |
+
return training_dataset
|
| 180 |
+
|
| 181 |
+
@property
|
| 182 |
+
def validation_dataset(self) -> TrainingDataset:
|
| 183 |
+
validation_dataset, _ = split(
|
| 184 |
+
self.gluonts_dataset, offset=-self.prediction_length * self.windows
|
| 185 |
+
)
|
| 186 |
+
return validation_dataset
|
| 187 |
+
|
| 188 |
+
@property
|
| 189 |
+
def test_data(self) -> TestData:
|
| 190 |
+
_, test_template = split(
|
| 191 |
+
self.gluonts_dataset, offset=-self.prediction_length * self.windows
|
| 192 |
+
)
|
| 193 |
+
return test_template.generate_instances(
|
| 194 |
+
prediction_length=self.prediction_length,
|
| 195 |
+
windows=self.windows,
|
| 196 |
+
distance=self.prediction_length,
|
| 197 |
+
)
|
tsfm_bench/data/interface.py
ADDED
|
@@ -0,0 +1,46 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
from dataclasses import dataclass
|
| 4 |
+
from typing import Iterator, Protocol, runtime_checkable
|
| 5 |
+
|
| 6 |
+
import numpy as np
|
| 7 |
+
|
| 8 |
+
|
| 9 |
+
@dataclass(frozen=True)
|
| 10 |
+
class DatasetMetadata:
|
| 11 |
+
"""Static metadata for a benchmark dataset configuration."""
|
| 12 |
+
|
| 13 |
+
name: str
|
| 14 |
+
domain: str
|
| 15 |
+
frequency: str
|
| 16 |
+
num_variates: int
|
| 17 |
+
terms: tuple[str, ...] = ("short",)
|
| 18 |
+
source_type: str = "synthetic"
|
| 19 |
+
|
| 20 |
+
|
| 21 |
+
@dataclass(frozen=True)
|
| 22 |
+
class TimeSeriesRecord:
|
| 23 |
+
"""One multivariate or univariate time series instance."""
|
| 24 |
+
|
| 25 |
+
item_id: str
|
| 26 |
+
target: np.ndarray
|
| 27 |
+
start: str
|
| 28 |
+
freq: str
|
| 29 |
+
|
| 30 |
+
|
| 31 |
+
@runtime_checkable
|
| 32 |
+
class TimeSeriesDataSource(Protocol):
|
| 33 |
+
"""Streaming data source for benchmark datasets.
|
| 34 |
+
|
| 35 |
+
Real-world backends (HF datasets, databases, APIs) should implement this
|
| 36 |
+
protocol and yield ``TimeSeriesRecord`` instances lazily via ``stream()``.
|
| 37 |
+
"""
|
| 38 |
+
|
| 39 |
+
def list_datasets(self) -> list[str]:
|
| 40 |
+
...
|
| 41 |
+
|
| 42 |
+
def get_metadata(self, name: str) -> DatasetMetadata:
|
| 43 |
+
...
|
| 44 |
+
|
| 45 |
+
def stream(self, name: str) -> Iterator[TimeSeriesRecord]:
|
| 46 |
+
...
|
tsfm_bench/data/open_meteo.py
ADDED
|
@@ -0,0 +1,194 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
import logging
|
| 4 |
+
from dataclasses import dataclass
|
| 5 |
+
from datetime import datetime, timedelta, timezone
|
| 6 |
+
from typing import Any, Iterator
|
| 7 |
+
from zoneinfo import ZoneInfo
|
| 8 |
+
|
| 9 |
+
import httpx
|
| 10 |
+
import numpy as np
|
| 11 |
+
import pandas as pd
|
| 12 |
+
|
| 13 |
+
from tsfm_bench.data.interface import DatasetMetadata, TimeSeriesRecord
|
| 14 |
+
|
| 15 |
+
logger = logging.getLogger(__name__)
|
| 16 |
+
|
| 17 |
+
FORECAST_API = "https://api.open-meteo.com/v1/forecast"
|
| 18 |
+
ARCHIVE_API = "https://archive-api.open-meteo.com/v1/archive"
|
| 19 |
+
|
| 20 |
+
|
| 21 |
+
@dataclass(frozen=True)
|
| 22 |
+
class _LocationSpec:
|
| 23 |
+
name: str
|
| 24 |
+
domain: str
|
| 25 |
+
latitude: float
|
| 26 |
+
longitude: float
|
| 27 |
+
variables: tuple[str, ...]
|
| 28 |
+
frequency: str
|
| 29 |
+
terms: tuple[str, ...]
|
| 30 |
+
timezone: str = "UTC"
|
| 31 |
+
city: str = ""
|
| 32 |
+
|
| 33 |
+
|
| 34 |
+
@dataclass(frozen=True)
|
| 35 |
+
class _OpenMeteoSettings:
|
| 36 |
+
past_days: int = 14
|
| 37 |
+
archive_days: int = 30
|
| 38 |
+
min_context_hours: int = 168
|
| 39 |
+
prediction_length: int = 24
|
| 40 |
+
request_timeout: float = 30.0
|
| 41 |
+
|
| 42 |
+
|
| 43 |
+
class OpenMeteoDataSource:
|
| 44 |
+
"""Live weather time series from Open-Meteo (https://open-meteo.com/)."""
|
| 45 |
+
|
| 46 |
+
def __init__(
|
| 47 |
+
self,
|
| 48 |
+
locations: tuple[_LocationSpec, ...],
|
| 49 |
+
settings: _OpenMeteoSettings | None = None,
|
| 50 |
+
):
|
| 51 |
+
self._locations = {spec.name: spec for spec in locations}
|
| 52 |
+
self._settings = settings or _OpenMeteoSettings()
|
| 53 |
+
|
| 54 |
+
@classmethod
|
| 55 |
+
def from_config(cls, payload: dict[str, Any]) -> OpenMeteoDataSource:
|
| 56 |
+
settings_raw = payload.get("settings", {})
|
| 57 |
+
settings = _OpenMeteoSettings(
|
| 58 |
+
past_days=int(settings_raw.get("past_days", 14)),
|
| 59 |
+
archive_days=int(settings_raw.get("archive_days", 30)),
|
| 60 |
+
min_context_hours=int(settings_raw.get("min_context_hours", 168)),
|
| 61 |
+
prediction_length=int(settings_raw.get("prediction_length", 24)),
|
| 62 |
+
request_timeout=float(settings_raw.get("request_timeout", 30.0)),
|
| 63 |
+
)
|
| 64 |
+
specs = tuple(_parse_location(entry) for entry in payload.get("locations", []))
|
| 65 |
+
if not specs:
|
| 66 |
+
raise ValueError("open_meteo config requires at least one location")
|
| 67 |
+
return cls(locations=specs, settings=settings)
|
| 68 |
+
|
| 69 |
+
def list_datasets(self) -> list[str]:
|
| 70 |
+
return sorted(self._locations.keys())
|
| 71 |
+
|
| 72 |
+
def get_metadata(self, name: str) -> DatasetMetadata:
|
| 73 |
+
spec = self._require_spec(name)
|
| 74 |
+
return DatasetMetadata(
|
| 75 |
+
name=spec.name,
|
| 76 |
+
domain=spec.domain,
|
| 77 |
+
frequency=spec.frequency,
|
| 78 |
+
num_variates=len(spec.variables),
|
| 79 |
+
terms=spec.terms,
|
| 80 |
+
source_type="open_meteo",
|
| 81 |
+
)
|
| 82 |
+
|
| 83 |
+
def stream(self, name: str) -> Iterator[TimeSeriesRecord]:
|
| 84 |
+
spec = self._require_spec(name)
|
| 85 |
+
frame = self.fetch_location_frame(spec)
|
| 86 |
+
if frame.empty:
|
| 87 |
+
raise RuntimeError(f"No Open-Meteo data returned for {name}")
|
| 88 |
+
|
| 89 |
+
values = frame[list(spec.variables)].to_numpy(dtype=np.float32).T
|
| 90 |
+
mask = ~np.isnan(values).any(axis=0)
|
| 91 |
+
values = values[:, mask]
|
| 92 |
+
if values.shape[-1] < self._settings.min_context_hours + self._settings.prediction_length:
|
| 93 |
+
raise RuntimeError(
|
| 94 |
+
f"Insufficient history for {name}: got {values.shape[-1]} hours, "
|
| 95 |
+
f"need at least {self._settings.min_context_hours + self._settings.prediction_length}"
|
| 96 |
+
)
|
| 97 |
+
|
| 98 |
+
start_ts = frame.index[mask][0]
|
| 99 |
+
freq = pd.tseries.frequencies.to_offset(spec.frequency)
|
| 100 |
+
yield TimeSeriesRecord(
|
| 101 |
+
item_id=spec.name.replace("/", "_"),
|
| 102 |
+
target=values,
|
| 103 |
+
start=str(pd.Period(start_ts, freq=freq)),
|
| 104 |
+
freq=spec.frequency,
|
| 105 |
+
)
|
| 106 |
+
|
| 107 |
+
def fetch_location_frame(self, spec: _LocationSpec) -> pd.DataFrame:
|
| 108 |
+
"""Fetch merged archive + recent forecast observations for one location."""
|
| 109 |
+
now = datetime.now(timezone.utc)
|
| 110 |
+
archive_start = (now - timedelta(days=self._settings.archive_days)).date()
|
| 111 |
+
archive_end = (now - timedelta(days=1)).date()
|
| 112 |
+
|
| 113 |
+
frames: list[pd.DataFrame] = []
|
| 114 |
+
if archive_start <= archive_end:
|
| 115 |
+
frames.append(
|
| 116 |
+
self._fetch_hourly(
|
| 117 |
+
ARCHIVE_API,
|
| 118 |
+
spec,
|
| 119 |
+
start_date=str(archive_start),
|
| 120 |
+
end_date=str(archive_end),
|
| 121 |
+
)
|
| 122 |
+
)
|
| 123 |
+
|
| 124 |
+
frames.append(
|
| 125 |
+
self._fetch_hourly(
|
| 126 |
+
FORECAST_API,
|
| 127 |
+
spec,
|
| 128 |
+
past_days=self._settings.past_days,
|
| 129 |
+
forecast_days=1,
|
| 130 |
+
)
|
| 131 |
+
)
|
| 132 |
+
|
| 133 |
+
merged = pd.concat(frames).sort_index()
|
| 134 |
+
merged = merged[~merged.index.duplicated(keep="last")]
|
| 135 |
+
merged = merged.dropna(how="all")
|
| 136 |
+
return merged
|
| 137 |
+
|
| 138 |
+
def fetched_at(self) -> str:
|
| 139 |
+
return datetime.now(timezone.utc).isoformat()
|
| 140 |
+
|
| 141 |
+
def _fetch_hourly(
|
| 142 |
+
self,
|
| 143 |
+
base_url: str,
|
| 144 |
+
spec: _LocationSpec,
|
| 145 |
+
**extra_params: Any,
|
| 146 |
+
) -> pd.DataFrame:
|
| 147 |
+
params: dict[str, Any] = {
|
| 148 |
+
"latitude": spec.latitude,
|
| 149 |
+
"longitude": spec.longitude,
|
| 150 |
+
"hourly": ",".join(spec.variables),
|
| 151 |
+
"timezone": spec.timezone,
|
| 152 |
+
**extra_params,
|
| 153 |
+
}
|
| 154 |
+
with httpx.Client(timeout=self._settings.request_timeout, trust_env=False) as client:
|
| 155 |
+
response = client.get(base_url, params=params)
|
| 156 |
+
response.raise_for_status()
|
| 157 |
+
payload = response.json()
|
| 158 |
+
|
| 159 |
+
hourly = payload.get("hourly", {})
|
| 160 |
+
times = hourly.get("time", [])
|
| 161 |
+
if not times:
|
| 162 |
+
return pd.DataFrame()
|
| 163 |
+
|
| 164 |
+
tz = ZoneInfo(spec.timezone)
|
| 165 |
+
index = pd.DatetimeIndex(
|
| 166 |
+
[datetime.fromisoformat(ts).replace(tzinfo=tz) for ts in times]
|
| 167 |
+
)
|
| 168 |
+
data = {var: hourly.get(var, []) for var in spec.variables}
|
| 169 |
+
frame = pd.DataFrame(data, index=index)
|
| 170 |
+
for col in spec.variables:
|
| 171 |
+
frame[col] = pd.to_numeric(frame[col], errors="coerce")
|
| 172 |
+
return frame
|
| 173 |
+
|
| 174 |
+
def _require_spec(self, name: str) -> _LocationSpec:
|
| 175 |
+
if name not in self._locations:
|
| 176 |
+
raise KeyError(f"Unknown Open-Meteo dataset: {name}")
|
| 177 |
+
return self._locations[name]
|
| 178 |
+
|
| 179 |
+
|
| 180 |
+
def _parse_location(entry: dict[str, Any]) -> _LocationSpec:
|
| 181 |
+
variables = entry.get("variables", ["temperature_2m"])
|
| 182 |
+
if isinstance(variables, str):
|
| 183 |
+
variables = [v.strip() for v in variables.split(",")]
|
| 184 |
+
return _LocationSpec(
|
| 185 |
+
name=entry["name"],
|
| 186 |
+
domain=entry.get("domain", "Nature"),
|
| 187 |
+
latitude=float(entry["latitude"]),
|
| 188 |
+
longitude=float(entry["longitude"]),
|
| 189 |
+
variables=tuple(variables),
|
| 190 |
+
frequency=entry.get("frequency", "H"),
|
| 191 |
+
terms=tuple(entry.get("terms", ["short"])),
|
| 192 |
+
timezone=entry.get("timezone", "UTC"),
|
| 193 |
+
city=entry.get("city", entry["name"].split("/")[0]),
|
| 194 |
+
)
|
tsfm_bench/data/registry.py
ADDED
|
@@ -0,0 +1,76 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
import json
|
| 4 |
+
from pathlib import Path
|
| 5 |
+
from typing import Any
|
| 6 |
+
|
| 7 |
+
import yaml
|
| 8 |
+
|
| 9 |
+
from tsfm_bench.data.interface import TimeSeriesDataSource
|
| 10 |
+
from tsfm_bench.data.open_meteo import OpenMeteoDataSource
|
| 11 |
+
from tsfm_bench.data.synthetic import DEFAULT_SYNTHETIC_SPECS, SyntheticDataSource, _SyntheticSpec
|
| 12 |
+
from tsfm_bench.data.ts_bench import TsBenchDataSource
|
| 13 |
+
|
| 14 |
+
|
| 15 |
+
def load_data_source(config_path: str | Path | None = None) -> TimeSeriesDataSource:
|
| 16 |
+
"""Load a data source from config. Defaults to built-in synthetic demo."""
|
| 17 |
+
if config_path is None:
|
| 18 |
+
return SyntheticDataSource()
|
| 19 |
+
|
| 20 |
+
path = Path(config_path)
|
| 21 |
+
if not path.exists():
|
| 22 |
+
raise FileNotFoundError(f"Dataset config not found: {path}")
|
| 23 |
+
|
| 24 |
+
if path.suffix in {".yaml", ".yml"}:
|
| 25 |
+
payload = yaml.safe_load(path.read_text())
|
| 26 |
+
elif path.suffix == ".json":
|
| 27 |
+
payload = json.loads(path.read_text())
|
| 28 |
+
else:
|
| 29 |
+
raise ValueError(f"Unsupported config format: {path.suffix}")
|
| 30 |
+
|
| 31 |
+
source_type = payload.get("source_type", "synthetic")
|
| 32 |
+
if source_type == "synthetic":
|
| 33 |
+
specs = tuple(_parse_spec(entry) for entry in payload.get("datasets", []))
|
| 34 |
+
return SyntheticDataSource(specs=specs or DEFAULT_SYNTHETIC_SPECS)
|
| 35 |
+
if source_type == "open_meteo":
|
| 36 |
+
return OpenMeteoDataSource.from_config(payload)
|
| 37 |
+
if source_type == "ts_bench":
|
| 38 |
+
return TsBenchDataSource.from_config(payload)
|
| 39 |
+
|
| 40 |
+
raise NotImplementedError(
|
| 41 |
+
f"Source type '{source_type}' is not implemented yet. "
|
| 42 |
+
"Implement TimeSeriesDataSource and register it in registry.py."
|
| 43 |
+
)
|
| 44 |
+
|
| 45 |
+
|
| 46 |
+
def load_dataset_properties(config_path: str | Path | None = None) -> dict[str, dict[str, Any]]:
|
| 47 |
+
"""Build dataset_properties map keyed by dataset stem (before frequency suffix)."""
|
| 48 |
+
source = load_data_source(config_path)
|
| 49 |
+
properties: dict[str, dict[str, Any]] = {}
|
| 50 |
+
|
| 51 |
+
for name in source.list_datasets():
|
| 52 |
+
meta = source.get_metadata(name)
|
| 53 |
+
if isinstance(source, TsBenchDataSource):
|
| 54 |
+
ds_key = source.leaderboard_dataset_name(name).split("/")[0].lower()
|
| 55 |
+
else:
|
| 56 |
+
ds_key = name.split("/")[0].lower()
|
| 57 |
+
properties[ds_key] = {
|
| 58 |
+
"domain": meta.domain,
|
| 59 |
+
"frequency": meta.frequency,
|
| 60 |
+
"num_variates": meta.num_variates,
|
| 61 |
+
"terms": list(meta.terms),
|
| 62 |
+
}
|
| 63 |
+
return properties
|
| 64 |
+
|
| 65 |
+
|
| 66 |
+
def _parse_spec(entry: dict[str, Any]) -> _SyntheticSpec:
|
| 67 |
+
return _SyntheticSpec(
|
| 68 |
+
name=entry["name"],
|
| 69 |
+
domain=entry["domain"],
|
| 70 |
+
frequency=entry["frequency"],
|
| 71 |
+
num_variates=int(entry["num_variates"]),
|
| 72 |
+
terms=tuple(entry.get("terms", ["short"])),
|
| 73 |
+
n_series=int(entry.get("n_series", 30)),
|
| 74 |
+
series_length=int(entry.get("series_length", 500)),
|
| 75 |
+
seed=int(entry.get("seed", 0)),
|
| 76 |
+
)
|
tsfm_bench/data/synthetic.py
ADDED
|
@@ -0,0 +1,151 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
import hashlib
|
| 4 |
+
from dataclasses import dataclass
|
| 5 |
+
from typing import Iterator
|
| 6 |
+
|
| 7 |
+
import numpy as np
|
| 8 |
+
import pandas as pd
|
| 9 |
+
|
| 10 |
+
from tsfm_bench.data.interface import DatasetMetadata, TimeSeriesRecord
|
| 11 |
+
|
| 12 |
+
|
| 13 |
+
@dataclass
|
| 14 |
+
class _SyntheticSpec:
|
| 15 |
+
name: str
|
| 16 |
+
domain: str
|
| 17 |
+
frequency: str
|
| 18 |
+
num_variates: int
|
| 19 |
+
terms: tuple[str, ...]
|
| 20 |
+
n_series: int
|
| 21 |
+
series_length: int
|
| 22 |
+
seed: int
|
| 23 |
+
|
| 24 |
+
|
| 25 |
+
DEFAULT_SYNTHETIC_SPECS: tuple[_SyntheticSpec, ...] = (
|
| 26 |
+
_SyntheticSpec(
|
| 27 |
+
name="synthetic_energy/H",
|
| 28 |
+
domain="Energy",
|
| 29 |
+
frequency="H",
|
| 30 |
+
num_variates=4,
|
| 31 |
+
terms=("short", "medium", "long"),
|
| 32 |
+
n_series=40,
|
| 33 |
+
series_length=5000,
|
| 34 |
+
seed=11,
|
| 35 |
+
),
|
| 36 |
+
_SyntheticSpec(
|
| 37 |
+
name="synthetic_finance/D",
|
| 38 |
+
domain="Econ/Fin",
|
| 39 |
+
frequency="D",
|
| 40 |
+
num_variates=2,
|
| 41 |
+
terms=("short",),
|
| 42 |
+
n_series=35,
|
| 43 |
+
series_length=1200,
|
| 44 |
+
seed=22,
|
| 45 |
+
),
|
| 46 |
+
_SyntheticSpec(
|
| 47 |
+
name="synthetic_healthcare/D",
|
| 48 |
+
domain="Healthcare",
|
| 49 |
+
frequency="D",
|
| 50 |
+
num_variates=1,
|
| 51 |
+
terms=("short", "medium"),
|
| 52 |
+
n_series=30,
|
| 53 |
+
series_length=2000,
|
| 54 |
+
seed=33,
|
| 55 |
+
),
|
| 56 |
+
_SyntheticSpec(
|
| 57 |
+
name="synthetic_retail/W",
|
| 58 |
+
domain="Sales",
|
| 59 |
+
frequency="W",
|
| 60 |
+
num_variates=3,
|
| 61 |
+
terms=("short",),
|
| 62 |
+
n_series=25,
|
| 63 |
+
series_length=800,
|
| 64 |
+
seed=44,
|
| 65 |
+
),
|
| 66 |
+
_SyntheticSpec(
|
| 67 |
+
name="synthetic_transport/15T",
|
| 68 |
+
domain="Transport",
|
| 69 |
+
frequency="15T",
|
| 70 |
+
num_variates=5,
|
| 71 |
+
terms=("short", "medium", "long"),
|
| 72 |
+
n_series=20,
|
| 73 |
+
series_length=6000,
|
| 74 |
+
seed=55,
|
| 75 |
+
),
|
| 76 |
+
_SyntheticSpec(
|
| 77 |
+
name="synthetic_cloudops/H",
|
| 78 |
+
domain="Web/CloudOps",
|
| 79 |
+
frequency="H",
|
| 80 |
+
num_variates=2,
|
| 81 |
+
terms=("short", "medium"),
|
| 82 |
+
n_series=30,
|
| 83 |
+
series_length=4000,
|
| 84 |
+
seed=66,
|
| 85 |
+
),
|
| 86 |
+
)
|
| 87 |
+
|
| 88 |
+
|
| 89 |
+
class SyntheticDataSource:
|
| 90 |
+
"""In-memory streaming source backed by deterministic synthetic series."""
|
| 91 |
+
|
| 92 |
+
def __init__(self, specs: tuple[_SyntheticSpec, ...] | None = None):
|
| 93 |
+
self._specs = {spec.name: spec for spec in (specs or DEFAULT_SYNTHETIC_SPECS)}
|
| 94 |
+
|
| 95 |
+
def list_datasets(self) -> list[str]:
|
| 96 |
+
return sorted(self._specs.keys())
|
| 97 |
+
|
| 98 |
+
def get_metadata(self, name: str) -> DatasetMetadata:
|
| 99 |
+
spec = self._require_spec(name)
|
| 100 |
+
return DatasetMetadata(
|
| 101 |
+
name=spec.name,
|
| 102 |
+
domain=spec.domain,
|
| 103 |
+
frequency=spec.frequency,
|
| 104 |
+
num_variates=spec.num_variates,
|
| 105 |
+
terms=spec.terms,
|
| 106 |
+
source_type="synthetic",
|
| 107 |
+
)
|
| 108 |
+
|
| 109 |
+
def stream(self, name: str) -> Iterator[TimeSeriesRecord]:
|
| 110 |
+
spec = self._require_spec(name)
|
| 111 |
+
freq = pd.tseries.frequencies.to_offset(spec.frequency)
|
| 112 |
+
start = pd.Timestamp("2020-01-01")
|
| 113 |
+
|
| 114 |
+
for series_idx in range(spec.n_series):
|
| 115 |
+
seed = int(
|
| 116 |
+
hashlib.sha256(f"{spec.seed}:{series_idx}".encode()).hexdigest()[:8],
|
| 117 |
+
16,
|
| 118 |
+
)
|
| 119 |
+
rng = np.random.default_rng(seed)
|
| 120 |
+
length = spec.series_length + series_idx % 7
|
| 121 |
+
|
| 122 |
+
t = np.arange(length, dtype=np.float64)
|
| 123 |
+
seasonal_period = max(2, min(24, length // 8))
|
| 124 |
+
seasonal = np.sin(2 * np.pi * t / seasonal_period)
|
| 125 |
+
trend = 0.02 * t + rng.normal(0, 0.5)
|
| 126 |
+
noise = rng.normal(0, 0.3, size=length)
|
| 127 |
+
|
| 128 |
+
if spec.num_variates == 1:
|
| 129 |
+
target = trend + seasonal + noise
|
| 130 |
+
else:
|
| 131 |
+
target = np.stack(
|
| 132 |
+
[
|
| 133 |
+
trend
|
| 134 |
+
+ (idx + 1) * 0.1 * seasonal
|
| 135 |
+
+ rng.normal(0, 0.25, size=length)
|
| 136 |
+
for idx in range(spec.num_variates)
|
| 137 |
+
],
|
| 138 |
+
axis=0,
|
| 139 |
+
)
|
| 140 |
+
|
| 141 |
+
yield TimeSeriesRecord(
|
| 142 |
+
item_id=f"{spec.name.replace('/', '_')}_{series_idx:04d}",
|
| 143 |
+
target=target.astype(np.float32),
|
| 144 |
+
start=str(pd.Period(start, freq=freq)),
|
| 145 |
+
freq=spec.frequency,
|
| 146 |
+
)
|
| 147 |
+
|
| 148 |
+
def _require_spec(self, name: str) -> _SyntheticSpec:
|
| 149 |
+
if name not in self._specs:
|
| 150 |
+
raise KeyError(f"Unknown synthetic dataset: {name}")
|
| 151 |
+
return self._specs[name]
|
tsfm_bench/data/ts_bench.py
ADDED
|
@@ -0,0 +1,304 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
import csv
|
| 4 |
+
import json
|
| 5 |
+
import logging
|
| 6 |
+
import os
|
| 7 |
+
import subprocess
|
| 8 |
+
import sys
|
| 9 |
+
from dataclasses import dataclass
|
| 10 |
+
from datetime import datetime, timezone
|
| 11 |
+
from pathlib import Path
|
| 12 |
+
from typing import Any, Iterator
|
| 13 |
+
|
| 14 |
+
import numpy as np
|
| 15 |
+
import pandas as pd
|
| 16 |
+
|
| 17 |
+
from tsfm_bench.data.interface import DatasetMetadata, TimeSeriesRecord
|
| 18 |
+
|
| 19 |
+
logger = logging.getLogger(__name__)
|
| 20 |
+
|
| 21 |
+
DEFAULT_SOURCE_DOMAINS: dict[str, str] = {
|
| 22 |
+
"open_meteo_weather": "Weather",
|
| 23 |
+
"open_meteo_air_quality": "Air Quality",
|
| 24 |
+
"nasa_power": "Weather",
|
| 25 |
+
"usgs_water": "Hydrology",
|
| 26 |
+
"noaa_coops": "Ocean",
|
| 27 |
+
"noaa_ndbc": "Ocean",
|
| 28 |
+
"gbfs_citibike": "Transport",
|
| 29 |
+
"binance": "Finance",
|
| 30 |
+
"coingecko": "Finance",
|
| 31 |
+
"wikimedia_pageviews": "Web",
|
| 32 |
+
"nws_observations": "Weather",
|
| 33 |
+
"usgs_earthquake": "Hazards",
|
| 34 |
+
"noaa_ncei": "Weather",
|
| 35 |
+
}
|
| 36 |
+
|
| 37 |
+
FREQ_ALIASES = {
|
| 38 |
+
"1h": "1h",
|
| 39 |
+
"6min": "6min",
|
| 40 |
+
"10min": "10min",
|
| 41 |
+
"15min": "15min",
|
| 42 |
+
"1d": "1D",
|
| 43 |
+
"snapshot": "1h",
|
| 44 |
+
}
|
| 45 |
+
|
| 46 |
+
|
| 47 |
+
@dataclass(frozen=True)
|
| 48 |
+
class _TsBenchSettings:
|
| 49 |
+
ts_bench_root: Path
|
| 50 |
+
output_subdir: str = "data/sample_week"
|
| 51 |
+
collect_days: int = 7
|
| 52 |
+
collect_timeout: int = 45
|
| 53 |
+
limit_entities: int = 10
|
| 54 |
+
task_limit: int = 20
|
| 55 |
+
max_tasks_per_dataset: int = 1
|
| 56 |
+
min_future_steps: int = 1
|
| 57 |
+
skip_frequencies: tuple[str, ...] = ("snapshot",)
|
| 58 |
+
auto_refresh: bool = True
|
| 59 |
+
|
| 60 |
+
|
| 61 |
+
@dataclass(frozen=True)
|
| 62 |
+
class _TaskSpec:
|
| 63 |
+
task_id: str
|
| 64 |
+
dataset_id: str
|
| 65 |
+
source_id: str
|
| 66 |
+
domain: str
|
| 67 |
+
target_variable_id: str
|
| 68 |
+
frequency: str
|
| 69 |
+
gluonts_freq: str
|
| 70 |
+
prediction_length: int
|
| 71 |
+
task_dir: Path
|
| 72 |
+
leaderboard_name: str
|
| 73 |
+
|
| 74 |
+
|
| 75 |
+
class TsBenchDataSource:
|
| 76 |
+
"""Live TS-Bench data: refresh via collect_weekly_data.py, read exported tasks."""
|
| 77 |
+
|
| 78 |
+
def __init__(
|
| 79 |
+
self,
|
| 80 |
+
settings: _TsBenchSettings,
|
| 81 |
+
source_domains: dict[str, str],
|
| 82 |
+
tasks: tuple[_TaskSpec, ...],
|
| 83 |
+
fetched_at: str = "",
|
| 84 |
+
):
|
| 85 |
+
self._settings = settings
|
| 86 |
+
self._source_domains = source_domains
|
| 87 |
+
self._tasks = {task.task_id: task for task in tasks}
|
| 88 |
+
self._fetched_at = fetched_at
|
| 89 |
+
|
| 90 |
+
@classmethod
|
| 91 |
+
def from_config(cls, payload: dict[str, Any]) -> TsBenchDataSource:
|
| 92 |
+
raw = payload.get("settings", {})
|
| 93 |
+
root_value = os.environ.get("TS_BENCH_ROOT") or raw.get("ts_bench_root", "../TS-Bench")
|
| 94 |
+
ts_bench_root = Path(root_value).expanduser().resolve()
|
| 95 |
+
settings = _TsBenchSettings(
|
| 96 |
+
ts_bench_root=ts_bench_root,
|
| 97 |
+
output_subdir=str(raw.get("output_subdir", "data/sample_week")),
|
| 98 |
+
collect_days=int(raw.get("collect_days", 7)),
|
| 99 |
+
collect_timeout=int(raw.get("collect_timeout", 45)),
|
| 100 |
+
limit_entities=int(raw.get("limit_entities", 10)),
|
| 101 |
+
task_limit=int(raw.get("task_limit", 20)),
|
| 102 |
+
max_tasks_per_dataset=int(raw.get("max_tasks_per_dataset", 1)),
|
| 103 |
+
min_future_steps=int(raw.get("min_future_steps", 1)),
|
| 104 |
+
skip_frequencies=tuple(raw.get("skip_frequencies", ["snapshot"])),
|
| 105 |
+
auto_refresh=bool(raw.get("auto_refresh", True)),
|
| 106 |
+
)
|
| 107 |
+
source_domains = {**DEFAULT_SOURCE_DOMAINS, **payload.get("source_domains", {})}
|
| 108 |
+
source = cls(settings=settings, source_domains=source_domains, tasks=())
|
| 109 |
+
source._load_tasks()
|
| 110 |
+
return source
|
| 111 |
+
|
| 112 |
+
@property
|
| 113 |
+
def data_root(self) -> Path:
|
| 114 |
+
return self._settings.ts_bench_root / self._settings.output_subdir
|
| 115 |
+
|
| 116 |
+
def refresh(self) -> None:
|
| 117 |
+
root = self._settings.ts_bench_root
|
| 118 |
+
output_dir = self.data_root
|
| 119 |
+
collect_script = root / "collect_weekly_data.py"
|
| 120 |
+
export_script = root / "export_model_inputs.py"
|
| 121 |
+
if not collect_script.exists():
|
| 122 |
+
raise FileNotFoundError(f"TS-Bench collect script not found: {collect_script}")
|
| 123 |
+
|
| 124 |
+
logger.info("Collecting TS-Bench data → %s", output_dir)
|
| 125 |
+
collect_cmd = [
|
| 126 |
+
sys.executable,
|
| 127 |
+
str(collect_script),
|
| 128 |
+
"--days",
|
| 129 |
+
str(self._settings.collect_days),
|
| 130 |
+
"--output",
|
| 131 |
+
self._settings.output_subdir,
|
| 132 |
+
"--timeout",
|
| 133 |
+
str(self._settings.collect_timeout),
|
| 134 |
+
"--limit-entities",
|
| 135 |
+
str(self._settings.limit_entities),
|
| 136 |
+
"--max-tasks-per-dataset",
|
| 137 |
+
str(self._settings.max_tasks_per_dataset),
|
| 138 |
+
]
|
| 139 |
+
subprocess.run(collect_cmd, cwd=root, check=True)
|
| 140 |
+
|
| 141 |
+
normalized = self._settings.output_subdir + "/normalized"
|
| 142 |
+
model_inputs = self._settings.output_subdir + "/model_inputs_latest"
|
| 143 |
+
if export_script.exists():
|
| 144 |
+
logger.info("Exporting TS-Bench model inputs → %s", self.data_root / "model_inputs_latest")
|
| 145 |
+
export_cmd = [
|
| 146 |
+
sys.executable,
|
| 147 |
+
str(export_script),
|
| 148 |
+
"--input",
|
| 149 |
+
normalized,
|
| 150 |
+
"--output",
|
| 151 |
+
model_inputs,
|
| 152 |
+
"--limit",
|
| 153 |
+
str(self._settings.task_limit),
|
| 154 |
+
]
|
| 155 |
+
subprocess.run(export_cmd, cwd=root, check=True)
|
| 156 |
+
|
| 157 |
+
self._load_tasks()
|
| 158 |
+
|
| 159 |
+
def _load_tasks(self) -> None:
|
| 160 |
+
summary_path = self.data_root / "summary.json"
|
| 161 |
+
if summary_path.exists():
|
| 162 |
+
summary = json.loads(summary_path.read_text())
|
| 163 |
+
self._fetched_at = summary.get("generated_at", "")
|
| 164 |
+
else:
|
| 165 |
+
self._fetched_at = datetime.now(timezone.utc).isoformat()
|
| 166 |
+
|
| 167 |
+
source_by_dataset = self._load_dataset_sources()
|
| 168 |
+
tasks = self._load_task_specs(source_by_dataset)
|
| 169 |
+
self._tasks = {task.task_id: task for task in tasks}
|
| 170 |
+
logger.info("Loaded %d TS-Bench tasks from %s", len(tasks), self.data_root)
|
| 171 |
+
|
| 172 |
+
def _load_dataset_sources(self) -> dict[str, str]:
|
| 173 |
+
entities_path = self.data_root / "normalized" / "entities.csv"
|
| 174 |
+
mapping: dict[str, str] = {}
|
| 175 |
+
if not entities_path.exists():
|
| 176 |
+
return mapping
|
| 177 |
+
with entities_path.open(newline="", encoding="utf-8") as handle:
|
| 178 |
+
for row in csv.DictReader(handle):
|
| 179 |
+
mapping.setdefault(row["dataset_id"], row["source_id"])
|
| 180 |
+
return mapping
|
| 181 |
+
|
| 182 |
+
def _load_task_specs(self, source_by_dataset: dict[str, str]) -> tuple[_TaskSpec, ...]:
|
| 183 |
+
manifest_path = self.data_root / "model_inputs_latest" / "manifest.json"
|
| 184 |
+
if not manifest_path.exists():
|
| 185 |
+
manifest_path = self.data_root / "model_inputs_latest" / "manifest.csv"
|
| 186 |
+
if not manifest_path.exists():
|
| 187 |
+
logger.warning(
|
| 188 |
+
"No TS-Bench manifest under %s — call refresh() to collect data",
|
| 189 |
+
manifest_path.parent,
|
| 190 |
+
)
|
| 191 |
+
return tuple()
|
| 192 |
+
|
| 193 |
+
entries: list[dict[str, Any]]
|
| 194 |
+
if manifest_path.suffix == ".json":
|
| 195 |
+
entries = json.loads(manifest_path.read_text())
|
| 196 |
+
else:
|
| 197 |
+
with manifest_path.open(newline="", encoding="utf-8") as handle:
|
| 198 |
+
entries = list(csv.DictReader(handle))
|
| 199 |
+
|
| 200 |
+
specs: list[_TaskSpec] = []
|
| 201 |
+
for entry in entries:
|
| 202 |
+
future_rows = int(entry.get("future_target_rows", 0))
|
| 203 |
+
if future_rows < self._settings.min_future_steps:
|
| 204 |
+
continue
|
| 205 |
+
|
| 206 |
+
task_id = entry["task_id"]
|
| 207 |
+
task_dir = self.data_root / "model_inputs_latest" / task_id
|
| 208 |
+
task_json = task_dir / "task.json"
|
| 209 |
+
if not task_json.exists():
|
| 210 |
+
continue
|
| 211 |
+
|
| 212 |
+
task = json.loads(task_json.read_text())
|
| 213 |
+
raw_freq = str(task.get("frequency", "1h"))
|
| 214 |
+
if raw_freq in self._settings.skip_frequencies:
|
| 215 |
+
continue
|
| 216 |
+
|
| 217 |
+
dataset_id = task["dataset_id"]
|
| 218 |
+
source_id = source_by_dataset.get(dataset_id, dataset_id.split("_")[0])
|
| 219 |
+
domain = self._source_domains.get(source_id, "Other")
|
| 220 |
+
gluonts_freq = FREQ_ALIASES.get(raw_freq, raw_freq)
|
| 221 |
+
pred_len = int(task.get("horizon_length", future_rows))
|
| 222 |
+
variable = task["target_variable_id"]
|
| 223 |
+
leaderboard_name = f"{dataset_id}_{variable}"
|
| 224 |
+
|
| 225 |
+
specs.append(
|
| 226 |
+
_TaskSpec(
|
| 227 |
+
task_id=task_id,
|
| 228 |
+
dataset_id=dataset_id,
|
| 229 |
+
source_id=source_id,
|
| 230 |
+
domain=domain,
|
| 231 |
+
target_variable_id=variable,
|
| 232 |
+
frequency=raw_freq,
|
| 233 |
+
gluonts_freq=gluonts_freq,
|
| 234 |
+
prediction_length=pred_len,
|
| 235 |
+
task_dir=task_dir,
|
| 236 |
+
leaderboard_name=leaderboard_name,
|
| 237 |
+
)
|
| 238 |
+
)
|
| 239 |
+
return tuple(specs)
|
| 240 |
+
|
| 241 |
+
def list_datasets(self) -> list[str]:
|
| 242 |
+
return sorted(self._tasks.keys())
|
| 243 |
+
|
| 244 |
+
def get_metadata(self, name: str) -> DatasetMetadata:
|
| 245 |
+
task = self._require_task(name)
|
| 246 |
+
return DatasetMetadata(
|
| 247 |
+
name=name,
|
| 248 |
+
domain=task.domain,
|
| 249 |
+
frequency=task.gluonts_freq,
|
| 250 |
+
num_variates=1,
|
| 251 |
+
terms=("short",),
|
| 252 |
+
source_type="ts_bench",
|
| 253 |
+
)
|
| 254 |
+
|
| 255 |
+
def get_prediction_length(self, name: str) -> int:
|
| 256 |
+
return self._require_task(name).prediction_length
|
| 257 |
+
|
| 258 |
+
def stream(self, name: str) -> Iterator[TimeSeriesRecord]:
|
| 259 |
+
task = self._require_task(name)
|
| 260 |
+
context_path = task.task_dir / "context.csv"
|
| 261 |
+
future_path = task.task_dir / "future_target.csv"
|
| 262 |
+
if not context_path.exists() or not future_path.exists():
|
| 263 |
+
raise FileNotFoundError(f"Missing task CSVs for {name}")
|
| 264 |
+
|
| 265 |
+
context = pd.read_csv(context_path)
|
| 266 |
+
future = pd.read_csv(future_path)
|
| 267 |
+
target_col = task.target_variable_id
|
| 268 |
+
if target_col not in context.columns:
|
| 269 |
+
raise RuntimeError(f"Target column {target_col} missing in {context_path}")
|
| 270 |
+
|
| 271 |
+
context_vals = pd.to_numeric(context[target_col], errors="coerce").to_numpy(dtype=np.float32)
|
| 272 |
+
future_vals = pd.to_numeric(future[target_col], errors="coerce").to_numpy(dtype=np.float32)
|
| 273 |
+
values = np.concatenate([context_vals, future_vals])
|
| 274 |
+
mask = ~np.isnan(values)
|
| 275 |
+
values = values[mask]
|
| 276 |
+
if len(values) < task.prediction_length + 2:
|
| 277 |
+
raise RuntimeError(
|
| 278 |
+
f"Insufficient data for {name}: {len(values)} points, "
|
| 279 |
+
f"need > {task.prediction_length + 2}"
|
| 280 |
+
)
|
| 281 |
+
|
| 282 |
+
timestamps = list(context["timestamp"]) + list(future["timestamp"])
|
| 283 |
+
timestamps = [timestamps[i] for i, keep in enumerate(mask) if keep]
|
| 284 |
+
start_ts = pd.Timestamp(timestamps[0])
|
| 285 |
+
freq = pd.tseries.frequencies.to_offset(task.gluonts_freq)
|
| 286 |
+
|
| 287 |
+
yield TimeSeriesRecord(
|
| 288 |
+
item_id=task.leaderboard_name.replace("/", "_"),
|
| 289 |
+
target=values,
|
| 290 |
+
start=str(pd.Period(start_ts, freq=freq)),
|
| 291 |
+
freq=task.gluonts_freq,
|
| 292 |
+
)
|
| 293 |
+
|
| 294 |
+
def leaderboard_dataset_name(self, name: str) -> str:
|
| 295 |
+
task = self._require_task(name)
|
| 296 |
+
return f"{task.leaderboard_name}/{task.gluonts_freq}/short"
|
| 297 |
+
|
| 298 |
+
def fetched_at(self) -> str:
|
| 299 |
+
return self._fetched_at or datetime.now(timezone.utc).isoformat()
|
| 300 |
+
|
| 301 |
+
def _require_task(self, name: str) -> _TaskSpec:
|
| 302 |
+
if name not in self._tasks:
|
| 303 |
+
raise KeyError(f"Unknown TS-Bench task: {name}")
|
| 304 |
+
return self._tasks[name]
|
tsfm_bench/eval/__init__.py
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
|
tsfm_bench/eval/api_predictor.py
ADDED
|
@@ -0,0 +1,166 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
import os
|
| 4 |
+
from dataclasses import dataclass
|
| 5 |
+
from typing import Any, Iterator, List, Optional
|
| 6 |
+
|
| 7 |
+
import httpx
|
| 8 |
+
import numpy as np
|
| 9 |
+
from gluonts.dataset import Dataset as GluonDataset
|
| 10 |
+
from gluonts.model import Forecast
|
| 11 |
+
from gluonts.model.forecast import QuantileForecast
|
| 12 |
+
from gluonts.model.predictor import RepresentablePredictor
|
| 13 |
+
|
| 14 |
+
from tsfm_bench.eval.predictors import _ForecastConfig
|
| 15 |
+
|
| 16 |
+
DEFAULT_API_URL = "https://api.tsfm.ai/v1/forecast"
|
| 17 |
+
DEFAULT_QUANTILES = [0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9]
|
| 18 |
+
|
| 19 |
+
|
| 20 |
+
@dataclass
|
| 21 |
+
class TsfmApiConfig:
|
| 22 |
+
model_id: str
|
| 23 |
+
api_url: str = DEFAULT_API_URL
|
| 24 |
+
api_key: str | None = None
|
| 25 |
+
timeout: float = 120.0
|
| 26 |
+
freq: str = "H"
|
| 27 |
+
|
| 28 |
+
|
| 29 |
+
class TsfmApiPredictor(RepresentablePredictor):
|
| 30 |
+
"""Zero-shot forecaster backed by the TSFM.ai hosted API."""
|
| 31 |
+
|
| 32 |
+
def __init__(
|
| 33 |
+
self,
|
| 34 |
+
config: TsfmApiConfig,
|
| 35 |
+
prediction_length: int,
|
| 36 |
+
quantile_levels: Optional[List[float]] = None,
|
| 37 |
+
):
|
| 38 |
+
super().__init__(prediction_length=prediction_length)
|
| 39 |
+
self.config = config
|
| 40 |
+
self.quantile_levels = quantile_levels or DEFAULT_QUANTILES
|
| 41 |
+
self.forecast_config = _ForecastConfig.from_quantiles(self.quantile_levels)
|
| 42 |
+
self._api_key = config.api_key or os.getenv("TSFM_API_KEY")
|
| 43 |
+
self.leaderboard_name = config.model_id.split("/")[-1]
|
| 44 |
+
|
| 45 |
+
def predict(self, dataset: GluonDataset, **kwargs) -> Iterator[Forecast]:
|
| 46 |
+
if not self._api_key:
|
| 47 |
+
raise RuntimeError(
|
| 48 |
+
"TSFM_API_KEY is required for online zero-shot evaluation. "
|
| 49 |
+
"Get a key at https://tsfm.ai/"
|
| 50 |
+
)
|
| 51 |
+
|
| 52 |
+
headers = {
|
| 53 |
+
"Authorization": f"Bearer {self._api_key}",
|
| 54 |
+
"Content-Type": "application/json",
|
| 55 |
+
}
|
| 56 |
+
|
| 57 |
+
with httpx.Client(timeout=self.config.timeout, trust_env=False) as client:
|
| 58 |
+
for entry in dataset:
|
| 59 |
+
target = np.asarray(entry["target"], dtype=np.float64)
|
| 60 |
+
univariate = target.ndim == 1
|
| 61 |
+
working = target.reshape(1, -1) if univariate else target
|
| 62 |
+
|
| 63 |
+
forecast_rows = []
|
| 64 |
+
for variate_idx in range(working.shape[0]):
|
| 65 |
+
series = working[variate_idx]
|
| 66 |
+
series = series[np.isfinite(series)]
|
| 67 |
+
payload = {
|
| 68 |
+
"model": self.config.model_id,
|
| 69 |
+
"inputs": [
|
| 70 |
+
{
|
| 71 |
+
"item_id": str(entry.get("item_id", variate_idx)),
|
| 72 |
+
"target": _to_api_target(series),
|
| 73 |
+
"start": _format_start(entry.get("start")),
|
| 74 |
+
}
|
| 75 |
+
],
|
| 76 |
+
"parameters": {
|
| 77 |
+
"prediction_length": self.prediction_length,
|
| 78 |
+
"freq": self.config.freq,
|
| 79 |
+
"quantiles": self.quantile_levels,
|
| 80 |
+
},
|
| 81 |
+
}
|
| 82 |
+
response = client.post(
|
| 83 |
+
self.config.api_url,
|
| 84 |
+
headers=headers,
|
| 85 |
+
json=payload,
|
| 86 |
+
)
|
| 87 |
+
response.raise_for_status()
|
| 88 |
+
forecast_rows.append(
|
| 89 |
+
_parse_api_forecast(
|
| 90 |
+
response.json(),
|
| 91 |
+
self.prediction_length,
|
| 92 |
+
self.quantile_levels,
|
| 93 |
+
)
|
| 94 |
+
)
|
| 95 |
+
|
| 96 |
+
stacked = np.stack(forecast_rows, axis=0)
|
| 97 |
+
if univariate:
|
| 98 |
+
forecast_arrays = stacked[0]
|
| 99 |
+
else:
|
| 100 |
+
forecast_arrays = stacked
|
| 101 |
+
|
| 102 |
+
yield QuantileForecast(
|
| 103 |
+
forecast_arrays=forecast_arrays,
|
| 104 |
+
forecast_keys=self.forecast_config.forecast_keys,
|
| 105 |
+
start_date=entry["start"] + len(entry["target"]),
|
| 106 |
+
item_id=entry["item_id"],
|
| 107 |
+
)
|
| 108 |
+
|
| 109 |
+
|
| 110 |
+
def _to_api_target(series: np.ndarray) -> list[list[float]]:
|
| 111 |
+
"""TSFM.ai expects target shaped [num_timesteps][num_channels]."""
|
| 112 |
+
return [[float(v)] for v in series.tolist()]
|
| 113 |
+
|
| 114 |
+
|
| 115 |
+
def _format_start(start: Any) -> str:
|
| 116 |
+
if hasattr(start, "to_timestamp"):
|
| 117 |
+
return start.to_timestamp().isoformat()
|
| 118 |
+
return str(start)
|
| 119 |
+
|
| 120 |
+
|
| 121 |
+
def _parse_api_forecast(
|
| 122 |
+
payload: dict[str, Any],
|
| 123 |
+
prediction_length: int,
|
| 124 |
+
quantile_levels: list[float],
|
| 125 |
+
) -> np.ndarray:
|
| 126 |
+
"""Parse TSFM.ai response into (num_outputs, prediction_length)."""
|
| 127 |
+
outputs_block = payload.get("outputs") or payload.get("forecasts") or []
|
| 128 |
+
if not outputs_block:
|
| 129 |
+
raise ValueError(f"Unrecognized TSFM API response: {payload}")
|
| 130 |
+
|
| 131 |
+
first = outputs_block[0]
|
| 132 |
+
rows: list[np.ndarray] = [ _flatten_forecast_values(first.get("mean"), prediction_length)]
|
| 133 |
+
|
| 134 |
+
quantile_map: dict[float, np.ndarray] = {}
|
| 135 |
+
for item in first.get("quantile_predictions", []):
|
| 136 |
+
level = float(item.get("level", item.get("quantile", 0.5)))
|
| 137 |
+
quantile_map[level] = _flatten_forecast_values(item.get("values"), prediction_length)
|
| 138 |
+
|
| 139 |
+
for q in quantile_levels:
|
| 140 |
+
if q in quantile_map:
|
| 141 |
+
rows.append(quantile_map[q])
|
| 142 |
+
else:
|
| 143 |
+
nearest = min(quantile_map.keys(), key=lambda k: abs(k - q), default=None)
|
| 144 |
+
if nearest is None:
|
| 145 |
+
rows.append(rows[0].copy())
|
| 146 |
+
else:
|
| 147 |
+
rows.append(quantile_map[nearest])
|
| 148 |
+
|
| 149 |
+
return np.stack(rows, axis=0).astype(np.float64)
|
| 150 |
+
|
| 151 |
+
|
| 152 |
+
def _flatten_forecast_values(values: Any, prediction_length: int) -> np.ndarray:
|
| 153 |
+
"""Convert nested API values like [[1.2], [1.3], ...] to 1D array."""
|
| 154 |
+
if values is None:
|
| 155 |
+
raise ValueError("Missing forecast values in TSFM API response")
|
| 156 |
+
|
| 157 |
+
arr = np.asarray(values, dtype=np.float64)
|
| 158 |
+
if arr.ndim == 2 and arr.shape[1] == 1:
|
| 159 |
+
arr = arr[:, 0]
|
| 160 |
+
elif arr.ndim > 1:
|
| 161 |
+
arr = arr.reshape(arr.shape[0], -1).mean(axis=1)
|
| 162 |
+
|
| 163 |
+
arr = arr.reshape(-1)
|
| 164 |
+
if arr.size < prediction_length:
|
| 165 |
+
raise ValueError(f"Forecast length {arr.size} < expected {prediction_length}")
|
| 166 |
+
return arr[:prediction_length]
|
tsfm_bench/eval/metrics.py
ADDED
|
@@ -0,0 +1,50 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from gluonts.ev.metrics import (
|
| 2 |
+
MAE,
|
| 3 |
+
MAPE,
|
| 4 |
+
MASE,
|
| 5 |
+
MeanWeightedSumQuantileLoss,
|
| 6 |
+
MSIS,
|
| 7 |
+
MSE,
|
| 8 |
+
ND,
|
| 9 |
+
NRMSE,
|
| 10 |
+
RMSE,
|
| 11 |
+
SMAPE,
|
| 12 |
+
)
|
| 13 |
+
|
| 14 |
+
|
| 15 |
+
def get_eval_metrics():
|
| 16 |
+
"""GIFT-Eval compatible metric set."""
|
| 17 |
+
return [
|
| 18 |
+
MSE(forecast_type="mean"),
|
| 19 |
+
MSE(forecast_type=0.5),
|
| 20 |
+
MAE(),
|
| 21 |
+
MASE(),
|
| 22 |
+
MAPE(),
|
| 23 |
+
SMAPE(),
|
| 24 |
+
MSIS(),
|
| 25 |
+
RMSE(),
|
| 26 |
+
NRMSE(),
|
| 27 |
+
ND(),
|
| 28 |
+
MeanWeightedSumQuantileLoss(
|
| 29 |
+
quantile_levels=[0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9]
|
| 30 |
+
),
|
| 31 |
+
]
|
| 32 |
+
|
| 33 |
+
|
| 34 |
+
RESULT_COLUMNS = [
|
| 35 |
+
"dataset",
|
| 36 |
+
"model",
|
| 37 |
+
"eval_metrics/MSE[mean]",
|
| 38 |
+
"eval_metrics/MSE[0.5]",
|
| 39 |
+
"eval_metrics/MAE[0.5]",
|
| 40 |
+
"eval_metrics/MASE[0.5]",
|
| 41 |
+
"eval_metrics/MAPE[0.5]",
|
| 42 |
+
"eval_metrics/sMAPE[0.5]",
|
| 43 |
+
"eval_metrics/MSIS",
|
| 44 |
+
"eval_metrics/RMSE[mean]",
|
| 45 |
+
"eval_metrics/NRMSE[mean]",
|
| 46 |
+
"eval_metrics/ND[0.5]",
|
| 47 |
+
"eval_metrics/mean_weighted_sum_quantile_loss",
|
| 48 |
+
"domain",
|
| 49 |
+
"num_variates",
|
| 50 |
+
]
|
tsfm_bench/eval/online_eval.py
ADDED
|
@@ -0,0 +1,154 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
from dataclasses import dataclass
|
| 4 |
+
from typing import Any, Iterable
|
| 5 |
+
|
| 6 |
+
import numpy as np
|
| 7 |
+
import pandas as pd
|
| 8 |
+
from gluonts.dataset.common import ListDataset
|
| 9 |
+
from gluonts.dataset.split import split
|
| 10 |
+
from gluonts.model import Predictor, evaluate_model
|
| 11 |
+
from gluonts.time_feature import get_seasonality
|
| 12 |
+
|
| 13 |
+
from tsfm_bench.data.interface import TimeSeriesDataSource
|
| 14 |
+
from tsfm_bench.data.open_meteo import OpenMeteoDataSource
|
| 15 |
+
from tsfm_bench.data.ts_bench import TsBenchDataSource
|
| 16 |
+
from tsfm_bench.eval.metrics import RESULT_COLUMNS, get_eval_metrics
|
| 17 |
+
|
| 18 |
+
|
| 19 |
+
@dataclass
|
| 20 |
+
class OnlineEvalResult:
|
| 21 |
+
dataset: str
|
| 22 |
+
model: str
|
| 23 |
+
metrics: dict[str, float]
|
| 24 |
+
domain: str
|
| 25 |
+
num_variates: int
|
| 26 |
+
data_fetched_at: str
|
| 27 |
+
context_length: int
|
| 28 |
+
prediction_length: int
|
| 29 |
+
|
| 30 |
+
|
| 31 |
+
def _records_to_entries(
|
| 32 |
+
record,
|
| 33 |
+
to_univariate: bool,
|
| 34 |
+
) -> list[dict[str, Any]]:
|
| 35 |
+
target = np.asarray(record.target, dtype=np.float32)
|
| 36 |
+
start = pd.Period(record.start, freq=record.freq)
|
| 37 |
+
entries: list[dict[str, Any]] = []
|
| 38 |
+
|
| 39 |
+
if target.ndim == 1 or not to_univariate:
|
| 40 |
+
entries.append(
|
| 41 |
+
{
|
| 42 |
+
"item_id": record.item_id,
|
| 43 |
+
"start": start,
|
| 44 |
+
"target": target if target.ndim == 1 else target,
|
| 45 |
+
"freq": record.freq,
|
| 46 |
+
}
|
| 47 |
+
)
|
| 48 |
+
return entries
|
| 49 |
+
|
| 50 |
+
for idx in range(target.shape[0]):
|
| 51 |
+
entries.append(
|
| 52 |
+
{
|
| 53 |
+
"item_id": f"{record.item_id}_v{idx}",
|
| 54 |
+
"start": start,
|
| 55 |
+
"target": target[idx],
|
| 56 |
+
"freq": record.freq,
|
| 57 |
+
}
|
| 58 |
+
)
|
| 59 |
+
return entries
|
| 60 |
+
|
| 61 |
+
|
| 62 |
+
def evaluate_online_window(
|
| 63 |
+
predictor: Predictor,
|
| 64 |
+
entries: list[dict[str, Any]],
|
| 65 |
+
prediction_length: int,
|
| 66 |
+
freq: str,
|
| 67 |
+
) -> dict[str, float]:
|
| 68 |
+
dataset = ListDataset(entries, freq=freq)
|
| 69 |
+
_, test_template = split(dataset, offset=-prediction_length)
|
| 70 |
+
test_data = test_template.generate_instances(
|
| 71 |
+
prediction_length=prediction_length,
|
| 72 |
+
windows=1,
|
| 73 |
+
distance=prediction_length,
|
| 74 |
+
)
|
| 75 |
+
seasonality = get_seasonality(freq)
|
| 76 |
+
evaluated = evaluate_model(
|
| 77 |
+
predictor,
|
| 78 |
+
test_data=test_data,
|
| 79 |
+
metrics=get_eval_metrics(),
|
| 80 |
+
batch_size=64,
|
| 81 |
+
axis=None,
|
| 82 |
+
mask_invalid_label=True,
|
| 83 |
+
allow_nan_forecast=False,
|
| 84 |
+
seasonality=seasonality,
|
| 85 |
+
)
|
| 86 |
+
return {key: float(value.iloc[0]) for key, value in evaluated.items()}
|
| 87 |
+
|
| 88 |
+
|
| 89 |
+
def run_online_eval_for_dataset(
|
| 90 |
+
source: TimeSeriesDataSource,
|
| 91 |
+
ds_name: str,
|
| 92 |
+
predictor: Predictor,
|
| 93 |
+
prediction_length: int | None = None,
|
| 94 |
+
) -> OnlineEvalResult:
|
| 95 |
+
meta = source.get_metadata(ds_name)
|
| 96 |
+
record = next(iter(source.stream(ds_name)))
|
| 97 |
+
|
| 98 |
+
if isinstance(source, OpenMeteoDataSource):
|
| 99 |
+
pred_len = prediction_length or source._settings.prediction_length
|
| 100 |
+
fetched_at = source.fetched_at()
|
| 101 |
+
elif isinstance(source, TsBenchDataSource):
|
| 102 |
+
pred_len = prediction_length or source.get_prediction_length(ds_name)
|
| 103 |
+
fetched_at = source.fetched_at()
|
| 104 |
+
else:
|
| 105 |
+
pred_len = prediction_length or 24
|
| 106 |
+
fetched_at = pd.Timestamp.utcnow().isoformat()
|
| 107 |
+
|
| 108 |
+
to_univariate = meta.num_variates > 1
|
| 109 |
+
entries = _records_to_entries(record, to_univariate=to_univariate)
|
| 110 |
+
metrics = evaluate_online_window(predictor, entries, pred_len, meta.frequency)
|
| 111 |
+
|
| 112 |
+
ds_key = ds_name.split("/")[0].lower()
|
| 113 |
+
if isinstance(source, TsBenchDataSource):
|
| 114 |
+
config_name = source.leaderboard_dataset_name(ds_name)
|
| 115 |
+
else:
|
| 116 |
+
config_name = f"{ds_key}/{meta.frequency}/short"
|
| 117 |
+
model_name = getattr(predictor, "leaderboard_name", predictor.__class__.__name__)
|
| 118 |
+
context_length = int(np.asarray(entries[0]["target"]).shape[-1]) - pred_len
|
| 119 |
+
|
| 120 |
+
return OnlineEvalResult(
|
| 121 |
+
dataset=config_name,
|
| 122 |
+
model=model_name,
|
| 123 |
+
metrics=metrics,
|
| 124 |
+
domain=meta.domain,
|
| 125 |
+
num_variates=1 if to_univariate else meta.num_variates,
|
| 126 |
+
data_fetched_at=fetched_at,
|
| 127 |
+
context_length=context_length,
|
| 128 |
+
prediction_length=pred_len,
|
| 129 |
+
)
|
| 130 |
+
|
| 131 |
+
|
| 132 |
+
def result_to_row(result: OnlineEvalResult) -> list[Any]:
|
| 133 |
+
return [
|
| 134 |
+
result.dataset,
|
| 135 |
+
result.model,
|
| 136 |
+
result.metrics["MSE[mean]"],
|
| 137 |
+
result.metrics["MSE[0.5]"],
|
| 138 |
+
result.metrics["MAE[0.5]"],
|
| 139 |
+
result.metrics["MASE[0.5]"],
|
| 140 |
+
result.metrics["MAPE[0.5]"],
|
| 141 |
+
result.metrics["sMAPE[0.5]"],
|
| 142 |
+
result.metrics["MSIS"],
|
| 143 |
+
result.metrics["RMSE[mean]"],
|
| 144 |
+
result.metrics["NRMSE[mean]"],
|
| 145 |
+
result.metrics["ND[0.5]"],
|
| 146 |
+
result.metrics["mean_weighted_sum_quantile_loss"],
|
| 147 |
+
result.domain,
|
| 148 |
+
result.num_variates,
|
| 149 |
+
]
|
| 150 |
+
|
| 151 |
+
|
| 152 |
+
def results_to_dataframe(results: Iterable[OnlineEvalResult]) -> pd.DataFrame:
|
| 153 |
+
rows = [result_to_row(result) for result in results]
|
| 154 |
+
return pd.DataFrame(rows, columns=RESULT_COLUMNS)
|
tsfm_bench/eval/predictors.py
ADDED
|
@@ -0,0 +1,85 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
from dataclasses import dataclass
|
| 4 |
+
from typing import Iterator, List, Optional
|
| 5 |
+
|
| 6 |
+
import numpy as np
|
| 7 |
+
from gluonts.dataset import Dataset as GluonDataset
|
| 8 |
+
from gluonts.model import Forecast
|
| 9 |
+
from gluonts.model.forecast import QuantileForecast
|
| 10 |
+
from gluonts.model.predictor import RepresentablePredictor
|
| 11 |
+
|
| 12 |
+
|
| 13 |
+
@dataclass
|
| 14 |
+
class _ForecastConfig:
|
| 15 |
+
quantile_levels: Optional[List[float]]
|
| 16 |
+
forecast_keys: List[str]
|
| 17 |
+
intervals: Optional[List[int]]
|
| 18 |
+
|
| 19 |
+
@classmethod
|
| 20 |
+
def from_quantiles(cls, quantile_levels: Optional[List[float]]):
|
| 21 |
+
forecast_keys = ["mean"]
|
| 22 |
+
intervals = None
|
| 23 |
+
if not quantile_levels:
|
| 24 |
+
return cls(quantile_levels=None, forecast_keys=forecast_keys, intervals=intervals)
|
| 25 |
+
|
| 26 |
+
intervals = sorted(
|
| 27 |
+
{
|
| 28 |
+
round(200 * (max(q, 1 - q) - 0.5))
|
| 29 |
+
for q in quantile_levels
|
| 30 |
+
}
|
| 31 |
+
)
|
| 32 |
+
forecast_keys = ["mean"] + [str(q) for q in quantile_levels]
|
| 33 |
+
return cls(
|
| 34 |
+
quantile_levels=quantile_levels,
|
| 35 |
+
forecast_keys=forecast_keys,
|
| 36 |
+
intervals=intervals,
|
| 37 |
+
)
|
| 38 |
+
|
| 39 |
+
|
| 40 |
+
class SeasonalNaivePredictor(RepresentablePredictor):
|
| 41 |
+
"""Lightweight seasonal naive baseline for pipeline smoke tests."""
|
| 42 |
+
|
| 43 |
+
def __init__(
|
| 44 |
+
self,
|
| 45 |
+
prediction_length: int,
|
| 46 |
+
season_length: int = 1,
|
| 47 |
+
quantile_levels: Optional[List[float]] = None,
|
| 48 |
+
):
|
| 49 |
+
super().__init__(prediction_length=prediction_length)
|
| 50 |
+
self.season_length = max(1, season_length)
|
| 51 |
+
self.config = _ForecastConfig.from_quantiles(quantile_levels)
|
| 52 |
+
|
| 53 |
+
def predict(self, dataset: GluonDataset, **kwargs) -> Iterator[Forecast]:
|
| 54 |
+
for entry in dataset:
|
| 55 |
+
target = np.asarray(entry["target"], dtype=np.float64)
|
| 56 |
+
univariate = target.ndim == 1
|
| 57 |
+
if univariate:
|
| 58 |
+
working = target.reshape(1, -1)
|
| 59 |
+
else:
|
| 60 |
+
working = target
|
| 61 |
+
|
| 62 |
+
history = working[:, -self.season_length :]
|
| 63 |
+
tiled = np.tile(
|
| 64 |
+
history, (1, int(np.ceil(self.prediction_length / self.season_length)))
|
| 65 |
+
)
|
| 66 |
+
mean = tiled[:, : self.prediction_length]
|
| 67 |
+
if univariate:
|
| 68 |
+
mean = mean[0]
|
| 69 |
+
|
| 70 |
+
arrays = [mean]
|
| 71 |
+
if self.config.quantile_levels:
|
| 72 |
+
for q in self.config.quantile_levels:
|
| 73 |
+
if q <= 0.5:
|
| 74 |
+
arrays.append(mean * (1 - (0.5 - q) * 0.05))
|
| 75 |
+
else:
|
| 76 |
+
arrays.append(mean * (1 + (q - 0.5) * 0.05))
|
| 77 |
+
|
| 78 |
+
forecast_arrays = np.stack(arrays, axis=0)
|
| 79 |
+
|
| 80 |
+
yield QuantileForecast(
|
| 81 |
+
forecast_arrays=forecast_arrays,
|
| 82 |
+
forecast_keys=self.config.forecast_keys,
|
| 83 |
+
start_date=entry["start"],
|
| 84 |
+
item_id=entry["item_id"],
|
| 85 |
+
)
|
vendor/ts_bench/collect_weekly_data.py
ADDED
|
@@ -0,0 +1,1457 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python3
|
| 2 |
+
"""Collect a small one-week TS-Bench data sample and normalize it.
|
| 3 |
+
|
| 4 |
+
The script is intentionally conservative:
|
| 5 |
+
- Raw API responses are saved first and never overwritten.
|
| 6 |
+
- Normalized storage uses long-format CSV tables.
|
| 7 |
+
- Model-facing CSV/tensor inputs should be exported later by a loader from
|
| 8 |
+
observations + forecast_tasks, not used as the source of truth.
|
| 9 |
+
|
| 10 |
+
Default collectors avoid API keys where possible. Some public services may be
|
| 11 |
+
temporarily unavailable or rate-limited; failures are captured in run_log.csv
|
| 12 |
+
instead of aborting the whole run.
|
| 13 |
+
"""
|
| 14 |
+
|
| 15 |
+
from __future__ import annotations
|
| 16 |
+
|
| 17 |
+
import argparse
|
| 18 |
+
import csv
|
| 19 |
+
import hashlib
|
| 20 |
+
import io
|
| 21 |
+
import json
|
| 22 |
+
import math
|
| 23 |
+
import os
|
| 24 |
+
import re
|
| 25 |
+
import sys
|
| 26 |
+
import zipfile
|
| 27 |
+
from collections import defaultdict
|
| 28 |
+
from dataclasses import dataclass, field
|
| 29 |
+
from datetime import datetime, date, timedelta, timezone
|
| 30 |
+
from pathlib import Path
|
| 31 |
+
from typing import Any, Callable, Dict, Iterable, List, Optional, Tuple
|
| 32 |
+
from urllib.parse import quote
|
| 33 |
+
|
| 34 |
+
import requests
|
| 35 |
+
|
| 36 |
+
|
| 37 |
+
UTC = timezone.utc
|
| 38 |
+
PARSER_VERSION = "tsbench_sample_v0.1"
|
| 39 |
+
|
| 40 |
+
|
| 41 |
+
RAW_SNAPSHOT_FIELDS = [
|
| 42 |
+
"raw_id",
|
| 43 |
+
"source_id",
|
| 44 |
+
"dataset_id",
|
| 45 |
+
"request_url",
|
| 46 |
+
"request_params",
|
| 47 |
+
"http_status",
|
| 48 |
+
"fetched_at",
|
| 49 |
+
"content_type",
|
| 50 |
+
"content_hash",
|
| 51 |
+
"raw_uri",
|
| 52 |
+
"parser_version",
|
| 53 |
+
"error",
|
| 54 |
+
]
|
| 55 |
+
|
| 56 |
+
ENTITY_FIELDS = [
|
| 57 |
+
"dataset_id",
|
| 58 |
+
"source_id",
|
| 59 |
+
"entity_id",
|
| 60 |
+
"entity_type",
|
| 61 |
+
"entity_name",
|
| 62 |
+
"lat",
|
| 63 |
+
"lon",
|
| 64 |
+
"region",
|
| 65 |
+
"country",
|
| 66 |
+
"timezone",
|
| 67 |
+
"metadata_json",
|
| 68 |
+
]
|
| 69 |
+
|
| 70 |
+
VARIABLE_FIELDS = [
|
| 71 |
+
"dataset_id",
|
| 72 |
+
"source_id",
|
| 73 |
+
"variable_id",
|
| 74 |
+
"variable_name",
|
| 75 |
+
"unit",
|
| 76 |
+
"value_type",
|
| 77 |
+
"frequency_hint",
|
| 78 |
+
"aggregation",
|
| 79 |
+
"is_target_default",
|
| 80 |
+
"metadata_json",
|
| 81 |
+
]
|
| 82 |
+
|
| 83 |
+
OBSERVATION_FIELDS = [
|
| 84 |
+
"dataset_id",
|
| 85 |
+
"source_id",
|
| 86 |
+
"entity_id",
|
| 87 |
+
"variable_id",
|
| 88 |
+
"timestamp",
|
| 89 |
+
"available_time",
|
| 90 |
+
"ingest_time",
|
| 91 |
+
"value",
|
| 92 |
+
"value_text",
|
| 93 |
+
"value_json",
|
| 94 |
+
"frequency",
|
| 95 |
+
"unit",
|
| 96 |
+
"quality_flag",
|
| 97 |
+
"revision_id",
|
| 98 |
+
"raw_id",
|
| 99 |
+
]
|
| 100 |
+
|
| 101 |
+
FORECAST_TASK_FIELDS = [
|
| 102 |
+
"task_id",
|
| 103 |
+
"dataset_id",
|
| 104 |
+
"target_entity_id",
|
| 105 |
+
"target_variable_id",
|
| 106 |
+
"forecast_issue_time",
|
| 107 |
+
"context_start",
|
| 108 |
+
"context_end",
|
| 109 |
+
"horizon_start",
|
| 110 |
+
"horizon_end",
|
| 111 |
+
"input_length",
|
| 112 |
+
"horizon_length",
|
| 113 |
+
"frequency",
|
| 114 |
+
"covariate_variable_ids",
|
| 115 |
+
"task_split",
|
| 116 |
+
"created_at",
|
| 117 |
+
]
|
| 118 |
+
|
| 119 |
+
QUALITY_REPORT_FIELDS = [
|
| 120 |
+
"dataset_id",
|
| 121 |
+
"source_id",
|
| 122 |
+
"entity_id",
|
| 123 |
+
"variable_id",
|
| 124 |
+
"window_start",
|
| 125 |
+
"window_end",
|
| 126 |
+
"observation_count",
|
| 127 |
+
"missing_ratio",
|
| 128 |
+
"duplicate_count",
|
| 129 |
+
"freshness_delay_hours",
|
| 130 |
+
"outlier_count",
|
| 131 |
+
"api_failure_rate",
|
| 132 |
+
"notes",
|
| 133 |
+
]
|
| 134 |
+
|
| 135 |
+
RUN_LOG_FIELDS = [
|
| 136 |
+
"source_id",
|
| 137 |
+
"dataset_id",
|
| 138 |
+
"status",
|
| 139 |
+
"started_at",
|
| 140 |
+
"finished_at",
|
| 141 |
+
"observation_count",
|
| 142 |
+
"error",
|
| 143 |
+
]
|
| 144 |
+
|
| 145 |
+
|
| 146 |
+
def utc_now() -> datetime:
|
| 147 |
+
return datetime.now(UTC)
|
| 148 |
+
|
| 149 |
+
|
| 150 |
+
def iso(dt: datetime) -> str:
|
| 151 |
+
if dt.tzinfo is None:
|
| 152 |
+
dt = dt.replace(tzinfo=UTC)
|
| 153 |
+
return dt.astimezone(UTC).isoformat().replace("+00:00", "Z")
|
| 154 |
+
|
| 155 |
+
|
| 156 |
+
def parse_iso_datetime(value: str) -> datetime:
|
| 157 |
+
text = str(value).strip()
|
| 158 |
+
if text.endswith("Z"):
|
| 159 |
+
text = text[:-1] + "+00:00"
|
| 160 |
+
dt = datetime.fromisoformat(text)
|
| 161 |
+
if dt.tzinfo is None:
|
| 162 |
+
dt = dt.replace(tzinfo=UTC)
|
| 163 |
+
return dt.astimezone(UTC)
|
| 164 |
+
|
| 165 |
+
|
| 166 |
+
def parse_float(value: Any) -> Optional[float]:
|
| 167 |
+
if value is None:
|
| 168 |
+
return None
|
| 169 |
+
if isinstance(value, (int, float)):
|
| 170 |
+
if isinstance(value, float) and (math.isnan(value) or math.isinf(value)):
|
| 171 |
+
return None
|
| 172 |
+
return float(value)
|
| 173 |
+
text = str(value).strip()
|
| 174 |
+
if not text or text.upper() in {"MM", "M", "NA", "N/A", "NULL", "NONE", "-9999", "-999.0"}:
|
| 175 |
+
return None
|
| 176 |
+
try:
|
| 177 |
+
parsed = float(text)
|
| 178 |
+
except ValueError:
|
| 179 |
+
return None
|
| 180 |
+
if math.isnan(parsed) or math.isinf(parsed):
|
| 181 |
+
return None
|
| 182 |
+
return parsed
|
| 183 |
+
|
| 184 |
+
|
| 185 |
+
def floor_hour(dt: datetime) -> datetime:
|
| 186 |
+
return dt.astimezone(UTC).replace(minute=0, second=0, microsecond=0)
|
| 187 |
+
|
| 188 |
+
|
| 189 |
+
def parse_epoch_ms_or_us(value: Any) -> datetime:
|
| 190 |
+
raw = int(value)
|
| 191 |
+
if raw > 10_000_000_000_000:
|
| 192 |
+
return datetime.fromtimestamp(raw / 1_000_000, tz=UTC)
|
| 193 |
+
return datetime.fromtimestamp(raw / 1000, tz=UTC)
|
| 194 |
+
|
| 195 |
+
|
| 196 |
+
def safe_id(text: str) -> str:
|
| 197 |
+
return re.sub(r"[^a-zA-Z0-9_.-]+", "_", text).strip("_").lower()
|
| 198 |
+
|
| 199 |
+
|
| 200 |
+
def dumps(value: Any) -> str:
|
| 201 |
+
if value in (None, ""):
|
| 202 |
+
return ""
|
| 203 |
+
if isinstance(value, str):
|
| 204 |
+
return value
|
| 205 |
+
return json.dumps(value, ensure_ascii=True, sort_keys=True)
|
| 206 |
+
|
| 207 |
+
|
| 208 |
+
def date_yyyymmdd(d: date) -> str:
|
| 209 |
+
return d.strftime("%Y%m%d")
|
| 210 |
+
|
| 211 |
+
|
| 212 |
+
def datetime_yyyymmddhhmmss(dt: datetime) -> str:
|
| 213 |
+
return dt.astimezone(UTC).strftime("%Y%m%d%H%M%S")
|
| 214 |
+
|
| 215 |
+
|
| 216 |
+
def frequency_delta(freq: str) -> Optional[timedelta]:
|
| 217 |
+
return {
|
| 218 |
+
"1min": timedelta(minutes=1),
|
| 219 |
+
"5min": timedelta(minutes=5),
|
| 220 |
+
"6min": timedelta(minutes=6),
|
| 221 |
+
"10min": timedelta(minutes=10),
|
| 222 |
+
"15min": timedelta(minutes=15),
|
| 223 |
+
"1h": timedelta(hours=1),
|
| 224 |
+
"1d": timedelta(days=1),
|
| 225 |
+
"snapshot": None,
|
| 226 |
+
}.get(freq)
|
| 227 |
+
|
| 228 |
+
|
| 229 |
+
@dataclass
|
| 230 |
+
class Store:
|
| 231 |
+
output_dir: Path
|
| 232 |
+
raw_snapshots: List[Dict[str, Any]] = field(default_factory=list)
|
| 233 |
+
entities: Dict[Tuple[str, str], Dict[str, Any]] = field(default_factory=dict)
|
| 234 |
+
variables: Dict[Tuple[str, str], Dict[str, Any]] = field(default_factory=dict)
|
| 235 |
+
observations: List[Dict[str, Any]] = field(default_factory=list)
|
| 236 |
+
forecast_tasks: List[Dict[str, Any]] = field(default_factory=list)
|
| 237 |
+
quality_reports: List[Dict[str, Any]] = field(default_factory=list)
|
| 238 |
+
run_log: List[Dict[str, Any]] = field(default_factory=list)
|
| 239 |
+
|
| 240 |
+
def add_entity(
|
| 241 |
+
self,
|
| 242 |
+
dataset_id: str,
|
| 243 |
+
source_id: str,
|
| 244 |
+
entity_id: str,
|
| 245 |
+
entity_type: str,
|
| 246 |
+
entity_name: str,
|
| 247 |
+
lat: Any = "",
|
| 248 |
+
lon: Any = "",
|
| 249 |
+
region: str = "",
|
| 250 |
+
country: str = "",
|
| 251 |
+
timezone_name: str = "UTC",
|
| 252 |
+
metadata: Optional[Dict[str, Any]] = None,
|
| 253 |
+
) -> None:
|
| 254 |
+
key = (dataset_id, entity_id)
|
| 255 |
+
if key in self.entities:
|
| 256 |
+
return
|
| 257 |
+
self.entities[key] = {
|
| 258 |
+
"dataset_id": dataset_id,
|
| 259 |
+
"source_id": source_id,
|
| 260 |
+
"entity_id": entity_id,
|
| 261 |
+
"entity_type": entity_type,
|
| 262 |
+
"entity_name": entity_name,
|
| 263 |
+
"lat": lat,
|
| 264 |
+
"lon": lon,
|
| 265 |
+
"region": region,
|
| 266 |
+
"country": country,
|
| 267 |
+
"timezone": timezone_name,
|
| 268 |
+
"metadata_json": metadata or {},
|
| 269 |
+
}
|
| 270 |
+
|
| 271 |
+
def add_variable(
|
| 272 |
+
self,
|
| 273 |
+
dataset_id: str,
|
| 274 |
+
source_id: str,
|
| 275 |
+
variable_id: str,
|
| 276 |
+
variable_name: str,
|
| 277 |
+
unit: str = "",
|
| 278 |
+
value_type: str = "numeric",
|
| 279 |
+
frequency_hint: str = "",
|
| 280 |
+
aggregation: str = "",
|
| 281 |
+
is_target_default: bool = False,
|
| 282 |
+
metadata: Optional[Dict[str, Any]] = None,
|
| 283 |
+
) -> None:
|
| 284 |
+
key = (dataset_id, variable_id)
|
| 285 |
+
if key in self.variables:
|
| 286 |
+
return
|
| 287 |
+
self.variables[key] = {
|
| 288 |
+
"dataset_id": dataset_id,
|
| 289 |
+
"source_id": source_id,
|
| 290 |
+
"variable_id": variable_id,
|
| 291 |
+
"variable_name": variable_name,
|
| 292 |
+
"unit": unit,
|
| 293 |
+
"value_type": value_type,
|
| 294 |
+
"frequency_hint": frequency_hint,
|
| 295 |
+
"aggregation": aggregation,
|
| 296 |
+
"is_target_default": str(bool(is_target_default)).lower(),
|
| 297 |
+
"metadata_json": metadata or {},
|
| 298 |
+
}
|
| 299 |
+
|
| 300 |
+
def add_observation(
|
| 301 |
+
self,
|
| 302 |
+
dataset_id: str,
|
| 303 |
+
source_id: str,
|
| 304 |
+
entity_id: str,
|
| 305 |
+
variable_id: str,
|
| 306 |
+
timestamp: datetime,
|
| 307 |
+
available_time: datetime,
|
| 308 |
+
ingest_time: datetime,
|
| 309 |
+
value: Optional[float],
|
| 310 |
+
frequency: str,
|
| 311 |
+
unit: str = "",
|
| 312 |
+
value_text: str = "",
|
| 313 |
+
value_json: Any = "",
|
| 314 |
+
quality_flag: Any = "",
|
| 315 |
+
revision_id: str = "v1",
|
| 316 |
+
raw_id: str = "",
|
| 317 |
+
) -> None:
|
| 318 |
+
self.observations.append(
|
| 319 |
+
{
|
| 320 |
+
"dataset_id": dataset_id,
|
| 321 |
+
"source_id": source_id,
|
| 322 |
+
"entity_id": entity_id,
|
| 323 |
+
"variable_id": variable_id,
|
| 324 |
+
"timestamp": iso(timestamp),
|
| 325 |
+
"available_time": iso(available_time),
|
| 326 |
+
"ingest_time": iso(ingest_time),
|
| 327 |
+
"value": "" if value is None else value,
|
| 328 |
+
"value_text": value_text,
|
| 329 |
+
"value_json": value_json,
|
| 330 |
+
"frequency": frequency,
|
| 331 |
+
"unit": unit,
|
| 332 |
+
"quality_flag": quality_flag,
|
| 333 |
+
"revision_id": revision_id,
|
| 334 |
+
"raw_id": raw_id,
|
| 335 |
+
}
|
| 336 |
+
)
|
| 337 |
+
|
| 338 |
+
|
| 339 |
+
class Fetcher:
|
| 340 |
+
def __init__(self, store: Store, timeout: int = 30) -> None:
|
| 341 |
+
self.store = store
|
| 342 |
+
self.timeout = timeout
|
| 343 |
+
self.session = requests.Session()
|
| 344 |
+
self.session.headers.update(
|
| 345 |
+
{
|
| 346 |
+
"User-Agent": (
|
| 347 |
+
"TS-Bench data-layer prototype "
|
| 348 |
+
"(research; contact: local-run@example.invalid)"
|
| 349 |
+
)
|
| 350 |
+
}
|
| 351 |
+
)
|
| 352 |
+
|
| 353 |
+
def get(
|
| 354 |
+
self,
|
| 355 |
+
source_id: str,
|
| 356 |
+
dataset_id: str,
|
| 357 |
+
url: str,
|
| 358 |
+
params: Optional[Dict[str, Any]] = None,
|
| 359 |
+
headers: Optional[Dict[str, str]] = None,
|
| 360 |
+
raw_ext: str = "json",
|
| 361 |
+
) -> Tuple[requests.Response, str, datetime]:
|
| 362 |
+
fetched_at = utc_now()
|
| 363 |
+
response: Optional[requests.Response] = None
|
| 364 |
+
error = ""
|
| 365 |
+
try:
|
| 366 |
+
response = self.session.get(
|
| 367 |
+
url,
|
| 368 |
+
params=params,
|
| 369 |
+
headers=headers,
|
| 370 |
+
timeout=self.timeout,
|
| 371 |
+
)
|
| 372 |
+
content = response.content
|
| 373 |
+
status = response.status_code
|
| 374 |
+
content_type = response.headers.get("content-type", "")
|
| 375 |
+
except Exception as exc: # noqa: BLE001
|
| 376 |
+
content = str(exc).encode("utf-8")
|
| 377 |
+
status = 0
|
| 378 |
+
content_type = "text/plain"
|
| 379 |
+
error = str(exc)
|
| 380 |
+
|
| 381 |
+
digest = hashlib.sha256(content).hexdigest()
|
| 382 |
+
raw_id = f"{source_id}_{fetched_at.strftime('%Y%m%dT%H%M%SZ')}_{digest[:12]}"
|
| 383 |
+
raw_dir = self.store.output_dir / "raw" / source_id
|
| 384 |
+
raw_dir.mkdir(parents=True, exist_ok=True)
|
| 385 |
+
raw_uri = raw_dir / f"{raw_id}.{raw_ext}"
|
| 386 |
+
raw_uri.write_bytes(content)
|
| 387 |
+
full_url = response.url if response is not None else url
|
| 388 |
+
|
| 389 |
+
self.store.raw_snapshots.append(
|
| 390 |
+
{
|
| 391 |
+
"raw_id": raw_id,
|
| 392 |
+
"source_id": source_id,
|
| 393 |
+
"dataset_id": dataset_id,
|
| 394 |
+
"request_url": full_url,
|
| 395 |
+
"request_params": params or {},
|
| 396 |
+
"http_status": status,
|
| 397 |
+
"fetched_at": iso(fetched_at),
|
| 398 |
+
"content_type": content_type,
|
| 399 |
+
"content_hash": f"sha256:{digest}",
|
| 400 |
+
"raw_uri": str(raw_uri),
|
| 401 |
+
"parser_version": PARSER_VERSION,
|
| 402 |
+
"error": error,
|
| 403 |
+
}
|
| 404 |
+
)
|
| 405 |
+
|
| 406 |
+
if response is None:
|
| 407 |
+
raise RuntimeError(error)
|
| 408 |
+
if response.status_code >= 400:
|
| 409 |
+
raise RuntimeError(f"HTTP {response.status_code}: {response.text[:300]}")
|
| 410 |
+
return response, raw_id, fetched_at
|
| 411 |
+
|
| 412 |
+
|
| 413 |
+
def add_time_series_observations(
|
| 414 |
+
store: Store,
|
| 415 |
+
dataset_id: str,
|
| 416 |
+
source_id: str,
|
| 417 |
+
entity_id: str,
|
| 418 |
+
raw_id: str,
|
| 419 |
+
fetched_at: datetime,
|
| 420 |
+
times: List[datetime],
|
| 421 |
+
variables: Dict[str, Tuple[List[Any], str, str]],
|
| 422 |
+
frequency: str,
|
| 423 |
+
is_target: Optional[str] = None,
|
| 424 |
+
end_dt: Optional[datetime] = None,
|
| 425 |
+
) -> int:
|
| 426 |
+
count = 0
|
| 427 |
+
for variable_id, (values, unit, variable_name) in variables.items():
|
| 428 |
+
store.add_variable(
|
| 429 |
+
dataset_id,
|
| 430 |
+
source_id,
|
| 431 |
+
variable_id,
|
| 432 |
+
variable_name,
|
| 433 |
+
unit=unit,
|
| 434 |
+
frequency_hint=frequency,
|
| 435 |
+
aggregation="instantaneous",
|
| 436 |
+
is_target_default=variable_id == is_target,
|
| 437 |
+
)
|
| 438 |
+
for idx, ts in enumerate(times):
|
| 439 |
+
if end_dt and ts > end_dt:
|
| 440 |
+
continue
|
| 441 |
+
if idx >= len(values):
|
| 442 |
+
continue
|
| 443 |
+
value = parse_float(values[idx])
|
| 444 |
+
if value is None:
|
| 445 |
+
continue
|
| 446 |
+
store.add_observation(
|
| 447 |
+
dataset_id,
|
| 448 |
+
source_id,
|
| 449 |
+
entity_id,
|
| 450 |
+
variable_id,
|
| 451 |
+
ts,
|
| 452 |
+
fetched_at,
|
| 453 |
+
fetched_at,
|
| 454 |
+
value,
|
| 455 |
+
frequency,
|
| 456 |
+
unit,
|
| 457 |
+
raw_id=raw_id,
|
| 458 |
+
)
|
| 459 |
+
count += 1
|
| 460 |
+
return count
|
| 461 |
+
|
| 462 |
+
|
| 463 |
+
def collect_open_meteo_weather(store: Store, fetcher: Fetcher, start_dt: datetime, end_dt: datetime, args: argparse.Namespace) -> int:
|
| 464 |
+
source_id = "open_meteo_weather"
|
| 465 |
+
dataset_id = "open_meteo_shanghai_hourly"
|
| 466 |
+
entity_id = "grid_shanghai_31.2304_121.4737"
|
| 467 |
+
store.add_entity(
|
| 468 |
+
dataset_id,
|
| 469 |
+
source_id,
|
| 470 |
+
entity_id,
|
| 471 |
+
"grid",
|
| 472 |
+
"Shanghai grid point",
|
| 473 |
+
lat=31.2304,
|
| 474 |
+
lon=121.4737,
|
| 475 |
+
region="Shanghai",
|
| 476 |
+
country="CN",
|
| 477 |
+
timezone_name="UTC",
|
| 478 |
+
)
|
| 479 |
+
params = {
|
| 480 |
+
"latitude": 31.2304,
|
| 481 |
+
"longitude": 121.4737,
|
| 482 |
+
"hourly": "temperature_2m,relative_humidity_2m,wind_speed_10m,precipitation",
|
| 483 |
+
"past_days": args.days,
|
| 484 |
+
"forecast_days": 1,
|
| 485 |
+
"timezone": "UTC",
|
| 486 |
+
}
|
| 487 |
+
response, raw_id, fetched_at = fetcher.get(source_id, dataset_id, "https://api.open-meteo.com/v1/forecast", params=params)
|
| 488 |
+
payload = response.json()
|
| 489 |
+
hourly = payload.get("hourly", {})
|
| 490 |
+
units = payload.get("hourly_units", {})
|
| 491 |
+
times = [parse_iso_datetime(t) for t in hourly.get("time", [])]
|
| 492 |
+
variables = {
|
| 493 |
+
"temperature_2m": (hourly.get("temperature_2m", []), units.get("temperature_2m", "celsius"), "temperature_2m"),
|
| 494 |
+
"relative_humidity_2m": (hourly.get("relative_humidity_2m", []), units.get("relative_humidity_2m", "%"), "relative_humidity_2m"),
|
| 495 |
+
"wind_speed_10m": (hourly.get("wind_speed_10m", []), units.get("wind_speed_10m", "km/h"), "wind_speed_10m"),
|
| 496 |
+
"precipitation": (hourly.get("precipitation", []), units.get("precipitation", "mm"), "precipitation"),
|
| 497 |
+
}
|
| 498 |
+
return add_time_series_observations(store, dataset_id, source_id, entity_id, raw_id, fetched_at, times, variables, "1h", "temperature_2m", end_dt)
|
| 499 |
+
|
| 500 |
+
|
| 501 |
+
def collect_open_meteo_air_quality(store: Store, fetcher: Fetcher, start_dt: datetime, end_dt: datetime, args: argparse.Namespace) -> int:
|
| 502 |
+
source_id = "open_meteo_air_quality"
|
| 503 |
+
dataset_id = "open_meteo_shanghai_air_quality_hourly"
|
| 504 |
+
entity_id = "grid_shanghai_31.2304_121.4737"
|
| 505 |
+
store.add_entity(
|
| 506 |
+
dataset_id,
|
| 507 |
+
source_id,
|
| 508 |
+
entity_id,
|
| 509 |
+
"grid",
|
| 510 |
+
"Shanghai air quality grid point",
|
| 511 |
+
lat=31.2304,
|
| 512 |
+
lon=121.4737,
|
| 513 |
+
region="Shanghai",
|
| 514 |
+
country="CN",
|
| 515 |
+
timezone_name="UTC",
|
| 516 |
+
)
|
| 517 |
+
params = {
|
| 518 |
+
"latitude": 31.2304,
|
| 519 |
+
"longitude": 121.4737,
|
| 520 |
+
"hourly": "pm10,pm2_5,nitrogen_dioxide,carbon_monoxide",
|
| 521 |
+
"past_days": args.days,
|
| 522 |
+
"forecast_days": 1,
|
| 523 |
+
"timezone": "UTC",
|
| 524 |
+
}
|
| 525 |
+
response, raw_id, fetched_at = fetcher.get(source_id, dataset_id, "https://air-quality-api.open-meteo.com/v1/air-quality", params=params)
|
| 526 |
+
payload = response.json()
|
| 527 |
+
hourly = payload.get("hourly", {})
|
| 528 |
+
units = payload.get("hourly_units", {})
|
| 529 |
+
times = [parse_iso_datetime(t) for t in hourly.get("time", [])]
|
| 530 |
+
variables = {
|
| 531 |
+
"pm10": (hourly.get("pm10", []), units.get("pm10", "ug/m3"), "pm10"),
|
| 532 |
+
"pm2_5": (hourly.get("pm2_5", []), units.get("pm2_5", "ug/m3"), "pm2_5"),
|
| 533 |
+
"nitrogen_dioxide": (hourly.get("nitrogen_dioxide", []), units.get("nitrogen_dioxide", "ug/m3"), "nitrogen_dioxide"),
|
| 534 |
+
"carbon_monoxide": (hourly.get("carbon_monoxide", []), units.get("carbon_monoxide", "ug/m3"), "carbon_monoxide"),
|
| 535 |
+
}
|
| 536 |
+
return add_time_series_observations(store, dataset_id, source_id, entity_id, raw_id, fetched_at, times, variables, "1h", "pm2_5", end_dt)
|
| 537 |
+
|
| 538 |
+
|
| 539 |
+
def collect_nasa_power(store: Store, fetcher: Fetcher, start_dt: datetime, end_dt: datetime, args: argparse.Namespace) -> int:
|
| 540 |
+
source_id = "nasa_power"
|
| 541 |
+
dataset_id = "nasa_power_shanghai_hourly"
|
| 542 |
+
entity_id = "point_shanghai_31.2304_121.4737"
|
| 543 |
+
store.add_entity(
|
| 544 |
+
dataset_id,
|
| 545 |
+
source_id,
|
| 546 |
+
entity_id,
|
| 547 |
+
"grid",
|
| 548 |
+
"NASA POWER Shanghai point",
|
| 549 |
+
lat=31.2304,
|
| 550 |
+
lon=121.4737,
|
| 551 |
+
region="Shanghai",
|
| 552 |
+
country="CN",
|
| 553 |
+
timezone_name="UTC",
|
| 554 |
+
)
|
| 555 |
+
params = {
|
| 556 |
+
"parameters": "T2M,RH2M,WS10M,PRECTOTCORR",
|
| 557 |
+
"community": "RE",
|
| 558 |
+
"longitude": 121.4737,
|
| 559 |
+
"latitude": 31.2304,
|
| 560 |
+
"start": date_yyyymmdd(start_dt.date()),
|
| 561 |
+
"end": date_yyyymmdd(min(end_dt.date(), (utc_now() - timedelta(days=1)).date())),
|
| 562 |
+
"format": "JSON",
|
| 563 |
+
}
|
| 564 |
+
response, raw_id, fetched_at = fetcher.get(source_id, dataset_id, "https://power.larc.nasa.gov/api/temporal/hourly/point", params=params)
|
| 565 |
+
payload = response.json()
|
| 566 |
+
parameter = payload.get("properties", {}).get("parameter", {})
|
| 567 |
+
variable_info = {
|
| 568 |
+
"T2M": ("T2M", "celsius", "temperature_2m"),
|
| 569 |
+
"RH2M": ("RH2M", "%", "relative_humidity_2m"),
|
| 570 |
+
"WS10M": ("WS10M", "m/s", "wind_speed_10m"),
|
| 571 |
+
"PRECTOTCORR": ("PRECTOTCORR", "mm/hour", "precipitation_corrected"),
|
| 572 |
+
}
|
| 573 |
+
count = 0
|
| 574 |
+
for param_name, (variable_id, unit, variable_name) in variable_info.items():
|
| 575 |
+
series = parameter.get(param_name, {})
|
| 576 |
+
store.add_variable(dataset_id, source_id, variable_id, variable_name, unit, "numeric", "1h", "instantaneous", param_name == "T2M")
|
| 577 |
+
for stamp, value_raw in series.items():
|
| 578 |
+
if len(str(stamp)) < 10:
|
| 579 |
+
continue
|
| 580 |
+
ts = datetime.strptime(str(stamp)[:10], "%Y%m%d%H").replace(tzinfo=UTC)
|
| 581 |
+
if ts < start_dt or ts > end_dt:
|
| 582 |
+
continue
|
| 583 |
+
value = parse_float(value_raw)
|
| 584 |
+
if value is None or value <= -900:
|
| 585 |
+
continue
|
| 586 |
+
store.add_observation(dataset_id, source_id, entity_id, variable_id, ts, fetched_at, fetched_at, value, "1h", unit, raw_id=raw_id)
|
| 587 |
+
count += 1
|
| 588 |
+
return count
|
| 589 |
+
|
| 590 |
+
|
| 591 |
+
def collect_usgs_water(store: Store, fetcher: Fetcher, start_dt: datetime, end_dt: datetime, args: argparse.Namespace) -> int:
|
| 592 |
+
source_id = "usgs_water"
|
| 593 |
+
dataset_id = "usgs_potomac_iv"
|
| 594 |
+
site_id = "01646500"
|
| 595 |
+
entity_id = f"usgs_site_{site_id}"
|
| 596 |
+
params = {
|
| 597 |
+
"format": "json",
|
| 598 |
+
"sites": site_id,
|
| 599 |
+
"period": f"P{args.days}D",
|
| 600 |
+
"parameterCd": "00060,00065",
|
| 601 |
+
"siteStatus": "all",
|
| 602 |
+
}
|
| 603 |
+
response, raw_id, fetched_at = fetcher.get(source_id, dataset_id, "https://waterservices.usgs.gov/nwis/iv/", params=params)
|
| 604 |
+
payload = response.json()
|
| 605 |
+
count = 0
|
| 606 |
+
for series in payload.get("value", {}).get("timeSeries", []):
|
| 607 |
+
source_info = series.get("sourceInfo", {})
|
| 608 |
+
site_code = source_info.get("siteCode", [{}])[0].get("value", site_id)
|
| 609 |
+
site_name = source_info.get("siteName", f"USGS site {site_code}")
|
| 610 |
+
geo = source_info.get("geoLocation", {}).get("geogLocation", {})
|
| 611 |
+
store.add_entity(
|
| 612 |
+
dataset_id,
|
| 613 |
+
source_id,
|
| 614 |
+
f"usgs_site_{site_code}",
|
| 615 |
+
"station",
|
| 616 |
+
site_name,
|
| 617 |
+
lat=geo.get("latitude", ""),
|
| 618 |
+
lon=geo.get("longitude", ""),
|
| 619 |
+
region="US",
|
| 620 |
+
country="US",
|
| 621 |
+
timezone_name="UTC",
|
| 622 |
+
metadata={"site_code": site_code},
|
| 623 |
+
)
|
| 624 |
+
variable = series.get("variable", {})
|
| 625 |
+
code = variable.get("variableCode", [{}])[0].get("value", "unknown")
|
| 626 |
+
variable_id = f"usgs_{code}"
|
| 627 |
+
unit = variable.get("unit", {}).get("unitCode", "")
|
| 628 |
+
variable_name = variable.get("variableName", variable_id)
|
| 629 |
+
store.add_variable(dataset_id, source_id, variable_id, variable_name, unit, "numeric", "15min", "instantaneous", code == "00060")
|
| 630 |
+
for value_block in series.get("values", []):
|
| 631 |
+
for point in value_block.get("value", []):
|
| 632 |
+
value = parse_float(point.get("value"))
|
| 633 |
+
if value is None:
|
| 634 |
+
continue
|
| 635 |
+
ts = parse_iso_datetime(point["dateTime"])
|
| 636 |
+
if ts < start_dt or ts > end_dt:
|
| 637 |
+
continue
|
| 638 |
+
qualifiers = point.get("qualifiers", [])
|
| 639 |
+
store.add_observation(
|
| 640 |
+
dataset_id,
|
| 641 |
+
source_id,
|
| 642 |
+
f"usgs_site_{site_code}",
|
| 643 |
+
variable_id,
|
| 644 |
+
ts,
|
| 645 |
+
fetched_at,
|
| 646 |
+
fetched_at,
|
| 647 |
+
value,
|
| 648 |
+
"15min",
|
| 649 |
+
unit,
|
| 650 |
+
quality_flag=qualifiers,
|
| 651 |
+
raw_id=raw_id,
|
| 652 |
+
)
|
| 653 |
+
count += 1
|
| 654 |
+
return count
|
| 655 |
+
|
| 656 |
+
|
| 657 |
+
def collect_noaa_coops(store: Store, fetcher: Fetcher, start_dt: datetime, end_dt: datetime, args: argparse.Namespace) -> int:
|
| 658 |
+
source_id = "noaa_coops"
|
| 659 |
+
dataset_id = "noaa_coops_sf_water_level"
|
| 660 |
+
station_id = "9414290"
|
| 661 |
+
entity_id = f"coops_station_{station_id}"
|
| 662 |
+
store.add_entity(
|
| 663 |
+
dataset_id,
|
| 664 |
+
source_id,
|
| 665 |
+
entity_id,
|
| 666 |
+
"station",
|
| 667 |
+
"NOAA CO-OPS San Francisco",
|
| 668 |
+
lat=37.8063,
|
| 669 |
+
lon=-122.4659,
|
| 670 |
+
region="CA",
|
| 671 |
+
country="US",
|
| 672 |
+
timezone_name="UTC",
|
| 673 |
+
metadata={"station": station_id},
|
| 674 |
+
)
|
| 675 |
+
variable_id = "water_level"
|
| 676 |
+
store.add_variable(dataset_id, source_id, variable_id, "water_level", "m", "numeric", "6min", "instantaneous", True)
|
| 677 |
+
params = {
|
| 678 |
+
"product": "water_level",
|
| 679 |
+
"application": "TSBench",
|
| 680 |
+
"begin_date": date_yyyymmdd(start_dt.date()),
|
| 681 |
+
"end_date": date_yyyymmdd(end_dt.date()),
|
| 682 |
+
"datum": "MLLW",
|
| 683 |
+
"station": station_id,
|
| 684 |
+
"time_zone": "gmt",
|
| 685 |
+
"units": "metric",
|
| 686 |
+
"format": "json",
|
| 687 |
+
}
|
| 688 |
+
response, raw_id, fetched_at = fetcher.get(source_id, dataset_id, "https://api.tidesandcurrents.noaa.gov/api/prod/datagetter", params=params)
|
| 689 |
+
payload = response.json()
|
| 690 |
+
if "error" in payload:
|
| 691 |
+
raise RuntimeError(payload["error"].get("message", str(payload["error"])))
|
| 692 |
+
count = 0
|
| 693 |
+
for point in payload.get("data", []):
|
| 694 |
+
value = parse_float(point.get("v"))
|
| 695 |
+
if value is None:
|
| 696 |
+
continue
|
| 697 |
+
ts = datetime.strptime(point["t"], "%Y-%m-%d %H:%M").replace(tzinfo=UTC)
|
| 698 |
+
if ts < start_dt or ts > end_dt:
|
| 699 |
+
continue
|
| 700 |
+
store.add_observation(
|
| 701 |
+
dataset_id,
|
| 702 |
+
source_id,
|
| 703 |
+
entity_id,
|
| 704 |
+
variable_id,
|
| 705 |
+
ts,
|
| 706 |
+
fetched_at,
|
| 707 |
+
fetched_at,
|
| 708 |
+
value,
|
| 709 |
+
"6min",
|
| 710 |
+
"m",
|
| 711 |
+
quality_flag=point.get("q", ""),
|
| 712 |
+
raw_id=raw_id,
|
| 713 |
+
)
|
| 714 |
+
count += 1
|
| 715 |
+
return count
|
| 716 |
+
|
| 717 |
+
|
| 718 |
+
def collect_noaa_ndbc(store: Store, fetcher: Fetcher, start_dt: datetime, end_dt: datetime, args: argparse.Namespace) -> int:
|
| 719 |
+
source_id = "noaa_ndbc"
|
| 720 |
+
dataset_id = "noaa_ndbc_46013_realtime"
|
| 721 |
+
station_id = "46013"
|
| 722 |
+
entity_id = f"ndbc_station_{station_id}"
|
| 723 |
+
store.add_entity(
|
| 724 |
+
dataset_id,
|
| 725 |
+
source_id,
|
| 726 |
+
entity_id,
|
| 727 |
+
"station",
|
| 728 |
+
"NDBC station 46013",
|
| 729 |
+
lat=38.235,
|
| 730 |
+
lon=-123.317,
|
| 731 |
+
region="CA offshore",
|
| 732 |
+
country="US",
|
| 733 |
+
timezone_name="UTC",
|
| 734 |
+
metadata={"station": station_id},
|
| 735 |
+
)
|
| 736 |
+
url = f"https://www.ndbc.noaa.gov/data/realtime2/{station_id}.txt"
|
| 737 |
+
response, raw_id, fetched_at = fetcher.get(source_id, dataset_id, url, raw_ext="txt")
|
| 738 |
+
lines = response.text.splitlines()
|
| 739 |
+
header: List[str] = []
|
| 740 |
+
data_lines: List[str] = []
|
| 741 |
+
for line in lines:
|
| 742 |
+
if line.startswith("#YY"):
|
| 743 |
+
header = line.lstrip("#").split()
|
| 744 |
+
elif line and not line.startswith("#"):
|
| 745 |
+
data_lines.append(line)
|
| 746 |
+
if not header:
|
| 747 |
+
raise RuntimeError("NDBC header not found")
|
| 748 |
+
variable_defs = {
|
| 749 |
+
"WSPD": ("wind_speed", "m/s"),
|
| 750 |
+
"WVHT": ("wave_height", "m"),
|
| 751 |
+
"PRES": ("pressure", "hPa"),
|
| 752 |
+
"ATMP": ("air_temperature", "celsius"),
|
| 753 |
+
"WTMP": ("water_temperature", "celsius"),
|
| 754 |
+
}
|
| 755 |
+
for column, (variable_id, unit) in variable_defs.items():
|
| 756 |
+
store.add_variable(dataset_id, source_id, variable_id, variable_id, unit, "numeric", "10min", "instantaneous", column == "WVHT")
|
| 757 |
+
count = 0
|
| 758 |
+
for line in data_lines:
|
| 759 |
+
values = line.split()
|
| 760 |
+
if len(values) < len(header):
|
| 761 |
+
continue
|
| 762 |
+
row = dict(zip(header, values))
|
| 763 |
+
try:
|
| 764 |
+
ts = datetime(
|
| 765 |
+
int(row["YY"]),
|
| 766 |
+
int(row["MM"]),
|
| 767 |
+
int(row["DD"]),
|
| 768 |
+
int(row["hh"]),
|
| 769 |
+
int(row["mm"]),
|
| 770 |
+
tzinfo=UTC,
|
| 771 |
+
)
|
| 772 |
+
except Exception:
|
| 773 |
+
continue
|
| 774 |
+
if ts < start_dt or ts > end_dt:
|
| 775 |
+
continue
|
| 776 |
+
for column, (variable_id, unit) in variable_defs.items():
|
| 777 |
+
value = parse_float(row.get(column))
|
| 778 |
+
if value is None:
|
| 779 |
+
continue
|
| 780 |
+
store.add_observation(dataset_id, source_id, entity_id, variable_id, ts, fetched_at, fetched_at, value, "10min", unit, raw_id=raw_id)
|
| 781 |
+
count += 1
|
| 782 |
+
return count
|
| 783 |
+
|
| 784 |
+
|
| 785 |
+
def collect_gbfs_citibike(store: Store, fetcher: Fetcher, start_dt: datetime, end_dt: datetime, args: argparse.Namespace) -> int:
|
| 786 |
+
source_id = "gbfs_citibike"
|
| 787 |
+
dataset_id = "gbfs_citibike_current"
|
| 788 |
+
info_url = "https://gbfs.citibikenyc.com/gbfs/en/station_information.json"
|
| 789 |
+
status_url = "https://gbfs.citibikenyc.com/gbfs/en/station_status.json"
|
| 790 |
+
info_response, info_raw_id, info_fetched = fetcher.get(source_id, dataset_id, info_url)
|
| 791 |
+
status_response, status_raw_id, fetched_at = fetcher.get(source_id, dataset_id, status_url)
|
| 792 |
+
info_payload = info_response.json()
|
| 793 |
+
status_payload = status_response.json()
|
| 794 |
+
stations = {
|
| 795 |
+
str(item["station_id"]): item
|
| 796 |
+
for item in info_payload.get("data", {}).get("stations", [])
|
| 797 |
+
}
|
| 798 |
+
statuses = status_payload.get("data", {}).get("stations", [])[: args.limit_entities]
|
| 799 |
+
variables = {
|
| 800 |
+
"num_bikes_available": ("bikes", True),
|
| 801 |
+
"num_docks_available": ("docks", False),
|
| 802 |
+
}
|
| 803 |
+
for variable_id, (unit, is_target) in variables.items():
|
| 804 |
+
store.add_variable(dataset_id, source_id, variable_id, variable_id, unit, "numeric", "snapshot", "instantaneous", is_target)
|
| 805 |
+
count = 0
|
| 806 |
+
for status in statuses:
|
| 807 |
+
station_id = str(status.get("station_id", "unknown"))
|
| 808 |
+
station = stations.get(station_id, {})
|
| 809 |
+
entity_id = f"citibike_station_{safe_id(station_id)}"
|
| 810 |
+
store.add_entity(
|
| 811 |
+
dataset_id,
|
| 812 |
+
source_id,
|
| 813 |
+
entity_id,
|
| 814 |
+
"station",
|
| 815 |
+
station.get("name", f"Citi Bike station {station_id}"),
|
| 816 |
+
lat=station.get("lat", ""),
|
| 817 |
+
lon=station.get("lon", ""),
|
| 818 |
+
region="New York City",
|
| 819 |
+
country="US",
|
| 820 |
+
timezone_name="America/New_York",
|
| 821 |
+
metadata={"station_id": station_id, "raw_info_id": info_raw_id},
|
| 822 |
+
)
|
| 823 |
+
ts_raw = status.get("last_reported") or status_payload.get("last_updated")
|
| 824 |
+
ts = datetime.fromtimestamp(int(ts_raw), tz=UTC) if ts_raw else fetched_at
|
| 825 |
+
if ts < start_dt:
|
| 826 |
+
ts = fetched_at
|
| 827 |
+
for variable_id, (unit, _) in variables.items():
|
| 828 |
+
value = parse_float(status.get(variable_id))
|
| 829 |
+
if value is None:
|
| 830 |
+
continue
|
| 831 |
+
store.add_observation(dataset_id, source_id, entity_id, variable_id, ts, fetched_at, fetched_at, value, "snapshot", unit, raw_id=status_raw_id)
|
| 832 |
+
count += 1
|
| 833 |
+
_ = info_fetched
|
| 834 |
+
return count
|
| 835 |
+
|
| 836 |
+
|
| 837 |
+
def collect_binance_btcusdt(store: Store, fetcher: Fetcher, start_dt: datetime, end_dt: datetime, args: argparse.Namespace) -> int:
|
| 838 |
+
source_id = "binance"
|
| 839 |
+
dataset_id = "binance_btcusdt_1h"
|
| 840 |
+
entity_id = "crypto_btcusdt"
|
| 841 |
+
store.add_entity(dataset_id, source_id, entity_id, "asset", "BTCUSDT", region="global", timezone_name="UTC")
|
| 842 |
+
variable_defs = {
|
| 843 |
+
"open": "USD",
|
| 844 |
+
"high": "USD",
|
| 845 |
+
"low": "USD",
|
| 846 |
+
"close": "USD",
|
| 847 |
+
"volume": "BTC",
|
| 848 |
+
}
|
| 849 |
+
for variable_id, unit in variable_defs.items():
|
| 850 |
+
store.add_variable(dataset_id, source_id, variable_id, variable_id, unit, "numeric", "1h", "ohlcv", variable_id == "close")
|
| 851 |
+
params = {
|
| 852 |
+
"symbol": "BTCUSDT",
|
| 853 |
+
"interval": "1h",
|
| 854 |
+
"startTime": int(start_dt.timestamp() * 1000),
|
| 855 |
+
"endTime": int(end_dt.timestamp() * 1000),
|
| 856 |
+
"limit": 1000,
|
| 857 |
+
}
|
| 858 |
+
count = 0
|
| 859 |
+
try:
|
| 860 |
+
response, raw_id, fetched_at = fetcher.get(source_id, dataset_id, "https://api.binance.com/api/v3/klines", params=params)
|
| 861 |
+
rows = response.json()
|
| 862 |
+
for row in rows:
|
| 863 |
+
ts = parse_epoch_ms_or_us(row[0])
|
| 864 |
+
values = {
|
| 865 |
+
"open": row[1],
|
| 866 |
+
"high": row[2],
|
| 867 |
+
"low": row[3],
|
| 868 |
+
"close": row[4],
|
| 869 |
+
"volume": row[5],
|
| 870 |
+
}
|
| 871 |
+
for variable_id, value_raw in values.items():
|
| 872 |
+
value = parse_float(value_raw)
|
| 873 |
+
if value is None:
|
| 874 |
+
continue
|
| 875 |
+
store.add_observation(dataset_id, source_id, entity_id, variable_id, ts, fetched_at, fetched_at, value, "1h", variable_defs[variable_id], raw_id=raw_id)
|
| 876 |
+
count += 1
|
| 877 |
+
return count
|
| 878 |
+
except Exception:
|
| 879 |
+
# Fallback to official daily zip files.
|
| 880 |
+
current = start_dt.date()
|
| 881 |
+
while current <= end_dt.date():
|
| 882 |
+
url = f"https://data.binance.vision/data/spot/daily/klines/BTCUSDT/1h/BTCUSDT-1h-{current.isoformat()}.zip"
|
| 883 |
+
try:
|
| 884 |
+
response, raw_id, fetched_at = fetcher.get(source_id, dataset_id, url, raw_ext="zip")
|
| 885 |
+
except Exception:
|
| 886 |
+
current += timedelta(days=1)
|
| 887 |
+
continue
|
| 888 |
+
with zipfile.ZipFile(io.BytesIO(response.content)) as archive:
|
| 889 |
+
name = archive.namelist()[0]
|
| 890 |
+
text = archive.read(name).decode("utf-8")
|
| 891 |
+
for row in csv.reader(io.StringIO(text)):
|
| 892 |
+
if len(row) < 6:
|
| 893 |
+
continue
|
| 894 |
+
try:
|
| 895 |
+
ts = parse_epoch_ms_or_us(row[0])
|
| 896 |
+
except ValueError:
|
| 897 |
+
continue
|
| 898 |
+
if ts < start_dt or ts > end_dt:
|
| 899 |
+
continue
|
| 900 |
+
values = {
|
| 901 |
+
"open": row[1],
|
| 902 |
+
"high": row[2],
|
| 903 |
+
"low": row[3],
|
| 904 |
+
"close": row[4],
|
| 905 |
+
"volume": row[5],
|
| 906 |
+
}
|
| 907 |
+
for variable_id, value_raw in values.items():
|
| 908 |
+
value = parse_float(value_raw)
|
| 909 |
+
if value is None:
|
| 910 |
+
continue
|
| 911 |
+
store.add_observation(dataset_id, source_id, entity_id, variable_id, ts, fetched_at, fetched_at, value, "1h", variable_defs[variable_id], raw_id=raw_id)
|
| 912 |
+
count += 1
|
| 913 |
+
current += timedelta(days=1)
|
| 914 |
+
return count
|
| 915 |
+
|
| 916 |
+
|
| 917 |
+
def collect_coingecko_bitcoin(store: Store, fetcher: Fetcher, start_dt: datetime, end_dt: datetime, args: argparse.Namespace) -> int:
|
| 918 |
+
source_id = "coingecko"
|
| 919 |
+
dataset_id = "coingecko_bitcoin_7d"
|
| 920 |
+
entity_id = "coin_bitcoin"
|
| 921 |
+
store.add_entity(dataset_id, source_id, entity_id, "asset", "bitcoin", region="global", timezone_name="UTC")
|
| 922 |
+
params = {"vs_currency": "usd", "days": args.days}
|
| 923 |
+
response, raw_id, fetched_at = fetcher.get(source_id, dataset_id, "https://api.coingecko.com/api/v3/coins/bitcoin/market_chart", params=params)
|
| 924 |
+
payload = response.json()
|
| 925 |
+
variable_defs = {
|
| 926 |
+
"price_usd": ("prices", "USD", True),
|
| 927 |
+
"market_cap_usd": ("market_caps", "USD", False),
|
| 928 |
+
"total_volume_usd": ("total_volumes", "USD", False),
|
| 929 |
+
}
|
| 930 |
+
count = 0
|
| 931 |
+
for variable_id, (payload_key, unit, is_target) in variable_defs.items():
|
| 932 |
+
store.add_variable(dataset_id, source_id, variable_id, variable_id, unit, "numeric", "1h", "instantaneous", is_target)
|
| 933 |
+
for ts_ms, value_raw in payload.get(payload_key, []):
|
| 934 |
+
ts = datetime.fromtimestamp(ts_ms / 1000, tz=UTC)
|
| 935 |
+
if ts < start_dt or ts > end_dt:
|
| 936 |
+
continue
|
| 937 |
+
value = parse_float(value_raw)
|
| 938 |
+
if value is None:
|
| 939 |
+
continue
|
| 940 |
+
store.add_observation(dataset_id, source_id, entity_id, variable_id, ts, fetched_at, fetched_at, value, "1h", unit, raw_id=raw_id)
|
| 941 |
+
count += 1
|
| 942 |
+
return count
|
| 943 |
+
|
| 944 |
+
|
| 945 |
+
def collect_wikimedia_pageviews(store: Store, fetcher: Fetcher, start_dt: datetime, end_dt: datetime, args: argparse.Namespace) -> int:
|
| 946 |
+
source_id = "wikimedia_pageviews"
|
| 947 |
+
dataset_id = "wikimedia_time_series_hourly"
|
| 948 |
+
article = "Time_series"
|
| 949 |
+
entity_id = f"enwiki_article_{safe_id(article)}"
|
| 950 |
+
store.add_entity(dataset_id, source_id, entity_id, "page", "en.wikipedia/Time_series", region="global", timezone_name="UTC")
|
| 951 |
+
variable_id = "pageviews"
|
| 952 |
+
store.add_variable(dataset_id, source_id, variable_id, "pageviews", "views", "numeric", "1d", "sum", True)
|
| 953 |
+
start = start_dt.strftime("%Y%m%d00")
|
| 954 |
+
end = end_dt.strftime("%Y%m%d23")
|
| 955 |
+
article_url = quote(article, safe="")
|
| 956 |
+
url = f"https://wikimedia.org/api/rest_v1/metrics/pageviews/per-article/en.wikipedia/all-access/user/{article_url}/daily/{start}/{end}"
|
| 957 |
+
response, raw_id, fetched_at = fetcher.get(source_id, dataset_id, url)
|
| 958 |
+
payload = response.json()
|
| 959 |
+
count = 0
|
| 960 |
+
for item in payload.get("items", []):
|
| 961 |
+
ts = datetime.strptime(item["timestamp"][:10], "%Y%m%d%H").replace(tzinfo=UTC)
|
| 962 |
+
if ts < start_dt or ts > end_dt:
|
| 963 |
+
continue
|
| 964 |
+
value = parse_float(item.get("views"))
|
| 965 |
+
if value is None:
|
| 966 |
+
continue
|
| 967 |
+
store.add_observation(dataset_id, source_id, entity_id, variable_id, ts, fetched_at, fetched_at, value, "1d", "views", raw_id=raw_id)
|
| 968 |
+
count += 1
|
| 969 |
+
return count
|
| 970 |
+
|
| 971 |
+
|
| 972 |
+
def collect_nws_observations(store: Store, fetcher: Fetcher, start_dt: datetime, end_dt: datetime, args: argparse.Namespace) -> int:
|
| 973 |
+
source_id = "nws_observations"
|
| 974 |
+
dataset_id = "nws_ksfo_observations"
|
| 975 |
+
station_id = "KSFO"
|
| 976 |
+
entity_id = f"nws_station_{station_id.lower()}"
|
| 977 |
+
store.add_entity(
|
| 978 |
+
dataset_id,
|
| 979 |
+
source_id,
|
| 980 |
+
entity_id,
|
| 981 |
+
"station",
|
| 982 |
+
"NWS KSFO observations",
|
| 983 |
+
lat=37.619,
|
| 984 |
+
lon=-122.365,
|
| 985 |
+
region="CA",
|
| 986 |
+
country="US",
|
| 987 |
+
timezone_name="UTC",
|
| 988 |
+
metadata={"station": station_id},
|
| 989 |
+
)
|
| 990 |
+
variable_defs = {
|
| 991 |
+
"temperature": ("temperature", "celsius", True),
|
| 992 |
+
"wind_speed": ("windSpeed", "km/h", False),
|
| 993 |
+
"relative_humidity": ("relativeHumidity", "%", False),
|
| 994 |
+
"barometric_pressure": ("barometricPressure", "Pa", False),
|
| 995 |
+
}
|
| 996 |
+
for variable_id, (_, unit, is_target) in variable_defs.items():
|
| 997 |
+
store.add_variable(dataset_id, source_id, variable_id, variable_id, unit, "numeric", "1h", "instantaneous", is_target)
|
| 998 |
+
params = {
|
| 999 |
+
"start": iso(start_dt),
|
| 1000 |
+
"end": iso(end_dt),
|
| 1001 |
+
}
|
| 1002 |
+
url = f"https://api.weather.gov/stations/{station_id}/observations"
|
| 1003 |
+
response, raw_id, fetched_at = fetcher.get(source_id, dataset_id, url, params=params)
|
| 1004 |
+
payload = response.json()
|
| 1005 |
+
count = 0
|
| 1006 |
+
for feature in payload.get("features", []):
|
| 1007 |
+
props = feature.get("properties", {})
|
| 1008 |
+
timestamp = props.get("timestamp")
|
| 1009 |
+
if not timestamp:
|
| 1010 |
+
continue
|
| 1011 |
+
ts = parse_iso_datetime(timestamp)
|
| 1012 |
+
if ts < start_dt or ts > end_dt:
|
| 1013 |
+
continue
|
| 1014 |
+
for variable_id, (property_name, unit, _) in variable_defs.items():
|
| 1015 |
+
raw_value = props.get(property_name, {})
|
| 1016 |
+
if not isinstance(raw_value, dict):
|
| 1017 |
+
continue
|
| 1018 |
+
value = parse_float(raw_value.get("value"))
|
| 1019 |
+
if value is None:
|
| 1020 |
+
continue
|
| 1021 |
+
store.add_observation(dataset_id, source_id, entity_id, variable_id, ts, fetched_at, fetched_at, value, "1h", unit, raw_id=raw_id)
|
| 1022 |
+
count += 1
|
| 1023 |
+
return count
|
| 1024 |
+
|
| 1025 |
+
|
| 1026 |
+
def collect_gdelt_timeline(store: Store, fetcher: Fetcher, start_dt: datetime, end_dt: datetime, args: argparse.Namespace) -> int:
|
| 1027 |
+
source_id = "gdelt"
|
| 1028 |
+
dataset_id = "gdelt_climate_timeline"
|
| 1029 |
+
entity_id = "gdelt_query_climate_change"
|
| 1030 |
+
store.add_entity(dataset_id, source_id, entity_id, "query", "climate change", region="global", timezone_name="UTC")
|
| 1031 |
+
variable_id = "document_volume"
|
| 1032 |
+
store.add_variable(dataset_id, source_id, variable_id, "document_volume", "relative_volume", "numeric", "15min", "mean", True)
|
| 1033 |
+
params = {
|
| 1034 |
+
"query": "climate change",
|
| 1035 |
+
"mode": "timelinevol",
|
| 1036 |
+
"format": "json",
|
| 1037 |
+
"startdatetime": datetime_yyyymmddhhmmss(start_dt),
|
| 1038 |
+
"enddatetime": datetime_yyyymmddhhmmss(end_dt),
|
| 1039 |
+
"timelinesmooth": 0,
|
| 1040 |
+
}
|
| 1041 |
+
try:
|
| 1042 |
+
response, raw_id, fetched_at = fetcher.get(source_id, dataset_id, "https://api.gdeltproject.org/api/v2/doc/doc", params=params)
|
| 1043 |
+
except Exception:
|
| 1044 |
+
response, raw_id, fetched_at = fetcher.get(source_id, dataset_id, "http://api.gdeltproject.org/api/v2/doc/doc", params=params)
|
| 1045 |
+
payload = response.json()
|
| 1046 |
+
candidates: List[Dict[str, Any]] = []
|
| 1047 |
+
for item in payload.get("timeline", []):
|
| 1048 |
+
if isinstance(item, dict) and isinstance(item.get("data"), list):
|
| 1049 |
+
candidates.extend(item["data"])
|
| 1050 |
+
if not candidates and isinstance(payload.get("data"), list):
|
| 1051 |
+
candidates = payload["data"]
|
| 1052 |
+
count = 0
|
| 1053 |
+
for item in candidates:
|
| 1054 |
+
stamp = item.get("date") or item.get("datetime")
|
| 1055 |
+
value_raw = item.get("value") or item.get("norm") or item.get("count")
|
| 1056 |
+
if not stamp:
|
| 1057 |
+
continue
|
| 1058 |
+
try:
|
| 1059 |
+
if "-" in str(stamp):
|
| 1060 |
+
ts = parse_iso_datetime(str(stamp))
|
| 1061 |
+
else:
|
| 1062 |
+
ts = datetime.strptime(str(stamp)[:14], "%Y%m%d%H%M%S").replace(tzinfo=UTC)
|
| 1063 |
+
except Exception:
|
| 1064 |
+
continue
|
| 1065 |
+
if ts < start_dt or ts > end_dt:
|
| 1066 |
+
continue
|
| 1067 |
+
value = parse_float(value_raw)
|
| 1068 |
+
if value is None:
|
| 1069 |
+
continue
|
| 1070 |
+
store.add_observation(dataset_id, source_id, entity_id, variable_id, ts, fetched_at, fetched_at, value, "15min", "relative_volume", raw_id=raw_id)
|
| 1071 |
+
count += 1
|
| 1072 |
+
return count
|
| 1073 |
+
|
| 1074 |
+
|
| 1075 |
+
def collect_usgs_earthquake(store: Store, fetcher: Fetcher, start_dt: datetime, end_dt: datetime, args: argparse.Namespace) -> int:
|
| 1076 |
+
source_id = "usgs_earthquake"
|
| 1077 |
+
dataset_id = "usgs_earthquake_all_week_hourly"
|
| 1078 |
+
entity_id = "earth_global"
|
| 1079 |
+
store.add_entity(dataset_id, source_id, entity_id, "global_region", "Global earthquakes", region="global", timezone_name="UTC")
|
| 1080 |
+
variables = {
|
| 1081 |
+
"earthquake_count": ("events", True),
|
| 1082 |
+
"earthquake_avg_magnitude": ("magnitude", False),
|
| 1083 |
+
"earthquake_max_magnitude": ("magnitude", False),
|
| 1084 |
+
}
|
| 1085 |
+
for variable_id, (unit, is_target) in variables.items():
|
| 1086 |
+
store.add_variable(dataset_id, source_id, variable_id, variable_id, unit, "numeric", "1h", "aggregate", is_target)
|
| 1087 |
+
response, raw_id, fetched_at = fetcher.get(source_id, dataset_id, "https://earthquake.usgs.gov/earthquakes/feed/v1.0/summary/all_week.geojson")
|
| 1088 |
+
payload = response.json()
|
| 1089 |
+
bucket: Dict[datetime, List[float]] = defaultdict(list)
|
| 1090 |
+
for feature in payload.get("features", []):
|
| 1091 |
+
props = feature.get("properties", {})
|
| 1092 |
+
ts_ms = props.get("time")
|
| 1093 |
+
mag = parse_float(props.get("mag"))
|
| 1094 |
+
if ts_ms is None or mag is None:
|
| 1095 |
+
continue
|
| 1096 |
+
ts = floor_hour(datetime.fromtimestamp(ts_ms / 1000, tz=UTC))
|
| 1097 |
+
if ts < start_dt or ts > end_dt:
|
| 1098 |
+
continue
|
| 1099 |
+
bucket[ts].append(mag)
|
| 1100 |
+
count = 0
|
| 1101 |
+
for ts, mags in sorted(bucket.items()):
|
| 1102 |
+
values = {
|
| 1103 |
+
"earthquake_count": float(len(mags)),
|
| 1104 |
+
"earthquake_avg_magnitude": sum(mags) / len(mags),
|
| 1105 |
+
"earthquake_max_magnitude": max(mags),
|
| 1106 |
+
}
|
| 1107 |
+
for variable_id, value in values.items():
|
| 1108 |
+
store.add_observation(dataset_id, source_id, entity_id, variable_id, ts, fetched_at, fetched_at, value, "1h", variables[variable_id][0], raw_id=raw_id)
|
| 1109 |
+
count += 1
|
| 1110 |
+
return count
|
| 1111 |
+
|
| 1112 |
+
|
| 1113 |
+
def collect_noaa_ncei_daily(store: Store, fetcher: Fetcher, start_dt: datetime, end_dt: datetime, args: argparse.Namespace) -> int:
|
| 1114 |
+
source_id = "noaa_ncei"
|
| 1115 |
+
dataset_id = "noaa_ncei_nyc_daily_summaries"
|
| 1116 |
+
station_id = "USW00014732"
|
| 1117 |
+
entity_id = f"ncei_station_{station_id}"
|
| 1118 |
+
store.add_entity(
|
| 1119 |
+
dataset_id,
|
| 1120 |
+
source_id,
|
| 1121 |
+
entity_id,
|
| 1122 |
+
"station",
|
| 1123 |
+
"NCEI daily summaries USW00014732",
|
| 1124 |
+
lat=40.779,
|
| 1125 |
+
lon=-73.880,
|
| 1126 |
+
region="NY",
|
| 1127 |
+
country="US",
|
| 1128 |
+
timezone_name="UTC",
|
| 1129 |
+
metadata={"station": station_id},
|
| 1130 |
+
)
|
| 1131 |
+
params = {
|
| 1132 |
+
"dataset": "daily-summaries",
|
| 1133 |
+
"stations": station_id,
|
| 1134 |
+
"startDate": start_dt.date().isoformat(),
|
| 1135 |
+
"endDate": end_dt.date().isoformat(),
|
| 1136 |
+
"dataTypes": "TAVG,TMAX,TMIN,PRCP",
|
| 1137 |
+
"format": "json",
|
| 1138 |
+
"units": "metric",
|
| 1139 |
+
"includeAttributes": "false",
|
| 1140 |
+
}
|
| 1141 |
+
response, raw_id, fetched_at = fetcher.get(source_id, dataset_id, "https://www.ncei.noaa.gov/access/services/data/v1", params=params)
|
| 1142 |
+
rows = response.json()
|
| 1143 |
+
variable_defs = {
|
| 1144 |
+
"TAVG": ("daily_avg_temperature", "celsius", True),
|
| 1145 |
+
"TMAX": ("daily_max_temperature", "celsius", False),
|
| 1146 |
+
"TMIN": ("daily_min_temperature", "celsius", False),
|
| 1147 |
+
"PRCP": ("daily_precipitation", "mm", False),
|
| 1148 |
+
}
|
| 1149 |
+
for code, (variable_id, unit, is_target) in variable_defs.items():
|
| 1150 |
+
store.add_variable(dataset_id, source_id, variable_id, variable_id, unit, "numeric", "1d", "daily", is_target)
|
| 1151 |
+
count = 0
|
| 1152 |
+
for row in rows:
|
| 1153 |
+
if "DATE" not in row:
|
| 1154 |
+
continue
|
| 1155 |
+
ts = datetime.fromisoformat(row["DATE"]).replace(tzinfo=UTC)
|
| 1156 |
+
for code, (variable_id, unit, _) in variable_defs.items():
|
| 1157 |
+
value = parse_float(row.get(code))
|
| 1158 |
+
if value is None:
|
| 1159 |
+
continue
|
| 1160 |
+
store.add_observation(dataset_id, source_id, entity_id, variable_id, ts, fetched_at, fetched_at, value, "1d", unit, raw_id=raw_id)
|
| 1161 |
+
count += 1
|
| 1162 |
+
return count
|
| 1163 |
+
|
| 1164 |
+
|
| 1165 |
+
COLLECTORS: Dict[str, Callable[[Store, Fetcher, datetime, datetime, argparse.Namespace], int]] = {
|
| 1166 |
+
"open_meteo_weather": collect_open_meteo_weather,
|
| 1167 |
+
"open_meteo_air_quality": collect_open_meteo_air_quality,
|
| 1168 |
+
"nasa_power": collect_nasa_power,
|
| 1169 |
+
"usgs_water": collect_usgs_water,
|
| 1170 |
+
"noaa_coops": collect_noaa_coops,
|
| 1171 |
+
"noaa_ndbc": collect_noaa_ndbc,
|
| 1172 |
+
"gbfs_citibike": collect_gbfs_citibike,
|
| 1173 |
+
"binance": collect_binance_btcusdt,
|
| 1174 |
+
"coingecko": collect_coingecko_bitcoin,
|
| 1175 |
+
"wikimedia_pageviews": collect_wikimedia_pageviews,
|
| 1176 |
+
"nws_observations": collect_nws_observations,
|
| 1177 |
+
"gdelt": collect_gdelt_timeline,
|
| 1178 |
+
"usgs_earthquake": collect_usgs_earthquake,
|
| 1179 |
+
"noaa_ncei": collect_noaa_ncei_daily,
|
| 1180 |
+
}
|
| 1181 |
+
|
| 1182 |
+
DEFAULT_COLLECTORS = [
|
| 1183 |
+
"open_meteo_weather",
|
| 1184 |
+
"open_meteo_air_quality",
|
| 1185 |
+
"nasa_power",
|
| 1186 |
+
"usgs_water",
|
| 1187 |
+
"noaa_coops",
|
| 1188 |
+
"noaa_ndbc",
|
| 1189 |
+
"gbfs_citibike",
|
| 1190 |
+
"binance",
|
| 1191 |
+
"coingecko",
|
| 1192 |
+
"wikimedia_pageviews",
|
| 1193 |
+
"nws_observations",
|
| 1194 |
+
"usgs_earthquake",
|
| 1195 |
+
"noaa_ncei",
|
| 1196 |
+
]
|
| 1197 |
+
|
| 1198 |
+
|
| 1199 |
+
def generate_forecast_tasks(store: Store, created_at: datetime, max_tasks_per_dataset: int = 3) -> None:
|
| 1200 |
+
grouped: Dict[Tuple[str, str, str], List[Dict[str, Any]]] = defaultdict(list)
|
| 1201 |
+
for obs in store.observations:
|
| 1202 |
+
grouped[(obs["dataset_id"], obs["entity_id"], obs["variable_id"])].append(obs)
|
| 1203 |
+
|
| 1204 |
+
variable_lookup = {
|
| 1205 |
+
(row["dataset_id"], row["variable_id"]): row
|
| 1206 |
+
for row in store.variables.values()
|
| 1207 |
+
}
|
| 1208 |
+
dataset_counts: Dict[str, int] = defaultdict(int)
|
| 1209 |
+
|
| 1210 |
+
for (dataset_id, entity_id, variable_id), rows in sorted(grouped.items()):
|
| 1211 |
+
if dataset_counts[dataset_id] >= max_tasks_per_dataset:
|
| 1212 |
+
continue
|
| 1213 |
+
var = variable_lookup.get((dataset_id, variable_id), {})
|
| 1214 |
+
if var.get("is_target_default") != "true":
|
| 1215 |
+
continue
|
| 1216 |
+
rows_sorted = sorted(rows, key=lambda row: row["timestamp"])
|
| 1217 |
+
timestamps = [parse_iso_datetime(row["timestamp"]) for row in rows_sorted]
|
| 1218 |
+
if not timestamps:
|
| 1219 |
+
continue
|
| 1220 |
+
freq = rows_sorted[-1]["frequency"]
|
| 1221 |
+
delta = frequency_delta(freq) or timedelta(hours=1)
|
| 1222 |
+
unique_timestamps = sorted(set(timestamps))
|
| 1223 |
+
if freq == "1d":
|
| 1224 |
+
horizon_length = 2
|
| 1225 |
+
input_length = min(5, max(1, len(unique_timestamps) - horizon_length))
|
| 1226 |
+
elif freq == "snapshot":
|
| 1227 |
+
input_length = 1
|
| 1228 |
+
horizon_length = 1
|
| 1229 |
+
else:
|
| 1230 |
+
input_length = min(96, max(1, len(unique_timestamps)))
|
| 1231 |
+
horizon_length = 24
|
| 1232 |
+
|
| 1233 |
+
if len(unique_timestamps) > horizon_length and freq != "snapshot":
|
| 1234 |
+
context_end = unique_timestamps[-horizon_length - 1]
|
| 1235 |
+
horizon_start = unique_timestamps[-horizon_length]
|
| 1236 |
+
horizon_end = unique_timestamps[-1]
|
| 1237 |
+
context_candidates = [ts for ts in unique_timestamps if ts <= context_end]
|
| 1238 |
+
context_start = context_candidates[max(0, len(context_candidates) - input_length)]
|
| 1239 |
+
input_length = len([ts for ts in context_candidates if ts >= context_start])
|
| 1240 |
+
horizon_length = len([ts for ts in unique_timestamps if horizon_start <= ts <= horizon_end])
|
| 1241 |
+
else:
|
| 1242 |
+
context_end = unique_timestamps[-1]
|
| 1243 |
+
context_start = unique_timestamps[max(0, len(unique_timestamps) - input_length)]
|
| 1244 |
+
horizon_start = context_end + delta
|
| 1245 |
+
horizon_end = horizon_start + delta * (horizon_length - 1)
|
| 1246 |
+
covariates = sorted(
|
| 1247 |
+
row["variable_id"]
|
| 1248 |
+
for (ds, vid), row in variable_lookup.items()
|
| 1249 |
+
if ds == dataset_id and vid != variable_id
|
| 1250 |
+
)
|
| 1251 |
+
|
| 1252 |
+
task_id = safe_id(f"task_{dataset_id}_{entity_id}_{variable_id}_{context_end.strftime('%Y%m%dT%H%M%SZ')}")
|
| 1253 |
+
store.forecast_tasks.append(
|
| 1254 |
+
{
|
| 1255 |
+
"task_id": task_id,
|
| 1256 |
+
"dataset_id": dataset_id,
|
| 1257 |
+
"target_entity_id": entity_id,
|
| 1258 |
+
"target_variable_id": variable_id,
|
| 1259 |
+
"forecast_issue_time": iso(created_at),
|
| 1260 |
+
"context_start": iso(context_start),
|
| 1261 |
+
"context_end": iso(context_end),
|
| 1262 |
+
"horizon_start": iso(horizon_start),
|
| 1263 |
+
"horizon_end": iso(horizon_end),
|
| 1264 |
+
"input_length": input_length,
|
| 1265 |
+
"horizon_length": horizon_length,
|
| 1266 |
+
"frequency": freq,
|
| 1267 |
+
"covariate_variable_ids": covariates,
|
| 1268 |
+
"task_split": "sample_week",
|
| 1269 |
+
"created_at": iso(created_at),
|
| 1270 |
+
}
|
| 1271 |
+
)
|
| 1272 |
+
dataset_counts[dataset_id] += 1
|
| 1273 |
+
|
| 1274 |
+
|
| 1275 |
+
def generate_quality_reports(store: Store, run_started: datetime, run_finished: datetime) -> None:
|
| 1276 |
+
grouped: Dict[Tuple[str, str, str, str], List[Dict[str, Any]]] = defaultdict(list)
|
| 1277 |
+
for obs in store.observations:
|
| 1278 |
+
grouped[(obs["dataset_id"], obs["source_id"], obs["entity_id"], obs["variable_id"])].append(obs)
|
| 1279 |
+
failures_by_dataset: Dict[str, int] = defaultdict(int)
|
| 1280 |
+
total_by_dataset: Dict[str, int] = defaultdict(int)
|
| 1281 |
+
for log in store.run_log:
|
| 1282 |
+
total_by_dataset[log["dataset_id"]] += 1
|
| 1283 |
+
if log["status"] != "ok":
|
| 1284 |
+
failures_by_dataset[log["dataset_id"]] += 1
|
| 1285 |
+
|
| 1286 |
+
for (dataset_id, source_id, entity_id, variable_id), rows in sorted(grouped.items()):
|
| 1287 |
+
timestamps = [parse_iso_datetime(row["timestamp"]) for row in rows]
|
| 1288 |
+
timestamp_strings = [row["timestamp"] for row in rows]
|
| 1289 |
+
duplicate_count = len(timestamp_strings) - len(set(timestamp_strings))
|
| 1290 |
+
latest = max(timestamps)
|
| 1291 |
+
earliest = min(timestamps)
|
| 1292 |
+
values = [parse_float(row["value"]) for row in rows if parse_float(row["value"]) is not None]
|
| 1293 |
+
outlier_count = 0
|
| 1294 |
+
if len(values) >= 5:
|
| 1295 |
+
mean = sum(values) / len(values)
|
| 1296 |
+
variance = sum((value - mean) ** 2 for value in values) / len(values)
|
| 1297 |
+
std = math.sqrt(variance)
|
| 1298 |
+
if std > 0:
|
| 1299 |
+
outlier_count = sum(1 for value in values if abs(value - mean) > 5 * std)
|
| 1300 |
+
freq = rows[-1]["frequency"]
|
| 1301 |
+
delta = frequency_delta(freq)
|
| 1302 |
+
missing_ratio = ""
|
| 1303 |
+
if delta is not None and len(timestamps) > 1:
|
| 1304 |
+
expected = int((latest - earliest) / delta) + 1
|
| 1305 |
+
if expected > 0:
|
| 1306 |
+
missing_ratio = max(0.0, min(1.0, 1.0 - len(set(timestamp_strings)) / expected))
|
| 1307 |
+
api_failure_rate = ""
|
| 1308 |
+
if total_by_dataset.get(dataset_id):
|
| 1309 |
+
api_failure_rate = failures_by_dataset[dataset_id] / total_by_dataset[dataset_id]
|
| 1310 |
+
store.quality_reports.append(
|
| 1311 |
+
{
|
| 1312 |
+
"dataset_id": dataset_id,
|
| 1313 |
+
"source_id": source_id,
|
| 1314 |
+
"entity_id": entity_id,
|
| 1315 |
+
"variable_id": variable_id,
|
| 1316 |
+
"window_start": iso(earliest),
|
| 1317 |
+
"window_end": iso(latest),
|
| 1318 |
+
"observation_count": len(rows),
|
| 1319 |
+
"missing_ratio": missing_ratio,
|
| 1320 |
+
"duplicate_count": duplicate_count,
|
| 1321 |
+
"freshness_delay_hours": round((run_finished - latest).total_seconds() / 3600, 3),
|
| 1322 |
+
"outlier_count": outlier_count,
|
| 1323 |
+
"api_failure_rate": api_failure_rate,
|
| 1324 |
+
"notes": "",
|
| 1325 |
+
}
|
| 1326 |
+
)
|
| 1327 |
+
|
| 1328 |
+
|
| 1329 |
+
def normalize_row(row: Dict[str, Any], fields: List[str]) -> Dict[str, Any]:
|
| 1330 |
+
normalized: Dict[str, Any] = {}
|
| 1331 |
+
for field_name in fields:
|
| 1332 |
+
value = row.get(field_name, "")
|
| 1333 |
+
if isinstance(value, (dict, list, tuple)):
|
| 1334 |
+
normalized[field_name] = dumps(value)
|
| 1335 |
+
else:
|
| 1336 |
+
normalized[field_name] = value
|
| 1337 |
+
return normalized
|
| 1338 |
+
|
| 1339 |
+
|
| 1340 |
+
def write_csv(path: Path, fields: List[str], rows: Iterable[Dict[str, Any]]) -> None:
|
| 1341 |
+
path.parent.mkdir(parents=True, exist_ok=True)
|
| 1342 |
+
with path.open("w", newline="", encoding="utf-8") as handle:
|
| 1343 |
+
writer = csv.DictWriter(handle, fieldnames=fields)
|
| 1344 |
+
writer.writeheader()
|
| 1345 |
+
for row in rows:
|
| 1346 |
+
writer.writerow(normalize_row(row, fields))
|
| 1347 |
+
|
| 1348 |
+
|
| 1349 |
+
def write_outputs(store: Store) -> None:
|
| 1350 |
+
normalized_dir = store.output_dir / "normalized"
|
| 1351 |
+
write_csv(normalized_dir / "raw_snapshots.csv", RAW_SNAPSHOT_FIELDS, store.raw_snapshots)
|
| 1352 |
+
write_csv(normalized_dir / "entities.csv", ENTITY_FIELDS, store.entities.values())
|
| 1353 |
+
write_csv(normalized_dir / "variables.csv", VARIABLE_FIELDS, store.variables.values())
|
| 1354 |
+
write_csv(normalized_dir / "observations.csv", OBSERVATION_FIELDS, store.observations)
|
| 1355 |
+
write_csv(normalized_dir / "forecast_tasks.csv", FORECAST_TASK_FIELDS, store.forecast_tasks)
|
| 1356 |
+
write_csv(normalized_dir / "quality_reports.csv", QUALITY_REPORT_FIELDS, store.quality_reports)
|
| 1357 |
+
write_csv(normalized_dir / "run_log.csv", RUN_LOG_FIELDS, store.run_log)
|
| 1358 |
+
|
| 1359 |
+
summary = {
|
| 1360 |
+
"generated_at": iso(utc_now()),
|
| 1361 |
+
"output_dir": str(store.output_dir),
|
| 1362 |
+
"raw_snapshots": len(store.raw_snapshots),
|
| 1363 |
+
"entities": len(store.entities),
|
| 1364 |
+
"variables": len(store.variables),
|
| 1365 |
+
"observations": len(store.observations),
|
| 1366 |
+
"forecast_tasks": len(store.forecast_tasks),
|
| 1367 |
+
"quality_reports": len(store.quality_reports),
|
| 1368 |
+
"run_log": store.run_log,
|
| 1369 |
+
"notes": [
|
| 1370 |
+
"CSV files are normalized storage artifacts for this minimal prototype.",
|
| 1371 |
+
"For production, keep the same schema but prefer Parquet, DuckDB, or PostgreSQL.",
|
| 1372 |
+
"Model-specific wide CSV files should be exported from observations + forecast_tasks.",
|
| 1373 |
+
],
|
| 1374 |
+
}
|
| 1375 |
+
(store.output_dir / "summary.json").write_text(json.dumps(summary, ensure_ascii=False, indent=2), encoding="utf-8")
|
| 1376 |
+
|
| 1377 |
+
|
| 1378 |
+
def parse_args(argv: List[str]) -> argparse.Namespace:
|
| 1379 |
+
parser = argparse.ArgumentParser(description="Collect and normalize a one-week TS-Bench data sample.")
|
| 1380 |
+
parser.add_argument("--days", type=int, default=7, help="Number of trailing days to collect.")
|
| 1381 |
+
parser.add_argument("--output", default="data/sample_week", help="Output directory.")
|
| 1382 |
+
parser.add_argument("--timeout", type=int, default=30, help="HTTP timeout in seconds.")
|
| 1383 |
+
parser.add_argument("--limit-entities", type=int, default=25, help="Entity cap for high-cardinality snapshot sources such as GBFS.")
|
| 1384 |
+
parser.add_argument("--max-tasks-per-dataset", type=int, default=3, help="Maximum forecast tasks generated per dataset.")
|
| 1385 |
+
parser.add_argument("--only", nargs="*", choices=sorted(COLLECTORS.keys()), help="Run only selected collectors.")
|
| 1386 |
+
parser.add_argument("--list", action="store_true", help="List collectors and exit.")
|
| 1387 |
+
parser.add_argument("--strict", action="store_true", help="Exit non-zero if any collector fails.")
|
| 1388 |
+
return parser.parse_args(argv)
|
| 1389 |
+
|
| 1390 |
+
|
| 1391 |
+
def main(argv: List[str]) -> int:
|
| 1392 |
+
args = parse_args(argv)
|
| 1393 |
+
if args.list:
|
| 1394 |
+
for name in sorted(COLLECTORS):
|
| 1395 |
+
print(name)
|
| 1396 |
+
return 0
|
| 1397 |
+
|
| 1398 |
+
output_dir = Path(args.output)
|
| 1399 |
+
output_dir.mkdir(parents=True, exist_ok=True)
|
| 1400 |
+
store = Store(output_dir=output_dir)
|
| 1401 |
+
fetcher = Fetcher(store=store, timeout=args.timeout)
|
| 1402 |
+
run_started = utc_now()
|
| 1403 |
+
end_dt = run_started
|
| 1404 |
+
start_dt = end_dt - timedelta(days=args.days)
|
| 1405 |
+
|
| 1406 |
+
selected = args.only or DEFAULT_COLLECTORS
|
| 1407 |
+
any_failed = False
|
| 1408 |
+
for name in selected:
|
| 1409 |
+
collector = COLLECTORS[name]
|
| 1410 |
+
started_at = utc_now()
|
| 1411 |
+
before = len(store.observations)
|
| 1412 |
+
dataset_id = ""
|
| 1413 |
+
try:
|
| 1414 |
+
added = collector(store, fetcher, start_dt, end_dt, args)
|
| 1415 |
+
status = "ok"
|
| 1416 |
+
error = ""
|
| 1417 |
+
after = len(store.observations)
|
| 1418 |
+
observation_count = after - before if added is None else added
|
| 1419 |
+
matching = [row for row in store.observations[before:after]]
|
| 1420 |
+
dataset_id = matching[0]["dataset_id"] if matching else name
|
| 1421 |
+
except Exception as exc: # noqa: BLE001
|
| 1422 |
+
any_failed = True
|
| 1423 |
+
status = "failed"
|
| 1424 |
+
error = str(exc)
|
| 1425 |
+
observation_count = 0
|
| 1426 |
+
dataset_id = name
|
| 1427 |
+
finished_at = utc_now()
|
| 1428 |
+
store.run_log.append(
|
| 1429 |
+
{
|
| 1430 |
+
"source_id": name,
|
| 1431 |
+
"dataset_id": dataset_id,
|
| 1432 |
+
"status": status,
|
| 1433 |
+
"started_at": iso(started_at),
|
| 1434 |
+
"finished_at": iso(finished_at),
|
| 1435 |
+
"observation_count": observation_count,
|
| 1436 |
+
"error": error,
|
| 1437 |
+
}
|
| 1438 |
+
)
|
| 1439 |
+
print(f"{name:24s} {status:7s} observations={observation_count} {error[:100]}")
|
| 1440 |
+
|
| 1441 |
+
run_finished = utc_now()
|
| 1442 |
+
generate_forecast_tasks(store, created_at=run_finished, max_tasks_per_dataset=args.max_tasks_per_dataset)
|
| 1443 |
+
generate_quality_reports(store, run_started, run_finished)
|
| 1444 |
+
write_outputs(store)
|
| 1445 |
+
print()
|
| 1446 |
+
print(f"Output: {output_dir}")
|
| 1447 |
+
print(f"Raw snapshots: {len(store.raw_snapshots)}")
|
| 1448 |
+
print(f"Entities: {len(store.entities)}")
|
| 1449 |
+
print(f"Variables: {len(store.variables)}")
|
| 1450 |
+
print(f"Observations: {len(store.observations)}")
|
| 1451 |
+
print(f"Forecast tasks: {len(store.forecast_tasks)}")
|
| 1452 |
+
print(f"Quality reports: {len(store.quality_reports)}")
|
| 1453 |
+
return 1 if (args.strict and any_failed) else 0
|
| 1454 |
+
|
| 1455 |
+
|
| 1456 |
+
if __name__ == "__main__":
|
| 1457 |
+
raise SystemExit(main(sys.argv[1:]))
|
vendor/ts_bench/data/sample_week/model_inputs_latest/manifest.csv
ADDED
|
@@ -0,0 +1,8 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
task_id,dataset_id,entity_id,target_variable_id,context_rows,future_target_rows,context_csv,future_target_csv
|
| 2 |
+
task_binance_btcusdt_1h_crypto_btcusdt_close_20260528t020000z,binance_btcusdt_1h,crypto_btcusdt,close,96,24,data/sample_week/model_inputs_latest/task_binance_btcusdt_1h_crypto_btcusdt_close_20260528t020000z/context.csv,data/sample_week/model_inputs_latest/task_binance_btcusdt_1h_crypto_btcusdt_close_20260528t020000z/future_target.csv
|
| 3 |
+
task_coingecko_bitcoin_7d_coin_bitcoin_price_usd_20260528t030012z,coingecko_bitcoin_7d,coin_bitcoin,price_usd,96,24,data/sample_week/model_inputs_latest/task_coingecko_bitcoin_7d_coin_bitcoin_price_usd_20260528t030012z/context.csv,data/sample_week/model_inputs_latest/task_coingecko_bitcoin_7d_coin_bitcoin_price_usd_20260528t030012z/future_target.csv
|
| 4 |
+
task_nasa_power_shanghai_hourly_point_shanghai_31.2304_121.4737_t2m_20260524t230000z,nasa_power_shanghai_hourly,point_shanghai_31.2304_121.4737,T2M,69,24,data/sample_week/model_inputs_latest/task_nasa_power_shanghai_hourly_point_shanghai_31.2304_121.4737_t2m_20260524t230000z/context.csv,data/sample_week/model_inputs_latest/task_nasa_power_shanghai_hourly_point_shanghai_31.2304_121.4737_t2m_20260524t230000z/future_target.csv
|
| 5 |
+
task_noaa_coops_sf_water_level_coops_station_9414290_water_level_20260529t001800z,noaa_coops_sf_water_level,coops_station_9414290,water_level,96,24,data/sample_week/model_inputs_latest/task_noaa_coops_sf_water_level_coops_station_9414290_water_level_20260529t001800z/context.csv,data/sample_week/model_inputs_latest/task_noaa_coops_sf_water_level_coops_station_9414290_water_level_20260529t001800z/future_target.csv
|
| 6 |
+
task_nws_ksfo_observations_nws_station_ksfo_temperature_20260529t004500z,nws_ksfo_observations,nws_station_ksfo,temperature,96,24,data/sample_week/model_inputs_latest/task_nws_ksfo_observations_nws_station_ksfo_temperature_20260529t004500z/context.csv,data/sample_week/model_inputs_latest/task_nws_ksfo_observations_nws_station_ksfo_temperature_20260529t004500z/future_target.csv
|
| 7 |
+
task_open_meteo_shanghai_air_quality_hourly_grid_shanghai_31.2304_121.4737_pm2_5_20260528t020000z,open_meteo_shanghai_air_quality_hourly,grid_shanghai_31.2304_121.4737,pm2_5,96,24,data/sample_week/model_inputs_latest/task_open_meteo_shanghai_air_quality_hourly_grid_shanghai_31.2304_121.4737_pm2_5_20260528t020000z/context.csv,data/sample_week/model_inputs_latest/task_open_meteo_shanghai_air_quality_hourly_grid_shanghai_31.2304_121.4737_pm2_5_20260528t020000z/future_target.csv
|
| 8 |
+
task_usgs_potomac_iv_usgs_site_01646500_usgs_00060_20260528t235000z,usgs_potomac_iv,usgs_site_01646500,usgs_00060,96,24,data/sample_week/model_inputs_latest/task_usgs_potomac_iv_usgs_site_01646500_usgs_00060_20260528t235000z/context.csv,data/sample_week/model_inputs_latest/task_usgs_potomac_iv_usgs_site_01646500_usgs_00060_20260528t235000z/future_target.csv
|
vendor/ts_bench/data/sample_week/model_inputs_latest/manifest.json
ADDED
|
@@ -0,0 +1,72 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
[
|
| 2 |
+
{
|
| 3 |
+
"task_id": "task_binance_btcusdt_1h_crypto_btcusdt_close_20260528t020000z",
|
| 4 |
+
"dataset_id": "binance_btcusdt_1h",
|
| 5 |
+
"entity_id": "crypto_btcusdt",
|
| 6 |
+
"target_variable_id": "close",
|
| 7 |
+
"context_rows": 96,
|
| 8 |
+
"future_target_rows": 24,
|
| 9 |
+
"context_csv": "data/sample_week/model_inputs_latest/task_binance_btcusdt_1h_crypto_btcusdt_close_20260528t020000z/context.csv",
|
| 10 |
+
"future_target_csv": "data/sample_week/model_inputs_latest/task_binance_btcusdt_1h_crypto_btcusdt_close_20260528t020000z/future_target.csv"
|
| 11 |
+
},
|
| 12 |
+
{
|
| 13 |
+
"task_id": "task_coingecko_bitcoin_7d_coin_bitcoin_price_usd_20260528t030012z",
|
| 14 |
+
"dataset_id": "coingecko_bitcoin_7d",
|
| 15 |
+
"entity_id": "coin_bitcoin",
|
| 16 |
+
"target_variable_id": "price_usd",
|
| 17 |
+
"context_rows": 96,
|
| 18 |
+
"future_target_rows": 24,
|
| 19 |
+
"context_csv": "data/sample_week/model_inputs_latest/task_coingecko_bitcoin_7d_coin_bitcoin_price_usd_20260528t030012z/context.csv",
|
| 20 |
+
"future_target_csv": "data/sample_week/model_inputs_latest/task_coingecko_bitcoin_7d_coin_bitcoin_price_usd_20260528t030012z/future_target.csv"
|
| 21 |
+
},
|
| 22 |
+
{
|
| 23 |
+
"task_id": "task_nasa_power_shanghai_hourly_point_shanghai_31.2304_121.4737_t2m_20260524t230000z",
|
| 24 |
+
"dataset_id": "nasa_power_shanghai_hourly",
|
| 25 |
+
"entity_id": "point_shanghai_31.2304_121.4737",
|
| 26 |
+
"target_variable_id": "T2M",
|
| 27 |
+
"context_rows": 69,
|
| 28 |
+
"future_target_rows": 24,
|
| 29 |
+
"context_csv": "data/sample_week/model_inputs_latest/task_nasa_power_shanghai_hourly_point_shanghai_31.2304_121.4737_t2m_20260524t230000z/context.csv",
|
| 30 |
+
"future_target_csv": "data/sample_week/model_inputs_latest/task_nasa_power_shanghai_hourly_point_shanghai_31.2304_121.4737_t2m_20260524t230000z/future_target.csv"
|
| 31 |
+
},
|
| 32 |
+
{
|
| 33 |
+
"task_id": "task_noaa_coops_sf_water_level_coops_station_9414290_water_level_20260529t001800z",
|
| 34 |
+
"dataset_id": "noaa_coops_sf_water_level",
|
| 35 |
+
"entity_id": "coops_station_9414290",
|
| 36 |
+
"target_variable_id": "water_level",
|
| 37 |
+
"context_rows": 96,
|
| 38 |
+
"future_target_rows": 24,
|
| 39 |
+
"context_csv": "data/sample_week/model_inputs_latest/task_noaa_coops_sf_water_level_coops_station_9414290_water_level_20260529t001800z/context.csv",
|
| 40 |
+
"future_target_csv": "data/sample_week/model_inputs_latest/task_noaa_coops_sf_water_level_coops_station_9414290_water_level_20260529t001800z/future_target.csv"
|
| 41 |
+
},
|
| 42 |
+
{
|
| 43 |
+
"task_id": "task_nws_ksfo_observations_nws_station_ksfo_temperature_20260529t004500z",
|
| 44 |
+
"dataset_id": "nws_ksfo_observations",
|
| 45 |
+
"entity_id": "nws_station_ksfo",
|
| 46 |
+
"target_variable_id": "temperature",
|
| 47 |
+
"context_rows": 96,
|
| 48 |
+
"future_target_rows": 24,
|
| 49 |
+
"context_csv": "data/sample_week/model_inputs_latest/task_nws_ksfo_observations_nws_station_ksfo_temperature_20260529t004500z/context.csv",
|
| 50 |
+
"future_target_csv": "data/sample_week/model_inputs_latest/task_nws_ksfo_observations_nws_station_ksfo_temperature_20260529t004500z/future_target.csv"
|
| 51 |
+
},
|
| 52 |
+
{
|
| 53 |
+
"task_id": "task_open_meteo_shanghai_air_quality_hourly_grid_shanghai_31.2304_121.4737_pm2_5_20260528t020000z",
|
| 54 |
+
"dataset_id": "open_meteo_shanghai_air_quality_hourly",
|
| 55 |
+
"entity_id": "grid_shanghai_31.2304_121.4737",
|
| 56 |
+
"target_variable_id": "pm2_5",
|
| 57 |
+
"context_rows": 96,
|
| 58 |
+
"future_target_rows": 24,
|
| 59 |
+
"context_csv": "data/sample_week/model_inputs_latest/task_open_meteo_shanghai_air_quality_hourly_grid_shanghai_31.2304_121.4737_pm2_5_20260528t020000z/context.csv",
|
| 60 |
+
"future_target_csv": "data/sample_week/model_inputs_latest/task_open_meteo_shanghai_air_quality_hourly_grid_shanghai_31.2304_121.4737_pm2_5_20260528t020000z/future_target.csv"
|
| 61 |
+
},
|
| 62 |
+
{
|
| 63 |
+
"task_id": "task_usgs_potomac_iv_usgs_site_01646500_usgs_00060_20260528t235000z",
|
| 64 |
+
"dataset_id": "usgs_potomac_iv",
|
| 65 |
+
"entity_id": "usgs_site_01646500",
|
| 66 |
+
"target_variable_id": "usgs_00060",
|
| 67 |
+
"context_rows": 96,
|
| 68 |
+
"future_target_rows": 24,
|
| 69 |
+
"context_csv": "data/sample_week/model_inputs_latest/task_usgs_potomac_iv_usgs_site_01646500_usgs_00060_20260528t235000z/context.csv",
|
| 70 |
+
"future_target_csv": "data/sample_week/model_inputs_latest/task_usgs_potomac_iv_usgs_site_01646500_usgs_00060_20260528t235000z/future_target.csv"
|
| 71 |
+
}
|
| 72 |
+
]
|
vendor/ts_bench/data/sample_week/model_inputs_latest/task_binance_btcusdt_1h_crypto_btcusdt_close_20260527t230000z/context.csv
ADDED
|
@@ -0,0 +1,97 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
timestamp,close,high,low,open,volume
|
| 2 |
+
2026-05-24T00:00:00Z,76835.16,76930.65,76629.02,76752.0,392.54327
|
| 3 |
+
2026-05-24T01:00:00Z,76890.04,76921.45,76754.53,76835.16,225.24128
|
| 4 |
+
2026-05-24T02:00:00Z,76830.0,76968.04,76704.32,76890.04,185.87406
|
| 5 |
+
2026-05-24T03:00:00Z,76750.0,77011.62,76738.5,76830.01,248.04543
|
| 6 |
+
2026-05-24T04:00:00Z,76786.5,76858.26,76730.0,76750.01,163.64327
|
| 7 |
+
2026-05-24T05:00:00Z,76822.23,76861.78,76651.59,76786.5,202.16815
|
| 8 |
+
2026-05-24T06:00:00Z,76987.58,77065.94,76716.02,76822.23,257.8917
|
| 9 |
+
2026-05-24T07:00:00Z,76799.99,77014.24,76786.2,76987.58,193.96514
|
| 10 |
+
2026-05-24T08:00:00Z,76830.69,76861.77,76734.42,76799.99,226.94949
|
| 11 |
+
2026-05-24T09:00:00Z,77020.53,77046.67,76821.86,76830.69,369.92973
|
| 12 |
+
2026-05-24T10:00:00Z,77190.74,77543.15,76927.04,77020.54,863.69953
|
| 13 |
+
2026-05-24T11:00:00Z,77320.06,77356.62,77190.73,77190.73,285.66801
|
| 14 |
+
2026-05-24T12:00:00Z,77118.73,77325.72,77044.73,77320.06,271.94108
|
| 15 |
+
2026-05-24T13:00:00Z,76802.23,77127.54,76688.89,77118.72,474.39799
|
| 16 |
+
2026-05-24T14:00:00Z,76465.16,76840.01,76195.36,76802.23,837.87651
|
| 17 |
+
2026-05-24T15:00:00Z,76371.59,76539.38,76362.78,76465.15,398.04105
|
| 18 |
+
2026-05-24T16:00:00Z,76671.7,76734.3,76371.59,76371.59,359.46109
|
| 19 |
+
2026-05-24T17:00:00Z,76693.34,76799.99,76566.15,76671.69,249.31214
|
| 20 |
+
2026-05-24T18:00:00Z,76665.73,76788.01,76596.0,76693.34,148.14196
|
| 21 |
+
2026-05-24T19:00:00Z,76773.34,76783.51,76611.17,76665.72,125.12918
|
| 22 |
+
2026-05-24T20:00:00Z,76668.4,76796.18,76596.37,76773.33,560.38206
|
| 23 |
+
2026-05-24T21:00:00Z,76151.14,76748.12,76108.0,76668.41,497.80071
|
| 24 |
+
2026-05-24T22:00:00Z,76803.99,77012.01,76132.0,76151.14,543.11534
|
| 25 |
+
2026-05-24T23:00:00Z,77064.96,77225.97,76803.99,76803.99,316.78339
|
| 26 |
+
2026-05-25T00:00:00Z,77153.01,77288.0,77064.95,77064.96,281.30951
|
| 27 |
+
2026-05-25T01:00:00Z,76945.27,77245.67,76914.25,77153.01,574.42489
|
| 28 |
+
2026-05-25T02:00:00Z,77108.77,77485.0,76945.28,76945.28,313.96478
|
| 29 |
+
2026-05-25T03:00:00Z,77026.67,77169.88,77002.3,77108.77,198.70522
|
| 30 |
+
2026-05-25T04:00:00Z,77391.07,77450.0,76990.78,77026.68,251.77406
|
| 31 |
+
2026-05-25T05:00:00Z,77407.0,77498.0,77324.0,77391.07,306.77798
|
| 32 |
+
2026-05-25T06:00:00Z,77419.87,77650.0,77222.22,77406.99,413.85927
|
| 33 |
+
2026-05-25T07:00:00Z,77401.12,77520.0,77184.62,77419.87,376.11814
|
| 34 |
+
2026-05-25T08:00:00Z,77456.94,77469.64,77280.32,77401.11,352.6298
|
| 35 |
+
2026-05-25T09:00:00Z,77604.56,77699.97,77425.17,77456.94,369.10396
|
| 36 |
+
2026-05-25T10:00:00Z,77545.65,77681.37,77490.37,77604.56,283.53522
|
| 37 |
+
2026-05-25T11:00:00Z,77357.5,77545.65,77333.33,77545.65,209.17805
|
| 38 |
+
2026-05-25T12:00:00Z,77367.98,77440.46,77286.5,77357.5,261.28583
|
| 39 |
+
2026-05-25T13:00:00Z,77341.05,77450.0,77286.5,77367.98,199.52985
|
| 40 |
+
2026-05-25T14:00:00Z,77714.0,77773.69,77304.84,77341.05,603.07815
|
| 41 |
+
2026-05-25T15:00:00Z,77628.82,77905.52,77557.0,77713.99,499.50313
|
| 42 |
+
2026-05-25T16:00:00Z,77735.69,77807.29,77508.88,77628.82,342.64397
|
| 43 |
+
2026-05-25T17:00:00Z,77564.01,77804.48,77550.39,77735.69,278.21782
|
| 44 |
+
2026-05-25T18:00:00Z,77543.39,77658.01,77543.38,77564.0,204.7646
|
| 45 |
+
2026-05-25T19:00:00Z,77441.86,77543.39,77343.61,77543.39,295.53937
|
| 46 |
+
2026-05-25T20:00:00Z,77273.06,77500.0,77220.0,77441.85,314.11527
|
| 47 |
+
2026-05-25T21:00:00Z,77367.03,77396.08,77273.05,77273.06,183.92616
|
| 48 |
+
2026-05-25T22:00:00Z,77116.21,77449.94,77019.82,77367.03,395.72227
|
| 49 |
+
2026-05-25T23:00:00Z,77322.01,77367.44,77072.36,77116.21,162.61467
|
| 50 |
+
2026-05-26T00:00:00Z,76792.71,77345.0,76575.49,77322.01,686.17714
|
| 51 |
+
2026-05-26T01:00:00Z,76756.98,76888.01,76590.9,76792.24,342.05131
|
| 52 |
+
2026-05-26T02:00:00Z,76660.0,76810.0,76512.98,76756.98,721.43857
|
| 53 |
+
2026-05-26T03:00:00Z,76664.01,76705.71,76475.59,76660.0,396.14567
|
| 54 |
+
2026-05-26T04:00:00Z,76749.5,76817.28,76606.79,76664.0,392.59093
|
| 55 |
+
2026-05-26T05:00:00Z,76930.01,76989.23,76704.8,76749.51,556.0432
|
| 56 |
+
2026-05-26T06:00:00Z,76908.01,77059.74,76775.0,76930.0,658.30833
|
| 57 |
+
2026-05-26T07:00:00Z,76730.84,76924.55,76603.73,76908.0,436.26184
|
| 58 |
+
2026-05-26T08:00:00Z,76726.37,76852.01,76693.21,76730.84,246.06951
|
| 59 |
+
2026-05-26T09:00:00Z,76672.19,76804.0,76649.79,76726.37,350.5237
|
| 60 |
+
2026-05-26T10:00:00Z,77430.0,77621.75,76586.94,76672.2,897.13516
|
| 61 |
+
2026-05-26T11:00:00Z,77183.55,77516.93,77174.94,77430.0,403.26795
|
| 62 |
+
2026-05-26T12:00:00Z,77132.5,77232.52,76877.94,77183.56,402.68245
|
| 63 |
+
2026-05-26T13:00:00Z,77231.56,77464.01,76691.88,77132.51,659.9668
|
| 64 |
+
2026-05-26T14:00:00Z,76908.01,78080.0,76724.63,77231.56,2020.17697
|
| 65 |
+
2026-05-26T15:00:00Z,76518.1,77012.0,76272.63,76908.01,1751.25838
|
| 66 |
+
2026-05-26T16:00:00Z,76558.0,76710.01,76209.58,76518.1,2070.02098
|
| 67 |
+
2026-05-26T17:00:00Z,75935.08,76649.91,75850.0,76558.0,1190.65802
|
| 68 |
+
2026-05-26T18:00:00Z,75840.55,76273.66,75779.3,75935.09,846.30902
|
| 69 |
+
2026-05-26T19:00:00Z,76052.0,76104.49,75805.23,75840.55,581.71052
|
| 70 |
+
2026-05-26T20:00:00Z,76106.86,76180.0,75982.64,76052.0,324.41575
|
| 71 |
+
2026-05-26T21:00:00Z,75890.0,76117.31,75850.0,76106.86,275.58323
|
| 72 |
+
2026-05-26T22:00:00Z,75749.02,75998.94,75677.97,75890.0,428.12169
|
| 73 |
+
2026-05-26T23:00:00Z,75930.01,75986.45,75749.02,75749.02,316.55917
|
| 74 |
+
2026-05-27T00:00:00Z,76085.01,76089.7,75872.71,75930.01,780.41682
|
| 75 |
+
2026-05-27T01:00:00Z,75877.44,76104.02,75824.0,76085.0,573.72878
|
| 76 |
+
2026-05-27T02:00:00Z,75879.0,75890.59,75682.99,75877.44,437.25803
|
| 77 |
+
2026-05-27T03:00:00Z,75683.0,75966.91,75607.62,75879.01,612.84746
|
| 78 |
+
2026-05-27T04:00:00Z,75579.01,75706.77,75310.9,75683.0,817.05359
|
| 79 |
+
2026-05-27T05:00:00Z,75697.99,75739.36,75579.0,75579.01,490.56114
|
| 80 |
+
2026-05-27T06:00:00Z,75838.65,75900.0,75690.01,75697.99,482.86494
|
| 81 |
+
2026-05-27T07:00:00Z,75977.17,75980.0,75764.0,75838.65,433.92885
|
| 82 |
+
2026-05-27T08:00:00Z,75873.24,76071.41,75843.02,75977.16,308.77238
|
| 83 |
+
2026-05-27T09:00:00Z,75912.6,75978.01,75710.76,75873.25,294.26595
|
| 84 |
+
2026-05-27T10:00:00Z,75909.82,76043.94,75880.0,75912.61,372.61961
|
| 85 |
+
2026-05-27T11:00:00Z,75769.18,76029.24,75754.1,75909.82,316.01081
|
| 86 |
+
2026-05-27T12:00:00Z,75688.0,76174.15,75575.75,75769.18,621.24106
|
| 87 |
+
2026-05-27T13:00:00Z,75073.4,75760.0,74866.18,75688.01,2200.23684
|
| 88 |
+
2026-05-27T14:00:00Z,75031.92,75274.11,74758.33,75073.4,1567.20208
|
| 89 |
+
2026-05-27T15:00:00Z,75326.01,75502.29,74983.77,75031.92,797.5234
|
| 90 |
+
2026-05-27T16:00:00Z,75304.01,75408.01,75000.0,75326.02,632.49097
|
| 91 |
+
2026-05-27T17:00:00Z,74860.42,75344.01,74685.58,75304.01,1103.99219
|
| 92 |
+
2026-05-27T18:00:00Z,75026.67,75032.01,74663.45,74860.42,549.25309
|
| 93 |
+
2026-05-27T19:00:00Z,75066.08,75105.17,74952.36,75026.67,879.80197
|
| 94 |
+
2026-05-27T20:00:00Z,75274.58,75415.81,75056.25,75066.09,389.41476
|
| 95 |
+
2026-05-27T21:00:00Z,74473.01,75274.58,74341.76,75274.58,653.21971
|
| 96 |
+
2026-05-27T22:00:00Z,74418.13,74775.5,74260.91,74473.01,624.64107
|
| 97 |
+
2026-05-27T23:00:00Z,74449.3,74572.0,74243.99,74418.12,938.42933
|