zhouziyu02 Cursor commited on
Commit
5063745
·
1 Parent(s): 1d5b407

Deploy live 1h benchmark: scheduler, tsfm_bench vendor, latest results.

Browse files

Space auto-collects TS-Bench data, runs TSFM.ai eval hourly, and reloads leaderboard.

Co-authored-by: Cursor <cursoragent@cursor.com>

This view is limited to 50 files because it contains too many changes.   See raw diff
Files changed (50) hide show
  1. README.md +18 -2
  2. app.py +29 -12
  3. configs/datasets/ts_bench.yaml +29 -0
  4. configs/models/online_tsfm.yaml +26 -0
  5. requirements.txt +8 -3
  6. results/chronos_2/all_results.csv +7 -10
  7. results/chronos_2/config.json +1 -1
  8. results/chronos_2/online_meta.json +15 -33
  9. results/chronos_bolt_base/all_results.csv +7 -10
  10. results/chronos_bolt_base/config.json +1 -1
  11. results/chronos_bolt_base/online_meta.json +15 -33
  12. results/chronos_bolt_mini/all_results.csv +7 -10
  13. results/chronos_bolt_mini/config.json +1 -1
  14. results/chronos_bolt_mini/online_meta.json +15 -33
  15. results/chronos_bolt_tiny/all_results.csv +7 -10
  16. results/chronos_bolt_tiny/config.json +1 -1
  17. results/chronos_bolt_tiny/online_meta.json +15 -33
  18. results/dataset_properties.csv +0 -3
  19. results/eval_state.json +10 -0
  20. results/moirai_1_1_small/all_results.csv +7 -10
  21. results/moirai_1_1_small/config.json +1 -1
  22. results/moirai_1_1_small/online_meta.json +15 -33
  23. results/online_status.json +94 -203
  24. results/timesfm_2_5/all_results.csv +7 -10
  25. results/timesfm_2_5/config.json +1 -1
  26. results/timesfm_2_5/online_meta.json +15 -33
  27. scripts/run_online_daemon.py +63 -0
  28. scripts/run_space_eval.py +281 -0
  29. src/about.py +33 -21
  30. src/benchmark_config.py +19 -0
  31. src/display/css_html_js.py +2 -2
  32. src/eval_scheduler.py +123 -0
  33. tsfm_bench/__about__.py +1 -0
  34. tsfm_bench/__init__.py +1 -0
  35. tsfm_bench/data/__init__.py +18 -0
  36. tsfm_bench/data/dataset.py +197 -0
  37. tsfm_bench/data/interface.py +46 -0
  38. tsfm_bench/data/open_meteo.py +194 -0
  39. tsfm_bench/data/registry.py +76 -0
  40. tsfm_bench/data/synthetic.py +151 -0
  41. tsfm_bench/data/ts_bench.py +304 -0
  42. tsfm_bench/eval/__init__.py +1 -0
  43. tsfm_bench/eval/api_predictor.py +166 -0
  44. tsfm_bench/eval/metrics.py +50 -0
  45. tsfm_bench/eval/online_eval.py +154 -0
  46. tsfm_bench/eval/predictors.py +85 -0
  47. vendor/ts_bench/collect_weekly_data.py +1457 -0
  48. vendor/ts_bench/data/sample_week/model_inputs_latest/manifest.csv +8 -0
  49. vendor/ts_bench/data/sample_week/model_inputs_latest/manifest.json +72 -0
  50. vendor/ts_bench/data/sample_week/model_inputs_latest/task_binance_btcusdt_1h_crypto_btcusdt_close_20260527t230000z/context.csv +97 -0
README.md CHANGED
@@ -13,6 +13,22 @@ python_version: 3.11.9
13
 
14
  # TSFM Realworld Bench Leaderboard
15
 
16
- Leaderboard for the **TSFM Realworld Benchmark** — evaluating time series foundation models on real-world multivariate forecasting tasks.
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
17
 
18
- Results are submitted via pull request to the [TSFM_Bench](https://github.com/zhouziyu02/TSFM_Bench) repository.
 
13
 
14
  # TSFM Realworld Bench Leaderboard
15
 
16
+ Live benchmark: every **1 hour** the Space collects TS-Bench data, runs all TSFM models via TSFM.ai, and updates the leaderboard.
17
+
18
+ ## Enable live evaluation (required)
19
+
20
+ In **Settings → Repository secrets**, add:
21
+
22
+ | Secret | Value |
23
+ |---|---|
24
+ | `TSFM_API_KEY` | Your key from [tsfm.ai](https://tsfm.ai/) |
25
+
26
+ Optional variables (Settings → Variables):
27
+
28
+ | Variable | Default | Purpose |
29
+ |---|---|---|
30
+ | `TSFM_BENCH_INTERVAL_SECONDS` | `3600` | Collect + eval cycle (1 hour) |
31
+ | `TSFM_REFRESH_SECONDS` | `60` | UI reload from disk (seconds) |
32
+
33
+ Without `TSFM_API_KEY`, the leaderboard only reloads static CSV files and **does not** call the API.
34
 
 
app.py CHANGED
@@ -16,6 +16,8 @@ from src.display.css_html_js import (
16
  refresh_countdown_head_js,
17
  refresh_countdown_markup,
18
  )
 
 
19
  from src.utils import (
20
  get_grouped_dfs,
21
  prepare_ranks_table,
@@ -23,25 +25,38 @@ from src.utils import (
23
  )
24
 
25
  RESULTS_PATH = os.getenv("TSFM_RESULTS_PATH", "results")
26
- REFRESH_SECONDS = int(os.getenv("TSFM_REFRESH_SECONDS", "300"))
 
27
 
28
 
29
  def load_online_status(results_path: str = RESULTS_PATH) -> dict:
30
  status_path = Path(results_path) / "online_status.json"
31
  if not status_path.exists():
32
- return {"status": "unknown", "message": "No online_status.json yet. Run run_online_eval.py."}
33
  return json.loads(status_path.read_text())
34
 
35
 
36
  def build_status_line(status: dict) -> str:
 
37
  finished = status.get("finished_at", "n/a")
38
- data_source = status.get("data_source", "n/a")
39
- ts_bench_root = status.get("ts_bench_root", "")
40
- root_hint = f" · TS-Bench: `{ts_bench_root}`" if ts_bench_root else ""
 
 
 
 
 
 
 
 
 
 
 
 
41
  return (
42
- f"**Live benchmark** · Data: `{data_source}`{root_hint} · "
43
- f"Last eval: `{finished}` · "
44
- f"Auto-refresh every {REFRESH_SECONDS}s"
45
  )
46
 
47
 
@@ -73,14 +88,14 @@ DOMAINS: list[str] = grouped_initial["domains"]
73
  INITIAL_OUTPUTS = refresh_leaderboard()
74
 
75
  demo = gr.Blocks(
76
- css=build_custom_css(REFRESH_SECONDS),
77
- head=refresh_countdown_head_js(REFRESH_SECONDS),
78
  )
79
  with demo:
80
  gr.HTML(TITLE)
81
  gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text")
82
  status = gr.Markdown(INITIAL_OUTPUTS[2], elem_classes="markdown-text")
83
- gr.HTML(refresh_countdown_markup(REFRESH_SECONDS))
84
 
85
  domain_tables: list[tuple[gr.Dataframe, gr.Dataframe]] = []
86
  with gr.Tabs():
@@ -130,6 +145,8 @@ with demo:
130
  refresh_outputs.extend([values_df, ranks_df])
131
 
132
  demo.load(refresh_leaderboard, inputs=[], outputs=refresh_outputs)
133
- gr.Timer(value=REFRESH_SECONDS).tick(refresh_leaderboard, inputs=[], outputs=refresh_outputs)
 
 
134
 
135
  demo.queue(default_concurrency_limit=20).launch()
 
16
  refresh_countdown_head_js,
17
  refresh_countdown_markup,
18
  )
19
+ from src.benchmark_config import BENCHMARK_INTERVAL_SECONDS, DISPLAY_REFRESH_SECONDS
20
+ from src.eval_scheduler import get_scheduler, start_live_eval_scheduler
21
  from src.utils import (
22
  get_grouped_dfs,
23
  prepare_ranks_table,
 
25
  )
26
 
27
  RESULTS_PATH = os.getenv("TSFM_RESULTS_PATH", "results")
28
+
29
+ start_live_eval_scheduler()
30
 
31
 
32
  def load_online_status(results_path: str = RESULTS_PATH) -> dict:
33
  status_path = Path(results_path) / "online_status.json"
34
  if not status_path.exists():
35
+ return {"status": "unknown", "message": "No online_status.json yet."}
36
  return json.loads(status_path.read_text())
37
 
38
 
39
  def build_status_line(status: dict) -> str:
40
+ eval_state = get_scheduler().snapshot()
41
  finished = status.get("finished_at", "n/a")
42
+ eval_status = eval_state.get("status", "unknown")
43
+ eval_message = eval_state.get("message", "")
44
+
45
+ if eval_status == "running":
46
+ live_hint = "**Evaluating now** — collecting TS-Bench data and calling TSFM.ai…"
47
+ elif eval_status == "disabled":
48
+ live_hint = f"**Live eval off** — {eval_message}"
49
+ else:
50
+ secs = get_scheduler().seconds_until_next_run()
51
+ next_hint = f" · Next eval in ~{secs}s" if secs is not None else ""
52
+ live_hint = (
53
+ f"**Live eval** · Last run `{finished}` · "
54
+ f"collect + eval every **{BENCHMARK_INTERVAL_SECONDS // 60} min**{next_hint}"
55
+ )
56
+
57
  return (
58
+ f"{live_hint} · "
59
+ f"Display reloads every {DISPLAY_REFRESH_SECONDS}s"
 
60
  )
61
 
62
 
 
88
  INITIAL_OUTPUTS = refresh_leaderboard()
89
 
90
  demo = gr.Blocks(
91
+ css=build_custom_css(DISPLAY_REFRESH_SECONDS),
92
+ head=refresh_countdown_head_js(DISPLAY_REFRESH_SECONDS),
93
  )
94
  with demo:
95
  gr.HTML(TITLE)
96
  gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text")
97
  status = gr.Markdown(INITIAL_OUTPUTS[2], elem_classes="markdown-text")
98
+ gr.HTML(refresh_countdown_markup(DISPLAY_REFRESH_SECONDS))
99
 
100
  domain_tables: list[tuple[gr.Dataframe, gr.Dataframe]] = []
101
  with gr.Tabs():
 
145
  refresh_outputs.extend([values_df, ranks_df])
146
 
147
  demo.load(refresh_leaderboard, inputs=[], outputs=refresh_outputs)
148
+ gr.Timer(value=DISPLAY_REFRESH_SECONDS).tick(
149
+ refresh_leaderboard, inputs=[], outputs=refresh_outputs
150
+ )
151
 
152
  demo.queue(default_concurrency_limit=20).launch()
configs/datasets/ts_bench.yaml ADDED
@@ -0,0 +1,29 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ source_type: ts_bench
2
+
3
+ settings:
4
+ ts_bench_root: vendor/ts_bench
5
+ output_subdir: data/sample_week
6
+ collect_days: 7
7
+ collect_timeout: 45
8
+ limit_entities: 10
9
+ task_limit: 20
10
+ max_tasks_per_dataset: 1
11
+ min_future_steps: 2
12
+ skip_frequencies:
13
+ - snapshot
14
+ auto_refresh: true
15
+
16
+ source_domains:
17
+ open_meteo_weather: Weather
18
+ open_meteo_air_quality: Air Quality
19
+ nasa_power: Weather
20
+ usgs_water: Hydrology
21
+ noaa_coops: Ocean
22
+ noaa_ndbc: Ocean
23
+ gbfs_citibike: Transport
24
+ binance: Finance
25
+ coingecko: Finance
26
+ wikimedia_pageviews: Web
27
+ nws_observations: Weather
28
+ usgs_earthquake: Hazards
29
+ noaa_ncei: Weather
configs/models/online_tsfm.yaml ADDED
@@ -0,0 +1,26 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ models:
2
+ - model_id: amazon/chronos-bolt-tiny
3
+ display_name: Chronos-Bolt-Tiny
4
+ model_type: zero-shot
5
+ org: Amazon / TSFM.ai
6
+ - model_id: amazon/chronos-bolt-mini
7
+ display_name: Chronos-Bolt-Mini
8
+ model_type: zero-shot
9
+ org: Amazon / TSFM.ai
10
+ - model_id: amazon/chronos-bolt-base
11
+ display_name: Chronos-Bolt-Base
12
+ model_type: zero-shot
13
+ org: Amazon / TSFM.ai
14
+ baseline: true
15
+ - model_id: amazon/chronos-2
16
+ display_name: Chronos-2
17
+ model_type: zero-shot
18
+ org: Amazon / TSFM.ai
19
+ - model_id: google/timesfm-2.5-200m-pytorch
20
+ display_name: TimesFM-2.5
21
+ model_type: zero-shot
22
+ org: Google / TSFM.ai
23
+ - model_id: Salesforce/moirai-1.1-R-small
24
+ display_name: Moirai-1.1-Small
25
+ model_type: zero-shot
26
+ org: Salesforce / TSFM.ai
requirements.txt CHANGED
@@ -4,6 +4,11 @@ starlette==0.37.2
4
  fastapi==0.112.4
5
  jinja2==3.1.4
6
  uvicorn==0.30.6
7
- pandas
8
- numpy
9
- scipy
 
 
 
 
 
 
4
  fastapi==0.112.4
5
  jinja2==3.1.4
6
  uvicorn==0.30.6
7
+ pandas>=2.0.0
8
+ numpy~=1.26.0
9
+ scipy~=1.11.3
10
+ gluonts~=0.15.1
11
+ httpx>=0.27.0
12
+ pyyaml>=6.0
13
+ python-dotenv==1.0.0
14
+ requests>=2.31.0
results/chronos_2/all_results.csv CHANGED
@@ -1,11 +1,8 @@
1
  dataset,model,eval_metrics/MSE[mean],eval_metrics/MSE[0.5],eval_metrics/MAE[0.5],eval_metrics/MASE[0.5],eval_metrics/MAPE[0.5],eval_metrics/sMAPE[0.5],eval_metrics/MSIS,eval_metrics/RMSE[mean],eval_metrics/NRMSE[mean],eval_metrics/ND[0.5],eval_metrics/mean_weighted_sum_quantile_loss,domain,num_variates
2
- binance_btcusdt_1h_close/1h/short,Chronos-2,4380093.155741374,4380093.155741374,1850.0364583333333,2.3838636016637054,0.025076568408860828,0.024678861943730448,66.88197607794382,2092.8672092947927,0.0282057298197708,0.024933081405608456,0.021564816825717217,Finance,1
3
- coingecko_bitcoin_7d_price_usd/1h/short,Chronos-2,3302102.223780314,3302102.223780314,1589.05078125,1.9819221083493386,0.02156161218573524,0.021260322353283848,46.8612864887045,1817.1687383895623,0.02450082462177794,0.021425117923286136,0.017372480048838304,Finance,1
4
- nasa_power_shanghai_hourly_T2M/1h/short,Chronos-2,16.264854419926603,16.264854419926603,3.1164464155832925,0.9942700533878908,0.1112300523205632,0.12196958213989656,21.94742601097775,4.032970917317233,0.15713129036201698,0.12142196327823566,0.1048847735590834,Weather,1
5
- noaa_coops_sf_water_level_water_level/6min/short,Chronos-2,0.06003561507700905,0.06003561507700905,0.213998693972826,8.070613535689246,1.013472949369841,0.597409693051528,96.8635045441028,0.24502166246478913,0.9537009413978706,0.832949845522854,0.5562847402156419,Ocean,1
6
- noaa_ndbc_46013_realtime_wave_height/10min/short,Chronos-2,0.009589138295815758,0.009589138295815758,0.07559254765510559,0.5440371647347783,0.0288622829034601,0.029130192091777325,4.1629569029245,0.09792414562208729,0.0375428043484963,0.028981169136490237,0.03618823474141707,Ocean,1
7
- nws_ksfo_observations_temperature/1h/short,Chronos-2,0.003463902878100574,0.003463902878100574,0.02116549015045166,0.4010301182731177,0.0016484734142401373,0.0016379871537986845,9.278132686642117,0.0588549307883424,0.00453311396422428,0.0016302046009655346,0.0018722179194657846,Weather,1
8
- open_meteo_shanghai_air_quality_hourly_pm2_5/1h/short,Chronos-2,121.62235993895926,121.62235993895926,9.762574990590414,0.5251049426581348,0.25052453302473104,0.21224658419640266,4.581651077587976,11.028252805361294,0.2247033380106543,0.1989148441445373,0.2278692478910483,Air Quality,1
9
- open_meteo_shanghai_hourly_temperature_2m/1h/short,Chronos-2,0.30751845326813054,0.30751845326813054,0.4332726796468099,0.23597305034316118,0.017424389083777924,0.017404320116509994,2.091343357081799,0.5545434638223865,0.0221301007506678,0.0172905618380289,0.022378411607540734,Weather,1
10
- usgs_potomac_iv_usgs_00060/15min/short,Chronos-2,5269.190091451009,5269.190091451009,65.57242838541667,nan,0.0025808082601376345,0.0025768785670678665,nan,72.58918715243344,0.0028545641351112607,0.002578630642716697,0.001904930492972491,Hydrology,1
11
- wikimedia_time_series_hourly_pageviews/1D/short,Chronos-2,4063.1116605430143,4063.1116605430143,63.214759826660156,1.3546019962855749,0.2031273434135618,0.22642697598080944,5.089582116263253,63.74254199938228,0.20528999033617482,0.20359020878151418,0.18650881592714202,Web,1
 
1
  dataset,model,eval_metrics/MSE[mean],eval_metrics/MSE[0.5],eval_metrics/MAE[0.5],eval_metrics/MASE[0.5],eval_metrics/MAPE[0.5],eval_metrics/sMAPE[0.5],eval_metrics/MSIS,eval_metrics/RMSE[mean],eval_metrics/NRMSE[mean],eval_metrics/ND[0.5],eval_metrics/mean_weighted_sum_quantile_loss,domain,num_variates
2
+ binance_btcusdt_1h_close/1h/short,Chronos-2,823714.9134292603,823714.9134292603,858.2594401041666,0.9497371860172586,0.011705204247515538,0.011628998753724112,11.651465649030945,907.5874136573624,0.01236475636114196,0.01169272371106779,0.008400224593419353,Finance,1
3
+ coingecko_bitcoin_7d_price_usd/1h/short,Chronos-2,855220.5976460775,855220.5976460775,874.1985677083334,0.9460587958444222,0.011942090149853535,0.011862712920852017,11.805964342783492,924.7813782976372,0.012618568650165292,0.011928370206599736,0.008589137695491324,Finance,1
4
+ nasa_power_shanghai_hourly_T2M/1h/short,Chronos-2,16.932999722573793,16.932999722573793,3.1624552408854165,0.9207458206411238,0.11271485949261918,0.12390656428505213,20.748803742282863,4.114972627196176,0.1603262140917773,0.12321454404213872,0.10622444551676062,Weather,1
5
+ noaa_coops_sf_water_level_water_level/6min/short,Chronos-2,0.0002494680528126736,0.0002494680528126736,0.013614242275555929,0.853698368574301,0.010921521056233332,0.01099381527380565,7.152836038203941,0.015794557696012686,0.012375351058861507,0.010667030429280775,0.014505464409588711,Ocean,1
6
+ nws_ksfo_observations_temperature/1h/short,Chronos-2,6.805250338098176,6.805250338098176,2.0215774377187095,2.4880953079614887,0.13046011501961127,0.11750503806375708,54.233638763427734,2.608687474209622,0.1540184486618227,0.11935512547416734,0.0937472313337025,Weather,1
7
+ open_meteo_shanghai_air_quality_hourly_pm2_5/1h/short,Chronos-2,787.8530339560867,787.8530339560867,22.037151257197063,0.9668951191457578,0.3169416718860694,0.39892875770053077,5.485232905888252,28.06871984890096,0.4783761537649721,0.3755799236696442,0.30724907240289284,Air Quality,1
8
+ usgs_potomac_iv_usgs_00060/15min/short,Chronos-2,12352.058849016825,12352.058849016825,90.12906901041667,nan,0.003257321805455632,0.0032514783083314887,nan,111.13981666809076,0.003992449633339587,0.0032376854606346356,0.0034832326643549696,Hydrology,1
 
 
 
results/chronos_2/config.json CHANGED
@@ -3,7 +3,7 @@
3
  "model_type": "zero-shot",
4
  "model_dtype": "float32",
5
  "model_link": "https://tsfm.ai/models/amazon/chronos-2",
6
- "code_link": "https://github.com/zhouziyu02/TSFM_Bench/blob/main/scripts/run_online_eval.py",
7
  "org": "Amazon / TSFM.ai",
8
  "testdata_leakage": "No",
9
  "replication_code_available": "Yes",
 
3
  "model_type": "zero-shot",
4
  "model_dtype": "float32",
5
  "model_link": "https://tsfm.ai/models/amazon/chronos-2",
6
+ "code_link": "https://github.com/zhouziyu02/TSFM_Bench/blob/main/space/scripts/run_space_eval.py",
7
  "org": "Amazon / TSFM.ai",
8
  "testdata_leakage": "No",
9
  "replication_code_available": "Yes",
results/chronos_2/online_meta.json CHANGED
@@ -1,67 +1,49 @@
1
  {
2
  "model": "Chronos-2",
3
  "api_model_id": "amazon/chronos-2",
4
- "evaluated_at": "2026-05-28T12:20:18.799459+00:00",
5
  "datasets": [
6
  {
7
- "dataset": "task_binance_btcusdt_1h_crypto_btcusdt_close_20260527t120000z",
8
- "data_fetched_at": "2026-05-28T12:16:39.187896Z",
9
  "context_length": 96,
10
  "prediction_length": 24
11
  },
12
  {
13
- "dataset": "task_coingecko_bitcoin_7d_coin_bitcoin_price_usd_20260527t120016z",
14
- "data_fetched_at": "2026-05-28T12:16:39.187896Z",
15
  "context_length": 96,
16
  "prediction_length": 24
17
  },
18
  {
19
  "dataset": "task_nasa_power_shanghai_hourly_point_shanghai_31.2304_121.4737_t2m_20260524t230000z",
20
- "data_fetched_at": "2026-05-28T12:16:39.187896Z",
21
- "context_length": 83,
22
  "prediction_length": 24
23
  },
24
  {
25
- "dataset": "task_noaa_coops_sf_water_level_coops_station_9414290_water_level_20260528t094200z",
26
- "data_fetched_at": "2026-05-28T12:16:39.187896Z",
27
  "context_length": 96,
28
  "prediction_length": 24
29
  },
30
  {
31
- "dataset": "task_noaa_ndbc_46013_realtime_ndbc_station_46013_wave_height_20260528t045000z",
32
- "data_fetched_at": "2026-05-28T12:16:39.187896Z",
33
  "context_length": 96,
34
  "prediction_length": 24
35
  },
36
  {
37
- "dataset": "task_nws_ksfo_observations_nws_station_ksfo_temperature_20260528t101000z",
38
- "data_fetched_at": "2026-05-28T12:16:39.187896Z",
39
  "context_length": 96,
40
  "prediction_length": 24
41
  },
42
  {
43
- "dataset": "task_open_meteo_shanghai_air_quality_hourly_grid_shanghai_31.2304_121.4737_pm2_5_20260527t120000z",
44
- "data_fetched_at": "2026-05-28T12:16:39.187896Z",
45
  "context_length": 96,
46
  "prediction_length": 24
47
- },
48
- {
49
- "dataset": "task_open_meteo_shanghai_hourly_grid_shanghai_31.2304_121.4737_temperature_2m_20260527t120000z",
50
- "data_fetched_at": "2026-05-28T12:16:39.187896Z",
51
- "context_length": 96,
52
- "prediction_length": 24
53
- },
54
- {
55
- "dataset": "task_usgs_potomac_iv_usgs_site_01646500_usgs_00060_20260528t095000z",
56
- "data_fetched_at": "2026-05-28T12:16:39.187896Z",
57
- "context_length": 96,
58
- "prediction_length": 24
59
- },
60
- {
61
- "dataset": "task_wikimedia_time_series_hourly_enwiki_article_time_series_pageviews_20260525t000000z",
62
- "data_fetched_at": "2026-05-28T12:16:39.187896Z",
63
- "context_length": 4,
64
- "prediction_length": 2
65
  }
66
  ]
67
  }
 
1
  {
2
  "model": "Chronos-2",
3
  "api_model_id": "amazon/chronos-2",
4
+ "evaluated_at": "2026-05-29T02:53:16.309267+00:00",
5
  "datasets": [
6
  {
7
+ "dataset": "task_binance_btcusdt_1h_crypto_btcusdt_close_20260528t020000z",
8
+ "data_fetched_at": "2026-05-29T02:50:05.083862Z",
9
  "context_length": 96,
10
  "prediction_length": 24
11
  },
12
  {
13
+ "dataset": "task_coingecko_bitcoin_7d_coin_bitcoin_price_usd_20260528t030012z",
14
+ "data_fetched_at": "2026-05-29T02:50:05.083862Z",
15
  "context_length": 96,
16
  "prediction_length": 24
17
  },
18
  {
19
  "dataset": "task_nasa_power_shanghai_hourly_point_shanghai_31.2304_121.4737_t2m_20260524t230000z",
20
+ "data_fetched_at": "2026-05-29T02:50:05.083862Z",
21
+ "context_length": 69,
22
  "prediction_length": 24
23
  },
24
  {
25
+ "dataset": "task_noaa_coops_sf_water_level_coops_station_9414290_water_level_20260529t001800z",
26
+ "data_fetched_at": "2026-05-29T02:50:05.083862Z",
27
  "context_length": 96,
28
  "prediction_length": 24
29
  },
30
  {
31
+ "dataset": "task_nws_ksfo_observations_nws_station_ksfo_temperature_20260529t004500z",
32
+ "data_fetched_at": "2026-05-29T02:50:05.083862Z",
33
  "context_length": 96,
34
  "prediction_length": 24
35
  },
36
  {
37
+ "dataset": "task_open_meteo_shanghai_air_quality_hourly_grid_shanghai_31.2304_121.4737_pm2_5_20260528t020000z",
38
+ "data_fetched_at": "2026-05-29T02:50:05.083862Z",
39
  "context_length": 96,
40
  "prediction_length": 24
41
  },
42
  {
43
+ "dataset": "task_usgs_potomac_iv_usgs_site_01646500_usgs_00060_20260528t235000z",
44
+ "data_fetched_at": "2026-05-29T02:50:05.083862Z",
45
  "context_length": 96,
46
  "prediction_length": 24
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
47
  }
48
  ]
49
  }
results/chronos_bolt_base/all_results.csv CHANGED
@@ -1,11 +1,8 @@
1
  dataset,model,eval_metrics/MSE[mean],eval_metrics/MSE[0.5],eval_metrics/MAE[0.5],eval_metrics/MASE[0.5],eval_metrics/MAPE[0.5],eval_metrics/sMAPE[0.5],eval_metrics/MSIS,eval_metrics/RMSE[mean],eval_metrics/NRMSE[mean],eval_metrics/ND[0.5],eval_metrics/mean_weighted_sum_quantile_loss,domain,num_variates
2
- binance_btcusdt_1h_close/1h/short,Chronos-Bolt-Base,4627089.7309748335,4627089.7309748335,1874.4404296875,2.4153093273872144,0.025415879545412644,0.024995872396728593,52.461912288031094,2151.067114474775,0.028990094347893916,0.025261975575046464,0.019576547766102168,Finance,1
3
- coingecko_bitcoin_7d_price_usd/1h/short,Chronos-Bolt-Base,4946482.715515137,4946482.715515137,1928.9296875,2.4058314801622274,0.026177410818833124,0.02572777590076968,48.45609391950561,2224.0689547572792,0.02998704647293313,0.02600769371756978,0.019592513629850913,Finance,1
4
- nasa_power_shanghai_hourly_T2M/1h/short,Chronos-Bolt-Base,11.688720341001195,11.688720341001195,2.729196389516195,0.8707219307033777,0.09833320637961653,0.10621373079641616,8.073066633299149,3.4188770584800494,0.13320516681172243,0.10633405475219919,0.08541267410550227,Weather,1
5
- noaa_coops_sf_water_level_water_level/6min/short,Chronos-Bolt-Base,0.03234704147219288,0.03234704147219288,0.13382656003038088,5.047051581319789,0.6706533574633848,0.41720830815031723,22.83294145173187,0.1798528328167029,0.7000434747071089,0.5208948262007407,0.4173842468196957,Ocean,1
6
- noaa_ndbc_46013_realtime_wave_height/10min/short,Chronos-Bolt-Base,0.04078452699226176,0.04078452699226176,0.17054141561190286,1.2273811519446596,0.06768574491893825,0.06480219526685438,7.064554223996088,0.20195179373370706,0.07742560970848361,0.06538329192416381,0.061584868474800714,Ocean,1
7
- nws_ksfo_observations_temperature/1h/short,Chronos-Bolt-Base,0.0037005670213829944,0.0037005670213829944,0.01999191443125407,0.3787939590286682,0.0015589197272527502,0.0015477403017569464,13.350006060847901,0.06083228601148402,0.004685413464949044,0.0015398136615912097,0.0017142914417336535,Weather,1
8
- open_meteo_shanghai_air_quality_hourly_pm2_5/1h/short,Chronos-Bolt-Base,38.0265807971964,38.0265807971964,4.774677435557048,0.25681817793213635,0.1013508384060875,0.09818999823413017,3.9438365445644235,6.166569613423366,0.12564535839599894,0.0972852161289089,0.17859769229816153,Air Quality,1
9
- open_meteo_shanghai_hourly_temperature_2m/1h/short,Chronos-Bolt-Base,0.6043374313377777,0.6043374313377777,0.6502586205800375,0.3541499785661307,0.026396943053871125,0.026084632371007877,3.108420925181368,0.7773914273631899,0.03102326820637271,0.02594979424739109,0.027723991507018,Weather,1
10
- usgs_potomac_iv_usgs_00060/15min/short,Chronos-Bolt-Base,4639.297880808513,4639.297880808513,62.649576822916664,nan,0.0024655696006461968,0.0024621813211091674,nan,68.11239153640483,0.002678514495942513,0.002463689732508602,0.0015585257620341544,Hydrology,1
11
- wikimedia_time_series_hourly_pageviews/1D/short,Chronos-Bolt-Base,4028.8496973256115,4028.8496973256115,63.05271911621094,1.3511296953473773,0.2026653811521117,0.22577298837325582,4.94127665928432,63.473220316331926,0.20442260971443454,0.20306833853852155,0.181499555051636,Web,1
 
1
  dataset,model,eval_metrics/MSE[mean],eval_metrics/MSE[0.5],eval_metrics/MAE[0.5],eval_metrics/MASE[0.5],eval_metrics/MAPE[0.5],eval_metrics/sMAPE[0.5],eval_metrics/MSIS,eval_metrics/RMSE[mean],eval_metrics/NRMSE[mean],eval_metrics/ND[0.5],eval_metrics/mean_weighted_sum_quantile_loss,domain,num_variates
2
+ binance_btcusdt_1h_close/1h/short,Chronos-Bolt-Base,1020807.1654103597,1020807.1654103597,978.3776041666666,1.082658167477634,0.013340172949584,0.013245844896014864,12.505715915951397,1010.3500212353933,0.013764769832701284,0.013329185181148515,0.009075649577973174,Finance,1
3
+ coingecko_bitcoin_7d_price_usd/1h/short,Chronos-Bolt-Base,1032518.6303456625,1032518.6303456625,981.5572916666666,1.0622425427220556,0.013405463836439183,0.013309736826454506,12.85205207248118,1016.1292389975118,0.01386500297327957,0.013393271490572569,0.009228772376426409,Finance,1
4
+ nasa_power_shanghai_hourly_T2M/1h/short,Chronos-Bolt-Base,14.357317871038353,14.357317871038353,3.029151995976766,0.8819347083000757,0.1091449078122792,0.11893372374924811,7.6038772586826555,3.7891051543917795,0.1476298724769894,0.1180208267276896,0.09493130206210582,Weather,1
5
+ noaa_coops_sf_water_level_water_level/6min/short,Chronos-Bolt-Base,0.007210553512415767,0.007210553512415767,0.07268983622392018,4.558108585133124,0.054106267823969455,0.055993512387166634,21.160505010226164,0.08491497813940581,0.06653257944006775,0.05695393685568374,0.049410255141733854,Ocean,1
6
+ nws_ksfo_observations_temperature/1h/short,Chronos-Bolt-Base,6.471037574445442,6.471037574445442,1.968206803003947,2.4224083729279346,0.1270358446325033,0.11470028206444982,37.253308711907806,2.543823416521957,0.15018883639982034,0.11620409169026993,0.08526762779478787,Weather,1
7
+ open_meteo_shanghai_air_quality_hourly_pm2_5/1h/short,Chronos-Bolt-Base,964.608039366991,964.608039366991,25.43541646003723,1.115996334626862,0.3828018459905362,0.4930053251111303,8.94359384637864,31.058139663653247,0.5293249380582461,0.4334966739154405,0.35288186109406783,Air Quality,1
8
+ usgs_potomac_iv_usgs_00060/15min/short,Chronos-Bolt-Base,31798.887013276417,31798.887013276417,151.68888346354166,nan,0.00542055609772574,0.005430840108326886,nan,178.32242431415185,0.006405834730638593,0.005449084273499476,0.0052516232677867584,Hydrology,1
 
 
 
results/chronos_bolt_base/config.json CHANGED
@@ -3,7 +3,7 @@
3
  "model_type": "zero-shot",
4
  "model_dtype": "float32",
5
  "model_link": "https://tsfm.ai/models/amazon/chronos-bolt-base",
6
- "code_link": "https://github.com/zhouziyu02/TSFM_Bench/blob/main/scripts/run_online_eval.py",
7
  "org": "Amazon / TSFM.ai",
8
  "testdata_leakage": "No",
9
  "replication_code_available": "Yes",
 
3
  "model_type": "zero-shot",
4
  "model_dtype": "float32",
5
  "model_link": "https://tsfm.ai/models/amazon/chronos-bolt-base",
6
+ "code_link": "https://github.com/zhouziyu02/TSFM_Bench/blob/main/space/scripts/run_space_eval.py",
7
  "org": "Amazon / TSFM.ai",
8
  "testdata_leakage": "No",
9
  "replication_code_available": "Yes",
results/chronos_bolt_base/online_meta.json CHANGED
@@ -1,67 +1,49 @@
1
  {
2
  "model": "Chronos-Bolt-Base",
3
  "api_model_id": "amazon/chronos-bolt-base",
4
- "evaluated_at": "2026-05-28T12:19:28.056546+00:00",
5
  "datasets": [
6
  {
7
- "dataset": "task_binance_btcusdt_1h_crypto_btcusdt_close_20260527t120000z",
8
- "data_fetched_at": "2026-05-28T12:16:39.187896Z",
9
  "context_length": 96,
10
  "prediction_length": 24
11
  },
12
  {
13
- "dataset": "task_coingecko_bitcoin_7d_coin_bitcoin_price_usd_20260527t120016z",
14
- "data_fetched_at": "2026-05-28T12:16:39.187896Z",
15
  "context_length": 96,
16
  "prediction_length": 24
17
  },
18
  {
19
  "dataset": "task_nasa_power_shanghai_hourly_point_shanghai_31.2304_121.4737_t2m_20260524t230000z",
20
- "data_fetched_at": "2026-05-28T12:16:39.187896Z",
21
- "context_length": 83,
22
  "prediction_length": 24
23
  },
24
  {
25
- "dataset": "task_noaa_coops_sf_water_level_coops_station_9414290_water_level_20260528t094200z",
26
- "data_fetched_at": "2026-05-28T12:16:39.187896Z",
27
  "context_length": 96,
28
  "prediction_length": 24
29
  },
30
  {
31
- "dataset": "task_noaa_ndbc_46013_realtime_ndbc_station_46013_wave_height_20260528t045000z",
32
- "data_fetched_at": "2026-05-28T12:16:39.187896Z",
33
  "context_length": 96,
34
  "prediction_length": 24
35
  },
36
  {
37
- "dataset": "task_nws_ksfo_observations_nws_station_ksfo_temperature_20260528t101000z",
38
- "data_fetched_at": "2026-05-28T12:16:39.187896Z",
39
  "context_length": 96,
40
  "prediction_length": 24
41
  },
42
  {
43
- "dataset": "task_open_meteo_shanghai_air_quality_hourly_grid_shanghai_31.2304_121.4737_pm2_5_20260527t120000z",
44
- "data_fetched_at": "2026-05-28T12:16:39.187896Z",
45
  "context_length": 96,
46
  "prediction_length": 24
47
- },
48
- {
49
- "dataset": "task_open_meteo_shanghai_hourly_grid_shanghai_31.2304_121.4737_temperature_2m_20260527t120000z",
50
- "data_fetched_at": "2026-05-28T12:16:39.187896Z",
51
- "context_length": 96,
52
- "prediction_length": 24
53
- },
54
- {
55
- "dataset": "task_usgs_potomac_iv_usgs_site_01646500_usgs_00060_20260528t095000z",
56
- "data_fetched_at": "2026-05-28T12:16:39.187896Z",
57
- "context_length": 96,
58
- "prediction_length": 24
59
- },
60
- {
61
- "dataset": "task_wikimedia_time_series_hourly_enwiki_article_time_series_pageviews_20260525t000000z",
62
- "data_fetched_at": "2026-05-28T12:16:39.187896Z",
63
- "context_length": 4,
64
- "prediction_length": 2
65
  }
66
  ]
67
  }
 
1
  {
2
  "model": "Chronos-Bolt-Base",
3
  "api_model_id": "amazon/chronos-bolt-base",
4
+ "evaluated_at": "2026-05-29T02:51:30.013825+00:00",
5
  "datasets": [
6
  {
7
+ "dataset": "task_binance_btcusdt_1h_crypto_btcusdt_close_20260528t020000z",
8
+ "data_fetched_at": "2026-05-29T02:50:05.083862Z",
9
  "context_length": 96,
10
  "prediction_length": 24
11
  },
12
  {
13
+ "dataset": "task_coingecko_bitcoin_7d_coin_bitcoin_price_usd_20260528t030012z",
14
+ "data_fetched_at": "2026-05-29T02:50:05.083862Z",
15
  "context_length": 96,
16
  "prediction_length": 24
17
  },
18
  {
19
  "dataset": "task_nasa_power_shanghai_hourly_point_shanghai_31.2304_121.4737_t2m_20260524t230000z",
20
+ "data_fetched_at": "2026-05-29T02:50:05.083862Z",
21
+ "context_length": 69,
22
  "prediction_length": 24
23
  },
24
  {
25
+ "dataset": "task_noaa_coops_sf_water_level_coops_station_9414290_water_level_20260529t001800z",
26
+ "data_fetched_at": "2026-05-29T02:50:05.083862Z",
27
  "context_length": 96,
28
  "prediction_length": 24
29
  },
30
  {
31
+ "dataset": "task_nws_ksfo_observations_nws_station_ksfo_temperature_20260529t004500z",
32
+ "data_fetched_at": "2026-05-29T02:50:05.083862Z",
33
  "context_length": 96,
34
  "prediction_length": 24
35
  },
36
  {
37
+ "dataset": "task_open_meteo_shanghai_air_quality_hourly_grid_shanghai_31.2304_121.4737_pm2_5_20260528t020000z",
38
+ "data_fetched_at": "2026-05-29T02:50:05.083862Z",
39
  "context_length": 96,
40
  "prediction_length": 24
41
  },
42
  {
43
+ "dataset": "task_usgs_potomac_iv_usgs_site_01646500_usgs_00060_20260528t235000z",
44
+ "data_fetched_at": "2026-05-29T02:50:05.083862Z",
45
  "context_length": 96,
46
  "prediction_length": 24
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
47
  }
48
  ]
49
  }
results/chronos_bolt_mini/all_results.csv CHANGED
@@ -1,11 +1,8 @@
1
  dataset,model,eval_metrics/MSE[mean],eval_metrics/MSE[0.5],eval_metrics/MAE[0.5],eval_metrics/MASE[0.5],eval_metrics/MAPE[0.5],eval_metrics/sMAPE[0.5],eval_metrics/MSIS,eval_metrics/RMSE[mean],eval_metrics/NRMSE[mean],eval_metrics/ND[0.5],eval_metrics/mean_weighted_sum_quantile_loss,domain,num_variates
2
- binance_btcusdt_1h_close/1h/short,Chronos-Bolt-Mini,3679822.6813456216,3679822.6813456216,1654.4537760416667,2.1318456290822945,0.022439195282325037,0.02210438779696162,36.894830953375646,1918.2863918991923,0.02585289092712348,0.022297198789814756,0.016141385649750175,Finance,1
3
- coingecko_bitcoin_7d_price_usd/1h/short,Chronos-Bolt-Mini,3435441.7622146606,3435441.7622146606,1580.2054036458333,1.9708898306919778,0.02145509434972552,0.021141595171611006,22.12416891501253,1853.4944732085555,0.024990603275380642,0.02130585599630342,0.013890711488634161,Finance,1
4
- nasa_power_shanghai_hourly_T2M/1h/short,Chronos-Bolt-Mini,9.13818853845593,9.13818853845593,2.457893212636312,0.784165453167993,0.08935921081957772,0.09565064369787313,7.269570607721318,3.0229436876091373,0.1177789406529396,0.09576362934213734,0.07988338981433957,Weather,1
5
- noaa_coops_sf_water_level_water_level/6min/short,Chronos-Bolt-Mini,0.02307176451407955,0.02307176451407955,0.1410230112572511,5.318454063280177,0.6473759859576461,0.45994482734131564,19.641841542644524,0.15189392520466233,0.5912186620687938,0.5489056650822877,0.38075484208964183,Ocean,1
6
- noaa_ndbc_46013_realtime_wave_height/10min/short,Chronos-Bolt-Mini,0.03395013763280493,0.03395013763280493,0.15618878602981567,1.1240857326666478,0.06198749864085912,0.05956084295321493,7.4643506648433195,0.18425563121056826,0.0706411382882342,0.059880686199460895,0.0619203976492256,Ocean,1
7
- nws_ksfo_observations_temperature/1h/short,Chronos-Bolt-Mini,0.003293288668335966,0.003293288668335966,0.019572933514912922,0.3708553776284693,0.0015254847226304179,0.0015155839450093345,12.664317868508965,0.05738718209091614,0.004420065286277438,0.0015075429883074389,0.0017640636910896848,Weather,1
8
- open_meteo_shanghai_air_quality_hourly_pm2_5/1h/short,Chronos-Bolt-Mini,58.57930447065731,58.57930447065731,6.592350959777832,0.35458637460433845,0.15445332516570973,0.14256432500281468,4.4046256992472514,7.65371181000809,0.1559462430027276,0.13432075707220698,0.20333559743785112,Air Quality,1
9
- open_meteo_shanghai_hourly_temperature_2m/1h/short,Chronos-Bolt-Mini,0.8828099140087033,0.8828099140087033,0.7188208103179932,0.39149096453335225,0.028733217601040043,0.028834013771296452,2.773892978739956,0.9395796475066408,0.03749569442091312,0.02868589748468996,0.02969891826111223,Weather,1
10
- usgs_potomac_iv_usgs_00060/15min/short,Chronos-Bolt-Mini,4843.679802894592,4843.679802894592,64.029296875,nan,0.002519860464793693,0.0025163275141648713,nan,69.59655022265538,0.002736878920766392,0.0025179471161723742,0.001616091514077776,Hydrology,1
11
- wikimedia_time_series_hourly_pageviews/1D/short,Chronos-Bolt-Mini,4577.245442453423,4577.245442453423,67.47705841064453,1.445936965942383,0.21708162745704251,0.24360223675984644,8.007688195364816,67.65534304438506,0.21789160400768137,0.21731741839176982,0.23123530744333173,Web,1
 
1
  dataset,model,eval_metrics/MSE[mean],eval_metrics/MSE[0.5],eval_metrics/MAE[0.5],eval_metrics/MASE[0.5],eval_metrics/MAPE[0.5],eval_metrics/sMAPE[0.5],eval_metrics/MSIS,eval_metrics/RMSE[mean],eval_metrics/NRMSE[mean],eval_metrics/ND[0.5],eval_metrics/mean_weighted_sum_quantile_loss,domain,num_variates
2
+ binance_btcusdt_1h_close/1h/short,Chronos-Bolt-Mini,851287.6321029663,851287.6321029663,882.4768880208334,0.976535855233298,0.012034191309566289,0.011955457296705658,10.656671858587634,922.6524980202278,0.012569999508968603,0.012022656496243319,0.008244497111170809,Finance,1
3
+ coingecko_bitcoin_7d_price_usd/1h/short,Chronos-Bolt-Mini,864730.2875595093,864730.2875595093,885.3583984375,0.9581359787768308,0.012093613190106718,0.012013367698897761,11.016269186291742,929.9087522760011,0.01268853126193431,0.012080645212871425,0.0083666342836689,Finance,1
4
+ nasa_power_shanghai_hourly_T2M/1h/short,Chronos-Bolt-Mini,9.548240780425354,9.548240780425354,2.5835858980814614,0.7522085647795073,0.09503784443039846,0.1018251998426517,3.8696742236218,3.090022779920134,0.12039245425054106,0.10066082653447453,0.08411044052817845,Weather,1
5
+ noaa_coops_sf_water_level_water_level/6min/short,Chronos-Bolt-Mini,0.01640126373424522,0.01640126373424522,0.0986495316028595,6.185944284331548,0.07142778184577067,0.07563235656687009,25.340507814660793,0.12806741870688743,0.10034337752300845,0.07729387608529449,0.0749025769287721,Ocean,1
6
+ nws_ksfo_observations_temperature/1h/short,Chronos-Bolt-Mini,6.4003104863786575,6.4003104863786575,1.9677637418111165,2.4218630668444514,0.1268957925752334,0.1146813703822039,35.354552146715996,2.5298834926491494,0.14936581506415642,0.11617793309585928,0.0828891098816994,Weather,1
7
+ open_meteo_shanghai_air_quality_hourly_pm2_5/1h/short,Chronos-Bolt-Mini,1257.403550816446,1257.403550816446,29.262885729471844,1.2839291727739017,0.4375496892711623,0.5885718970399296,15.36094857308285,35.45988650315235,0.6043440602078141,0.4987283637688504,0.40799980188904916,Air Quality,1
8
+ usgs_potomac_iv_usgs_00060/15min/short,Chronos-Bolt-Mini,68632.59780025482,68632.59780025482,211.42757161458334,nan,0.007524633240620554,0.007560872203681779,nan,261.9782391731321,0.00941098299678966,0.007595063192261637,0.006913356011138552,Hydrology,1
 
 
 
results/chronos_bolt_mini/config.json CHANGED
@@ -3,7 +3,7 @@
3
  "model_type": "zero-shot",
4
  "model_dtype": "float32",
5
  "model_link": "https://tsfm.ai/models/amazon/chronos-bolt-mini",
6
- "code_link": "https://github.com/zhouziyu02/TSFM_Bench/blob/main/scripts/run_online_eval.py",
7
  "org": "Amazon / TSFM.ai",
8
  "testdata_leakage": "No",
9
  "replication_code_available": "Yes",
 
3
  "model_type": "zero-shot",
4
  "model_dtype": "float32",
5
  "model_link": "https://tsfm.ai/models/amazon/chronos-bolt-mini",
6
+ "code_link": "https://github.com/zhouziyu02/TSFM_Bench/blob/main/space/scripts/run_space_eval.py",
7
  "org": "Amazon / TSFM.ai",
8
  "testdata_leakage": "No",
9
  "replication_code_available": "Yes",
results/chronos_bolt_mini/online_meta.json CHANGED
@@ -1,67 +1,49 @@
1
  {
2
  "model": "Chronos-Bolt-Mini",
3
  "api_model_id": "amazon/chronos-bolt-mini",
4
- "evaluated_at": "2026-05-28T12:18:55.358936+00:00",
5
  "datasets": [
6
  {
7
- "dataset": "task_binance_btcusdt_1h_crypto_btcusdt_close_20260527t120000z",
8
- "data_fetched_at": "2026-05-28T12:16:39.187896Z",
9
  "context_length": 96,
10
  "prediction_length": 24
11
  },
12
  {
13
- "dataset": "task_coingecko_bitcoin_7d_coin_bitcoin_price_usd_20260527t120016z",
14
- "data_fetched_at": "2026-05-28T12:16:39.187896Z",
15
  "context_length": 96,
16
  "prediction_length": 24
17
  },
18
  {
19
  "dataset": "task_nasa_power_shanghai_hourly_point_shanghai_31.2304_121.4737_t2m_20260524t230000z",
20
- "data_fetched_at": "2026-05-28T12:16:39.187896Z",
21
- "context_length": 83,
22
  "prediction_length": 24
23
  },
24
  {
25
- "dataset": "task_noaa_coops_sf_water_level_coops_station_9414290_water_level_20260528t094200z",
26
- "data_fetched_at": "2026-05-28T12:16:39.187896Z",
27
  "context_length": 96,
28
  "prediction_length": 24
29
  },
30
  {
31
- "dataset": "task_noaa_ndbc_46013_realtime_ndbc_station_46013_wave_height_20260528t045000z",
32
- "data_fetched_at": "2026-05-28T12:16:39.187896Z",
33
  "context_length": 96,
34
  "prediction_length": 24
35
  },
36
  {
37
- "dataset": "task_nws_ksfo_observations_nws_station_ksfo_temperature_20260528t101000z",
38
- "data_fetched_at": "2026-05-28T12:16:39.187896Z",
39
  "context_length": 96,
40
  "prediction_length": 24
41
  },
42
  {
43
- "dataset": "task_open_meteo_shanghai_air_quality_hourly_grid_shanghai_31.2304_121.4737_pm2_5_20260527t120000z",
44
- "data_fetched_at": "2026-05-28T12:16:39.187896Z",
45
  "context_length": 96,
46
  "prediction_length": 24
47
- },
48
- {
49
- "dataset": "task_open_meteo_shanghai_hourly_grid_shanghai_31.2304_121.4737_temperature_2m_20260527t120000z",
50
- "data_fetched_at": "2026-05-28T12:16:39.187896Z",
51
- "context_length": 96,
52
- "prediction_length": 24
53
- },
54
- {
55
- "dataset": "task_usgs_potomac_iv_usgs_site_01646500_usgs_00060_20260528t095000z",
56
- "data_fetched_at": "2026-05-28T12:16:39.187896Z",
57
- "context_length": 96,
58
- "prediction_length": 24
59
- },
60
- {
61
- "dataset": "task_wikimedia_time_series_hourly_enwiki_article_time_series_pageviews_20260525t000000z",
62
- "data_fetched_at": "2026-05-28T12:16:39.187896Z",
63
- "context_length": 4,
64
- "prediction_length": 2
65
  }
66
  ]
67
  }
 
1
  {
2
  "model": "Chronos-Bolt-Mini",
3
  "api_model_id": "amazon/chronos-bolt-mini",
4
+ "evaluated_at": "2026-05-29T02:51:10.536027+00:00",
5
  "datasets": [
6
  {
7
+ "dataset": "task_binance_btcusdt_1h_crypto_btcusdt_close_20260528t020000z",
8
+ "data_fetched_at": "2026-05-29T02:50:05.083862Z",
9
  "context_length": 96,
10
  "prediction_length": 24
11
  },
12
  {
13
+ "dataset": "task_coingecko_bitcoin_7d_coin_bitcoin_price_usd_20260528t030012z",
14
+ "data_fetched_at": "2026-05-29T02:50:05.083862Z",
15
  "context_length": 96,
16
  "prediction_length": 24
17
  },
18
  {
19
  "dataset": "task_nasa_power_shanghai_hourly_point_shanghai_31.2304_121.4737_t2m_20260524t230000z",
20
+ "data_fetched_at": "2026-05-29T02:50:05.083862Z",
21
+ "context_length": 69,
22
  "prediction_length": 24
23
  },
24
  {
25
+ "dataset": "task_noaa_coops_sf_water_level_coops_station_9414290_water_level_20260529t001800z",
26
+ "data_fetched_at": "2026-05-29T02:50:05.083862Z",
27
  "context_length": 96,
28
  "prediction_length": 24
29
  },
30
  {
31
+ "dataset": "task_nws_ksfo_observations_nws_station_ksfo_temperature_20260529t004500z",
32
+ "data_fetched_at": "2026-05-29T02:50:05.083862Z",
33
  "context_length": 96,
34
  "prediction_length": 24
35
  },
36
  {
37
+ "dataset": "task_open_meteo_shanghai_air_quality_hourly_grid_shanghai_31.2304_121.4737_pm2_5_20260528t020000z",
38
+ "data_fetched_at": "2026-05-29T02:50:05.083862Z",
39
  "context_length": 96,
40
  "prediction_length": 24
41
  },
42
  {
43
+ "dataset": "task_usgs_potomac_iv_usgs_site_01646500_usgs_00060_20260528t235000z",
44
+ "data_fetched_at": "2026-05-29T02:50:05.083862Z",
45
  "context_length": 96,
46
  "prediction_length": 24
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
47
  }
48
  ]
49
  }
results/chronos_bolt_tiny/all_results.csv CHANGED
@@ -1,11 +1,8 @@
1
  dataset,model,eval_metrics/MSE[mean],eval_metrics/MSE[0.5],eval_metrics/MAE[0.5],eval_metrics/MASE[0.5],eval_metrics/MAPE[0.5],eval_metrics/sMAPE[0.5],eval_metrics/MSIS,eval_metrics/RMSE[mean],eval_metrics/NRMSE[mean],eval_metrics/ND[0.5],eval_metrics/mean_weighted_sum_quantile_loss,domain,num_variates
2
- binance_btcusdt_1h_close/1h/short,Chronos-Bolt-Tiny,4495006.537943522,4495006.537943522,1834.72265625,2.3641310092463494,0.024881456748435934,0.02447329243393806,42.788615799160766,2120.143046575755,0.028573328343718278,0.024726696135602966,0.018015038439380838,Finance,1
3
- coingecko_bitcoin_7d_price_usd/1h/short,Chronos-Bolt-Tiny,4406332.5283203125,4406332.5283203125,1807.1061197916667,2.2538886819785158,0.024529248640723117,0.02412824693875963,37.43977008949355,2099.1266108361147,0.028302453076857404,0.02436515067565812,0.017333996733501764,Finance,1
4
- nasa_power_shanghai_hourly_T2M/1h/short,Chronos-Bolt-Tiny,7.912413253015529,7.912413253015529,2.2673874696095786,0.7233865627167552,0.08212307663246772,0.0874627393188528,2.2266680874402787,2.812901216362837,0.10959533476675763,0.088341207054233,0.07361846408709673,Weather,1
5
- noaa_coops_sf_water_level_water_level/6min/short,Chronos-Bolt-Tiny,0.0028608058548383144,0.0028608058548383144,0.049041210363308586,1.8495096807224467,0.225792045612945,0.19658526579294686,13.967965661021097,0.05348650161338199,0.20818619230490099,0.19088372848461752,0.20820748121803842,Ocean,1
6
- noaa_ndbc_46013_realtime_wave_height/10min/short,Chronos-Bolt-Tiny,0.04937945652725754,0.04937945652725754,0.18734135230382284,1.348289762768264,0.07458714345199068,0.07090175042748963,7.0800555112695145,0.22221488817641705,0.0851942084062309,0.07182416237838214,0.06322130203989622,Ocean,1
7
- nws_ksfo_observations_temperature/1h/short,Chronos-Bolt-Tiny,0.003276212487397364,0.003276212487397364,0.02068618933359782,0.39194863412581116,0.001611035489054791,0.0016012503189502235,12.192836604810884,0.057238208282556886,0.004408591052225484,0.0015932879790811604,0.001700157311234707,Weather,1
8
- open_meteo_shanghai_air_quality_hourly_pm2_5/1h/short,Chronos-Bolt-Tiny,63.19570465734099,63.19570465734099,6.434090296427409,0.34607392203585513,0.14174756363352167,0.1365677615769388,4.458499596558531,7.949572608470281,0.1619744788596924,0.13109615749526107,0.20445340630382589,Air Quality,1
9
- open_meteo_shanghai_hourly_temperature_2m/1h/short,Chronos-Bolt-Tiny,0.5948811111434225,0.5948811111434225,0.6484154860178629,0.35314615324951076,0.026197955231929627,0.025978455964565026,2.995656525072318,0.7712853629775573,0.030779594213512538,0.025876240493323176,0.02777387188201828,Weather,1
10
- usgs_potomac_iv_usgs_00060/15min/short,Chronos-Bolt-Tiny,3149.2314387957254,3149.2314387957254,54.302978515625,nan,0.0021364303572376296,0.0021344827808550478,nan,56.11801349652111,0.002206836513053427,0.002135460403694904,0.0014279900766016713,Hydrology,1
11
- wikimedia_time_series_hourly_pageviews/1D/short,Chronos-Bolt-Tiny,4641.08992965892,4641.08992965892,67.62164306640625,1.4490352085658482,0.21732069855015004,0.24415806200991483,8.605713272094727,68.125545353112,0.21940594316622222,0.21778306945702497,0.2325466885637525,Web,1
 
1
  dataset,model,eval_metrics/MSE[mean],eval_metrics/MSE[0.5],eval_metrics/MAE[0.5],eval_metrics/MASE[0.5],eval_metrics/MAPE[0.5],eval_metrics/sMAPE[0.5],eval_metrics/MSIS,eval_metrics/RMSE[mean],eval_metrics/NRMSE[mean],eval_metrics/ND[0.5],eval_metrics/mean_weighted_sum_quantile_loss,domain,num_variates
2
+ binance_btcusdt_1h_close/1h/short,Chronos-Bolt-Tiny,1029025.1603978475,1029025.1603978475,984.4554036458334,1.0893837703720461,0.01342267473206749,0.013327594824025202,11.50025719460482,1014.4087738174624,0.013820065318350654,0.013411987684401543,0.008981437172713323,Finance,1
3
+ coingecko_bitcoin_7d_price_usd/1h/short,Chronos-Bolt-Tiny,1049676.785677592,1049676.785677592,991.6748046875,1.073191728111963,0.013543286102919765,0.013445979208987893,11.929663258797348,1024.537352016798,0.0139797310093777,0.013531324154281413,0.009152163608285595,Finance,1
4
+ nasa_power_shanghai_hourly_T2M/1h/short,Chronos-Bolt-Tiny,9.325186076793065,9.325186076793065,2.57205859820048,0.7488524024372003,0.09445699084526753,0.10095493097514746,2.3972798950588126,3.0537167643370373,0.11897791117710119,0.10021170365662015,0.08079254781825497,Weather,1
5
+ noaa_coops_sf_water_level_water_level/6min/short,Chronos-Bolt-Tiny,0.015442472707887234,0.015442472707887234,0.08998485902945201,5.64261497591828,0.06416666330862907,0.06807652745844346,21.565446339911507,0.12426774604814893,0.09736625819078952,0.07050493226238033,0.06671574848406818,Ocean,1
6
+ nws_ksfo_observations_temperature/1h/short,Chronos-Bolt-Tiny,7.537212424866539,7.537212424866539,2.111232042312622,2.5984394366924577,0.13644075438988787,0.12216720632877787,52.66092036320614,2.745398409132368,0.16208994297460477,0.12464838626199982,0.09697055688920332,Weather,1
7
+ open_meteo_shanghai_air_quality_hourly_pm2_5/1h/short,Chronos-Bolt-Tiny,1461.9665035634837,1461.9665035634837,32.67136557896932,1.4334785628798237,0.5069763645738149,0.7007815976613591,18.987324775681852,38.23567056510823,0.6516518430489527,0.5568192025874885,0.44639292146149856,Air Quality,1
8
+ usgs_potomac_iv_usgs_00060/15min/short,Chronos-Bolt-Tiny,78316.96788072586,78316.96788072586,221.53458658854166,nan,0.007875539639408598,0.007920176132274283,nan,279.8516890796371,0.010053046756340802,0.00795813512666517,0.007018508513986596,Hydrology,1
 
 
 
results/chronos_bolt_tiny/config.json CHANGED
@@ -3,7 +3,7 @@
3
  "model_type": "zero-shot",
4
  "model_dtype": "float32",
5
  "model_link": "https://tsfm.ai/models/amazon/chronos-bolt-tiny",
6
- "code_link": "https://github.com/zhouziyu02/TSFM_Bench/blob/main/scripts/run_online_eval.py",
7
  "org": "Amazon / TSFM.ai",
8
  "testdata_leakage": "No",
9
  "replication_code_available": "Yes",
 
3
  "model_type": "zero-shot",
4
  "model_dtype": "float32",
5
  "model_link": "https://tsfm.ai/models/amazon/chronos-bolt-tiny",
6
+ "code_link": "https://github.com/zhouziyu02/TSFM_Bench/blob/main/space/scripts/run_space_eval.py",
7
  "org": "Amazon / TSFM.ai",
8
  "testdata_leakage": "No",
9
  "replication_code_available": "Yes",
results/chronos_bolt_tiny/online_meta.json CHANGED
@@ -1,67 +1,49 @@
1
  {
2
  "model": "Chronos-Bolt-Tiny",
3
  "api_model_id": "amazon/chronos-bolt-tiny",
4
- "evaluated_at": "2026-05-28T12:18:05.053451+00:00",
5
  "datasets": [
6
  {
7
- "dataset": "task_binance_btcusdt_1h_crypto_btcusdt_close_20260527t120000z",
8
- "data_fetched_at": "2026-05-28T12:16:39.187896Z",
9
  "context_length": 96,
10
  "prediction_length": 24
11
  },
12
  {
13
- "dataset": "task_coingecko_bitcoin_7d_coin_bitcoin_price_usd_20260527t120016z",
14
- "data_fetched_at": "2026-05-28T12:16:39.187896Z",
15
  "context_length": 96,
16
  "prediction_length": 24
17
  },
18
  {
19
  "dataset": "task_nasa_power_shanghai_hourly_point_shanghai_31.2304_121.4737_t2m_20260524t230000z",
20
- "data_fetched_at": "2026-05-28T12:16:39.187896Z",
21
- "context_length": 83,
22
  "prediction_length": 24
23
  },
24
  {
25
- "dataset": "task_noaa_coops_sf_water_level_coops_station_9414290_water_level_20260528t094200z",
26
- "data_fetched_at": "2026-05-28T12:16:39.187896Z",
27
  "context_length": 96,
28
  "prediction_length": 24
29
  },
30
  {
31
- "dataset": "task_noaa_ndbc_46013_realtime_ndbc_station_46013_wave_height_20260528t045000z",
32
- "data_fetched_at": "2026-05-28T12:16:39.187896Z",
33
  "context_length": 96,
34
  "prediction_length": 24
35
  },
36
  {
37
- "dataset": "task_nws_ksfo_observations_nws_station_ksfo_temperature_20260528t101000z",
38
- "data_fetched_at": "2026-05-28T12:16:39.187896Z",
39
  "context_length": 96,
40
  "prediction_length": 24
41
  },
42
  {
43
- "dataset": "task_open_meteo_shanghai_air_quality_hourly_grid_shanghai_31.2304_121.4737_pm2_5_20260527t120000z",
44
- "data_fetched_at": "2026-05-28T12:16:39.187896Z",
45
  "context_length": 96,
46
  "prediction_length": 24
47
- },
48
- {
49
- "dataset": "task_open_meteo_shanghai_hourly_grid_shanghai_31.2304_121.4737_temperature_2m_20260527t120000z",
50
- "data_fetched_at": "2026-05-28T12:16:39.187896Z",
51
- "context_length": 96,
52
- "prediction_length": 24
53
- },
54
- {
55
- "dataset": "task_usgs_potomac_iv_usgs_site_01646500_usgs_00060_20260528t095000z",
56
- "data_fetched_at": "2026-05-28T12:16:39.187896Z",
57
- "context_length": 96,
58
- "prediction_length": 24
59
- },
60
- {
61
- "dataset": "task_wikimedia_time_series_hourly_enwiki_article_time_series_pageviews_20260525t000000z",
62
- "data_fetched_at": "2026-05-28T12:16:39.187896Z",
63
- "context_length": 4,
64
- "prediction_length": 2
65
  }
66
  ]
67
  }
 
1
  {
2
  "model": "Chronos-Bolt-Tiny",
3
  "api_model_id": "amazon/chronos-bolt-tiny",
4
+ "evaluated_at": "2026-05-29T02:50:52.849106+00:00",
5
  "datasets": [
6
  {
7
+ "dataset": "task_binance_btcusdt_1h_crypto_btcusdt_close_20260528t020000z",
8
+ "data_fetched_at": "2026-05-29T02:50:05.083862Z",
9
  "context_length": 96,
10
  "prediction_length": 24
11
  },
12
  {
13
+ "dataset": "task_coingecko_bitcoin_7d_coin_bitcoin_price_usd_20260528t030012z",
14
+ "data_fetched_at": "2026-05-29T02:50:05.083862Z",
15
  "context_length": 96,
16
  "prediction_length": 24
17
  },
18
  {
19
  "dataset": "task_nasa_power_shanghai_hourly_point_shanghai_31.2304_121.4737_t2m_20260524t230000z",
20
+ "data_fetched_at": "2026-05-29T02:50:05.083862Z",
21
+ "context_length": 69,
22
  "prediction_length": 24
23
  },
24
  {
25
+ "dataset": "task_noaa_coops_sf_water_level_coops_station_9414290_water_level_20260529t001800z",
26
+ "data_fetched_at": "2026-05-29T02:50:05.083862Z",
27
  "context_length": 96,
28
  "prediction_length": 24
29
  },
30
  {
31
+ "dataset": "task_nws_ksfo_observations_nws_station_ksfo_temperature_20260529t004500z",
32
+ "data_fetched_at": "2026-05-29T02:50:05.083862Z",
33
  "context_length": 96,
34
  "prediction_length": 24
35
  },
36
  {
37
+ "dataset": "task_open_meteo_shanghai_air_quality_hourly_grid_shanghai_31.2304_121.4737_pm2_5_20260528t020000z",
38
+ "data_fetched_at": "2026-05-29T02:50:05.083862Z",
39
  "context_length": 96,
40
  "prediction_length": 24
41
  },
42
  {
43
+ "dataset": "task_usgs_potomac_iv_usgs_site_01646500_usgs_00060_20260528t235000z",
44
+ "data_fetched_at": "2026-05-29T02:50:05.083862Z",
45
  "context_length": 96,
46
  "prediction_length": 24
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
47
  }
48
  ]
49
  }
results/dataset_properties.csv CHANGED
@@ -3,9 +3,6 @@ binance_btcusdt_1h_close,Finance,1h,1
3
  coingecko_bitcoin_7d_price_usd,Finance,1h,1
4
  nasa_power_shanghai_hourly_t2m,Weather,1h,1
5
  noaa_coops_sf_water_level_water_level,Ocean,6min,1
6
- noaa_ndbc_46013_realtime_wave_height,Ocean,10min,1
7
  nws_ksfo_observations_temperature,Weather,1h,1
8
  open_meteo_shanghai_air_quality_hourly_pm2_5,Air Quality,1h,1
9
- open_meteo_shanghai_hourly_temperature_2m,Weather,1h,1
10
  usgs_potomac_iv_usgs_00060,Hydrology,15min,1
11
- wikimedia_time_series_hourly_pageviews,Web,1D,1
 
3
  coingecko_bitcoin_7d_price_usd,Finance,1h,1
4
  nasa_power_shanghai_hourly_t2m,Weather,1h,1
5
  noaa_coops_sf_water_level_water_level,Ocean,6min,1
 
6
  nws_ksfo_observations_temperature,Weather,1h,1
7
  open_meteo_shanghai_air_quality_hourly_pm2_5,Air Quality,1h,1
 
8
  usgs_potomac_iv_usgs_00060,Hydrology,15min,1
 
results/eval_state.json ADDED
@@ -0,0 +1,10 @@
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "status": "ok",
3
+ "started_at": "2026-05-29T02:49:52.974580+00:00",
4
+ "finished_at": "2026-05-29T02:55:13.595519+00:00",
5
+ "failed_models": [],
6
+ "task_count": 7,
7
+ "model_count": 6,
8
+ "message": "Evaluation complete",
9
+ "updated_at": "2026-05-29T02:55:13.595519+00:00"
10
+ }
results/moirai_1_1_small/all_results.csv CHANGED
@@ -1,11 +1,8 @@
1
  dataset,model,eval_metrics/MSE[mean],eval_metrics/MSE[0.5],eval_metrics/MAE[0.5],eval_metrics/MASE[0.5],eval_metrics/MAPE[0.5],eval_metrics/sMAPE[0.5],eval_metrics/MSIS,eval_metrics/RMSE[mean],eval_metrics/NRMSE[mean],eval_metrics/ND[0.5],eval_metrics/mean_weighted_sum_quantile_loss,domain,num_variates
2
- binance_btcusdt_1h_close/1h/short,Moirai-1.1-Small,8869335.108332315,7410361.406759898,2589.4026692708335,3.3365735823857965,0.035022303348886334,0.0343546582322679,33.181524624215,2978.1428958886972,0.04013665726762046,0.034897575803988286,0.022626002293161866,Finance,1
3
- coingecko_bitcoin_7d_price_usd/1h/short,Moirai-1.1-Small,3513908.5660069785,3177372.7269617715,1577.9192708333333,1.9680384824423314,0.021403528918522274,0.021113633378368957,50.807128803692336,1874.5422283872344,0.02527438944637319,0.021275032144936267,0.0176526858138831,Finance,1
4
- nasa_power_shanghai_hourly_T2M/1h/short,Moirai-1.1-Small,16.46809493020707,17.762606350991785,3.2289379835128784,1.03015932672555,0.1150119196766957,0.12686051090764325,15.835757291519093,4.058090059400736,0.15810997413863923,0.12580482285892922,0.10702413857726764,Weather,1
5
- noaa_coops_sf_water_level_water_level/6min/short,Moirai-1.1-Small,0.005106911324939271,0.005023066128869232,0.061788178980350494,2.3302409204783667,0.26746783672360525,0.22650807762100097,22.07225150197471,0.071462656856146,0.2781550105933736,0.24049891698571166,0.309963690852684,Ocean,1
6
- noaa_ndbc_46013_realtime_wave_height/10min/short,Moirai-1.1-Small,0.011788767184261436,0.009929538068242039,0.07437371214230855,0.5352652442580991,0.029046001335686772,0.02862860060252284,5.27525164074997,0.10857608937635135,0.04162661674999375,0.028513883944475068,0.039676113202017554,Ocean,1
7
- nws_ksfo_observations_temperature/1h/short,Moirai-1.1-Small,0.009181221596918476,0.0033276888748332567,0.017116785049438477,0.3243178534509565,0.0013366897324967708,0.001326615176037771,15.141958610055587,0.09581869127116314,0.007380130120228421,0.0013183659600123755,0.002086598079706136,Weather,1
8
- open_meteo_shanghai_air_quality_hourly_pm2_5/1h/short,Moirai-1.1-Small,187.9900532822021,186.3770823168015,9.616911133130392,0.5172700413546742,0.1997350052588335,0.24096720588809792,6.676616536919641,13.710946476527509,0.2793638752154284,0.1959469075568973,0.21197237663449722,Air Quality,1
9
- open_meteo_shanghai_hourly_temperature_2m/1h/short,Moirai-1.1-Small,0.7451148641512191,0.7336263607690702,0.7014484405517578,0.38202946077832767,0.028982091534334033,0.02845294724981401,3.3017538202957617,0.8632003615332996,0.03444763524408459,0.027992620368853683,0.026208127846003952,Weather,1
10
- usgs_potomac_iv_usgs_00060/15min/short,Moirai-1.1-Small,10798.155609925589,6040.659053166707,68.469970703125,nan,0.0026951572531489633,0.0026905473764017203,nan,103.91417424935632,0.004086416814655992,0.002692576268843192,0.0026380449775952625,Hydrology,1
11
- wikimedia_time_series_hourly_pageviews/1D/short,Moirai-1.1-Small,3089.0964928199537,2815.892555135797,52.98386764526367,1.1353685923985073,0.17051746000903956,0.18643314794028967,2.6726661137172156,55.579640992183045,0.17900045408110482,0.17064047550809555,0.13484585527060686,Web,1
 
1
  dataset,model,eval_metrics/MSE[mean],eval_metrics/MSE[0.5],eval_metrics/MAE[0.5],eval_metrics/MASE[0.5],eval_metrics/MAPE[0.5],eval_metrics/sMAPE[0.5],eval_metrics/MSIS,eval_metrics/RMSE[mean],eval_metrics/NRMSE[mean],eval_metrics/ND[0.5],eval_metrics/mean_weighted_sum_quantile_loss,domain,num_variates
2
+ binance_btcusdt_1h_close/1h/short,Moirai-1.1-Small,952374.008026123,920160.1939131418,925.12646484375,1.0237312453261904,0.012614736997089234,0.012529663657250362,10.904908958873357,975.8965150189456,0.013295383409153661,0.012603704248102358,0.008648531284962633,Finance,1
3
+ coingecko_bitcoin_7d_price_usd/1h/short,Moirai-1.1-Small,980882.1796239218,946688.7947781881,936.5074869791666,1.0134895871007623,0.012790573136249405,0.012702768980917836,11.206580901590142,990.3949614289855,0.013513860794401974,0.012778570474239172,0.008733789549838429,Finance,1
4
+ nasa_power_shanghai_hourly_T2M/1h/short,Moirai-1.1-Small,22.11944576046153,23.27638368696455,3.802117625872294,1.106982912627002,0.1363775798751412,0.15233264027581442,7.944486966482941,4.703131484496423,0.18324186661696704,0.14813686012367747,0.13622131038191412,Weather,1
5
+ noaa_coops_sf_water_level_water_level/6min/short,Moirai-1.1-Small,0.07894352857742426,0.08123021915922779,0.24160874634981155,15.1503835775314,0.17799479341783764,0.2016127114400773,229.84756618905703,0.2809689103396037,0.2201447466272617,0.18930527289948496,0.1584442261775357,Ocean,1
6
+ nws_ksfo_observations_temperature/1h/short,Moirai-1.1-Small,7.333502640536911,7.098776238255065,2.0732465585072837,2.551688072008964,0.13368541403511328,0.12020856200648522,67.84784727830153,2.708044061779075,0.1598845202526391,0.12240570087127874,0.1052653917054816,Weather,1
7
+ open_meteo_shanghai_air_quality_hourly_pm2_5/1h/short,Moirai-1.1-Small,973.8622120598106,1023.9657259982699,23.962568004926045,1.051374098936426,0.3256804509855051,0.4334222612374714,12.957339274599958,31.206765485384906,0.5318579730285935,0.4083948672485306,0.3651104991221374,Air Quality,1
8
+ usgs_potomac_iv_usgs_00060/15min/short,Moirai-1.1-Small,96855.45377286275,112976.57937427361,257.6378173828125,nan,0.009145275820772208,0.009214959719790085,nan,311.2160885508054,0.011179742740935982,0.009255063040244723,0.008606347849519367,Hydrology,1
 
 
 
results/moirai_1_1_small/config.json CHANGED
@@ -3,7 +3,7 @@
3
  "model_type": "zero-shot",
4
  "model_dtype": "float32",
5
  "model_link": "https://tsfm.ai/models/Salesforce/moirai-1.1-R-small",
6
- "code_link": "https://github.com/zhouziyu02/TSFM_Bench/blob/main/scripts/run_online_eval.py",
7
  "org": "Salesforce / TSFM.ai",
8
  "testdata_leakage": "No",
9
  "replication_code_available": "Yes",
 
3
  "model_type": "zero-shot",
4
  "model_dtype": "float32",
5
  "model_link": "https://tsfm.ai/models/Salesforce/moirai-1.1-R-small",
6
+ "code_link": "https://github.com/zhouziyu02/TSFM_Bench/blob/main/space/scripts/run_space_eval.py",
7
  "org": "Salesforce / TSFM.ai",
8
  "testdata_leakage": "No",
9
  "replication_code_available": "Yes",
results/moirai_1_1_small/online_meta.json CHANGED
@@ -1,67 +1,49 @@
1
  {
2
  "model": "Moirai-1.1-Small",
3
  "api_model_id": "Salesforce/moirai-1.1-R-small",
4
- "evaluated_at": "2026-05-28T12:21:49.771336+00:00",
5
  "datasets": [
6
  {
7
- "dataset": "task_binance_btcusdt_1h_crypto_btcusdt_close_20260527t120000z",
8
- "data_fetched_at": "2026-05-28T12:16:39.187896Z",
9
  "context_length": 96,
10
  "prediction_length": 24
11
  },
12
  {
13
- "dataset": "task_coingecko_bitcoin_7d_coin_bitcoin_price_usd_20260527t120016z",
14
- "data_fetched_at": "2026-05-28T12:16:39.187896Z",
15
  "context_length": 96,
16
  "prediction_length": 24
17
  },
18
  {
19
  "dataset": "task_nasa_power_shanghai_hourly_point_shanghai_31.2304_121.4737_t2m_20260524t230000z",
20
- "data_fetched_at": "2026-05-28T12:16:39.187896Z",
21
- "context_length": 83,
22
  "prediction_length": 24
23
  },
24
  {
25
- "dataset": "task_noaa_coops_sf_water_level_coops_station_9414290_water_level_20260528t094200z",
26
- "data_fetched_at": "2026-05-28T12:16:39.187896Z",
27
  "context_length": 96,
28
  "prediction_length": 24
29
  },
30
  {
31
- "dataset": "task_noaa_ndbc_46013_realtime_ndbc_station_46013_wave_height_20260528t045000z",
32
- "data_fetched_at": "2026-05-28T12:16:39.187896Z",
33
  "context_length": 96,
34
  "prediction_length": 24
35
  },
36
  {
37
- "dataset": "task_nws_ksfo_observations_nws_station_ksfo_temperature_20260528t101000z",
38
- "data_fetched_at": "2026-05-28T12:16:39.187896Z",
39
  "context_length": 96,
40
  "prediction_length": 24
41
  },
42
  {
43
- "dataset": "task_open_meteo_shanghai_air_quality_hourly_grid_shanghai_31.2304_121.4737_pm2_5_20260527t120000z",
44
- "data_fetched_at": "2026-05-28T12:16:39.187896Z",
45
  "context_length": 96,
46
  "prediction_length": 24
47
- },
48
- {
49
- "dataset": "task_open_meteo_shanghai_hourly_grid_shanghai_31.2304_121.4737_temperature_2m_20260527t120000z",
50
- "data_fetched_at": "2026-05-28T12:16:39.187896Z",
51
- "context_length": 96,
52
- "prediction_length": 24
53
- },
54
- {
55
- "dataset": "task_usgs_potomac_iv_usgs_site_01646500_usgs_00060_20260528t095000z",
56
- "data_fetched_at": "2026-05-28T12:16:39.187896Z",
57
- "context_length": 96,
58
- "prediction_length": 24
59
- },
60
- {
61
- "dataset": "task_wikimedia_time_series_hourly_enwiki_article_time_series_pageviews_20260525t000000z",
62
- "data_fetched_at": "2026-05-28T12:16:39.187896Z",
63
- "context_length": 4,
64
- "prediction_length": 2
65
  }
66
  ]
67
  }
 
1
  {
2
  "model": "Moirai-1.1-Small",
3
  "api_model_id": "Salesforce/moirai-1.1-R-small",
4
+ "evaluated_at": "2026-05-29T02:55:13.592899+00:00",
5
  "datasets": [
6
  {
7
+ "dataset": "task_binance_btcusdt_1h_crypto_btcusdt_close_20260528t020000z",
8
+ "data_fetched_at": "2026-05-29T02:50:05.083862Z",
9
  "context_length": 96,
10
  "prediction_length": 24
11
  },
12
  {
13
+ "dataset": "task_coingecko_bitcoin_7d_coin_bitcoin_price_usd_20260528t030012z",
14
+ "data_fetched_at": "2026-05-29T02:50:05.083862Z",
15
  "context_length": 96,
16
  "prediction_length": 24
17
  },
18
  {
19
  "dataset": "task_nasa_power_shanghai_hourly_point_shanghai_31.2304_121.4737_t2m_20260524t230000z",
20
+ "data_fetched_at": "2026-05-29T02:50:05.083862Z",
21
+ "context_length": 69,
22
  "prediction_length": 24
23
  },
24
  {
25
+ "dataset": "task_noaa_coops_sf_water_level_coops_station_9414290_water_level_20260529t001800z",
26
+ "data_fetched_at": "2026-05-29T02:50:05.083862Z",
27
  "context_length": 96,
28
  "prediction_length": 24
29
  },
30
  {
31
+ "dataset": "task_nws_ksfo_observations_nws_station_ksfo_temperature_20260529t004500z",
32
+ "data_fetched_at": "2026-05-29T02:50:05.083862Z",
33
  "context_length": 96,
34
  "prediction_length": 24
35
  },
36
  {
37
+ "dataset": "task_open_meteo_shanghai_air_quality_hourly_grid_shanghai_31.2304_121.4737_pm2_5_20260528t020000z",
38
+ "data_fetched_at": "2026-05-29T02:50:05.083862Z",
39
  "context_length": 96,
40
  "prediction_length": 24
41
  },
42
  {
43
+ "dataset": "task_usgs_potomac_iv_usgs_site_01646500_usgs_00060_20260528t235000z",
44
+ "data_fetched_at": "2026-05-29T02:50:05.083862Z",
45
  "context_length": 96,
46
  "prediction_length": 24
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
47
  }
48
  ]
49
  }
results/online_status.json CHANGED
@@ -1,411 +1,302 @@
1
  {
2
  "status": "ok",
3
- "started_at": "2026-05-28T12:16:39.312039+00:00",
4
- "finished_at": "2026-05-28T12:21:49.773564+00:00",
5
  "data_source": "ts_bench",
6
- "data_config": "configs/datasets/ts_bench.yaml",
7
- "ts_bench_root": "/Users/zhouziyu/Desktop/TS-Bench",
8
- "baseline_model": "Chronos-Bolt-Base",
9
  "models": {
10
  "Chronos-Bolt-Tiny": {
11
  "model": "Chronos-Bolt-Tiny",
12
  "api_model_id": "amazon/chronos-bolt-tiny",
13
- "evaluated_at": "2026-05-28T12:18:05.053451+00:00",
14
  "datasets": [
15
  {
16
- "dataset": "task_binance_btcusdt_1h_crypto_btcusdt_close_20260527t120000z",
17
- "data_fetched_at": "2026-05-28T12:16:39.187896Z",
18
  "context_length": 96,
19
  "prediction_length": 24
20
  },
21
  {
22
- "dataset": "task_coingecko_bitcoin_7d_coin_bitcoin_price_usd_20260527t120016z",
23
- "data_fetched_at": "2026-05-28T12:16:39.187896Z",
24
  "context_length": 96,
25
  "prediction_length": 24
26
  },
27
  {
28
  "dataset": "task_nasa_power_shanghai_hourly_point_shanghai_31.2304_121.4737_t2m_20260524t230000z",
29
- "data_fetched_at": "2026-05-28T12:16:39.187896Z",
30
- "context_length": 83,
31
  "prediction_length": 24
32
  },
33
  {
34
- "dataset": "task_noaa_coops_sf_water_level_coops_station_9414290_water_level_20260528t094200z",
35
- "data_fetched_at": "2026-05-28T12:16:39.187896Z",
36
  "context_length": 96,
37
  "prediction_length": 24
38
  },
39
  {
40
- "dataset": "task_noaa_ndbc_46013_realtime_ndbc_station_46013_wave_height_20260528t045000z",
41
- "data_fetched_at": "2026-05-28T12:16:39.187896Z",
42
  "context_length": 96,
43
  "prediction_length": 24
44
  },
45
  {
46
- "dataset": "task_nws_ksfo_observations_nws_station_ksfo_temperature_20260528t101000z",
47
- "data_fetched_at": "2026-05-28T12:16:39.187896Z",
48
  "context_length": 96,
49
  "prediction_length": 24
50
  },
51
  {
52
- "dataset": "task_open_meteo_shanghai_air_quality_hourly_grid_shanghai_31.2304_121.4737_pm2_5_20260527t120000z",
53
- "data_fetched_at": "2026-05-28T12:16:39.187896Z",
54
  "context_length": 96,
55
  "prediction_length": 24
56
- },
57
- {
58
- "dataset": "task_open_meteo_shanghai_hourly_grid_shanghai_31.2304_121.4737_temperature_2m_20260527t120000z",
59
- "data_fetched_at": "2026-05-28T12:16:39.187896Z",
60
- "context_length": 96,
61
- "prediction_length": 24
62
- },
63
- {
64
- "dataset": "task_usgs_potomac_iv_usgs_site_01646500_usgs_00060_20260528t095000z",
65
- "data_fetched_at": "2026-05-28T12:16:39.187896Z",
66
- "context_length": 96,
67
- "prediction_length": 24
68
- },
69
- {
70
- "dataset": "task_wikimedia_time_series_hourly_enwiki_article_time_series_pageviews_20260525t000000z",
71
- "data_fetched_at": "2026-05-28T12:16:39.187896Z",
72
- "context_length": 4,
73
- "prediction_length": 2
74
  }
75
  ]
76
  },
77
  "Chronos-Bolt-Mini": {
78
  "model": "Chronos-Bolt-Mini",
79
  "api_model_id": "amazon/chronos-bolt-mini",
80
- "evaluated_at": "2026-05-28T12:18:55.358936+00:00",
81
  "datasets": [
82
  {
83
- "dataset": "task_binance_btcusdt_1h_crypto_btcusdt_close_20260527t120000z",
84
- "data_fetched_at": "2026-05-28T12:16:39.187896Z",
85
  "context_length": 96,
86
  "prediction_length": 24
87
  },
88
  {
89
- "dataset": "task_coingecko_bitcoin_7d_coin_bitcoin_price_usd_20260527t120016z",
90
- "data_fetched_at": "2026-05-28T12:16:39.187896Z",
91
  "context_length": 96,
92
  "prediction_length": 24
93
  },
94
  {
95
  "dataset": "task_nasa_power_shanghai_hourly_point_shanghai_31.2304_121.4737_t2m_20260524t230000z",
96
- "data_fetched_at": "2026-05-28T12:16:39.187896Z",
97
- "context_length": 83,
98
- "prediction_length": 24
99
- },
100
- {
101
- "dataset": "task_noaa_coops_sf_water_level_coops_station_9414290_water_level_20260528t094200z",
102
- "data_fetched_at": "2026-05-28T12:16:39.187896Z",
103
- "context_length": 96,
104
- "prediction_length": 24
105
- },
106
- {
107
- "dataset": "task_noaa_ndbc_46013_realtime_ndbc_station_46013_wave_height_20260528t045000z",
108
- "data_fetched_at": "2026-05-28T12:16:39.187896Z",
109
- "context_length": 96,
110
  "prediction_length": 24
111
  },
112
  {
113
- "dataset": "task_nws_ksfo_observations_nws_station_ksfo_temperature_20260528t101000z",
114
- "data_fetched_at": "2026-05-28T12:16:39.187896Z",
115
  "context_length": 96,
116
  "prediction_length": 24
117
  },
118
  {
119
- "dataset": "task_open_meteo_shanghai_air_quality_hourly_grid_shanghai_31.2304_121.4737_pm2_5_20260527t120000z",
120
- "data_fetched_at": "2026-05-28T12:16:39.187896Z",
121
  "context_length": 96,
122
  "prediction_length": 24
123
  },
124
  {
125
- "dataset": "task_open_meteo_shanghai_hourly_grid_shanghai_31.2304_121.4737_temperature_2m_20260527t120000z",
126
- "data_fetched_at": "2026-05-28T12:16:39.187896Z",
127
  "context_length": 96,
128
  "prediction_length": 24
129
  },
130
  {
131
- "dataset": "task_usgs_potomac_iv_usgs_site_01646500_usgs_00060_20260528t095000z",
132
- "data_fetched_at": "2026-05-28T12:16:39.187896Z",
133
  "context_length": 96,
134
  "prediction_length": 24
135
- },
136
- {
137
- "dataset": "task_wikimedia_time_series_hourly_enwiki_article_time_series_pageviews_20260525t000000z",
138
- "data_fetched_at": "2026-05-28T12:16:39.187896Z",
139
- "context_length": 4,
140
- "prediction_length": 2
141
  }
142
  ]
143
  },
144
  "Chronos-Bolt-Base": {
145
  "model": "Chronos-Bolt-Base",
146
  "api_model_id": "amazon/chronos-bolt-base",
147
- "evaluated_at": "2026-05-28T12:19:28.056546+00:00",
148
  "datasets": [
149
  {
150
- "dataset": "task_binance_btcusdt_1h_crypto_btcusdt_close_20260527t120000z",
151
- "data_fetched_at": "2026-05-28T12:16:39.187896Z",
152
  "context_length": 96,
153
  "prediction_length": 24
154
  },
155
  {
156
- "dataset": "task_coingecko_bitcoin_7d_coin_bitcoin_price_usd_20260527t120016z",
157
- "data_fetched_at": "2026-05-28T12:16:39.187896Z",
158
  "context_length": 96,
159
  "prediction_length": 24
160
  },
161
  {
162
  "dataset": "task_nasa_power_shanghai_hourly_point_shanghai_31.2304_121.4737_t2m_20260524t230000z",
163
- "data_fetched_at": "2026-05-28T12:16:39.187896Z",
164
- "context_length": 83,
165
  "prediction_length": 24
166
  },
167
  {
168
- "dataset": "task_noaa_coops_sf_water_level_coops_station_9414290_water_level_20260528t094200z",
169
- "data_fetched_at": "2026-05-28T12:16:39.187896Z",
170
  "context_length": 96,
171
  "prediction_length": 24
172
  },
173
  {
174
- "dataset": "task_noaa_ndbc_46013_realtime_ndbc_station_46013_wave_height_20260528t045000z",
175
- "data_fetched_at": "2026-05-28T12:16:39.187896Z",
176
  "context_length": 96,
177
  "prediction_length": 24
178
  },
179
  {
180
- "dataset": "task_nws_ksfo_observations_nws_station_ksfo_temperature_20260528t101000z",
181
- "data_fetched_at": "2026-05-28T12:16:39.187896Z",
182
  "context_length": 96,
183
  "prediction_length": 24
184
  },
185
  {
186
- "dataset": "task_open_meteo_shanghai_air_quality_hourly_grid_shanghai_31.2304_121.4737_pm2_5_20260527t120000z",
187
- "data_fetched_at": "2026-05-28T12:16:39.187896Z",
188
  "context_length": 96,
189
  "prediction_length": 24
190
- },
191
- {
192
- "dataset": "task_open_meteo_shanghai_hourly_grid_shanghai_31.2304_121.4737_temperature_2m_20260527t120000z",
193
- "data_fetched_at": "2026-05-28T12:16:39.187896Z",
194
- "context_length": 96,
195
- "prediction_length": 24
196
- },
197
- {
198
- "dataset": "task_usgs_potomac_iv_usgs_site_01646500_usgs_00060_20260528t095000z",
199
- "data_fetched_at": "2026-05-28T12:16:39.187896Z",
200
- "context_length": 96,
201
- "prediction_length": 24
202
- },
203
- {
204
- "dataset": "task_wikimedia_time_series_hourly_enwiki_article_time_series_pageviews_20260525t000000z",
205
- "data_fetched_at": "2026-05-28T12:16:39.187896Z",
206
- "context_length": 4,
207
- "prediction_length": 2
208
  }
209
  ]
210
  },
211
  "Chronos-2": {
212
  "model": "Chronos-2",
213
  "api_model_id": "amazon/chronos-2",
214
- "evaluated_at": "2026-05-28T12:20:18.799459+00:00",
215
  "datasets": [
216
  {
217
- "dataset": "task_binance_btcusdt_1h_crypto_btcusdt_close_20260527t120000z",
218
- "data_fetched_at": "2026-05-28T12:16:39.187896Z",
219
  "context_length": 96,
220
  "prediction_length": 24
221
  },
222
  {
223
- "dataset": "task_coingecko_bitcoin_7d_coin_bitcoin_price_usd_20260527t120016z",
224
- "data_fetched_at": "2026-05-28T12:16:39.187896Z",
225
  "context_length": 96,
226
  "prediction_length": 24
227
  },
228
  {
229
  "dataset": "task_nasa_power_shanghai_hourly_point_shanghai_31.2304_121.4737_t2m_20260524t230000z",
230
- "data_fetched_at": "2026-05-28T12:16:39.187896Z",
231
- "context_length": 83,
232
  "prediction_length": 24
233
  },
234
  {
235
- "dataset": "task_noaa_coops_sf_water_level_coops_station_9414290_water_level_20260528t094200z",
236
- "data_fetched_at": "2026-05-28T12:16:39.187896Z",
237
  "context_length": 96,
238
  "prediction_length": 24
239
  },
240
  {
241
- "dataset": "task_noaa_ndbc_46013_realtime_ndbc_station_46013_wave_height_20260528t045000z",
242
- "data_fetched_at": "2026-05-28T12:16:39.187896Z",
243
  "context_length": 96,
244
  "prediction_length": 24
245
  },
246
  {
247
- "dataset": "task_nws_ksfo_observations_nws_station_ksfo_temperature_20260528t101000z",
248
- "data_fetched_at": "2026-05-28T12:16:39.187896Z",
249
  "context_length": 96,
250
  "prediction_length": 24
251
  },
252
  {
253
- "dataset": "task_open_meteo_shanghai_air_quality_hourly_grid_shanghai_31.2304_121.4737_pm2_5_20260527t120000z",
254
- "data_fetched_at": "2026-05-28T12:16:39.187896Z",
255
  "context_length": 96,
256
  "prediction_length": 24
257
- },
258
- {
259
- "dataset": "task_open_meteo_shanghai_hourly_grid_shanghai_31.2304_121.4737_temperature_2m_20260527t120000z",
260
- "data_fetched_at": "2026-05-28T12:16:39.187896Z",
261
- "context_length": 96,
262
- "prediction_length": 24
263
- },
264
- {
265
- "dataset": "task_usgs_potomac_iv_usgs_site_01646500_usgs_00060_20260528t095000z",
266
- "data_fetched_at": "2026-05-28T12:16:39.187896Z",
267
- "context_length": 96,
268
- "prediction_length": 24
269
- },
270
- {
271
- "dataset": "task_wikimedia_time_series_hourly_enwiki_article_time_series_pageviews_20260525t000000z",
272
- "data_fetched_at": "2026-05-28T12:16:39.187896Z",
273
- "context_length": 4,
274
- "prediction_length": 2
275
  }
276
  ]
277
  },
278
  "TimesFM-2.5": {
279
  "model": "TimesFM-2.5",
280
  "api_model_id": "google/timesfm-2.5-200m-pytorch",
281
- "evaluated_at": "2026-05-28T12:21:19.114949+00:00",
282
  "datasets": [
283
  {
284
- "dataset": "task_binance_btcusdt_1h_crypto_btcusdt_close_20260527t120000z",
285
- "data_fetched_at": "2026-05-28T12:16:39.187896Z",
286
  "context_length": 96,
287
  "prediction_length": 24
288
  },
289
  {
290
- "dataset": "task_coingecko_bitcoin_7d_coin_bitcoin_price_usd_20260527t120016z",
291
- "data_fetched_at": "2026-05-28T12:16:39.187896Z",
292
  "context_length": 96,
293
  "prediction_length": 24
294
  },
295
  {
296
  "dataset": "task_nasa_power_shanghai_hourly_point_shanghai_31.2304_121.4737_t2m_20260524t230000z",
297
- "data_fetched_at": "2026-05-28T12:16:39.187896Z",
298
- "context_length": 83,
299
- "prediction_length": 24
300
- },
301
- {
302
- "dataset": "task_noaa_coops_sf_water_level_coops_station_9414290_water_level_20260528t094200z",
303
- "data_fetched_at": "2026-05-28T12:16:39.187896Z",
304
- "context_length": 96,
305
- "prediction_length": 24
306
- },
307
- {
308
- "dataset": "task_noaa_ndbc_46013_realtime_ndbc_station_46013_wave_height_20260528t045000z",
309
- "data_fetched_at": "2026-05-28T12:16:39.187896Z",
310
- "context_length": 96,
311
  "prediction_length": 24
312
  },
313
  {
314
- "dataset": "task_nws_ksfo_observations_nws_station_ksfo_temperature_20260528t101000z",
315
- "data_fetched_at": "2026-05-28T12:16:39.187896Z",
316
  "context_length": 96,
317
  "prediction_length": 24
318
  },
319
  {
320
- "dataset": "task_open_meteo_shanghai_air_quality_hourly_grid_shanghai_31.2304_121.4737_pm2_5_20260527t120000z",
321
- "data_fetched_at": "2026-05-28T12:16:39.187896Z",
322
  "context_length": 96,
323
  "prediction_length": 24
324
  },
325
  {
326
- "dataset": "task_open_meteo_shanghai_hourly_grid_shanghai_31.2304_121.4737_temperature_2m_20260527t120000z",
327
- "data_fetched_at": "2026-05-28T12:16:39.187896Z",
328
  "context_length": 96,
329
  "prediction_length": 24
330
  },
331
  {
332
- "dataset": "task_usgs_potomac_iv_usgs_site_01646500_usgs_00060_20260528t095000z",
333
- "data_fetched_at": "2026-05-28T12:16:39.187896Z",
334
  "context_length": 96,
335
  "prediction_length": 24
336
- },
337
- {
338
- "dataset": "task_wikimedia_time_series_hourly_enwiki_article_time_series_pageviews_20260525t000000z",
339
- "data_fetched_at": "2026-05-28T12:16:39.187896Z",
340
- "context_length": 4,
341
- "prediction_length": 2
342
  }
343
  ]
344
  },
345
  "Moirai-1.1-Small": {
346
  "model": "Moirai-1.1-Small",
347
  "api_model_id": "Salesforce/moirai-1.1-R-small",
348
- "evaluated_at": "2026-05-28T12:21:49.771336+00:00",
349
  "datasets": [
350
  {
351
- "dataset": "task_binance_btcusdt_1h_crypto_btcusdt_close_20260527t120000z",
352
- "data_fetched_at": "2026-05-28T12:16:39.187896Z",
353
  "context_length": 96,
354
  "prediction_length": 24
355
  },
356
  {
357
- "dataset": "task_coingecko_bitcoin_7d_coin_bitcoin_price_usd_20260527t120016z",
358
- "data_fetched_at": "2026-05-28T12:16:39.187896Z",
359
  "context_length": 96,
360
  "prediction_length": 24
361
  },
362
  {
363
  "dataset": "task_nasa_power_shanghai_hourly_point_shanghai_31.2304_121.4737_t2m_20260524t230000z",
364
- "data_fetched_at": "2026-05-28T12:16:39.187896Z",
365
- "context_length": 83,
366
  "prediction_length": 24
367
  },
368
  {
369
- "dataset": "task_noaa_coops_sf_water_level_coops_station_9414290_water_level_20260528t094200z",
370
- "data_fetched_at": "2026-05-28T12:16:39.187896Z",
371
  "context_length": 96,
372
  "prediction_length": 24
373
  },
374
  {
375
- "dataset": "task_noaa_ndbc_46013_realtime_ndbc_station_46013_wave_height_20260528t045000z",
376
- "data_fetched_at": "2026-05-28T12:16:39.187896Z",
377
  "context_length": 96,
378
  "prediction_length": 24
379
  },
380
  {
381
- "dataset": "task_nws_ksfo_observations_nws_station_ksfo_temperature_20260528t101000z",
382
- "data_fetched_at": "2026-05-28T12:16:39.187896Z",
383
  "context_length": 96,
384
  "prediction_length": 24
385
  },
386
  {
387
- "dataset": "task_open_meteo_shanghai_air_quality_hourly_grid_shanghai_31.2304_121.4737_pm2_5_20260527t120000z",
388
- "data_fetched_at": "2026-05-28T12:16:39.187896Z",
389
  "context_length": 96,
390
  "prediction_length": 24
391
- },
392
- {
393
- "dataset": "task_open_meteo_shanghai_hourly_grid_shanghai_31.2304_121.4737_temperature_2m_20260527t120000z",
394
- "data_fetched_at": "2026-05-28T12:16:39.187896Z",
395
- "context_length": 96,
396
- "prediction_length": 24
397
- },
398
- {
399
- "dataset": "task_usgs_potomac_iv_usgs_site_01646500_usgs_00060_20260528t095000z",
400
- "data_fetched_at": "2026-05-28T12:16:39.187896Z",
401
- "context_length": 96,
402
- "prediction_length": 24
403
- },
404
- {
405
- "dataset": "task_wikimedia_time_series_hourly_enwiki_article_time_series_pageviews_20260525t000000z",
406
- "data_fetched_at": "2026-05-28T12:16:39.187896Z",
407
- "context_length": 4,
408
- "prediction_length": 2
409
  }
410
  ]
411
  }
 
1
  {
2
  "status": "ok",
3
+ "started_at": "2026-05-29T02:49:52.974580+00:00",
4
+ "finished_at": "2026-05-29T02:55:13.595519+00:00",
5
  "data_source": "ts_bench",
6
+ "data_config": "/Users/zhouziyu/Desktop/TSFM_Bench/space/configs/datasets/ts_bench.yaml",
7
+ "ts_bench_root": "/Users/zhouziyu/Desktop/TSFM_Bench/space/vendor/ts_bench",
 
8
  "models": {
9
  "Chronos-Bolt-Tiny": {
10
  "model": "Chronos-Bolt-Tiny",
11
  "api_model_id": "amazon/chronos-bolt-tiny",
12
+ "evaluated_at": "2026-05-29T02:50:52.849106+00:00",
13
  "datasets": [
14
  {
15
+ "dataset": "task_binance_btcusdt_1h_crypto_btcusdt_close_20260528t020000z",
16
+ "data_fetched_at": "2026-05-29T02:50:05.083862Z",
17
  "context_length": 96,
18
  "prediction_length": 24
19
  },
20
  {
21
+ "dataset": "task_coingecko_bitcoin_7d_coin_bitcoin_price_usd_20260528t030012z",
22
+ "data_fetched_at": "2026-05-29T02:50:05.083862Z",
23
  "context_length": 96,
24
  "prediction_length": 24
25
  },
26
  {
27
  "dataset": "task_nasa_power_shanghai_hourly_point_shanghai_31.2304_121.4737_t2m_20260524t230000z",
28
+ "data_fetched_at": "2026-05-29T02:50:05.083862Z",
29
+ "context_length": 69,
30
  "prediction_length": 24
31
  },
32
  {
33
+ "dataset": "task_noaa_coops_sf_water_level_coops_station_9414290_water_level_20260529t001800z",
34
+ "data_fetched_at": "2026-05-29T02:50:05.083862Z",
35
  "context_length": 96,
36
  "prediction_length": 24
37
  },
38
  {
39
+ "dataset": "task_nws_ksfo_observations_nws_station_ksfo_temperature_20260529t004500z",
40
+ "data_fetched_at": "2026-05-29T02:50:05.083862Z",
41
  "context_length": 96,
42
  "prediction_length": 24
43
  },
44
  {
45
+ "dataset": "task_open_meteo_shanghai_air_quality_hourly_grid_shanghai_31.2304_121.4737_pm2_5_20260528t020000z",
46
+ "data_fetched_at": "2026-05-29T02:50:05.083862Z",
47
  "context_length": 96,
48
  "prediction_length": 24
49
  },
50
  {
51
+ "dataset": "task_usgs_potomac_iv_usgs_site_01646500_usgs_00060_20260528t235000z",
52
+ "data_fetched_at": "2026-05-29T02:50:05.083862Z",
53
  "context_length": 96,
54
  "prediction_length": 24
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
55
  }
56
  ]
57
  },
58
  "Chronos-Bolt-Mini": {
59
  "model": "Chronos-Bolt-Mini",
60
  "api_model_id": "amazon/chronos-bolt-mini",
61
+ "evaluated_at": "2026-05-29T02:51:10.536027+00:00",
62
  "datasets": [
63
  {
64
+ "dataset": "task_binance_btcusdt_1h_crypto_btcusdt_close_20260528t020000z",
65
+ "data_fetched_at": "2026-05-29T02:50:05.083862Z",
66
  "context_length": 96,
67
  "prediction_length": 24
68
  },
69
  {
70
+ "dataset": "task_coingecko_bitcoin_7d_coin_bitcoin_price_usd_20260528t030012z",
71
+ "data_fetched_at": "2026-05-29T02:50:05.083862Z",
72
  "context_length": 96,
73
  "prediction_length": 24
74
  },
75
  {
76
  "dataset": "task_nasa_power_shanghai_hourly_point_shanghai_31.2304_121.4737_t2m_20260524t230000z",
77
+ "data_fetched_at": "2026-05-29T02:50:05.083862Z",
78
+ "context_length": 69,
 
 
 
 
 
 
 
 
 
 
 
 
79
  "prediction_length": 24
80
  },
81
  {
82
+ "dataset": "task_noaa_coops_sf_water_level_coops_station_9414290_water_level_20260529t001800z",
83
+ "data_fetched_at": "2026-05-29T02:50:05.083862Z",
84
  "context_length": 96,
85
  "prediction_length": 24
86
  },
87
  {
88
+ "dataset": "task_nws_ksfo_observations_nws_station_ksfo_temperature_20260529t004500z",
89
+ "data_fetched_at": "2026-05-29T02:50:05.083862Z",
90
  "context_length": 96,
91
  "prediction_length": 24
92
  },
93
  {
94
+ "dataset": "task_open_meteo_shanghai_air_quality_hourly_grid_shanghai_31.2304_121.4737_pm2_5_20260528t020000z",
95
+ "data_fetched_at": "2026-05-29T02:50:05.083862Z",
96
  "context_length": 96,
97
  "prediction_length": 24
98
  },
99
  {
100
+ "dataset": "task_usgs_potomac_iv_usgs_site_01646500_usgs_00060_20260528t235000z",
101
+ "data_fetched_at": "2026-05-29T02:50:05.083862Z",
102
  "context_length": 96,
103
  "prediction_length": 24
 
 
 
 
 
 
104
  }
105
  ]
106
  },
107
  "Chronos-Bolt-Base": {
108
  "model": "Chronos-Bolt-Base",
109
  "api_model_id": "amazon/chronos-bolt-base",
110
+ "evaluated_at": "2026-05-29T02:51:30.013825+00:00",
111
  "datasets": [
112
  {
113
+ "dataset": "task_binance_btcusdt_1h_crypto_btcusdt_close_20260528t020000z",
114
+ "data_fetched_at": "2026-05-29T02:50:05.083862Z",
115
  "context_length": 96,
116
  "prediction_length": 24
117
  },
118
  {
119
+ "dataset": "task_coingecko_bitcoin_7d_coin_bitcoin_price_usd_20260528t030012z",
120
+ "data_fetched_at": "2026-05-29T02:50:05.083862Z",
121
  "context_length": 96,
122
  "prediction_length": 24
123
  },
124
  {
125
  "dataset": "task_nasa_power_shanghai_hourly_point_shanghai_31.2304_121.4737_t2m_20260524t230000z",
126
+ "data_fetched_at": "2026-05-29T02:50:05.083862Z",
127
+ "context_length": 69,
128
  "prediction_length": 24
129
  },
130
  {
131
+ "dataset": "task_noaa_coops_sf_water_level_coops_station_9414290_water_level_20260529t001800z",
132
+ "data_fetched_at": "2026-05-29T02:50:05.083862Z",
133
  "context_length": 96,
134
  "prediction_length": 24
135
  },
136
  {
137
+ "dataset": "task_nws_ksfo_observations_nws_station_ksfo_temperature_20260529t004500z",
138
+ "data_fetched_at": "2026-05-29T02:50:05.083862Z",
139
  "context_length": 96,
140
  "prediction_length": 24
141
  },
142
  {
143
+ "dataset": "task_open_meteo_shanghai_air_quality_hourly_grid_shanghai_31.2304_121.4737_pm2_5_20260528t020000z",
144
+ "data_fetched_at": "2026-05-29T02:50:05.083862Z",
145
  "context_length": 96,
146
  "prediction_length": 24
147
  },
148
  {
149
+ "dataset": "task_usgs_potomac_iv_usgs_site_01646500_usgs_00060_20260528t235000z",
150
+ "data_fetched_at": "2026-05-29T02:50:05.083862Z",
151
  "context_length": 96,
152
  "prediction_length": 24
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
153
  }
154
  ]
155
  },
156
  "Chronos-2": {
157
  "model": "Chronos-2",
158
  "api_model_id": "amazon/chronos-2",
159
+ "evaluated_at": "2026-05-29T02:53:16.309267+00:00",
160
  "datasets": [
161
  {
162
+ "dataset": "task_binance_btcusdt_1h_crypto_btcusdt_close_20260528t020000z",
163
+ "data_fetched_at": "2026-05-29T02:50:05.083862Z",
164
  "context_length": 96,
165
  "prediction_length": 24
166
  },
167
  {
168
+ "dataset": "task_coingecko_bitcoin_7d_coin_bitcoin_price_usd_20260528t030012z",
169
+ "data_fetched_at": "2026-05-29T02:50:05.083862Z",
170
  "context_length": 96,
171
  "prediction_length": 24
172
  },
173
  {
174
  "dataset": "task_nasa_power_shanghai_hourly_point_shanghai_31.2304_121.4737_t2m_20260524t230000z",
175
+ "data_fetched_at": "2026-05-29T02:50:05.083862Z",
176
+ "context_length": 69,
177
  "prediction_length": 24
178
  },
179
  {
180
+ "dataset": "task_noaa_coops_sf_water_level_coops_station_9414290_water_level_20260529t001800z",
181
+ "data_fetched_at": "2026-05-29T02:50:05.083862Z",
182
  "context_length": 96,
183
  "prediction_length": 24
184
  },
185
  {
186
+ "dataset": "task_nws_ksfo_observations_nws_station_ksfo_temperature_20260529t004500z",
187
+ "data_fetched_at": "2026-05-29T02:50:05.083862Z",
188
  "context_length": 96,
189
  "prediction_length": 24
190
  },
191
  {
192
+ "dataset": "task_open_meteo_shanghai_air_quality_hourly_grid_shanghai_31.2304_121.4737_pm2_5_20260528t020000z",
193
+ "data_fetched_at": "2026-05-29T02:50:05.083862Z",
194
  "context_length": 96,
195
  "prediction_length": 24
196
  },
197
  {
198
+ "dataset": "task_usgs_potomac_iv_usgs_site_01646500_usgs_00060_20260528t235000z",
199
+ "data_fetched_at": "2026-05-29T02:50:05.083862Z",
200
  "context_length": 96,
201
  "prediction_length": 24
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
202
  }
203
  ]
204
  },
205
  "TimesFM-2.5": {
206
  "model": "TimesFM-2.5",
207
  "api_model_id": "google/timesfm-2.5-200m-pytorch",
208
+ "evaluated_at": "2026-05-29T02:54:46.508756+00:00",
209
  "datasets": [
210
  {
211
+ "dataset": "task_binance_btcusdt_1h_crypto_btcusdt_close_20260528t020000z",
212
+ "data_fetched_at": "2026-05-29T02:50:05.083862Z",
213
  "context_length": 96,
214
  "prediction_length": 24
215
  },
216
  {
217
+ "dataset": "task_coingecko_bitcoin_7d_coin_bitcoin_price_usd_20260528t030012z",
218
+ "data_fetched_at": "2026-05-29T02:50:05.083862Z",
219
  "context_length": 96,
220
  "prediction_length": 24
221
  },
222
  {
223
  "dataset": "task_nasa_power_shanghai_hourly_point_shanghai_31.2304_121.4737_t2m_20260524t230000z",
224
+ "data_fetched_at": "2026-05-29T02:50:05.083862Z",
225
+ "context_length": 69,
 
 
 
 
 
 
 
 
 
 
 
 
226
  "prediction_length": 24
227
  },
228
  {
229
+ "dataset": "task_noaa_coops_sf_water_level_coops_station_9414290_water_level_20260529t001800z",
230
+ "data_fetched_at": "2026-05-29T02:50:05.083862Z",
231
  "context_length": 96,
232
  "prediction_length": 24
233
  },
234
  {
235
+ "dataset": "task_nws_ksfo_observations_nws_station_ksfo_temperature_20260529t004500z",
236
+ "data_fetched_at": "2026-05-29T02:50:05.083862Z",
237
  "context_length": 96,
238
  "prediction_length": 24
239
  },
240
  {
241
+ "dataset": "task_open_meteo_shanghai_air_quality_hourly_grid_shanghai_31.2304_121.4737_pm2_5_20260528t020000z",
242
+ "data_fetched_at": "2026-05-29T02:50:05.083862Z",
243
  "context_length": 96,
244
  "prediction_length": 24
245
  },
246
  {
247
+ "dataset": "task_usgs_potomac_iv_usgs_site_01646500_usgs_00060_20260528t235000z",
248
+ "data_fetched_at": "2026-05-29T02:50:05.083862Z",
249
  "context_length": 96,
250
  "prediction_length": 24
 
 
 
 
 
 
251
  }
252
  ]
253
  },
254
  "Moirai-1.1-Small": {
255
  "model": "Moirai-1.1-Small",
256
  "api_model_id": "Salesforce/moirai-1.1-R-small",
257
+ "evaluated_at": "2026-05-29T02:55:13.592899+00:00",
258
  "datasets": [
259
  {
260
+ "dataset": "task_binance_btcusdt_1h_crypto_btcusdt_close_20260528t020000z",
261
+ "data_fetched_at": "2026-05-29T02:50:05.083862Z",
262
  "context_length": 96,
263
  "prediction_length": 24
264
  },
265
  {
266
+ "dataset": "task_coingecko_bitcoin_7d_coin_bitcoin_price_usd_20260528t030012z",
267
+ "data_fetched_at": "2026-05-29T02:50:05.083862Z",
268
  "context_length": 96,
269
  "prediction_length": 24
270
  },
271
  {
272
  "dataset": "task_nasa_power_shanghai_hourly_point_shanghai_31.2304_121.4737_t2m_20260524t230000z",
273
+ "data_fetched_at": "2026-05-29T02:50:05.083862Z",
274
+ "context_length": 69,
275
  "prediction_length": 24
276
  },
277
  {
278
+ "dataset": "task_noaa_coops_sf_water_level_coops_station_9414290_water_level_20260529t001800z",
279
+ "data_fetched_at": "2026-05-29T02:50:05.083862Z",
280
  "context_length": 96,
281
  "prediction_length": 24
282
  },
283
  {
284
+ "dataset": "task_nws_ksfo_observations_nws_station_ksfo_temperature_20260529t004500z",
285
+ "data_fetched_at": "2026-05-29T02:50:05.083862Z",
286
  "context_length": 96,
287
  "prediction_length": 24
288
  },
289
  {
290
+ "dataset": "task_open_meteo_shanghai_air_quality_hourly_grid_shanghai_31.2304_121.4737_pm2_5_20260528t020000z",
291
+ "data_fetched_at": "2026-05-29T02:50:05.083862Z",
292
  "context_length": 96,
293
  "prediction_length": 24
294
  },
295
  {
296
+ "dataset": "task_usgs_potomac_iv_usgs_site_01646500_usgs_00060_20260528t235000z",
297
+ "data_fetched_at": "2026-05-29T02:50:05.083862Z",
298
  "context_length": 96,
299
  "prediction_length": 24
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
300
  }
301
  ]
302
  }
results/timesfm_2_5/all_results.csv CHANGED
@@ -1,11 +1,8 @@
1
  dataset,model,eval_metrics/MSE[mean],eval_metrics/MSE[0.5],eval_metrics/MAE[0.5],eval_metrics/MASE[0.5],eval_metrics/MAPE[0.5],eval_metrics/sMAPE[0.5],eval_metrics/MSIS,eval_metrics/RMSE[mean],eval_metrics/NRMSE[mean],eval_metrics/ND[0.5],eval_metrics/mean_weighted_sum_quantile_loss,domain,num_variates
2
- binance_btcusdt_1h_close/1h/short,TimesFM-2.5,3184300.868250529,2877341.8473765054,1502.9791666666667,1.9366630929550903,0.020371679342378764,0.0201094796089761,86.07571948209704,1784.4609461264567,0.024049315263214986,0.02025576401191205,0.021796525719852494,Finance,1
3
- coingecko_bitcoin_7d_price_usd/1h/short,TimesFM-2.5,2978472.513104757,2691326.2771759033,1431.572265625,1.7855091583102543,0.01942591517249738,0.019179939216192834,78.84789791391036,1725.8251687539955,0.023269242362690887,0.01930184042488197,0.020704134741179717,Finance,1
4
- nasa_power_shanghai_hourly_T2M/1h/short,TimesFM-2.5,7.424815583824056,11.590662498969019,2.684919834136963,0.8565959528394198,0.09649247329369882,0.10426336051909445,5.473512568019115,2.7248514792230525,0.1061647697821695,0.10460896612097444,0.06960420118115515,Weather,1
5
- noaa_coops_sf_water_level_water_level/6min/short,TimesFM-2.5,0.014191201840556963,0.008006066655363927,0.08215828488270442,3.0984664146014134,0.37820721429781673,0.3032819543290635,207.6676558778752,0.11912683090117425,0.4636788830559884,0.31978573995484816,0.46509102471709085,Ocean,1
6
- noaa_ndbc_46013_realtime_wave_height/10min/short,TimesFM-2.5,0.05512604274710261,0.031698439646326904,0.15231487154960632,1.09620529318399,0.06015022141194603,0.05812306444413484,72.78888463699143,0.23478935824926694,0.09001509161876256,0.05839547933378565,0.08648648191844961,Ocean,1
7
- nws_ksfo_observations_temperature/1h/short,TimesFM-2.5,0.0034255919382151965,0.0033221443641195947,0.0194094975789388,0.36775869844608894,0.0015129974061875602,0.0015030149122856155,15.604036901152464,0.05852855660457719,0.0045079760301519585,0.0014949548548461198,0.0016153432104989958,Weather,1
8
- open_meteo_shanghai_air_quality_hourly_pm2_5/1h/short,TimesFM-2.5,128.3803764339212,199.8811043637435,9.946486552556356,0.5349970940929124,0.19267528355159025,0.23408765568966516,7.410677562806579,11.330506450901531,0.2308618296844907,0.20266208703077312,0.166486311960706,Air Quality,1
9
- open_meteo_shanghai_hourly_temperature_2m/1h/short,TimesFM-2.5,0.4488093384068937,0.42550756879063556,0.5227727890014648,0.28471744342993616,0.021116368406687125,0.020967622620989168,11.867516613268592,0.6699323386782382,0.026734911000282555,0.020862232169443492,0.02480197235359902,Weather,1
10
- usgs_potomac_iv_usgs_00060/15min/short,TimesFM-2.5,3964.9586008389792,2807.6309466362,52.009847005208336,nan,0.002045911466877187,0.0020444056954015345,nan,62.9679172344058,0.0024762084444137954,0.002045283185523513,0.002015193888024105,Hydrology,1
11
- wikimedia_time_series_hourly_pageviews/1D/short,TimesFM-2.5,4207.680999754579,5861.192346271244,75.43997955322266,1.6165709904262,0.24218997554725216,0.2765631520841845,22.796233912876673,64.86664011458107,0.20891027412103405,0.2429628971118282,0.177518524153541,Web,1
 
1
  dataset,model,eval_metrics/MSE[mean],eval_metrics/MSE[0.5],eval_metrics/MAE[0.5],eval_metrics/MASE[0.5],eval_metrics/MAPE[0.5],eval_metrics/sMAPE[0.5],eval_metrics/MSIS,eval_metrics/RMSE[mean],eval_metrics/NRMSE[mean],eval_metrics/ND[0.5],eval_metrics/mean_weighted_sum_quantile_loss,domain,num_variates
2
+ binance_btcusdt_1h_close/1h/short,TimesFM-2.5,1219064.9854685466,835565.3181101481,884.322265625,0.97857792276093,0.012057590756120376,0.011980322700327062,53.58648546814461,1104.1127593993951,0.015042171211020878,0.0120477974844572,0.015083192708183289,Finance,1
3
+ coingecko_bitcoin_7d_price_usd/1h/short,TimesFM-2.5,1230621.6017862957,849700.2018839518,888.9869791666666,0.9620628334321281,0.012141146014077656,0.012062308830606919,52.43584047207273,1109.3338549716652,0.015136772575028212,0.012130156909482296,0.015134561186691181,Finance,1
4
+ nasa_power_shanghai_hourly_T2M/1h/short,TimesFM-2.5,7.933177144345336,11.399316419632365,2.7634520530700684,0.8045764238845047,0.10092765997658189,0.10842496844555281,9.535674024816261,2.8165896300926296,0.10973904153296879,0.10766871268224146,0.07858026268608374,Weather,1
5
+ noaa_coops_sf_water_level_water_level/6min/short,TimesFM-2.5,0.0015351705280940802,0.0024778745030360483,0.04266408085823059,2.675305425612182,0.03174085850119802,0.03239086232536759,12.171508226109887,0.03918125225275578,0.030699292812490016,0.03342815850788563,0.021909742273566946,Ocean,1
6
+ nws_ksfo_observations_temperature/1h/short,TimesFM-2.5,7.230681060140189,6.612742760992963,2.0171178181966147,2.4826065454727564,0.12987325798582047,0.11728228771068923,97.8729589413374,2.68899257346319,0.15875970913435808,0.11909182690459717,0.12032840390405451,Weather,1
7
+ open_meteo_shanghai_air_quality_hourly_pm2_5/1h/short,TimesFM-2.5,1370.1968184538339,1568.2607702783378,32.57473762830099,1.429238945300226,0.48036513469849956,0.6789653743383695,33.04442361416721,37.016169689121455,0.6308678478504538,0.5551723691146495,0.45629084448376683,Air Quality,1
8
+ usgs_potomac_iv_usgs_00060/15min/short,TimesFM-2.5,203315.88810078302,267129.3128611247,408.2639973958333,nan,0.014499470105041025,0.014667487665793741,nan,450.90563103689783,0.01619777749571254,0.01466597206630744,0.011492126628270052,Hydrology,1
 
 
 
results/timesfm_2_5/config.json CHANGED
@@ -3,7 +3,7 @@
3
  "model_type": "zero-shot",
4
  "model_dtype": "float32",
5
  "model_link": "https://tsfm.ai/models/google/timesfm-2.5-200m-pytorch",
6
- "code_link": "https://github.com/zhouziyu02/TSFM_Bench/blob/main/scripts/run_online_eval.py",
7
  "org": "Google / TSFM.ai",
8
  "testdata_leakage": "No",
9
  "replication_code_available": "Yes",
 
3
  "model_type": "zero-shot",
4
  "model_dtype": "float32",
5
  "model_link": "https://tsfm.ai/models/google/timesfm-2.5-200m-pytorch",
6
+ "code_link": "https://github.com/zhouziyu02/TSFM_Bench/blob/main/space/scripts/run_space_eval.py",
7
  "org": "Google / TSFM.ai",
8
  "testdata_leakage": "No",
9
  "replication_code_available": "Yes",
results/timesfm_2_5/online_meta.json CHANGED
@@ -1,67 +1,49 @@
1
  {
2
  "model": "TimesFM-2.5",
3
  "api_model_id": "google/timesfm-2.5-200m-pytorch",
4
- "evaluated_at": "2026-05-28T12:21:19.114949+00:00",
5
  "datasets": [
6
  {
7
- "dataset": "task_binance_btcusdt_1h_crypto_btcusdt_close_20260527t120000z",
8
- "data_fetched_at": "2026-05-28T12:16:39.187896Z",
9
  "context_length": 96,
10
  "prediction_length": 24
11
  },
12
  {
13
- "dataset": "task_coingecko_bitcoin_7d_coin_bitcoin_price_usd_20260527t120016z",
14
- "data_fetched_at": "2026-05-28T12:16:39.187896Z",
15
  "context_length": 96,
16
  "prediction_length": 24
17
  },
18
  {
19
  "dataset": "task_nasa_power_shanghai_hourly_point_shanghai_31.2304_121.4737_t2m_20260524t230000z",
20
- "data_fetched_at": "2026-05-28T12:16:39.187896Z",
21
- "context_length": 83,
22
  "prediction_length": 24
23
  },
24
  {
25
- "dataset": "task_noaa_coops_sf_water_level_coops_station_9414290_water_level_20260528t094200z",
26
- "data_fetched_at": "2026-05-28T12:16:39.187896Z",
27
  "context_length": 96,
28
  "prediction_length": 24
29
  },
30
  {
31
- "dataset": "task_noaa_ndbc_46013_realtime_ndbc_station_46013_wave_height_20260528t045000z",
32
- "data_fetched_at": "2026-05-28T12:16:39.187896Z",
33
  "context_length": 96,
34
  "prediction_length": 24
35
  },
36
  {
37
- "dataset": "task_nws_ksfo_observations_nws_station_ksfo_temperature_20260528t101000z",
38
- "data_fetched_at": "2026-05-28T12:16:39.187896Z",
39
  "context_length": 96,
40
  "prediction_length": 24
41
  },
42
  {
43
- "dataset": "task_open_meteo_shanghai_air_quality_hourly_grid_shanghai_31.2304_121.4737_pm2_5_20260527t120000z",
44
- "data_fetched_at": "2026-05-28T12:16:39.187896Z",
45
  "context_length": 96,
46
  "prediction_length": 24
47
- },
48
- {
49
- "dataset": "task_open_meteo_shanghai_hourly_grid_shanghai_31.2304_121.4737_temperature_2m_20260527t120000z",
50
- "data_fetched_at": "2026-05-28T12:16:39.187896Z",
51
- "context_length": 96,
52
- "prediction_length": 24
53
- },
54
- {
55
- "dataset": "task_usgs_potomac_iv_usgs_site_01646500_usgs_00060_20260528t095000z",
56
- "data_fetched_at": "2026-05-28T12:16:39.187896Z",
57
- "context_length": 96,
58
- "prediction_length": 24
59
- },
60
- {
61
- "dataset": "task_wikimedia_time_series_hourly_enwiki_article_time_series_pageviews_20260525t000000z",
62
- "data_fetched_at": "2026-05-28T12:16:39.187896Z",
63
- "context_length": 4,
64
- "prediction_length": 2
65
  }
66
  ]
67
  }
 
1
  {
2
  "model": "TimesFM-2.5",
3
  "api_model_id": "google/timesfm-2.5-200m-pytorch",
4
+ "evaluated_at": "2026-05-29T02:54:46.508756+00:00",
5
  "datasets": [
6
  {
7
+ "dataset": "task_binance_btcusdt_1h_crypto_btcusdt_close_20260528t020000z",
8
+ "data_fetched_at": "2026-05-29T02:50:05.083862Z",
9
  "context_length": 96,
10
  "prediction_length": 24
11
  },
12
  {
13
+ "dataset": "task_coingecko_bitcoin_7d_coin_bitcoin_price_usd_20260528t030012z",
14
+ "data_fetched_at": "2026-05-29T02:50:05.083862Z",
15
  "context_length": 96,
16
  "prediction_length": 24
17
  },
18
  {
19
  "dataset": "task_nasa_power_shanghai_hourly_point_shanghai_31.2304_121.4737_t2m_20260524t230000z",
20
+ "data_fetched_at": "2026-05-29T02:50:05.083862Z",
21
+ "context_length": 69,
22
  "prediction_length": 24
23
  },
24
  {
25
+ "dataset": "task_noaa_coops_sf_water_level_coops_station_9414290_water_level_20260529t001800z",
26
+ "data_fetched_at": "2026-05-29T02:50:05.083862Z",
27
  "context_length": 96,
28
  "prediction_length": 24
29
  },
30
  {
31
+ "dataset": "task_nws_ksfo_observations_nws_station_ksfo_temperature_20260529t004500z",
32
+ "data_fetched_at": "2026-05-29T02:50:05.083862Z",
33
  "context_length": 96,
34
  "prediction_length": 24
35
  },
36
  {
37
+ "dataset": "task_open_meteo_shanghai_air_quality_hourly_grid_shanghai_31.2304_121.4737_pm2_5_20260528t020000z",
38
+ "data_fetched_at": "2026-05-29T02:50:05.083862Z",
39
  "context_length": 96,
40
  "prediction_length": 24
41
  },
42
  {
43
+ "dataset": "task_usgs_potomac_iv_usgs_site_01646500_usgs_00060_20260528t235000z",
44
+ "data_fetched_at": "2026-05-29T02:50:05.083862Z",
45
  "context_length": 96,
46
  "prediction_length": 24
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
47
  }
48
  ]
49
  }
scripts/run_online_daemon.py ADDED
@@ -0,0 +1,63 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env python3
2
+ """Periodically run space/scripts/run_space_eval.py (1-hour live benchmark loop)."""
3
+
4
+ from __future__ import annotations
5
+
6
+ import argparse
7
+ import logging
8
+ import os
9
+ import subprocess
10
+ import sys
11
+ import time
12
+ from pathlib import Path
13
+
14
+ logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
15
+ logger = logging.getLogger(__name__)
16
+
17
+ SPACE_ROOT = Path(__file__).resolve().parents[1]
18
+ EVAL_SCRIPT = SPACE_ROOT / "scripts" / "run_space_eval.py"
19
+ DEFAULT_INTERVAL_SECONDS = int(
20
+ os.getenv(
21
+ "TSFM_BENCH_INTERVAL_SECONDS",
22
+ os.getenv("TSFM_EVAL_INTERVAL_SECONDS", "3600"),
23
+ )
24
+ )
25
+
26
+
27
+ def parse_args() -> argparse.Namespace:
28
+ parser = argparse.ArgumentParser(description=__doc__)
29
+ parser.add_argument(
30
+ "--interval-minutes",
31
+ type=int,
32
+ default=max(1, DEFAULT_INTERVAL_SECONDS // 60),
33
+ help="Minutes between collect + eval cycles (default: 60)",
34
+ )
35
+ parser.add_argument(
36
+ "--once",
37
+ action="store_true",
38
+ help="Run a single evaluation and exit",
39
+ )
40
+ return parser.parse_args()
41
+
42
+
43
+ def run_eval() -> int:
44
+ cmd = [sys.executable, str(EVAL_SCRIPT)]
45
+ logger.info("Running: %s", " ".join(cmd))
46
+ return subprocess.run(cmd, cwd=str(SPACE_ROOT), check=False).returncode
47
+
48
+
49
+ def main() -> None:
50
+ args = parse_args()
51
+ if args.once:
52
+ raise SystemExit(run_eval())
53
+
54
+ while True:
55
+ code = run_eval()
56
+ if code != 0:
57
+ logger.error("Evaluation failed with exit code %s", code)
58
+ logger.info("Sleeping %s minutes until next cycle", args.interval_minutes)
59
+ time.sleep(args.interval_minutes * 60)
60
+
61
+
62
+ if __name__ == "__main__":
63
+ main()
scripts/run_space_eval.py ADDED
@@ -0,0 +1,281 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env python3
2
+ """Run live TS-Bench + TSFM.ai evaluation inside the HF Space (or locally)."""
3
+
4
+ from __future__ import annotations
5
+
6
+ import csv
7
+ import json
8
+ import logging
9
+ import os
10
+ import sys
11
+ from datetime import datetime, timezone
12
+ from pathlib import Path
13
+ from typing import Any
14
+
15
+ import yaml
16
+
17
+ SPACE_ROOT = Path(__file__).resolve().parents[1]
18
+ if str(SPACE_ROOT) not in sys.path:
19
+ sys.path.insert(0, str(SPACE_ROOT))
20
+
21
+ os.environ.setdefault("TS_BENCH_ROOT", str(SPACE_ROOT / "vendor" / "ts_bench"))
22
+
23
+ from tsfm_bench.data.registry import load_data_source, load_dataset_properties
24
+ from tsfm_bench.data.ts_bench import TsBenchDataSource
25
+ from tsfm_bench.eval.api_predictor import DEFAULT_QUANTILES, TsfmApiConfig, TsfmApiPredictor
26
+ from tsfm_bench.eval.online_eval import run_online_eval_for_dataset
27
+
28
+ logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
29
+ logger = logging.getLogger(__name__)
30
+
31
+ DATA_CONFIG = SPACE_ROOT / "configs" / "datasets" / "ts_bench.yaml"
32
+ MODEL_CONFIG = SPACE_ROOT / "configs" / "models" / "online_tsfm.yaml"
33
+ OUTPUT_ROOT = Path(os.getenv("TSFM_RESULTS_PATH", str(SPACE_ROOT / "results")))
34
+ EVAL_STATE_PATH = OUTPUT_ROOT / "eval_state.json"
35
+
36
+
37
+ def model_display_name(model_spec: dict[str, Any]) -> str:
38
+ return model_spec.get("display_name") or model_spec["model_id"]
39
+
40
+
41
+ def model_output_slug(model_spec: dict[str, Any]) -> str:
42
+ import re
43
+
44
+ name = model_spec.get("display_name") or model_spec["model_id"]
45
+ return re.sub(r"[^a-zA-Z0-9]+", "_", name).strip("_").lower()
46
+
47
+
48
+ def load_model_specs(path: Path) -> list[dict[str, Any]]:
49
+ return yaml.safe_load(path.read_text()).get("models", [])
50
+
51
+
52
+ def write_model_results(
53
+ model_spec: dict[str, Any],
54
+ rows: list[list[Any]],
55
+ output_root: Path,
56
+ meta: dict[str, Any],
57
+ ) -> None:
58
+ model_name = model_display_name(model_spec)
59
+ out_dir = output_root / model_output_slug(model_spec)
60
+ out_dir.mkdir(parents=True, exist_ok=True)
61
+
62
+ csv_path = out_dir / "all_results.csv"
63
+ with csv_path.open("w", newline="") as handle:
64
+ writer = csv.writer(handle)
65
+ writer.writerow(
66
+ [
67
+ "dataset",
68
+ "model",
69
+ "eval_metrics/MSE[mean]",
70
+ "eval_metrics/MSE[0.5]",
71
+ "eval_metrics/MAE[0.5]",
72
+ "eval_metrics/MASE[0.5]",
73
+ "eval_metrics/MAPE[0.5]",
74
+ "eval_metrics/sMAPE[0.5]",
75
+ "eval_metrics/MSIS",
76
+ "eval_metrics/RMSE[mean]",
77
+ "eval_metrics/NRMSE[mean]",
78
+ "eval_metrics/ND[0.5]",
79
+ "eval_metrics/mean_weighted_sum_quantile_loss",
80
+ "domain",
81
+ "num_variates",
82
+ ]
83
+ )
84
+ writer.writerows(rows)
85
+
86
+ config = {
87
+ "model": model_name,
88
+ "model_type": model_spec.get("model_type", "zero-shot"),
89
+ "model_dtype": "float32",
90
+ "model_link": model_spec.get(
91
+ "model_link", f"https://tsfm.ai/models/{model_spec['model_id']}"
92
+ ),
93
+ "code_link": "https://github.com/zhouziyu02/TSFM_Bench/blob/main/space/scripts/run_space_eval.py",
94
+ "org": model_spec.get("org", "TSFM.ai"),
95
+ "testdata_leakage": "No",
96
+ "replication_code_available": "Yes",
97
+ "api_model_id": model_spec["model_id"],
98
+ }
99
+ (out_dir / "config.json").write_text(json.dumps(config, indent=4) + "\n")
100
+ (out_dir / "online_meta.json").write_text(json.dumps(meta, indent=4) + "\n")
101
+
102
+
103
+ def write_dataset_properties(properties: dict[str, dict[str, Any]], output_root: Path) -> None:
104
+ rows = [
105
+ {
106
+ "dataset": key,
107
+ "domain": values["domain"],
108
+ "frequency": values["frequency"],
109
+ "num_variates": values["num_variates"],
110
+ }
111
+ for key, values in sorted(properties.items())
112
+ ]
113
+ csv_path = output_root / "dataset_properties.csv"
114
+ with csv_path.open("w", newline="") as handle:
115
+ writer = csv.DictWriter(
116
+ handle,
117
+ fieldnames=["dataset", "domain", "frequency", "num_variates"],
118
+ )
119
+ writer.writeheader()
120
+ writer.writerows(rows)
121
+
122
+
123
+ def write_eval_state(payload: dict[str, Any]) -> None:
124
+ OUTPUT_ROOT.mkdir(parents=True, exist_ok=True)
125
+ EVAL_STATE_PATH.write_text(json.dumps(payload, indent=4) + "\n")
126
+
127
+
128
+ def write_run_metadata(output_root: Path, payload: dict[str, Any]) -> None:
129
+ (output_root / "online_status.json").write_text(json.dumps(payload, indent=4) + "\n")
130
+
131
+
132
+ def make_predictor(model_spec: dict[str, Any], pred_len: int) -> TsfmApiPredictor:
133
+ predictor = TsfmApiPredictor(
134
+ config=TsfmApiConfig(model_id=model_spec["model_id"]),
135
+ prediction_length=pred_len,
136
+ quantile_levels=DEFAULT_QUANTILES,
137
+ )
138
+ predictor.leaderboard_name = model_display_name(model_spec)
139
+ return predictor
140
+
141
+
142
+ def run_evaluation() -> int:
143
+ if not os.getenv("TSFM_API_KEY"):
144
+ write_eval_state(
145
+ {
146
+ "status": "disabled",
147
+ "message": "TSFM_API_KEY not configured — set it in HF Space secrets.",
148
+ "updated_at": datetime.now(timezone.utc).isoformat(),
149
+ }
150
+ )
151
+ logger.error("TSFM_API_KEY is missing; skipping evaluation")
152
+ return 1
153
+
154
+ started = datetime.now(timezone.utc).isoformat()
155
+ write_eval_state(
156
+ {
157
+ "status": "running",
158
+ "started_at": started,
159
+ "message": "Collecting TS-Bench data and calling TSFM.ai API…",
160
+ "updated_at": started,
161
+ }
162
+ )
163
+
164
+ try:
165
+ source = load_data_source(DATA_CONFIG)
166
+ if isinstance(source, TsBenchDataSource) and source._settings.auto_refresh:
167
+ source.refresh()
168
+
169
+ datasets = source.list_datasets()
170
+ if not datasets:
171
+ raise RuntimeError("No TS-Bench tasks available after data collection")
172
+
173
+ properties = load_dataset_properties(DATA_CONFIG)
174
+ model_specs = load_model_specs(MODEL_CONFIG)
175
+ OUTPUT_ROOT.mkdir(parents=True, exist_ok=True)
176
+ write_dataset_properties(properties, OUTPUT_ROOT)
177
+
178
+ failed_models: list[str] = []
179
+ all_model_meta: dict[str, Any] = {}
180
+
181
+ for model_spec in model_specs:
182
+ model_name = model_display_name(model_spec)
183
+ rows: list[list[Any]] = []
184
+ dataset_meta: list[dict[str, Any]] = []
185
+
186
+ try:
187
+ for ds_name in datasets:
188
+ pred_len = source.get_prediction_length(ds_name)
189
+ predictor = make_predictor(model_spec, pred_len)
190
+ try:
191
+ logger.info("Evaluating %s on %s", model_name, ds_name)
192
+ result = run_online_eval_for_dataset(source, ds_name, predictor)
193
+ except Exception:
194
+ logger.exception("Skipping %s on %s", model_name, ds_name)
195
+ continue
196
+ rows.append(
197
+ [
198
+ result.dataset,
199
+ model_name,
200
+ result.metrics["MSE[mean]"],
201
+ result.metrics["MSE[0.5]"],
202
+ result.metrics["MAE[0.5]"],
203
+ result.metrics["MASE[0.5]"],
204
+ result.metrics["MAPE[0.5]"],
205
+ result.metrics["sMAPE[0.5]"],
206
+ result.metrics["MSIS"],
207
+ result.metrics["RMSE[mean]"],
208
+ result.metrics["NRMSE[mean]"],
209
+ result.metrics["ND[0.5]"],
210
+ result.metrics["mean_weighted_sum_quantile_loss"],
211
+ result.domain,
212
+ result.num_variates,
213
+ ]
214
+ )
215
+ dataset_meta.append(
216
+ {
217
+ "dataset": ds_name,
218
+ "data_fetched_at": result.data_fetched_at,
219
+ "context_length": result.context_length,
220
+ "prediction_length": result.prediction_length,
221
+ }
222
+ )
223
+ if not rows:
224
+ raise RuntimeError(f"No successful datasets for {model_name}")
225
+ model_meta = {
226
+ "model": model_name,
227
+ "api_model_id": model_spec["model_id"],
228
+ "evaluated_at": datetime.now(timezone.utc).isoformat(),
229
+ "datasets": dataset_meta,
230
+ }
231
+ write_model_results(model_spec, rows, OUTPUT_ROOT, model_meta)
232
+ all_model_meta[model_name] = model_meta
233
+ except Exception:
234
+ logger.exception("Failed evaluating %s", model_name)
235
+ failed_models.append(model_name)
236
+
237
+ finished = datetime.now(timezone.utc).isoformat()
238
+ status = "ok" if not failed_models else "partial"
239
+ write_run_metadata(
240
+ OUTPUT_ROOT,
241
+ {
242
+ "status": status,
243
+ "started_at": started,
244
+ "finished_at": finished,
245
+ "data_source": "ts_bench",
246
+ "data_config": str(DATA_CONFIG),
247
+ "ts_bench_root": os.environ.get("TS_BENCH_ROOT", ""),
248
+ "models": all_model_meta,
249
+ "failed_models": failed_models,
250
+ },
251
+ )
252
+ write_eval_state(
253
+ {
254
+ "status": status,
255
+ "started_at": started,
256
+ "finished_at": finished,
257
+ "failed_models": failed_models,
258
+ "task_count": len(datasets),
259
+ "model_count": len(model_specs) - len(failed_models),
260
+ "message": "Evaluation complete",
261
+ "updated_at": finished,
262
+ }
263
+ )
264
+ return 0 if status == "ok" else 2
265
+ except Exception as exc:
266
+ finished = datetime.now(timezone.utc).isoformat()
267
+ logger.exception("Evaluation run failed")
268
+ write_eval_state(
269
+ {
270
+ "status": "error",
271
+ "started_at": started,
272
+ "finished_at": finished,
273
+ "message": str(exc),
274
+ "updated_at": finished,
275
+ }
276
+ )
277
+ return 1
278
+
279
+
280
+ if __name__ == "__main__":
281
+ raise SystemExit(run_evaluation())
src/about.py CHANGED
@@ -3,42 +3,54 @@ TITLE = """ TSFM Realworld Bench Leaderboard """
3
  INTRODUCTION_TEXT = """
4
  **TSFM Realworld Bench** evaluates time series foundation models on **live TS-Bench
5
  data** with **zero-shot API inference** via [TSFM.ai](https://tsfm.ai/).
6
- Following [GIFT-Eval](https://github.com/SalesforceAIResearch/gift-eval), each model's
7
- **raw metric values** are aggregated with a **geometric mean** across datasets.
8
- Per-dataset **ranks** are averaged for ranking (lower is better).
 
9
  """
10
 
11
  LLM_BENCHMARKS_TEXT = """
12
- ## How to participate
 
 
 
 
 
 
 
13
 
14
- 1. Clone [TSFM_Bench](https://github.com/zhouziyu02/TSFM_Bench)
15
- 2. Implement your model against the streaming data interface (`tsfm_bench.data`)
16
- 3. Run evaluation and write `results/<MODEL_NAME>/all_results.csv`
17
- 4. Open a pull request; results sync to this leaderboard
 
 
 
 
 
 
 
 
18
 
19
  ## Metrics
20
 
21
  - **MASE** — Mean Absolute Scaled Error
22
- - **CRPS** — Mean Weighted Sum Quantile Loss (probabilistic forecast quality)
23
- - **CRPS_Rank** / **MASE_Rank** — average per-dataset rank on raw metrics (lower is better)
24
 
25
  ## Aggregation (GIFT-Eval style)
26
 
27
- 1. Compute evaluation metrics on each dataset configuration (raw values).
28
- 2. Rank models per dataset on each metric.
29
- 3. Aggregate metric values with a **geometric mean** across datasets.
30
- 4. Aggregate ranks with an **arithmetic mean** across datasets.
31
-
32
- ## Data interface
33
 
34
- Live series are collected from **TS-Bench** (`TsBenchDataSource`). Each evaluation
35
- holds out the most recent forecast horizon as ground truth and calls TSFM models through
36
- the **TSFM.ai** API for zero-shot forecasts.
37
 
38
  ```bash
39
  export TSFM_API_KEY=your_key
40
- python scripts/run_online_eval.py
41
- python scripts/run_online_daemon.py --interval-minutes 60
 
42
  ```
43
  """
44
 
 
3
  INTRODUCTION_TEXT = """
4
  **TSFM Realworld Bench** evaluates time series foundation models on **live TS-Bench
5
  data** with **zero-shot API inference** via [TSFM.ai](https://tsfm.ai/).
6
+
7
+ The Space runs a **unified 1-hour loop**: collect TS-Bench data → call TSFM.ai for all models →
8
+ write new `results/` CSVs → leaderboard reloads. Set **`TSFM_API_KEY`** in Space Settings →
9
+ Repository secrets to enable it.
10
  """
11
 
12
  LLM_BENCHMARKS_TEXT = """
13
+ ## How it works (live benchmark)
14
+
15
+ 1. **Background scheduler** runs every **`TSFM_BENCH_INTERVAL_SECONDS`** (default **3600 = 1 hour**):
16
+ TS-Bench collect → TSFM.ai eval → update `results/`.
17
+ 2. Each run calls **TSFM.ai** for all configured models.
18
+ 3. The Gradio UI reloads tables every **`TSFM_REFRESH_SECONDS`** (default 60s) from disk.
19
+
20
+ ### Required HF Space secret
21
 
22
+ | Variable | Purpose |
23
+ |---|---|
24
+ | `TSFM_API_KEY` | TSFM.ai API key — **required** for live evaluation |
25
+
26
+ ### Optional environment variables
27
+
28
+ | Variable | Default | Purpose |
29
+ |---|---|---|
30
+ | `TSFM_BENCH_INTERVAL_SECONDS` | `3600` | **Main knob**: seconds between collect+eval cycles (1 h) |
31
+ | `TSFM_EVAL_INTERVAL_SECONDS` | (alias) | Same as above if `TSFM_BENCH_INTERVAL_SECONDS` unset |
32
+ | `TSFM_REFRESH_SECONDS` | `60` | UI table reload interval (display only) |
33
+ | `TS_BENCH_ROOT` | `vendor/ts_bench` | Path to TS-Bench collect scripts |
34
 
35
  ## Metrics
36
 
37
  - **MASE** — Mean Absolute Scaled Error
38
+ - **CRPS** — Mean Weighted Sum Quantile Loss
39
+ - Ranks are averaged per-dataset on raw metrics (lower is better)
40
 
41
  ## Aggregation (GIFT-Eval style)
42
 
43
+ 1. Compute evaluation metrics on each dataset (raw values).
44
+ 2. Rank models per dataset.
45
+ 3. Aggregate with **geometric mean** (metrics) and **arithmetic mean** (ranks).
 
 
 
46
 
47
+ ## Local development
 
 
48
 
49
  ```bash
50
  export TSFM_API_KEY=your_key
51
+ cd space
52
+ python scripts/run_space_eval.py # single eval run
53
+ python app.py # leaderboard + scheduler
54
  ```
55
  """
56
 
src/benchmark_config.py ADDED
@@ -0,0 +1,19 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Unified timing for the live benchmark loop."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import os
6
+
7
+ # One full cycle: TS-Bench collect → TSFM.ai eval → write results → leaderboard reload.
8
+ # Override with TSFM_BENCH_INTERVAL_SECONDS (preferred) or TSFM_EVAL_INTERVAL_SECONDS.
9
+ BENCHMARK_INTERVAL_SECONDS = int(
10
+ os.getenv(
11
+ "TSFM_BENCH_INTERVAL_SECONDS",
12
+ os.getenv("TSFM_EVAL_INTERVAL_SECONDS", "3600"),
13
+ )
14
+ )
15
+
16
+ # UI table reload (reads results/ from disk; does not call TSFM.ai).
17
+ DISPLAY_REFRESH_SECONDS = int(os.getenv("TSFM_REFRESH_SECONDS", "60"))
18
+
19
+ BENCHMARK_INTERVAL_MINUTES = BENCHMARK_INTERVAL_SECONDS // 60
src/display/css_html_js.py CHANGED
@@ -37,7 +37,7 @@ def refresh_countdown_markup(seconds: int) -> str:
37
  """Static HTML only — Gradio on HF Spaces strips <script> inside gr.HTML."""
38
  return f"""
39
  <div class="auto-refresh-wrap" id="tsfm-refresh-widget">
40
- <div class="auto-refresh-label" id="tsfm-refresh-label">Next auto-refresh in {seconds}s</div>
41
  <div class="auto-refresh-track">
42
  <div class="auto-refresh-fill" id="tsfm-refresh-fill"></div>
43
  </div>
@@ -55,7 +55,7 @@ def refresh_countdown_head_js(seconds: int) -> str:
55
  function render(remaining) {{
56
  const label = document.getElementById("tsfm-refresh-label");
57
  if (label) {{
58
- label.textContent = "Next auto-refresh in " + remaining + "s";
59
  }}
60
  }}
61
 
 
37
  """Static HTML only — Gradio on HF Spaces strips <script> inside gr.HTML."""
38
  return f"""
39
  <div class="auto-refresh-wrap" id="tsfm-refresh-widget">
40
+ <div class="auto-refresh-label" id="tsfm-refresh-label">Next display reload in {seconds}s</div>
41
  <div class="auto-refresh-track">
42
  <div class="auto-refresh-fill" id="tsfm-refresh-fill"></div>
43
  </div>
 
55
  function render(remaining) {{
56
  const label = document.getElementById("tsfm-refresh-label");
57
  if (label) {{
58
+ label.textContent = "Next display reload in " + remaining + "s";
59
  }}
60
  }}
61
 
src/eval_scheduler.py ADDED
@@ -0,0 +1,123 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Background scheduler: periodically runs live TS-Bench + TSFM.ai evaluation."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import json
6
+ import logging
7
+ import os
8
+ import subprocess
9
+ import sys
10
+ import threading
11
+ import time
12
+ from datetime import datetime, timezone
13
+ from pathlib import Path
14
+
15
+ from src.benchmark_config import BENCHMARK_INTERVAL_SECONDS, DISPLAY_REFRESH_SECONDS
16
+
17
+ logger = logging.getLogger(__name__)
18
+
19
+ SPACE_ROOT = Path(__file__).resolve().parents[1]
20
+ EVAL_SCRIPT = SPACE_ROOT / "scripts" / "run_space_eval.py"
21
+ DEFAULT_EVAL_INTERVAL = BENCHMARK_INTERVAL_SECONDS
22
+
23
+
24
+ class LiveEvalScheduler:
25
+ def __init__(
26
+ self,
27
+ interval_seconds: int = DEFAULT_EVAL_INTERVAL,
28
+ run_on_startup: bool = True,
29
+ ):
30
+ self.interval_seconds = max(300, interval_seconds)
31
+ self.run_on_startup = run_on_startup
32
+ self._lock = threading.Lock()
33
+ self._stop = threading.Event()
34
+ self._thread: threading.Thread | None = None
35
+ self._next_run_at: float | None = None
36
+
37
+ @property
38
+ def enabled(self) -> bool:
39
+ return bool(os.getenv("TSFM_API_KEY"))
40
+
41
+ def start(self) -> None:
42
+ if self._thread is not None:
43
+ return
44
+ if not self.enabled:
45
+ logger.warning(
46
+ "Live eval disabled: TSFM_API_KEY not set. "
47
+ "Leaderboard will only show static CSV results."
48
+ )
49
+ self._write_idle_state(
50
+ "TSFM_API_KEY not set — configure it in HF Space Settings → Secrets."
51
+ )
52
+ return
53
+
54
+ self._thread = threading.Thread(target=self._loop, name="live-eval-scheduler", daemon=True)
55
+ self._thread.start()
56
+ logger.info(
57
+ "Live eval scheduler started (interval=%ss, script=%s)",
58
+ self.interval_seconds,
59
+ EVAL_SCRIPT,
60
+ )
61
+
62
+ def snapshot(self) -> dict:
63
+ state_path = SPACE_ROOT / "results" / "eval_state.json"
64
+ if state_path.exists():
65
+ try:
66
+ return json.loads(state_path.read_text())
67
+ except json.JSONDecodeError:
68
+ pass
69
+ return {
70
+ "status": "idle" if self.enabled else "disabled",
71
+ "message": "Waiting for first evaluation run…" if self.enabled else "No API key",
72
+ }
73
+
74
+ def seconds_until_next_run(self) -> int | None:
75
+ if self._next_run_at is None:
76
+ return None
77
+ return max(0, int(self._next_run_at - time.monotonic()))
78
+
79
+ def _loop(self) -> None:
80
+ if self.run_on_startup:
81
+ self._run_once()
82
+ while not self._stop.wait(self.interval_seconds):
83
+ self._run_once()
84
+
85
+ def _run_once(self) -> None:
86
+ if not self._lock.acquire(blocking=False):
87
+ logger.info("Evaluation already in progress, skipping tick")
88
+ return
89
+ try:
90
+ logger.info("Starting scheduled live evaluation")
91
+ cmd = [sys.executable, str(EVAL_SCRIPT)]
92
+ completed = subprocess.run(cmd, cwd=str(SPACE_ROOT), check=False)
93
+ if completed.returncode != 0:
94
+ logger.error("Evaluation exited with code %s", completed.returncode)
95
+ finally:
96
+ self._lock.release()
97
+ self._next_run_at = time.monotonic() + self.interval_seconds
98
+
99
+ def _write_idle_state(self, message: str) -> None:
100
+ results = SPACE_ROOT / "results"
101
+ results.mkdir(parents=True, exist_ok=True)
102
+ payload = {
103
+ "status": "disabled",
104
+ "message": message,
105
+ "updated_at": datetime.now(timezone.utc).isoformat(),
106
+ }
107
+ (results / "eval_state.json").write_text(json.dumps(payload, indent=4) + "\n")
108
+
109
+
110
+ _scheduler: LiveEvalScheduler | None = None
111
+
112
+
113
+ def get_scheduler() -> LiveEvalScheduler:
114
+ global _scheduler
115
+ if _scheduler is None:
116
+ _scheduler = LiveEvalScheduler()
117
+ return _scheduler
118
+
119
+
120
+ def start_live_eval_scheduler() -> LiveEvalScheduler:
121
+ scheduler = get_scheduler()
122
+ scheduler.start()
123
+ return scheduler
tsfm_bench/__about__.py ADDED
@@ -0,0 +1 @@
 
 
1
+ __version__ = "0.1.0"
tsfm_bench/__init__.py ADDED
@@ -0,0 +1 @@
 
 
1
+ from tsfm_bench.__about__ import __version__
tsfm_bench/data/__init__.py ADDED
@@ -0,0 +1,18 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from tsfm_bench.data.dataset import Dataset, Term
2
+ from tsfm_bench.data.interface import DatasetMetadata, TimeSeriesDataSource, TimeSeriesRecord
3
+ from tsfm_bench.data.open_meteo import OpenMeteoDataSource
4
+ from tsfm_bench.data.registry import load_data_source
5
+ from tsfm_bench.data.synthetic import SyntheticDataSource
6
+ from tsfm_bench.data.ts_bench import TsBenchDataSource
7
+
8
+ __all__ = [
9
+ "Dataset",
10
+ "Term",
11
+ "DatasetMetadata",
12
+ "TimeSeriesDataSource",
13
+ "TimeSeriesRecord",
14
+ "SyntheticDataSource",
15
+ "OpenMeteoDataSource",
16
+ "TsBenchDataSource",
17
+ "load_data_source",
18
+ ]
tsfm_bench/data/dataset.py ADDED
@@ -0,0 +1,197 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from __future__ import annotations
2
+
3
+ import math
4
+ import os
5
+ from enum import Enum
6
+ from functools import cached_property
7
+ from pathlib import Path
8
+ from typing import Iterable, Iterator
9
+
10
+ from dotenv import load_dotenv
11
+ from gluonts.dataset import DataEntry
12
+ from gluonts.dataset.common import ProcessDataEntry
13
+ from gluonts.dataset.split import TestData, TrainingDataset, split
14
+ from gluonts.itertools import Map
15
+ from gluonts.time_feature import norm_freq_str
16
+ from gluonts.transform import Transformation
17
+ from pandas.tseries.frequencies import to_offset
18
+ from toolz import compose
19
+
20
+ from tsfm_bench.data.interface import TimeSeriesDataSource, TimeSeriesRecord
21
+ from tsfm_bench.data.registry import load_data_source
22
+
23
+ TEST_SPLIT = 0.1
24
+ MAX_WINDOW = 20
25
+
26
+ M4_PRED_LENGTH_MAP = {
27
+ "A": 6,
28
+ "Q": 8,
29
+ "M": 18,
30
+ "W": 13,
31
+ "D": 14,
32
+ "H": 48,
33
+ }
34
+
35
+ PRED_LENGTH_MAP = {
36
+ "M": 12,
37
+ "W": 8,
38
+ "D": 30,
39
+ "H": 48,
40
+ "T": 48,
41
+ "S": 60,
42
+ }
43
+
44
+
45
+ class Term(Enum):
46
+ SHORT = "short"
47
+ MEDIUM = "medium"
48
+ LONG = "long"
49
+
50
+ @property
51
+ def multiplier(self) -> int:
52
+ if self == Term.SHORT:
53
+ return 1
54
+ if self == Term.MEDIUM:
55
+ return 10
56
+ return 15
57
+
58
+
59
+ def itemize_start(data_entry: DataEntry) -> DataEntry:
60
+ start = data_entry["start"]
61
+ if hasattr(start, "item"):
62
+ data_entry["start"] = start.item()
63
+ return data_entry
64
+
65
+
66
+ def maybe_reconvert_freq(freq: str) -> str:
67
+ deprecated_map = {
68
+ "Y": "A",
69
+ "YE": "A",
70
+ "QE": "Q",
71
+ "ME": "M",
72
+ "h": "H",
73
+ "min": "T",
74
+ "s": "S",
75
+ "us": "U",
76
+ }
77
+ return deprecated_map.get(freq, freq)
78
+
79
+
80
+ class MultivariateToUnivariate(Transformation):
81
+ def __init__(self, field: str):
82
+ self.field = field
83
+
84
+ def __call__(
85
+ self, data_it: Iterable[DataEntry], is_train: bool = False
86
+ ) -> Iterator[DataEntry]:
87
+ for data_entry in data_it:
88
+ item_id = data_entry["item_id"]
89
+ val_ls = list(data_entry[self.field])
90
+ for dim_id, val in enumerate(val_ls):
91
+ univariate_entry = data_entry.copy()
92
+ univariate_entry[self.field] = val
93
+ univariate_entry["item_id"] = f"{item_id}_dim{dim_id}"
94
+ yield univariate_entry
95
+
96
+
97
+ def _record_to_entry(record: TimeSeriesRecord) -> dict:
98
+ import pandas as pd
99
+
100
+ return {
101
+ "item_id": record.item_id,
102
+ "start": pd.Period(record.start),
103
+ "target": record.target,
104
+ "freq": record.freq,
105
+ }
106
+
107
+
108
+ class Dataset:
109
+ """GluonTS-compatible dataset backed by a streaming data source."""
110
+
111
+ def __init__(
112
+ self,
113
+ name: str,
114
+ term: Term | str = Term.SHORT,
115
+ to_univariate: bool = False,
116
+ source: TimeSeriesDataSource | None = None,
117
+ config_path: str | Path | None = None,
118
+ ):
119
+ load_dotenv()
120
+ if source is None:
121
+ env_config = os.getenv("TSFM_BENCH_CONFIG")
122
+ source = load_data_source(config_path or env_config)
123
+
124
+ self.source = source
125
+ self.metadata = source.get_metadata(name)
126
+ self.name = name
127
+ self.term = Term(term)
128
+
129
+ records = list(source.stream(name))
130
+ self._raw_entries = [_record_to_entry(record) for record in records]
131
+
132
+ first_target = self._raw_entries[0]["target"]
133
+ target_dim = first_target.shape[0] if getattr(first_target, "ndim", 1) > 1 else 1
134
+
135
+ process = ProcessDataEntry(
136
+ self.freq,
137
+ one_dim_target=target_dim == 1,
138
+ )
139
+ self.gluonts_dataset = Map(compose(process, itemize_start), self._raw_entries)
140
+ if to_univariate:
141
+ self.gluonts_dataset = MultivariateToUnivariate("target").apply(
142
+ self.gluonts_dataset
143
+ )
144
+
145
+ @cached_property
146
+ def prediction_length(self) -> int:
147
+ freq = norm_freq_str(to_offset(self.freq).name)
148
+ freq = maybe_reconvert_freq(freq)
149
+ pred_len = PRED_LENGTH_MAP.get(freq, 48)
150
+ return self.term.multiplier * pred_len
151
+
152
+ @cached_property
153
+ def freq(self) -> str:
154
+ return self.metadata.frequency
155
+
156
+ @cached_property
157
+ def target_dim(self) -> int:
158
+ target = self._raw_entries[0]["target"]
159
+ return target.shape[0] if getattr(target, "ndim", 1) > 1 else 1
160
+
161
+ @cached_property
162
+ def windows(self) -> int:
163
+ w = math.ceil(TEST_SPLIT * self._min_series_length / self.prediction_length)
164
+ return min(max(1, w), MAX_WINDOW)
165
+
166
+ @cached_property
167
+ def _min_series_length(self) -> int:
168
+ lengths = []
169
+ for entry in self._raw_entries:
170
+ target = entry["target"]
171
+ lengths.append(target.shape[-1] if getattr(target, "ndim", 1) > 1 else len(target))
172
+ return min(lengths)
173
+
174
+ @property
175
+ def training_dataset(self) -> TrainingDataset:
176
+ training_dataset, _ = split(
177
+ self.gluonts_dataset, offset=-self.prediction_length * (self.windows + 1)
178
+ )
179
+ return training_dataset
180
+
181
+ @property
182
+ def validation_dataset(self) -> TrainingDataset:
183
+ validation_dataset, _ = split(
184
+ self.gluonts_dataset, offset=-self.prediction_length * self.windows
185
+ )
186
+ return validation_dataset
187
+
188
+ @property
189
+ def test_data(self) -> TestData:
190
+ _, test_template = split(
191
+ self.gluonts_dataset, offset=-self.prediction_length * self.windows
192
+ )
193
+ return test_template.generate_instances(
194
+ prediction_length=self.prediction_length,
195
+ windows=self.windows,
196
+ distance=self.prediction_length,
197
+ )
tsfm_bench/data/interface.py ADDED
@@ -0,0 +1,46 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from __future__ import annotations
2
+
3
+ from dataclasses import dataclass
4
+ from typing import Iterator, Protocol, runtime_checkable
5
+
6
+ import numpy as np
7
+
8
+
9
+ @dataclass(frozen=True)
10
+ class DatasetMetadata:
11
+ """Static metadata for a benchmark dataset configuration."""
12
+
13
+ name: str
14
+ domain: str
15
+ frequency: str
16
+ num_variates: int
17
+ terms: tuple[str, ...] = ("short",)
18
+ source_type: str = "synthetic"
19
+
20
+
21
+ @dataclass(frozen=True)
22
+ class TimeSeriesRecord:
23
+ """One multivariate or univariate time series instance."""
24
+
25
+ item_id: str
26
+ target: np.ndarray
27
+ start: str
28
+ freq: str
29
+
30
+
31
+ @runtime_checkable
32
+ class TimeSeriesDataSource(Protocol):
33
+ """Streaming data source for benchmark datasets.
34
+
35
+ Real-world backends (HF datasets, databases, APIs) should implement this
36
+ protocol and yield ``TimeSeriesRecord`` instances lazily via ``stream()``.
37
+ """
38
+
39
+ def list_datasets(self) -> list[str]:
40
+ ...
41
+
42
+ def get_metadata(self, name: str) -> DatasetMetadata:
43
+ ...
44
+
45
+ def stream(self, name: str) -> Iterator[TimeSeriesRecord]:
46
+ ...
tsfm_bench/data/open_meteo.py ADDED
@@ -0,0 +1,194 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from __future__ import annotations
2
+
3
+ import logging
4
+ from dataclasses import dataclass
5
+ from datetime import datetime, timedelta, timezone
6
+ from typing import Any, Iterator
7
+ from zoneinfo import ZoneInfo
8
+
9
+ import httpx
10
+ import numpy as np
11
+ import pandas as pd
12
+
13
+ from tsfm_bench.data.interface import DatasetMetadata, TimeSeriesRecord
14
+
15
+ logger = logging.getLogger(__name__)
16
+
17
+ FORECAST_API = "https://api.open-meteo.com/v1/forecast"
18
+ ARCHIVE_API = "https://archive-api.open-meteo.com/v1/archive"
19
+
20
+
21
+ @dataclass(frozen=True)
22
+ class _LocationSpec:
23
+ name: str
24
+ domain: str
25
+ latitude: float
26
+ longitude: float
27
+ variables: tuple[str, ...]
28
+ frequency: str
29
+ terms: tuple[str, ...]
30
+ timezone: str = "UTC"
31
+ city: str = ""
32
+
33
+
34
+ @dataclass(frozen=True)
35
+ class _OpenMeteoSettings:
36
+ past_days: int = 14
37
+ archive_days: int = 30
38
+ min_context_hours: int = 168
39
+ prediction_length: int = 24
40
+ request_timeout: float = 30.0
41
+
42
+
43
+ class OpenMeteoDataSource:
44
+ """Live weather time series from Open-Meteo (https://open-meteo.com/)."""
45
+
46
+ def __init__(
47
+ self,
48
+ locations: tuple[_LocationSpec, ...],
49
+ settings: _OpenMeteoSettings | None = None,
50
+ ):
51
+ self._locations = {spec.name: spec for spec in locations}
52
+ self._settings = settings or _OpenMeteoSettings()
53
+
54
+ @classmethod
55
+ def from_config(cls, payload: dict[str, Any]) -> OpenMeteoDataSource:
56
+ settings_raw = payload.get("settings", {})
57
+ settings = _OpenMeteoSettings(
58
+ past_days=int(settings_raw.get("past_days", 14)),
59
+ archive_days=int(settings_raw.get("archive_days", 30)),
60
+ min_context_hours=int(settings_raw.get("min_context_hours", 168)),
61
+ prediction_length=int(settings_raw.get("prediction_length", 24)),
62
+ request_timeout=float(settings_raw.get("request_timeout", 30.0)),
63
+ )
64
+ specs = tuple(_parse_location(entry) for entry in payload.get("locations", []))
65
+ if not specs:
66
+ raise ValueError("open_meteo config requires at least one location")
67
+ return cls(locations=specs, settings=settings)
68
+
69
+ def list_datasets(self) -> list[str]:
70
+ return sorted(self._locations.keys())
71
+
72
+ def get_metadata(self, name: str) -> DatasetMetadata:
73
+ spec = self._require_spec(name)
74
+ return DatasetMetadata(
75
+ name=spec.name,
76
+ domain=spec.domain,
77
+ frequency=spec.frequency,
78
+ num_variates=len(spec.variables),
79
+ terms=spec.terms,
80
+ source_type="open_meteo",
81
+ )
82
+
83
+ def stream(self, name: str) -> Iterator[TimeSeriesRecord]:
84
+ spec = self._require_spec(name)
85
+ frame = self.fetch_location_frame(spec)
86
+ if frame.empty:
87
+ raise RuntimeError(f"No Open-Meteo data returned for {name}")
88
+
89
+ values = frame[list(spec.variables)].to_numpy(dtype=np.float32).T
90
+ mask = ~np.isnan(values).any(axis=0)
91
+ values = values[:, mask]
92
+ if values.shape[-1] < self._settings.min_context_hours + self._settings.prediction_length:
93
+ raise RuntimeError(
94
+ f"Insufficient history for {name}: got {values.shape[-1]} hours, "
95
+ f"need at least {self._settings.min_context_hours + self._settings.prediction_length}"
96
+ )
97
+
98
+ start_ts = frame.index[mask][0]
99
+ freq = pd.tseries.frequencies.to_offset(spec.frequency)
100
+ yield TimeSeriesRecord(
101
+ item_id=spec.name.replace("/", "_"),
102
+ target=values,
103
+ start=str(pd.Period(start_ts, freq=freq)),
104
+ freq=spec.frequency,
105
+ )
106
+
107
+ def fetch_location_frame(self, spec: _LocationSpec) -> pd.DataFrame:
108
+ """Fetch merged archive + recent forecast observations for one location."""
109
+ now = datetime.now(timezone.utc)
110
+ archive_start = (now - timedelta(days=self._settings.archive_days)).date()
111
+ archive_end = (now - timedelta(days=1)).date()
112
+
113
+ frames: list[pd.DataFrame] = []
114
+ if archive_start <= archive_end:
115
+ frames.append(
116
+ self._fetch_hourly(
117
+ ARCHIVE_API,
118
+ spec,
119
+ start_date=str(archive_start),
120
+ end_date=str(archive_end),
121
+ )
122
+ )
123
+
124
+ frames.append(
125
+ self._fetch_hourly(
126
+ FORECAST_API,
127
+ spec,
128
+ past_days=self._settings.past_days,
129
+ forecast_days=1,
130
+ )
131
+ )
132
+
133
+ merged = pd.concat(frames).sort_index()
134
+ merged = merged[~merged.index.duplicated(keep="last")]
135
+ merged = merged.dropna(how="all")
136
+ return merged
137
+
138
+ def fetched_at(self) -> str:
139
+ return datetime.now(timezone.utc).isoformat()
140
+
141
+ def _fetch_hourly(
142
+ self,
143
+ base_url: str,
144
+ spec: _LocationSpec,
145
+ **extra_params: Any,
146
+ ) -> pd.DataFrame:
147
+ params: dict[str, Any] = {
148
+ "latitude": spec.latitude,
149
+ "longitude": spec.longitude,
150
+ "hourly": ",".join(spec.variables),
151
+ "timezone": spec.timezone,
152
+ **extra_params,
153
+ }
154
+ with httpx.Client(timeout=self._settings.request_timeout, trust_env=False) as client:
155
+ response = client.get(base_url, params=params)
156
+ response.raise_for_status()
157
+ payload = response.json()
158
+
159
+ hourly = payload.get("hourly", {})
160
+ times = hourly.get("time", [])
161
+ if not times:
162
+ return pd.DataFrame()
163
+
164
+ tz = ZoneInfo(spec.timezone)
165
+ index = pd.DatetimeIndex(
166
+ [datetime.fromisoformat(ts).replace(tzinfo=tz) for ts in times]
167
+ )
168
+ data = {var: hourly.get(var, []) for var in spec.variables}
169
+ frame = pd.DataFrame(data, index=index)
170
+ for col in spec.variables:
171
+ frame[col] = pd.to_numeric(frame[col], errors="coerce")
172
+ return frame
173
+
174
+ def _require_spec(self, name: str) -> _LocationSpec:
175
+ if name not in self._locations:
176
+ raise KeyError(f"Unknown Open-Meteo dataset: {name}")
177
+ return self._locations[name]
178
+
179
+
180
+ def _parse_location(entry: dict[str, Any]) -> _LocationSpec:
181
+ variables = entry.get("variables", ["temperature_2m"])
182
+ if isinstance(variables, str):
183
+ variables = [v.strip() for v in variables.split(",")]
184
+ return _LocationSpec(
185
+ name=entry["name"],
186
+ domain=entry.get("domain", "Nature"),
187
+ latitude=float(entry["latitude"]),
188
+ longitude=float(entry["longitude"]),
189
+ variables=tuple(variables),
190
+ frequency=entry.get("frequency", "H"),
191
+ terms=tuple(entry.get("terms", ["short"])),
192
+ timezone=entry.get("timezone", "UTC"),
193
+ city=entry.get("city", entry["name"].split("/")[0]),
194
+ )
tsfm_bench/data/registry.py ADDED
@@ -0,0 +1,76 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from __future__ import annotations
2
+
3
+ import json
4
+ from pathlib import Path
5
+ from typing import Any
6
+
7
+ import yaml
8
+
9
+ from tsfm_bench.data.interface import TimeSeriesDataSource
10
+ from tsfm_bench.data.open_meteo import OpenMeteoDataSource
11
+ from tsfm_bench.data.synthetic import DEFAULT_SYNTHETIC_SPECS, SyntheticDataSource, _SyntheticSpec
12
+ from tsfm_bench.data.ts_bench import TsBenchDataSource
13
+
14
+
15
+ def load_data_source(config_path: str | Path | None = None) -> TimeSeriesDataSource:
16
+ """Load a data source from config. Defaults to built-in synthetic demo."""
17
+ if config_path is None:
18
+ return SyntheticDataSource()
19
+
20
+ path = Path(config_path)
21
+ if not path.exists():
22
+ raise FileNotFoundError(f"Dataset config not found: {path}")
23
+
24
+ if path.suffix in {".yaml", ".yml"}:
25
+ payload = yaml.safe_load(path.read_text())
26
+ elif path.suffix == ".json":
27
+ payload = json.loads(path.read_text())
28
+ else:
29
+ raise ValueError(f"Unsupported config format: {path.suffix}")
30
+
31
+ source_type = payload.get("source_type", "synthetic")
32
+ if source_type == "synthetic":
33
+ specs = tuple(_parse_spec(entry) for entry in payload.get("datasets", []))
34
+ return SyntheticDataSource(specs=specs or DEFAULT_SYNTHETIC_SPECS)
35
+ if source_type == "open_meteo":
36
+ return OpenMeteoDataSource.from_config(payload)
37
+ if source_type == "ts_bench":
38
+ return TsBenchDataSource.from_config(payload)
39
+
40
+ raise NotImplementedError(
41
+ f"Source type '{source_type}' is not implemented yet. "
42
+ "Implement TimeSeriesDataSource and register it in registry.py."
43
+ )
44
+
45
+
46
+ def load_dataset_properties(config_path: str | Path | None = None) -> dict[str, dict[str, Any]]:
47
+ """Build dataset_properties map keyed by dataset stem (before frequency suffix)."""
48
+ source = load_data_source(config_path)
49
+ properties: dict[str, dict[str, Any]] = {}
50
+
51
+ for name in source.list_datasets():
52
+ meta = source.get_metadata(name)
53
+ if isinstance(source, TsBenchDataSource):
54
+ ds_key = source.leaderboard_dataset_name(name).split("/")[0].lower()
55
+ else:
56
+ ds_key = name.split("/")[0].lower()
57
+ properties[ds_key] = {
58
+ "domain": meta.domain,
59
+ "frequency": meta.frequency,
60
+ "num_variates": meta.num_variates,
61
+ "terms": list(meta.terms),
62
+ }
63
+ return properties
64
+
65
+
66
+ def _parse_spec(entry: dict[str, Any]) -> _SyntheticSpec:
67
+ return _SyntheticSpec(
68
+ name=entry["name"],
69
+ domain=entry["domain"],
70
+ frequency=entry["frequency"],
71
+ num_variates=int(entry["num_variates"]),
72
+ terms=tuple(entry.get("terms", ["short"])),
73
+ n_series=int(entry.get("n_series", 30)),
74
+ series_length=int(entry.get("series_length", 500)),
75
+ seed=int(entry.get("seed", 0)),
76
+ )
tsfm_bench/data/synthetic.py ADDED
@@ -0,0 +1,151 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from __future__ import annotations
2
+
3
+ import hashlib
4
+ from dataclasses import dataclass
5
+ from typing import Iterator
6
+
7
+ import numpy as np
8
+ import pandas as pd
9
+
10
+ from tsfm_bench.data.interface import DatasetMetadata, TimeSeriesRecord
11
+
12
+
13
+ @dataclass
14
+ class _SyntheticSpec:
15
+ name: str
16
+ domain: str
17
+ frequency: str
18
+ num_variates: int
19
+ terms: tuple[str, ...]
20
+ n_series: int
21
+ series_length: int
22
+ seed: int
23
+
24
+
25
+ DEFAULT_SYNTHETIC_SPECS: tuple[_SyntheticSpec, ...] = (
26
+ _SyntheticSpec(
27
+ name="synthetic_energy/H",
28
+ domain="Energy",
29
+ frequency="H",
30
+ num_variates=4,
31
+ terms=("short", "medium", "long"),
32
+ n_series=40,
33
+ series_length=5000,
34
+ seed=11,
35
+ ),
36
+ _SyntheticSpec(
37
+ name="synthetic_finance/D",
38
+ domain="Econ/Fin",
39
+ frequency="D",
40
+ num_variates=2,
41
+ terms=("short",),
42
+ n_series=35,
43
+ series_length=1200,
44
+ seed=22,
45
+ ),
46
+ _SyntheticSpec(
47
+ name="synthetic_healthcare/D",
48
+ domain="Healthcare",
49
+ frequency="D",
50
+ num_variates=1,
51
+ terms=("short", "medium"),
52
+ n_series=30,
53
+ series_length=2000,
54
+ seed=33,
55
+ ),
56
+ _SyntheticSpec(
57
+ name="synthetic_retail/W",
58
+ domain="Sales",
59
+ frequency="W",
60
+ num_variates=3,
61
+ terms=("short",),
62
+ n_series=25,
63
+ series_length=800,
64
+ seed=44,
65
+ ),
66
+ _SyntheticSpec(
67
+ name="synthetic_transport/15T",
68
+ domain="Transport",
69
+ frequency="15T",
70
+ num_variates=5,
71
+ terms=("short", "medium", "long"),
72
+ n_series=20,
73
+ series_length=6000,
74
+ seed=55,
75
+ ),
76
+ _SyntheticSpec(
77
+ name="synthetic_cloudops/H",
78
+ domain="Web/CloudOps",
79
+ frequency="H",
80
+ num_variates=2,
81
+ terms=("short", "medium"),
82
+ n_series=30,
83
+ series_length=4000,
84
+ seed=66,
85
+ ),
86
+ )
87
+
88
+
89
+ class SyntheticDataSource:
90
+ """In-memory streaming source backed by deterministic synthetic series."""
91
+
92
+ def __init__(self, specs: tuple[_SyntheticSpec, ...] | None = None):
93
+ self._specs = {spec.name: spec for spec in (specs or DEFAULT_SYNTHETIC_SPECS)}
94
+
95
+ def list_datasets(self) -> list[str]:
96
+ return sorted(self._specs.keys())
97
+
98
+ def get_metadata(self, name: str) -> DatasetMetadata:
99
+ spec = self._require_spec(name)
100
+ return DatasetMetadata(
101
+ name=spec.name,
102
+ domain=spec.domain,
103
+ frequency=spec.frequency,
104
+ num_variates=spec.num_variates,
105
+ terms=spec.terms,
106
+ source_type="synthetic",
107
+ )
108
+
109
+ def stream(self, name: str) -> Iterator[TimeSeriesRecord]:
110
+ spec = self._require_spec(name)
111
+ freq = pd.tseries.frequencies.to_offset(spec.frequency)
112
+ start = pd.Timestamp("2020-01-01")
113
+
114
+ for series_idx in range(spec.n_series):
115
+ seed = int(
116
+ hashlib.sha256(f"{spec.seed}:{series_idx}".encode()).hexdigest()[:8],
117
+ 16,
118
+ )
119
+ rng = np.random.default_rng(seed)
120
+ length = spec.series_length + series_idx % 7
121
+
122
+ t = np.arange(length, dtype=np.float64)
123
+ seasonal_period = max(2, min(24, length // 8))
124
+ seasonal = np.sin(2 * np.pi * t / seasonal_period)
125
+ trend = 0.02 * t + rng.normal(0, 0.5)
126
+ noise = rng.normal(0, 0.3, size=length)
127
+
128
+ if spec.num_variates == 1:
129
+ target = trend + seasonal + noise
130
+ else:
131
+ target = np.stack(
132
+ [
133
+ trend
134
+ + (idx + 1) * 0.1 * seasonal
135
+ + rng.normal(0, 0.25, size=length)
136
+ for idx in range(spec.num_variates)
137
+ ],
138
+ axis=0,
139
+ )
140
+
141
+ yield TimeSeriesRecord(
142
+ item_id=f"{spec.name.replace('/', '_')}_{series_idx:04d}",
143
+ target=target.astype(np.float32),
144
+ start=str(pd.Period(start, freq=freq)),
145
+ freq=spec.frequency,
146
+ )
147
+
148
+ def _require_spec(self, name: str) -> _SyntheticSpec:
149
+ if name not in self._specs:
150
+ raise KeyError(f"Unknown synthetic dataset: {name}")
151
+ return self._specs[name]
tsfm_bench/data/ts_bench.py ADDED
@@ -0,0 +1,304 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from __future__ import annotations
2
+
3
+ import csv
4
+ import json
5
+ import logging
6
+ import os
7
+ import subprocess
8
+ import sys
9
+ from dataclasses import dataclass
10
+ from datetime import datetime, timezone
11
+ from pathlib import Path
12
+ from typing import Any, Iterator
13
+
14
+ import numpy as np
15
+ import pandas as pd
16
+
17
+ from tsfm_bench.data.interface import DatasetMetadata, TimeSeriesRecord
18
+
19
+ logger = logging.getLogger(__name__)
20
+
21
+ DEFAULT_SOURCE_DOMAINS: dict[str, str] = {
22
+ "open_meteo_weather": "Weather",
23
+ "open_meteo_air_quality": "Air Quality",
24
+ "nasa_power": "Weather",
25
+ "usgs_water": "Hydrology",
26
+ "noaa_coops": "Ocean",
27
+ "noaa_ndbc": "Ocean",
28
+ "gbfs_citibike": "Transport",
29
+ "binance": "Finance",
30
+ "coingecko": "Finance",
31
+ "wikimedia_pageviews": "Web",
32
+ "nws_observations": "Weather",
33
+ "usgs_earthquake": "Hazards",
34
+ "noaa_ncei": "Weather",
35
+ }
36
+
37
+ FREQ_ALIASES = {
38
+ "1h": "1h",
39
+ "6min": "6min",
40
+ "10min": "10min",
41
+ "15min": "15min",
42
+ "1d": "1D",
43
+ "snapshot": "1h",
44
+ }
45
+
46
+
47
+ @dataclass(frozen=True)
48
+ class _TsBenchSettings:
49
+ ts_bench_root: Path
50
+ output_subdir: str = "data/sample_week"
51
+ collect_days: int = 7
52
+ collect_timeout: int = 45
53
+ limit_entities: int = 10
54
+ task_limit: int = 20
55
+ max_tasks_per_dataset: int = 1
56
+ min_future_steps: int = 1
57
+ skip_frequencies: tuple[str, ...] = ("snapshot",)
58
+ auto_refresh: bool = True
59
+
60
+
61
+ @dataclass(frozen=True)
62
+ class _TaskSpec:
63
+ task_id: str
64
+ dataset_id: str
65
+ source_id: str
66
+ domain: str
67
+ target_variable_id: str
68
+ frequency: str
69
+ gluonts_freq: str
70
+ prediction_length: int
71
+ task_dir: Path
72
+ leaderboard_name: str
73
+
74
+
75
+ class TsBenchDataSource:
76
+ """Live TS-Bench data: refresh via collect_weekly_data.py, read exported tasks."""
77
+
78
+ def __init__(
79
+ self,
80
+ settings: _TsBenchSettings,
81
+ source_domains: dict[str, str],
82
+ tasks: tuple[_TaskSpec, ...],
83
+ fetched_at: str = "",
84
+ ):
85
+ self._settings = settings
86
+ self._source_domains = source_domains
87
+ self._tasks = {task.task_id: task for task in tasks}
88
+ self._fetched_at = fetched_at
89
+
90
+ @classmethod
91
+ def from_config(cls, payload: dict[str, Any]) -> TsBenchDataSource:
92
+ raw = payload.get("settings", {})
93
+ root_value = os.environ.get("TS_BENCH_ROOT") or raw.get("ts_bench_root", "../TS-Bench")
94
+ ts_bench_root = Path(root_value).expanduser().resolve()
95
+ settings = _TsBenchSettings(
96
+ ts_bench_root=ts_bench_root,
97
+ output_subdir=str(raw.get("output_subdir", "data/sample_week")),
98
+ collect_days=int(raw.get("collect_days", 7)),
99
+ collect_timeout=int(raw.get("collect_timeout", 45)),
100
+ limit_entities=int(raw.get("limit_entities", 10)),
101
+ task_limit=int(raw.get("task_limit", 20)),
102
+ max_tasks_per_dataset=int(raw.get("max_tasks_per_dataset", 1)),
103
+ min_future_steps=int(raw.get("min_future_steps", 1)),
104
+ skip_frequencies=tuple(raw.get("skip_frequencies", ["snapshot"])),
105
+ auto_refresh=bool(raw.get("auto_refresh", True)),
106
+ )
107
+ source_domains = {**DEFAULT_SOURCE_DOMAINS, **payload.get("source_domains", {})}
108
+ source = cls(settings=settings, source_domains=source_domains, tasks=())
109
+ source._load_tasks()
110
+ return source
111
+
112
+ @property
113
+ def data_root(self) -> Path:
114
+ return self._settings.ts_bench_root / self._settings.output_subdir
115
+
116
+ def refresh(self) -> None:
117
+ root = self._settings.ts_bench_root
118
+ output_dir = self.data_root
119
+ collect_script = root / "collect_weekly_data.py"
120
+ export_script = root / "export_model_inputs.py"
121
+ if not collect_script.exists():
122
+ raise FileNotFoundError(f"TS-Bench collect script not found: {collect_script}")
123
+
124
+ logger.info("Collecting TS-Bench data → %s", output_dir)
125
+ collect_cmd = [
126
+ sys.executable,
127
+ str(collect_script),
128
+ "--days",
129
+ str(self._settings.collect_days),
130
+ "--output",
131
+ self._settings.output_subdir,
132
+ "--timeout",
133
+ str(self._settings.collect_timeout),
134
+ "--limit-entities",
135
+ str(self._settings.limit_entities),
136
+ "--max-tasks-per-dataset",
137
+ str(self._settings.max_tasks_per_dataset),
138
+ ]
139
+ subprocess.run(collect_cmd, cwd=root, check=True)
140
+
141
+ normalized = self._settings.output_subdir + "/normalized"
142
+ model_inputs = self._settings.output_subdir + "/model_inputs_latest"
143
+ if export_script.exists():
144
+ logger.info("Exporting TS-Bench model inputs → %s", self.data_root / "model_inputs_latest")
145
+ export_cmd = [
146
+ sys.executable,
147
+ str(export_script),
148
+ "--input",
149
+ normalized,
150
+ "--output",
151
+ model_inputs,
152
+ "--limit",
153
+ str(self._settings.task_limit),
154
+ ]
155
+ subprocess.run(export_cmd, cwd=root, check=True)
156
+
157
+ self._load_tasks()
158
+
159
+ def _load_tasks(self) -> None:
160
+ summary_path = self.data_root / "summary.json"
161
+ if summary_path.exists():
162
+ summary = json.loads(summary_path.read_text())
163
+ self._fetched_at = summary.get("generated_at", "")
164
+ else:
165
+ self._fetched_at = datetime.now(timezone.utc).isoformat()
166
+
167
+ source_by_dataset = self._load_dataset_sources()
168
+ tasks = self._load_task_specs(source_by_dataset)
169
+ self._tasks = {task.task_id: task for task in tasks}
170
+ logger.info("Loaded %d TS-Bench tasks from %s", len(tasks), self.data_root)
171
+
172
+ def _load_dataset_sources(self) -> dict[str, str]:
173
+ entities_path = self.data_root / "normalized" / "entities.csv"
174
+ mapping: dict[str, str] = {}
175
+ if not entities_path.exists():
176
+ return mapping
177
+ with entities_path.open(newline="", encoding="utf-8") as handle:
178
+ for row in csv.DictReader(handle):
179
+ mapping.setdefault(row["dataset_id"], row["source_id"])
180
+ return mapping
181
+
182
+ def _load_task_specs(self, source_by_dataset: dict[str, str]) -> tuple[_TaskSpec, ...]:
183
+ manifest_path = self.data_root / "model_inputs_latest" / "manifest.json"
184
+ if not manifest_path.exists():
185
+ manifest_path = self.data_root / "model_inputs_latest" / "manifest.csv"
186
+ if not manifest_path.exists():
187
+ logger.warning(
188
+ "No TS-Bench manifest under %s — call refresh() to collect data",
189
+ manifest_path.parent,
190
+ )
191
+ return tuple()
192
+
193
+ entries: list[dict[str, Any]]
194
+ if manifest_path.suffix == ".json":
195
+ entries = json.loads(manifest_path.read_text())
196
+ else:
197
+ with manifest_path.open(newline="", encoding="utf-8") as handle:
198
+ entries = list(csv.DictReader(handle))
199
+
200
+ specs: list[_TaskSpec] = []
201
+ for entry in entries:
202
+ future_rows = int(entry.get("future_target_rows", 0))
203
+ if future_rows < self._settings.min_future_steps:
204
+ continue
205
+
206
+ task_id = entry["task_id"]
207
+ task_dir = self.data_root / "model_inputs_latest" / task_id
208
+ task_json = task_dir / "task.json"
209
+ if not task_json.exists():
210
+ continue
211
+
212
+ task = json.loads(task_json.read_text())
213
+ raw_freq = str(task.get("frequency", "1h"))
214
+ if raw_freq in self._settings.skip_frequencies:
215
+ continue
216
+
217
+ dataset_id = task["dataset_id"]
218
+ source_id = source_by_dataset.get(dataset_id, dataset_id.split("_")[0])
219
+ domain = self._source_domains.get(source_id, "Other")
220
+ gluonts_freq = FREQ_ALIASES.get(raw_freq, raw_freq)
221
+ pred_len = int(task.get("horizon_length", future_rows))
222
+ variable = task["target_variable_id"]
223
+ leaderboard_name = f"{dataset_id}_{variable}"
224
+
225
+ specs.append(
226
+ _TaskSpec(
227
+ task_id=task_id,
228
+ dataset_id=dataset_id,
229
+ source_id=source_id,
230
+ domain=domain,
231
+ target_variable_id=variable,
232
+ frequency=raw_freq,
233
+ gluonts_freq=gluonts_freq,
234
+ prediction_length=pred_len,
235
+ task_dir=task_dir,
236
+ leaderboard_name=leaderboard_name,
237
+ )
238
+ )
239
+ return tuple(specs)
240
+
241
+ def list_datasets(self) -> list[str]:
242
+ return sorted(self._tasks.keys())
243
+
244
+ def get_metadata(self, name: str) -> DatasetMetadata:
245
+ task = self._require_task(name)
246
+ return DatasetMetadata(
247
+ name=name,
248
+ domain=task.domain,
249
+ frequency=task.gluonts_freq,
250
+ num_variates=1,
251
+ terms=("short",),
252
+ source_type="ts_bench",
253
+ )
254
+
255
+ def get_prediction_length(self, name: str) -> int:
256
+ return self._require_task(name).prediction_length
257
+
258
+ def stream(self, name: str) -> Iterator[TimeSeriesRecord]:
259
+ task = self._require_task(name)
260
+ context_path = task.task_dir / "context.csv"
261
+ future_path = task.task_dir / "future_target.csv"
262
+ if not context_path.exists() or not future_path.exists():
263
+ raise FileNotFoundError(f"Missing task CSVs for {name}")
264
+
265
+ context = pd.read_csv(context_path)
266
+ future = pd.read_csv(future_path)
267
+ target_col = task.target_variable_id
268
+ if target_col not in context.columns:
269
+ raise RuntimeError(f"Target column {target_col} missing in {context_path}")
270
+
271
+ context_vals = pd.to_numeric(context[target_col], errors="coerce").to_numpy(dtype=np.float32)
272
+ future_vals = pd.to_numeric(future[target_col], errors="coerce").to_numpy(dtype=np.float32)
273
+ values = np.concatenate([context_vals, future_vals])
274
+ mask = ~np.isnan(values)
275
+ values = values[mask]
276
+ if len(values) < task.prediction_length + 2:
277
+ raise RuntimeError(
278
+ f"Insufficient data for {name}: {len(values)} points, "
279
+ f"need > {task.prediction_length + 2}"
280
+ )
281
+
282
+ timestamps = list(context["timestamp"]) + list(future["timestamp"])
283
+ timestamps = [timestamps[i] for i, keep in enumerate(mask) if keep]
284
+ start_ts = pd.Timestamp(timestamps[0])
285
+ freq = pd.tseries.frequencies.to_offset(task.gluonts_freq)
286
+
287
+ yield TimeSeriesRecord(
288
+ item_id=task.leaderboard_name.replace("/", "_"),
289
+ target=values,
290
+ start=str(pd.Period(start_ts, freq=freq)),
291
+ freq=task.gluonts_freq,
292
+ )
293
+
294
+ def leaderboard_dataset_name(self, name: str) -> str:
295
+ task = self._require_task(name)
296
+ return f"{task.leaderboard_name}/{task.gluonts_freq}/short"
297
+
298
+ def fetched_at(self) -> str:
299
+ return self._fetched_at or datetime.now(timezone.utc).isoformat()
300
+
301
+ def _require_task(self, name: str) -> _TaskSpec:
302
+ if name not in self._tasks:
303
+ raise KeyError(f"Unknown TS-Bench task: {name}")
304
+ return self._tasks[name]
tsfm_bench/eval/__init__.py ADDED
@@ -0,0 +1 @@
 
 
1
+
tsfm_bench/eval/api_predictor.py ADDED
@@ -0,0 +1,166 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from __future__ import annotations
2
+
3
+ import os
4
+ from dataclasses import dataclass
5
+ from typing import Any, Iterator, List, Optional
6
+
7
+ import httpx
8
+ import numpy as np
9
+ from gluonts.dataset import Dataset as GluonDataset
10
+ from gluonts.model import Forecast
11
+ from gluonts.model.forecast import QuantileForecast
12
+ from gluonts.model.predictor import RepresentablePredictor
13
+
14
+ from tsfm_bench.eval.predictors import _ForecastConfig
15
+
16
+ DEFAULT_API_URL = "https://api.tsfm.ai/v1/forecast"
17
+ DEFAULT_QUANTILES = [0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9]
18
+
19
+
20
+ @dataclass
21
+ class TsfmApiConfig:
22
+ model_id: str
23
+ api_url: str = DEFAULT_API_URL
24
+ api_key: str | None = None
25
+ timeout: float = 120.0
26
+ freq: str = "H"
27
+
28
+
29
+ class TsfmApiPredictor(RepresentablePredictor):
30
+ """Zero-shot forecaster backed by the TSFM.ai hosted API."""
31
+
32
+ def __init__(
33
+ self,
34
+ config: TsfmApiConfig,
35
+ prediction_length: int,
36
+ quantile_levels: Optional[List[float]] = None,
37
+ ):
38
+ super().__init__(prediction_length=prediction_length)
39
+ self.config = config
40
+ self.quantile_levels = quantile_levels or DEFAULT_QUANTILES
41
+ self.forecast_config = _ForecastConfig.from_quantiles(self.quantile_levels)
42
+ self._api_key = config.api_key or os.getenv("TSFM_API_KEY")
43
+ self.leaderboard_name = config.model_id.split("/")[-1]
44
+
45
+ def predict(self, dataset: GluonDataset, **kwargs) -> Iterator[Forecast]:
46
+ if not self._api_key:
47
+ raise RuntimeError(
48
+ "TSFM_API_KEY is required for online zero-shot evaluation. "
49
+ "Get a key at https://tsfm.ai/"
50
+ )
51
+
52
+ headers = {
53
+ "Authorization": f"Bearer {self._api_key}",
54
+ "Content-Type": "application/json",
55
+ }
56
+
57
+ with httpx.Client(timeout=self.config.timeout, trust_env=False) as client:
58
+ for entry in dataset:
59
+ target = np.asarray(entry["target"], dtype=np.float64)
60
+ univariate = target.ndim == 1
61
+ working = target.reshape(1, -1) if univariate else target
62
+
63
+ forecast_rows = []
64
+ for variate_idx in range(working.shape[0]):
65
+ series = working[variate_idx]
66
+ series = series[np.isfinite(series)]
67
+ payload = {
68
+ "model": self.config.model_id,
69
+ "inputs": [
70
+ {
71
+ "item_id": str(entry.get("item_id", variate_idx)),
72
+ "target": _to_api_target(series),
73
+ "start": _format_start(entry.get("start")),
74
+ }
75
+ ],
76
+ "parameters": {
77
+ "prediction_length": self.prediction_length,
78
+ "freq": self.config.freq,
79
+ "quantiles": self.quantile_levels,
80
+ },
81
+ }
82
+ response = client.post(
83
+ self.config.api_url,
84
+ headers=headers,
85
+ json=payload,
86
+ )
87
+ response.raise_for_status()
88
+ forecast_rows.append(
89
+ _parse_api_forecast(
90
+ response.json(),
91
+ self.prediction_length,
92
+ self.quantile_levels,
93
+ )
94
+ )
95
+
96
+ stacked = np.stack(forecast_rows, axis=0)
97
+ if univariate:
98
+ forecast_arrays = stacked[0]
99
+ else:
100
+ forecast_arrays = stacked
101
+
102
+ yield QuantileForecast(
103
+ forecast_arrays=forecast_arrays,
104
+ forecast_keys=self.forecast_config.forecast_keys,
105
+ start_date=entry["start"] + len(entry["target"]),
106
+ item_id=entry["item_id"],
107
+ )
108
+
109
+
110
+ def _to_api_target(series: np.ndarray) -> list[list[float]]:
111
+ """TSFM.ai expects target shaped [num_timesteps][num_channels]."""
112
+ return [[float(v)] for v in series.tolist()]
113
+
114
+
115
+ def _format_start(start: Any) -> str:
116
+ if hasattr(start, "to_timestamp"):
117
+ return start.to_timestamp().isoformat()
118
+ return str(start)
119
+
120
+
121
+ def _parse_api_forecast(
122
+ payload: dict[str, Any],
123
+ prediction_length: int,
124
+ quantile_levels: list[float],
125
+ ) -> np.ndarray:
126
+ """Parse TSFM.ai response into (num_outputs, prediction_length)."""
127
+ outputs_block = payload.get("outputs") or payload.get("forecasts") or []
128
+ if not outputs_block:
129
+ raise ValueError(f"Unrecognized TSFM API response: {payload}")
130
+
131
+ first = outputs_block[0]
132
+ rows: list[np.ndarray] = [ _flatten_forecast_values(first.get("mean"), prediction_length)]
133
+
134
+ quantile_map: dict[float, np.ndarray] = {}
135
+ for item in first.get("quantile_predictions", []):
136
+ level = float(item.get("level", item.get("quantile", 0.5)))
137
+ quantile_map[level] = _flatten_forecast_values(item.get("values"), prediction_length)
138
+
139
+ for q in quantile_levels:
140
+ if q in quantile_map:
141
+ rows.append(quantile_map[q])
142
+ else:
143
+ nearest = min(quantile_map.keys(), key=lambda k: abs(k - q), default=None)
144
+ if nearest is None:
145
+ rows.append(rows[0].copy())
146
+ else:
147
+ rows.append(quantile_map[nearest])
148
+
149
+ return np.stack(rows, axis=0).astype(np.float64)
150
+
151
+
152
+ def _flatten_forecast_values(values: Any, prediction_length: int) -> np.ndarray:
153
+ """Convert nested API values like [[1.2], [1.3], ...] to 1D array."""
154
+ if values is None:
155
+ raise ValueError("Missing forecast values in TSFM API response")
156
+
157
+ arr = np.asarray(values, dtype=np.float64)
158
+ if arr.ndim == 2 and arr.shape[1] == 1:
159
+ arr = arr[:, 0]
160
+ elif arr.ndim > 1:
161
+ arr = arr.reshape(arr.shape[0], -1).mean(axis=1)
162
+
163
+ arr = arr.reshape(-1)
164
+ if arr.size < prediction_length:
165
+ raise ValueError(f"Forecast length {arr.size} < expected {prediction_length}")
166
+ return arr[:prediction_length]
tsfm_bench/eval/metrics.py ADDED
@@ -0,0 +1,50 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from gluonts.ev.metrics import (
2
+ MAE,
3
+ MAPE,
4
+ MASE,
5
+ MeanWeightedSumQuantileLoss,
6
+ MSIS,
7
+ MSE,
8
+ ND,
9
+ NRMSE,
10
+ RMSE,
11
+ SMAPE,
12
+ )
13
+
14
+
15
+ def get_eval_metrics():
16
+ """GIFT-Eval compatible metric set."""
17
+ return [
18
+ MSE(forecast_type="mean"),
19
+ MSE(forecast_type=0.5),
20
+ MAE(),
21
+ MASE(),
22
+ MAPE(),
23
+ SMAPE(),
24
+ MSIS(),
25
+ RMSE(),
26
+ NRMSE(),
27
+ ND(),
28
+ MeanWeightedSumQuantileLoss(
29
+ quantile_levels=[0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9]
30
+ ),
31
+ ]
32
+
33
+
34
+ RESULT_COLUMNS = [
35
+ "dataset",
36
+ "model",
37
+ "eval_metrics/MSE[mean]",
38
+ "eval_metrics/MSE[0.5]",
39
+ "eval_metrics/MAE[0.5]",
40
+ "eval_metrics/MASE[0.5]",
41
+ "eval_metrics/MAPE[0.5]",
42
+ "eval_metrics/sMAPE[0.5]",
43
+ "eval_metrics/MSIS",
44
+ "eval_metrics/RMSE[mean]",
45
+ "eval_metrics/NRMSE[mean]",
46
+ "eval_metrics/ND[0.5]",
47
+ "eval_metrics/mean_weighted_sum_quantile_loss",
48
+ "domain",
49
+ "num_variates",
50
+ ]
tsfm_bench/eval/online_eval.py ADDED
@@ -0,0 +1,154 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from __future__ import annotations
2
+
3
+ from dataclasses import dataclass
4
+ from typing import Any, Iterable
5
+
6
+ import numpy as np
7
+ import pandas as pd
8
+ from gluonts.dataset.common import ListDataset
9
+ from gluonts.dataset.split import split
10
+ from gluonts.model import Predictor, evaluate_model
11
+ from gluonts.time_feature import get_seasonality
12
+
13
+ from tsfm_bench.data.interface import TimeSeriesDataSource
14
+ from tsfm_bench.data.open_meteo import OpenMeteoDataSource
15
+ from tsfm_bench.data.ts_bench import TsBenchDataSource
16
+ from tsfm_bench.eval.metrics import RESULT_COLUMNS, get_eval_metrics
17
+
18
+
19
+ @dataclass
20
+ class OnlineEvalResult:
21
+ dataset: str
22
+ model: str
23
+ metrics: dict[str, float]
24
+ domain: str
25
+ num_variates: int
26
+ data_fetched_at: str
27
+ context_length: int
28
+ prediction_length: int
29
+
30
+
31
+ def _records_to_entries(
32
+ record,
33
+ to_univariate: bool,
34
+ ) -> list[dict[str, Any]]:
35
+ target = np.asarray(record.target, dtype=np.float32)
36
+ start = pd.Period(record.start, freq=record.freq)
37
+ entries: list[dict[str, Any]] = []
38
+
39
+ if target.ndim == 1 or not to_univariate:
40
+ entries.append(
41
+ {
42
+ "item_id": record.item_id,
43
+ "start": start,
44
+ "target": target if target.ndim == 1 else target,
45
+ "freq": record.freq,
46
+ }
47
+ )
48
+ return entries
49
+
50
+ for idx in range(target.shape[0]):
51
+ entries.append(
52
+ {
53
+ "item_id": f"{record.item_id}_v{idx}",
54
+ "start": start,
55
+ "target": target[idx],
56
+ "freq": record.freq,
57
+ }
58
+ )
59
+ return entries
60
+
61
+
62
+ def evaluate_online_window(
63
+ predictor: Predictor,
64
+ entries: list[dict[str, Any]],
65
+ prediction_length: int,
66
+ freq: str,
67
+ ) -> dict[str, float]:
68
+ dataset = ListDataset(entries, freq=freq)
69
+ _, test_template = split(dataset, offset=-prediction_length)
70
+ test_data = test_template.generate_instances(
71
+ prediction_length=prediction_length,
72
+ windows=1,
73
+ distance=prediction_length,
74
+ )
75
+ seasonality = get_seasonality(freq)
76
+ evaluated = evaluate_model(
77
+ predictor,
78
+ test_data=test_data,
79
+ metrics=get_eval_metrics(),
80
+ batch_size=64,
81
+ axis=None,
82
+ mask_invalid_label=True,
83
+ allow_nan_forecast=False,
84
+ seasonality=seasonality,
85
+ )
86
+ return {key: float(value.iloc[0]) for key, value in evaluated.items()}
87
+
88
+
89
+ def run_online_eval_for_dataset(
90
+ source: TimeSeriesDataSource,
91
+ ds_name: str,
92
+ predictor: Predictor,
93
+ prediction_length: int | None = None,
94
+ ) -> OnlineEvalResult:
95
+ meta = source.get_metadata(ds_name)
96
+ record = next(iter(source.stream(ds_name)))
97
+
98
+ if isinstance(source, OpenMeteoDataSource):
99
+ pred_len = prediction_length or source._settings.prediction_length
100
+ fetched_at = source.fetched_at()
101
+ elif isinstance(source, TsBenchDataSource):
102
+ pred_len = prediction_length or source.get_prediction_length(ds_name)
103
+ fetched_at = source.fetched_at()
104
+ else:
105
+ pred_len = prediction_length or 24
106
+ fetched_at = pd.Timestamp.utcnow().isoformat()
107
+
108
+ to_univariate = meta.num_variates > 1
109
+ entries = _records_to_entries(record, to_univariate=to_univariate)
110
+ metrics = evaluate_online_window(predictor, entries, pred_len, meta.frequency)
111
+
112
+ ds_key = ds_name.split("/")[0].lower()
113
+ if isinstance(source, TsBenchDataSource):
114
+ config_name = source.leaderboard_dataset_name(ds_name)
115
+ else:
116
+ config_name = f"{ds_key}/{meta.frequency}/short"
117
+ model_name = getattr(predictor, "leaderboard_name", predictor.__class__.__name__)
118
+ context_length = int(np.asarray(entries[0]["target"]).shape[-1]) - pred_len
119
+
120
+ return OnlineEvalResult(
121
+ dataset=config_name,
122
+ model=model_name,
123
+ metrics=metrics,
124
+ domain=meta.domain,
125
+ num_variates=1 if to_univariate else meta.num_variates,
126
+ data_fetched_at=fetched_at,
127
+ context_length=context_length,
128
+ prediction_length=pred_len,
129
+ )
130
+
131
+
132
+ def result_to_row(result: OnlineEvalResult) -> list[Any]:
133
+ return [
134
+ result.dataset,
135
+ result.model,
136
+ result.metrics["MSE[mean]"],
137
+ result.metrics["MSE[0.5]"],
138
+ result.metrics["MAE[0.5]"],
139
+ result.metrics["MASE[0.5]"],
140
+ result.metrics["MAPE[0.5]"],
141
+ result.metrics["sMAPE[0.5]"],
142
+ result.metrics["MSIS"],
143
+ result.metrics["RMSE[mean]"],
144
+ result.metrics["NRMSE[mean]"],
145
+ result.metrics["ND[0.5]"],
146
+ result.metrics["mean_weighted_sum_quantile_loss"],
147
+ result.domain,
148
+ result.num_variates,
149
+ ]
150
+
151
+
152
+ def results_to_dataframe(results: Iterable[OnlineEvalResult]) -> pd.DataFrame:
153
+ rows = [result_to_row(result) for result in results]
154
+ return pd.DataFrame(rows, columns=RESULT_COLUMNS)
tsfm_bench/eval/predictors.py ADDED
@@ -0,0 +1,85 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from __future__ import annotations
2
+
3
+ from dataclasses import dataclass
4
+ from typing import Iterator, List, Optional
5
+
6
+ import numpy as np
7
+ from gluonts.dataset import Dataset as GluonDataset
8
+ from gluonts.model import Forecast
9
+ from gluonts.model.forecast import QuantileForecast
10
+ from gluonts.model.predictor import RepresentablePredictor
11
+
12
+
13
+ @dataclass
14
+ class _ForecastConfig:
15
+ quantile_levels: Optional[List[float]]
16
+ forecast_keys: List[str]
17
+ intervals: Optional[List[int]]
18
+
19
+ @classmethod
20
+ def from_quantiles(cls, quantile_levels: Optional[List[float]]):
21
+ forecast_keys = ["mean"]
22
+ intervals = None
23
+ if not quantile_levels:
24
+ return cls(quantile_levels=None, forecast_keys=forecast_keys, intervals=intervals)
25
+
26
+ intervals = sorted(
27
+ {
28
+ round(200 * (max(q, 1 - q) - 0.5))
29
+ for q in quantile_levels
30
+ }
31
+ )
32
+ forecast_keys = ["mean"] + [str(q) for q in quantile_levels]
33
+ return cls(
34
+ quantile_levels=quantile_levels,
35
+ forecast_keys=forecast_keys,
36
+ intervals=intervals,
37
+ )
38
+
39
+
40
+ class SeasonalNaivePredictor(RepresentablePredictor):
41
+ """Lightweight seasonal naive baseline for pipeline smoke tests."""
42
+
43
+ def __init__(
44
+ self,
45
+ prediction_length: int,
46
+ season_length: int = 1,
47
+ quantile_levels: Optional[List[float]] = None,
48
+ ):
49
+ super().__init__(prediction_length=prediction_length)
50
+ self.season_length = max(1, season_length)
51
+ self.config = _ForecastConfig.from_quantiles(quantile_levels)
52
+
53
+ def predict(self, dataset: GluonDataset, **kwargs) -> Iterator[Forecast]:
54
+ for entry in dataset:
55
+ target = np.asarray(entry["target"], dtype=np.float64)
56
+ univariate = target.ndim == 1
57
+ if univariate:
58
+ working = target.reshape(1, -1)
59
+ else:
60
+ working = target
61
+
62
+ history = working[:, -self.season_length :]
63
+ tiled = np.tile(
64
+ history, (1, int(np.ceil(self.prediction_length / self.season_length)))
65
+ )
66
+ mean = tiled[:, : self.prediction_length]
67
+ if univariate:
68
+ mean = mean[0]
69
+
70
+ arrays = [mean]
71
+ if self.config.quantile_levels:
72
+ for q in self.config.quantile_levels:
73
+ if q <= 0.5:
74
+ arrays.append(mean * (1 - (0.5 - q) * 0.05))
75
+ else:
76
+ arrays.append(mean * (1 + (q - 0.5) * 0.05))
77
+
78
+ forecast_arrays = np.stack(arrays, axis=0)
79
+
80
+ yield QuantileForecast(
81
+ forecast_arrays=forecast_arrays,
82
+ forecast_keys=self.config.forecast_keys,
83
+ start_date=entry["start"],
84
+ item_id=entry["item_id"],
85
+ )
vendor/ts_bench/collect_weekly_data.py ADDED
@@ -0,0 +1,1457 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env python3
2
+ """Collect a small one-week TS-Bench data sample and normalize it.
3
+
4
+ The script is intentionally conservative:
5
+ - Raw API responses are saved first and never overwritten.
6
+ - Normalized storage uses long-format CSV tables.
7
+ - Model-facing CSV/tensor inputs should be exported later by a loader from
8
+ observations + forecast_tasks, not used as the source of truth.
9
+
10
+ Default collectors avoid API keys where possible. Some public services may be
11
+ temporarily unavailable or rate-limited; failures are captured in run_log.csv
12
+ instead of aborting the whole run.
13
+ """
14
+
15
+ from __future__ import annotations
16
+
17
+ import argparse
18
+ import csv
19
+ import hashlib
20
+ import io
21
+ import json
22
+ import math
23
+ import os
24
+ import re
25
+ import sys
26
+ import zipfile
27
+ from collections import defaultdict
28
+ from dataclasses import dataclass, field
29
+ from datetime import datetime, date, timedelta, timezone
30
+ from pathlib import Path
31
+ from typing import Any, Callable, Dict, Iterable, List, Optional, Tuple
32
+ from urllib.parse import quote
33
+
34
+ import requests
35
+
36
+
37
+ UTC = timezone.utc
38
+ PARSER_VERSION = "tsbench_sample_v0.1"
39
+
40
+
41
+ RAW_SNAPSHOT_FIELDS = [
42
+ "raw_id",
43
+ "source_id",
44
+ "dataset_id",
45
+ "request_url",
46
+ "request_params",
47
+ "http_status",
48
+ "fetched_at",
49
+ "content_type",
50
+ "content_hash",
51
+ "raw_uri",
52
+ "parser_version",
53
+ "error",
54
+ ]
55
+
56
+ ENTITY_FIELDS = [
57
+ "dataset_id",
58
+ "source_id",
59
+ "entity_id",
60
+ "entity_type",
61
+ "entity_name",
62
+ "lat",
63
+ "lon",
64
+ "region",
65
+ "country",
66
+ "timezone",
67
+ "metadata_json",
68
+ ]
69
+
70
+ VARIABLE_FIELDS = [
71
+ "dataset_id",
72
+ "source_id",
73
+ "variable_id",
74
+ "variable_name",
75
+ "unit",
76
+ "value_type",
77
+ "frequency_hint",
78
+ "aggregation",
79
+ "is_target_default",
80
+ "metadata_json",
81
+ ]
82
+
83
+ OBSERVATION_FIELDS = [
84
+ "dataset_id",
85
+ "source_id",
86
+ "entity_id",
87
+ "variable_id",
88
+ "timestamp",
89
+ "available_time",
90
+ "ingest_time",
91
+ "value",
92
+ "value_text",
93
+ "value_json",
94
+ "frequency",
95
+ "unit",
96
+ "quality_flag",
97
+ "revision_id",
98
+ "raw_id",
99
+ ]
100
+
101
+ FORECAST_TASK_FIELDS = [
102
+ "task_id",
103
+ "dataset_id",
104
+ "target_entity_id",
105
+ "target_variable_id",
106
+ "forecast_issue_time",
107
+ "context_start",
108
+ "context_end",
109
+ "horizon_start",
110
+ "horizon_end",
111
+ "input_length",
112
+ "horizon_length",
113
+ "frequency",
114
+ "covariate_variable_ids",
115
+ "task_split",
116
+ "created_at",
117
+ ]
118
+
119
+ QUALITY_REPORT_FIELDS = [
120
+ "dataset_id",
121
+ "source_id",
122
+ "entity_id",
123
+ "variable_id",
124
+ "window_start",
125
+ "window_end",
126
+ "observation_count",
127
+ "missing_ratio",
128
+ "duplicate_count",
129
+ "freshness_delay_hours",
130
+ "outlier_count",
131
+ "api_failure_rate",
132
+ "notes",
133
+ ]
134
+
135
+ RUN_LOG_FIELDS = [
136
+ "source_id",
137
+ "dataset_id",
138
+ "status",
139
+ "started_at",
140
+ "finished_at",
141
+ "observation_count",
142
+ "error",
143
+ ]
144
+
145
+
146
+ def utc_now() -> datetime:
147
+ return datetime.now(UTC)
148
+
149
+
150
+ def iso(dt: datetime) -> str:
151
+ if dt.tzinfo is None:
152
+ dt = dt.replace(tzinfo=UTC)
153
+ return dt.astimezone(UTC).isoformat().replace("+00:00", "Z")
154
+
155
+
156
+ def parse_iso_datetime(value: str) -> datetime:
157
+ text = str(value).strip()
158
+ if text.endswith("Z"):
159
+ text = text[:-1] + "+00:00"
160
+ dt = datetime.fromisoformat(text)
161
+ if dt.tzinfo is None:
162
+ dt = dt.replace(tzinfo=UTC)
163
+ return dt.astimezone(UTC)
164
+
165
+
166
+ def parse_float(value: Any) -> Optional[float]:
167
+ if value is None:
168
+ return None
169
+ if isinstance(value, (int, float)):
170
+ if isinstance(value, float) and (math.isnan(value) or math.isinf(value)):
171
+ return None
172
+ return float(value)
173
+ text = str(value).strip()
174
+ if not text or text.upper() in {"MM", "M", "NA", "N/A", "NULL", "NONE", "-9999", "-999.0"}:
175
+ return None
176
+ try:
177
+ parsed = float(text)
178
+ except ValueError:
179
+ return None
180
+ if math.isnan(parsed) or math.isinf(parsed):
181
+ return None
182
+ return parsed
183
+
184
+
185
+ def floor_hour(dt: datetime) -> datetime:
186
+ return dt.astimezone(UTC).replace(minute=0, second=0, microsecond=0)
187
+
188
+
189
+ def parse_epoch_ms_or_us(value: Any) -> datetime:
190
+ raw = int(value)
191
+ if raw > 10_000_000_000_000:
192
+ return datetime.fromtimestamp(raw / 1_000_000, tz=UTC)
193
+ return datetime.fromtimestamp(raw / 1000, tz=UTC)
194
+
195
+
196
+ def safe_id(text: str) -> str:
197
+ return re.sub(r"[^a-zA-Z0-9_.-]+", "_", text).strip("_").lower()
198
+
199
+
200
+ def dumps(value: Any) -> str:
201
+ if value in (None, ""):
202
+ return ""
203
+ if isinstance(value, str):
204
+ return value
205
+ return json.dumps(value, ensure_ascii=True, sort_keys=True)
206
+
207
+
208
+ def date_yyyymmdd(d: date) -> str:
209
+ return d.strftime("%Y%m%d")
210
+
211
+
212
+ def datetime_yyyymmddhhmmss(dt: datetime) -> str:
213
+ return dt.astimezone(UTC).strftime("%Y%m%d%H%M%S")
214
+
215
+
216
+ def frequency_delta(freq: str) -> Optional[timedelta]:
217
+ return {
218
+ "1min": timedelta(minutes=1),
219
+ "5min": timedelta(minutes=5),
220
+ "6min": timedelta(minutes=6),
221
+ "10min": timedelta(minutes=10),
222
+ "15min": timedelta(minutes=15),
223
+ "1h": timedelta(hours=1),
224
+ "1d": timedelta(days=1),
225
+ "snapshot": None,
226
+ }.get(freq)
227
+
228
+
229
+ @dataclass
230
+ class Store:
231
+ output_dir: Path
232
+ raw_snapshots: List[Dict[str, Any]] = field(default_factory=list)
233
+ entities: Dict[Tuple[str, str], Dict[str, Any]] = field(default_factory=dict)
234
+ variables: Dict[Tuple[str, str], Dict[str, Any]] = field(default_factory=dict)
235
+ observations: List[Dict[str, Any]] = field(default_factory=list)
236
+ forecast_tasks: List[Dict[str, Any]] = field(default_factory=list)
237
+ quality_reports: List[Dict[str, Any]] = field(default_factory=list)
238
+ run_log: List[Dict[str, Any]] = field(default_factory=list)
239
+
240
+ def add_entity(
241
+ self,
242
+ dataset_id: str,
243
+ source_id: str,
244
+ entity_id: str,
245
+ entity_type: str,
246
+ entity_name: str,
247
+ lat: Any = "",
248
+ lon: Any = "",
249
+ region: str = "",
250
+ country: str = "",
251
+ timezone_name: str = "UTC",
252
+ metadata: Optional[Dict[str, Any]] = None,
253
+ ) -> None:
254
+ key = (dataset_id, entity_id)
255
+ if key in self.entities:
256
+ return
257
+ self.entities[key] = {
258
+ "dataset_id": dataset_id,
259
+ "source_id": source_id,
260
+ "entity_id": entity_id,
261
+ "entity_type": entity_type,
262
+ "entity_name": entity_name,
263
+ "lat": lat,
264
+ "lon": lon,
265
+ "region": region,
266
+ "country": country,
267
+ "timezone": timezone_name,
268
+ "metadata_json": metadata or {},
269
+ }
270
+
271
+ def add_variable(
272
+ self,
273
+ dataset_id: str,
274
+ source_id: str,
275
+ variable_id: str,
276
+ variable_name: str,
277
+ unit: str = "",
278
+ value_type: str = "numeric",
279
+ frequency_hint: str = "",
280
+ aggregation: str = "",
281
+ is_target_default: bool = False,
282
+ metadata: Optional[Dict[str, Any]] = None,
283
+ ) -> None:
284
+ key = (dataset_id, variable_id)
285
+ if key in self.variables:
286
+ return
287
+ self.variables[key] = {
288
+ "dataset_id": dataset_id,
289
+ "source_id": source_id,
290
+ "variable_id": variable_id,
291
+ "variable_name": variable_name,
292
+ "unit": unit,
293
+ "value_type": value_type,
294
+ "frequency_hint": frequency_hint,
295
+ "aggregation": aggregation,
296
+ "is_target_default": str(bool(is_target_default)).lower(),
297
+ "metadata_json": metadata or {},
298
+ }
299
+
300
+ def add_observation(
301
+ self,
302
+ dataset_id: str,
303
+ source_id: str,
304
+ entity_id: str,
305
+ variable_id: str,
306
+ timestamp: datetime,
307
+ available_time: datetime,
308
+ ingest_time: datetime,
309
+ value: Optional[float],
310
+ frequency: str,
311
+ unit: str = "",
312
+ value_text: str = "",
313
+ value_json: Any = "",
314
+ quality_flag: Any = "",
315
+ revision_id: str = "v1",
316
+ raw_id: str = "",
317
+ ) -> None:
318
+ self.observations.append(
319
+ {
320
+ "dataset_id": dataset_id,
321
+ "source_id": source_id,
322
+ "entity_id": entity_id,
323
+ "variable_id": variable_id,
324
+ "timestamp": iso(timestamp),
325
+ "available_time": iso(available_time),
326
+ "ingest_time": iso(ingest_time),
327
+ "value": "" if value is None else value,
328
+ "value_text": value_text,
329
+ "value_json": value_json,
330
+ "frequency": frequency,
331
+ "unit": unit,
332
+ "quality_flag": quality_flag,
333
+ "revision_id": revision_id,
334
+ "raw_id": raw_id,
335
+ }
336
+ )
337
+
338
+
339
+ class Fetcher:
340
+ def __init__(self, store: Store, timeout: int = 30) -> None:
341
+ self.store = store
342
+ self.timeout = timeout
343
+ self.session = requests.Session()
344
+ self.session.headers.update(
345
+ {
346
+ "User-Agent": (
347
+ "TS-Bench data-layer prototype "
348
+ "(research; contact: local-run@example.invalid)"
349
+ )
350
+ }
351
+ )
352
+
353
+ def get(
354
+ self,
355
+ source_id: str,
356
+ dataset_id: str,
357
+ url: str,
358
+ params: Optional[Dict[str, Any]] = None,
359
+ headers: Optional[Dict[str, str]] = None,
360
+ raw_ext: str = "json",
361
+ ) -> Tuple[requests.Response, str, datetime]:
362
+ fetched_at = utc_now()
363
+ response: Optional[requests.Response] = None
364
+ error = ""
365
+ try:
366
+ response = self.session.get(
367
+ url,
368
+ params=params,
369
+ headers=headers,
370
+ timeout=self.timeout,
371
+ )
372
+ content = response.content
373
+ status = response.status_code
374
+ content_type = response.headers.get("content-type", "")
375
+ except Exception as exc: # noqa: BLE001
376
+ content = str(exc).encode("utf-8")
377
+ status = 0
378
+ content_type = "text/plain"
379
+ error = str(exc)
380
+
381
+ digest = hashlib.sha256(content).hexdigest()
382
+ raw_id = f"{source_id}_{fetched_at.strftime('%Y%m%dT%H%M%SZ')}_{digest[:12]}"
383
+ raw_dir = self.store.output_dir / "raw" / source_id
384
+ raw_dir.mkdir(parents=True, exist_ok=True)
385
+ raw_uri = raw_dir / f"{raw_id}.{raw_ext}"
386
+ raw_uri.write_bytes(content)
387
+ full_url = response.url if response is not None else url
388
+
389
+ self.store.raw_snapshots.append(
390
+ {
391
+ "raw_id": raw_id,
392
+ "source_id": source_id,
393
+ "dataset_id": dataset_id,
394
+ "request_url": full_url,
395
+ "request_params": params or {},
396
+ "http_status": status,
397
+ "fetched_at": iso(fetched_at),
398
+ "content_type": content_type,
399
+ "content_hash": f"sha256:{digest}",
400
+ "raw_uri": str(raw_uri),
401
+ "parser_version": PARSER_VERSION,
402
+ "error": error,
403
+ }
404
+ )
405
+
406
+ if response is None:
407
+ raise RuntimeError(error)
408
+ if response.status_code >= 400:
409
+ raise RuntimeError(f"HTTP {response.status_code}: {response.text[:300]}")
410
+ return response, raw_id, fetched_at
411
+
412
+
413
+ def add_time_series_observations(
414
+ store: Store,
415
+ dataset_id: str,
416
+ source_id: str,
417
+ entity_id: str,
418
+ raw_id: str,
419
+ fetched_at: datetime,
420
+ times: List[datetime],
421
+ variables: Dict[str, Tuple[List[Any], str, str]],
422
+ frequency: str,
423
+ is_target: Optional[str] = None,
424
+ end_dt: Optional[datetime] = None,
425
+ ) -> int:
426
+ count = 0
427
+ for variable_id, (values, unit, variable_name) in variables.items():
428
+ store.add_variable(
429
+ dataset_id,
430
+ source_id,
431
+ variable_id,
432
+ variable_name,
433
+ unit=unit,
434
+ frequency_hint=frequency,
435
+ aggregation="instantaneous",
436
+ is_target_default=variable_id == is_target,
437
+ )
438
+ for idx, ts in enumerate(times):
439
+ if end_dt and ts > end_dt:
440
+ continue
441
+ if idx >= len(values):
442
+ continue
443
+ value = parse_float(values[idx])
444
+ if value is None:
445
+ continue
446
+ store.add_observation(
447
+ dataset_id,
448
+ source_id,
449
+ entity_id,
450
+ variable_id,
451
+ ts,
452
+ fetched_at,
453
+ fetched_at,
454
+ value,
455
+ frequency,
456
+ unit,
457
+ raw_id=raw_id,
458
+ )
459
+ count += 1
460
+ return count
461
+
462
+
463
+ def collect_open_meteo_weather(store: Store, fetcher: Fetcher, start_dt: datetime, end_dt: datetime, args: argparse.Namespace) -> int:
464
+ source_id = "open_meteo_weather"
465
+ dataset_id = "open_meteo_shanghai_hourly"
466
+ entity_id = "grid_shanghai_31.2304_121.4737"
467
+ store.add_entity(
468
+ dataset_id,
469
+ source_id,
470
+ entity_id,
471
+ "grid",
472
+ "Shanghai grid point",
473
+ lat=31.2304,
474
+ lon=121.4737,
475
+ region="Shanghai",
476
+ country="CN",
477
+ timezone_name="UTC",
478
+ )
479
+ params = {
480
+ "latitude": 31.2304,
481
+ "longitude": 121.4737,
482
+ "hourly": "temperature_2m,relative_humidity_2m,wind_speed_10m,precipitation",
483
+ "past_days": args.days,
484
+ "forecast_days": 1,
485
+ "timezone": "UTC",
486
+ }
487
+ response, raw_id, fetched_at = fetcher.get(source_id, dataset_id, "https://api.open-meteo.com/v1/forecast", params=params)
488
+ payload = response.json()
489
+ hourly = payload.get("hourly", {})
490
+ units = payload.get("hourly_units", {})
491
+ times = [parse_iso_datetime(t) for t in hourly.get("time", [])]
492
+ variables = {
493
+ "temperature_2m": (hourly.get("temperature_2m", []), units.get("temperature_2m", "celsius"), "temperature_2m"),
494
+ "relative_humidity_2m": (hourly.get("relative_humidity_2m", []), units.get("relative_humidity_2m", "%"), "relative_humidity_2m"),
495
+ "wind_speed_10m": (hourly.get("wind_speed_10m", []), units.get("wind_speed_10m", "km/h"), "wind_speed_10m"),
496
+ "precipitation": (hourly.get("precipitation", []), units.get("precipitation", "mm"), "precipitation"),
497
+ }
498
+ return add_time_series_observations(store, dataset_id, source_id, entity_id, raw_id, fetched_at, times, variables, "1h", "temperature_2m", end_dt)
499
+
500
+
501
+ def collect_open_meteo_air_quality(store: Store, fetcher: Fetcher, start_dt: datetime, end_dt: datetime, args: argparse.Namespace) -> int:
502
+ source_id = "open_meteo_air_quality"
503
+ dataset_id = "open_meteo_shanghai_air_quality_hourly"
504
+ entity_id = "grid_shanghai_31.2304_121.4737"
505
+ store.add_entity(
506
+ dataset_id,
507
+ source_id,
508
+ entity_id,
509
+ "grid",
510
+ "Shanghai air quality grid point",
511
+ lat=31.2304,
512
+ lon=121.4737,
513
+ region="Shanghai",
514
+ country="CN",
515
+ timezone_name="UTC",
516
+ )
517
+ params = {
518
+ "latitude": 31.2304,
519
+ "longitude": 121.4737,
520
+ "hourly": "pm10,pm2_5,nitrogen_dioxide,carbon_monoxide",
521
+ "past_days": args.days,
522
+ "forecast_days": 1,
523
+ "timezone": "UTC",
524
+ }
525
+ response, raw_id, fetched_at = fetcher.get(source_id, dataset_id, "https://air-quality-api.open-meteo.com/v1/air-quality", params=params)
526
+ payload = response.json()
527
+ hourly = payload.get("hourly", {})
528
+ units = payload.get("hourly_units", {})
529
+ times = [parse_iso_datetime(t) for t in hourly.get("time", [])]
530
+ variables = {
531
+ "pm10": (hourly.get("pm10", []), units.get("pm10", "ug/m3"), "pm10"),
532
+ "pm2_5": (hourly.get("pm2_5", []), units.get("pm2_5", "ug/m3"), "pm2_5"),
533
+ "nitrogen_dioxide": (hourly.get("nitrogen_dioxide", []), units.get("nitrogen_dioxide", "ug/m3"), "nitrogen_dioxide"),
534
+ "carbon_monoxide": (hourly.get("carbon_monoxide", []), units.get("carbon_monoxide", "ug/m3"), "carbon_monoxide"),
535
+ }
536
+ return add_time_series_observations(store, dataset_id, source_id, entity_id, raw_id, fetched_at, times, variables, "1h", "pm2_5", end_dt)
537
+
538
+
539
+ def collect_nasa_power(store: Store, fetcher: Fetcher, start_dt: datetime, end_dt: datetime, args: argparse.Namespace) -> int:
540
+ source_id = "nasa_power"
541
+ dataset_id = "nasa_power_shanghai_hourly"
542
+ entity_id = "point_shanghai_31.2304_121.4737"
543
+ store.add_entity(
544
+ dataset_id,
545
+ source_id,
546
+ entity_id,
547
+ "grid",
548
+ "NASA POWER Shanghai point",
549
+ lat=31.2304,
550
+ lon=121.4737,
551
+ region="Shanghai",
552
+ country="CN",
553
+ timezone_name="UTC",
554
+ )
555
+ params = {
556
+ "parameters": "T2M,RH2M,WS10M,PRECTOTCORR",
557
+ "community": "RE",
558
+ "longitude": 121.4737,
559
+ "latitude": 31.2304,
560
+ "start": date_yyyymmdd(start_dt.date()),
561
+ "end": date_yyyymmdd(min(end_dt.date(), (utc_now() - timedelta(days=1)).date())),
562
+ "format": "JSON",
563
+ }
564
+ response, raw_id, fetched_at = fetcher.get(source_id, dataset_id, "https://power.larc.nasa.gov/api/temporal/hourly/point", params=params)
565
+ payload = response.json()
566
+ parameter = payload.get("properties", {}).get("parameter", {})
567
+ variable_info = {
568
+ "T2M": ("T2M", "celsius", "temperature_2m"),
569
+ "RH2M": ("RH2M", "%", "relative_humidity_2m"),
570
+ "WS10M": ("WS10M", "m/s", "wind_speed_10m"),
571
+ "PRECTOTCORR": ("PRECTOTCORR", "mm/hour", "precipitation_corrected"),
572
+ }
573
+ count = 0
574
+ for param_name, (variable_id, unit, variable_name) in variable_info.items():
575
+ series = parameter.get(param_name, {})
576
+ store.add_variable(dataset_id, source_id, variable_id, variable_name, unit, "numeric", "1h", "instantaneous", param_name == "T2M")
577
+ for stamp, value_raw in series.items():
578
+ if len(str(stamp)) < 10:
579
+ continue
580
+ ts = datetime.strptime(str(stamp)[:10], "%Y%m%d%H").replace(tzinfo=UTC)
581
+ if ts < start_dt or ts > end_dt:
582
+ continue
583
+ value = parse_float(value_raw)
584
+ if value is None or value <= -900:
585
+ continue
586
+ store.add_observation(dataset_id, source_id, entity_id, variable_id, ts, fetched_at, fetched_at, value, "1h", unit, raw_id=raw_id)
587
+ count += 1
588
+ return count
589
+
590
+
591
+ def collect_usgs_water(store: Store, fetcher: Fetcher, start_dt: datetime, end_dt: datetime, args: argparse.Namespace) -> int:
592
+ source_id = "usgs_water"
593
+ dataset_id = "usgs_potomac_iv"
594
+ site_id = "01646500"
595
+ entity_id = f"usgs_site_{site_id}"
596
+ params = {
597
+ "format": "json",
598
+ "sites": site_id,
599
+ "period": f"P{args.days}D",
600
+ "parameterCd": "00060,00065",
601
+ "siteStatus": "all",
602
+ }
603
+ response, raw_id, fetched_at = fetcher.get(source_id, dataset_id, "https://waterservices.usgs.gov/nwis/iv/", params=params)
604
+ payload = response.json()
605
+ count = 0
606
+ for series in payload.get("value", {}).get("timeSeries", []):
607
+ source_info = series.get("sourceInfo", {})
608
+ site_code = source_info.get("siteCode", [{}])[0].get("value", site_id)
609
+ site_name = source_info.get("siteName", f"USGS site {site_code}")
610
+ geo = source_info.get("geoLocation", {}).get("geogLocation", {})
611
+ store.add_entity(
612
+ dataset_id,
613
+ source_id,
614
+ f"usgs_site_{site_code}",
615
+ "station",
616
+ site_name,
617
+ lat=geo.get("latitude", ""),
618
+ lon=geo.get("longitude", ""),
619
+ region="US",
620
+ country="US",
621
+ timezone_name="UTC",
622
+ metadata={"site_code": site_code},
623
+ )
624
+ variable = series.get("variable", {})
625
+ code = variable.get("variableCode", [{}])[0].get("value", "unknown")
626
+ variable_id = f"usgs_{code}"
627
+ unit = variable.get("unit", {}).get("unitCode", "")
628
+ variable_name = variable.get("variableName", variable_id)
629
+ store.add_variable(dataset_id, source_id, variable_id, variable_name, unit, "numeric", "15min", "instantaneous", code == "00060")
630
+ for value_block in series.get("values", []):
631
+ for point in value_block.get("value", []):
632
+ value = parse_float(point.get("value"))
633
+ if value is None:
634
+ continue
635
+ ts = parse_iso_datetime(point["dateTime"])
636
+ if ts < start_dt or ts > end_dt:
637
+ continue
638
+ qualifiers = point.get("qualifiers", [])
639
+ store.add_observation(
640
+ dataset_id,
641
+ source_id,
642
+ f"usgs_site_{site_code}",
643
+ variable_id,
644
+ ts,
645
+ fetched_at,
646
+ fetched_at,
647
+ value,
648
+ "15min",
649
+ unit,
650
+ quality_flag=qualifiers,
651
+ raw_id=raw_id,
652
+ )
653
+ count += 1
654
+ return count
655
+
656
+
657
+ def collect_noaa_coops(store: Store, fetcher: Fetcher, start_dt: datetime, end_dt: datetime, args: argparse.Namespace) -> int:
658
+ source_id = "noaa_coops"
659
+ dataset_id = "noaa_coops_sf_water_level"
660
+ station_id = "9414290"
661
+ entity_id = f"coops_station_{station_id}"
662
+ store.add_entity(
663
+ dataset_id,
664
+ source_id,
665
+ entity_id,
666
+ "station",
667
+ "NOAA CO-OPS San Francisco",
668
+ lat=37.8063,
669
+ lon=-122.4659,
670
+ region="CA",
671
+ country="US",
672
+ timezone_name="UTC",
673
+ metadata={"station": station_id},
674
+ )
675
+ variable_id = "water_level"
676
+ store.add_variable(dataset_id, source_id, variable_id, "water_level", "m", "numeric", "6min", "instantaneous", True)
677
+ params = {
678
+ "product": "water_level",
679
+ "application": "TSBench",
680
+ "begin_date": date_yyyymmdd(start_dt.date()),
681
+ "end_date": date_yyyymmdd(end_dt.date()),
682
+ "datum": "MLLW",
683
+ "station": station_id,
684
+ "time_zone": "gmt",
685
+ "units": "metric",
686
+ "format": "json",
687
+ }
688
+ response, raw_id, fetched_at = fetcher.get(source_id, dataset_id, "https://api.tidesandcurrents.noaa.gov/api/prod/datagetter", params=params)
689
+ payload = response.json()
690
+ if "error" in payload:
691
+ raise RuntimeError(payload["error"].get("message", str(payload["error"])))
692
+ count = 0
693
+ for point in payload.get("data", []):
694
+ value = parse_float(point.get("v"))
695
+ if value is None:
696
+ continue
697
+ ts = datetime.strptime(point["t"], "%Y-%m-%d %H:%M").replace(tzinfo=UTC)
698
+ if ts < start_dt or ts > end_dt:
699
+ continue
700
+ store.add_observation(
701
+ dataset_id,
702
+ source_id,
703
+ entity_id,
704
+ variable_id,
705
+ ts,
706
+ fetched_at,
707
+ fetched_at,
708
+ value,
709
+ "6min",
710
+ "m",
711
+ quality_flag=point.get("q", ""),
712
+ raw_id=raw_id,
713
+ )
714
+ count += 1
715
+ return count
716
+
717
+
718
+ def collect_noaa_ndbc(store: Store, fetcher: Fetcher, start_dt: datetime, end_dt: datetime, args: argparse.Namespace) -> int:
719
+ source_id = "noaa_ndbc"
720
+ dataset_id = "noaa_ndbc_46013_realtime"
721
+ station_id = "46013"
722
+ entity_id = f"ndbc_station_{station_id}"
723
+ store.add_entity(
724
+ dataset_id,
725
+ source_id,
726
+ entity_id,
727
+ "station",
728
+ "NDBC station 46013",
729
+ lat=38.235,
730
+ lon=-123.317,
731
+ region="CA offshore",
732
+ country="US",
733
+ timezone_name="UTC",
734
+ metadata={"station": station_id},
735
+ )
736
+ url = f"https://www.ndbc.noaa.gov/data/realtime2/{station_id}.txt"
737
+ response, raw_id, fetched_at = fetcher.get(source_id, dataset_id, url, raw_ext="txt")
738
+ lines = response.text.splitlines()
739
+ header: List[str] = []
740
+ data_lines: List[str] = []
741
+ for line in lines:
742
+ if line.startswith("#YY"):
743
+ header = line.lstrip("#").split()
744
+ elif line and not line.startswith("#"):
745
+ data_lines.append(line)
746
+ if not header:
747
+ raise RuntimeError("NDBC header not found")
748
+ variable_defs = {
749
+ "WSPD": ("wind_speed", "m/s"),
750
+ "WVHT": ("wave_height", "m"),
751
+ "PRES": ("pressure", "hPa"),
752
+ "ATMP": ("air_temperature", "celsius"),
753
+ "WTMP": ("water_temperature", "celsius"),
754
+ }
755
+ for column, (variable_id, unit) in variable_defs.items():
756
+ store.add_variable(dataset_id, source_id, variable_id, variable_id, unit, "numeric", "10min", "instantaneous", column == "WVHT")
757
+ count = 0
758
+ for line in data_lines:
759
+ values = line.split()
760
+ if len(values) < len(header):
761
+ continue
762
+ row = dict(zip(header, values))
763
+ try:
764
+ ts = datetime(
765
+ int(row["YY"]),
766
+ int(row["MM"]),
767
+ int(row["DD"]),
768
+ int(row["hh"]),
769
+ int(row["mm"]),
770
+ tzinfo=UTC,
771
+ )
772
+ except Exception:
773
+ continue
774
+ if ts < start_dt or ts > end_dt:
775
+ continue
776
+ for column, (variable_id, unit) in variable_defs.items():
777
+ value = parse_float(row.get(column))
778
+ if value is None:
779
+ continue
780
+ store.add_observation(dataset_id, source_id, entity_id, variable_id, ts, fetched_at, fetched_at, value, "10min", unit, raw_id=raw_id)
781
+ count += 1
782
+ return count
783
+
784
+
785
+ def collect_gbfs_citibike(store: Store, fetcher: Fetcher, start_dt: datetime, end_dt: datetime, args: argparse.Namespace) -> int:
786
+ source_id = "gbfs_citibike"
787
+ dataset_id = "gbfs_citibike_current"
788
+ info_url = "https://gbfs.citibikenyc.com/gbfs/en/station_information.json"
789
+ status_url = "https://gbfs.citibikenyc.com/gbfs/en/station_status.json"
790
+ info_response, info_raw_id, info_fetched = fetcher.get(source_id, dataset_id, info_url)
791
+ status_response, status_raw_id, fetched_at = fetcher.get(source_id, dataset_id, status_url)
792
+ info_payload = info_response.json()
793
+ status_payload = status_response.json()
794
+ stations = {
795
+ str(item["station_id"]): item
796
+ for item in info_payload.get("data", {}).get("stations", [])
797
+ }
798
+ statuses = status_payload.get("data", {}).get("stations", [])[: args.limit_entities]
799
+ variables = {
800
+ "num_bikes_available": ("bikes", True),
801
+ "num_docks_available": ("docks", False),
802
+ }
803
+ for variable_id, (unit, is_target) in variables.items():
804
+ store.add_variable(dataset_id, source_id, variable_id, variable_id, unit, "numeric", "snapshot", "instantaneous", is_target)
805
+ count = 0
806
+ for status in statuses:
807
+ station_id = str(status.get("station_id", "unknown"))
808
+ station = stations.get(station_id, {})
809
+ entity_id = f"citibike_station_{safe_id(station_id)}"
810
+ store.add_entity(
811
+ dataset_id,
812
+ source_id,
813
+ entity_id,
814
+ "station",
815
+ station.get("name", f"Citi Bike station {station_id}"),
816
+ lat=station.get("lat", ""),
817
+ lon=station.get("lon", ""),
818
+ region="New York City",
819
+ country="US",
820
+ timezone_name="America/New_York",
821
+ metadata={"station_id": station_id, "raw_info_id": info_raw_id},
822
+ )
823
+ ts_raw = status.get("last_reported") or status_payload.get("last_updated")
824
+ ts = datetime.fromtimestamp(int(ts_raw), tz=UTC) if ts_raw else fetched_at
825
+ if ts < start_dt:
826
+ ts = fetched_at
827
+ for variable_id, (unit, _) in variables.items():
828
+ value = parse_float(status.get(variable_id))
829
+ if value is None:
830
+ continue
831
+ store.add_observation(dataset_id, source_id, entity_id, variable_id, ts, fetched_at, fetched_at, value, "snapshot", unit, raw_id=status_raw_id)
832
+ count += 1
833
+ _ = info_fetched
834
+ return count
835
+
836
+
837
+ def collect_binance_btcusdt(store: Store, fetcher: Fetcher, start_dt: datetime, end_dt: datetime, args: argparse.Namespace) -> int:
838
+ source_id = "binance"
839
+ dataset_id = "binance_btcusdt_1h"
840
+ entity_id = "crypto_btcusdt"
841
+ store.add_entity(dataset_id, source_id, entity_id, "asset", "BTCUSDT", region="global", timezone_name="UTC")
842
+ variable_defs = {
843
+ "open": "USD",
844
+ "high": "USD",
845
+ "low": "USD",
846
+ "close": "USD",
847
+ "volume": "BTC",
848
+ }
849
+ for variable_id, unit in variable_defs.items():
850
+ store.add_variable(dataset_id, source_id, variable_id, variable_id, unit, "numeric", "1h", "ohlcv", variable_id == "close")
851
+ params = {
852
+ "symbol": "BTCUSDT",
853
+ "interval": "1h",
854
+ "startTime": int(start_dt.timestamp() * 1000),
855
+ "endTime": int(end_dt.timestamp() * 1000),
856
+ "limit": 1000,
857
+ }
858
+ count = 0
859
+ try:
860
+ response, raw_id, fetched_at = fetcher.get(source_id, dataset_id, "https://api.binance.com/api/v3/klines", params=params)
861
+ rows = response.json()
862
+ for row in rows:
863
+ ts = parse_epoch_ms_or_us(row[0])
864
+ values = {
865
+ "open": row[1],
866
+ "high": row[2],
867
+ "low": row[3],
868
+ "close": row[4],
869
+ "volume": row[5],
870
+ }
871
+ for variable_id, value_raw in values.items():
872
+ value = parse_float(value_raw)
873
+ if value is None:
874
+ continue
875
+ store.add_observation(dataset_id, source_id, entity_id, variable_id, ts, fetched_at, fetched_at, value, "1h", variable_defs[variable_id], raw_id=raw_id)
876
+ count += 1
877
+ return count
878
+ except Exception:
879
+ # Fallback to official daily zip files.
880
+ current = start_dt.date()
881
+ while current <= end_dt.date():
882
+ url = f"https://data.binance.vision/data/spot/daily/klines/BTCUSDT/1h/BTCUSDT-1h-{current.isoformat()}.zip"
883
+ try:
884
+ response, raw_id, fetched_at = fetcher.get(source_id, dataset_id, url, raw_ext="zip")
885
+ except Exception:
886
+ current += timedelta(days=1)
887
+ continue
888
+ with zipfile.ZipFile(io.BytesIO(response.content)) as archive:
889
+ name = archive.namelist()[0]
890
+ text = archive.read(name).decode("utf-8")
891
+ for row in csv.reader(io.StringIO(text)):
892
+ if len(row) < 6:
893
+ continue
894
+ try:
895
+ ts = parse_epoch_ms_or_us(row[0])
896
+ except ValueError:
897
+ continue
898
+ if ts < start_dt or ts > end_dt:
899
+ continue
900
+ values = {
901
+ "open": row[1],
902
+ "high": row[2],
903
+ "low": row[3],
904
+ "close": row[4],
905
+ "volume": row[5],
906
+ }
907
+ for variable_id, value_raw in values.items():
908
+ value = parse_float(value_raw)
909
+ if value is None:
910
+ continue
911
+ store.add_observation(dataset_id, source_id, entity_id, variable_id, ts, fetched_at, fetched_at, value, "1h", variable_defs[variable_id], raw_id=raw_id)
912
+ count += 1
913
+ current += timedelta(days=1)
914
+ return count
915
+
916
+
917
+ def collect_coingecko_bitcoin(store: Store, fetcher: Fetcher, start_dt: datetime, end_dt: datetime, args: argparse.Namespace) -> int:
918
+ source_id = "coingecko"
919
+ dataset_id = "coingecko_bitcoin_7d"
920
+ entity_id = "coin_bitcoin"
921
+ store.add_entity(dataset_id, source_id, entity_id, "asset", "bitcoin", region="global", timezone_name="UTC")
922
+ params = {"vs_currency": "usd", "days": args.days}
923
+ response, raw_id, fetched_at = fetcher.get(source_id, dataset_id, "https://api.coingecko.com/api/v3/coins/bitcoin/market_chart", params=params)
924
+ payload = response.json()
925
+ variable_defs = {
926
+ "price_usd": ("prices", "USD", True),
927
+ "market_cap_usd": ("market_caps", "USD", False),
928
+ "total_volume_usd": ("total_volumes", "USD", False),
929
+ }
930
+ count = 0
931
+ for variable_id, (payload_key, unit, is_target) in variable_defs.items():
932
+ store.add_variable(dataset_id, source_id, variable_id, variable_id, unit, "numeric", "1h", "instantaneous", is_target)
933
+ for ts_ms, value_raw in payload.get(payload_key, []):
934
+ ts = datetime.fromtimestamp(ts_ms / 1000, tz=UTC)
935
+ if ts < start_dt or ts > end_dt:
936
+ continue
937
+ value = parse_float(value_raw)
938
+ if value is None:
939
+ continue
940
+ store.add_observation(dataset_id, source_id, entity_id, variable_id, ts, fetched_at, fetched_at, value, "1h", unit, raw_id=raw_id)
941
+ count += 1
942
+ return count
943
+
944
+
945
+ def collect_wikimedia_pageviews(store: Store, fetcher: Fetcher, start_dt: datetime, end_dt: datetime, args: argparse.Namespace) -> int:
946
+ source_id = "wikimedia_pageviews"
947
+ dataset_id = "wikimedia_time_series_hourly"
948
+ article = "Time_series"
949
+ entity_id = f"enwiki_article_{safe_id(article)}"
950
+ store.add_entity(dataset_id, source_id, entity_id, "page", "en.wikipedia/Time_series", region="global", timezone_name="UTC")
951
+ variable_id = "pageviews"
952
+ store.add_variable(dataset_id, source_id, variable_id, "pageviews", "views", "numeric", "1d", "sum", True)
953
+ start = start_dt.strftime("%Y%m%d00")
954
+ end = end_dt.strftime("%Y%m%d23")
955
+ article_url = quote(article, safe="")
956
+ url = f"https://wikimedia.org/api/rest_v1/metrics/pageviews/per-article/en.wikipedia/all-access/user/{article_url}/daily/{start}/{end}"
957
+ response, raw_id, fetched_at = fetcher.get(source_id, dataset_id, url)
958
+ payload = response.json()
959
+ count = 0
960
+ for item in payload.get("items", []):
961
+ ts = datetime.strptime(item["timestamp"][:10], "%Y%m%d%H").replace(tzinfo=UTC)
962
+ if ts < start_dt or ts > end_dt:
963
+ continue
964
+ value = parse_float(item.get("views"))
965
+ if value is None:
966
+ continue
967
+ store.add_observation(dataset_id, source_id, entity_id, variable_id, ts, fetched_at, fetched_at, value, "1d", "views", raw_id=raw_id)
968
+ count += 1
969
+ return count
970
+
971
+
972
+ def collect_nws_observations(store: Store, fetcher: Fetcher, start_dt: datetime, end_dt: datetime, args: argparse.Namespace) -> int:
973
+ source_id = "nws_observations"
974
+ dataset_id = "nws_ksfo_observations"
975
+ station_id = "KSFO"
976
+ entity_id = f"nws_station_{station_id.lower()}"
977
+ store.add_entity(
978
+ dataset_id,
979
+ source_id,
980
+ entity_id,
981
+ "station",
982
+ "NWS KSFO observations",
983
+ lat=37.619,
984
+ lon=-122.365,
985
+ region="CA",
986
+ country="US",
987
+ timezone_name="UTC",
988
+ metadata={"station": station_id},
989
+ )
990
+ variable_defs = {
991
+ "temperature": ("temperature", "celsius", True),
992
+ "wind_speed": ("windSpeed", "km/h", False),
993
+ "relative_humidity": ("relativeHumidity", "%", False),
994
+ "barometric_pressure": ("barometricPressure", "Pa", False),
995
+ }
996
+ for variable_id, (_, unit, is_target) in variable_defs.items():
997
+ store.add_variable(dataset_id, source_id, variable_id, variable_id, unit, "numeric", "1h", "instantaneous", is_target)
998
+ params = {
999
+ "start": iso(start_dt),
1000
+ "end": iso(end_dt),
1001
+ }
1002
+ url = f"https://api.weather.gov/stations/{station_id}/observations"
1003
+ response, raw_id, fetched_at = fetcher.get(source_id, dataset_id, url, params=params)
1004
+ payload = response.json()
1005
+ count = 0
1006
+ for feature in payload.get("features", []):
1007
+ props = feature.get("properties", {})
1008
+ timestamp = props.get("timestamp")
1009
+ if not timestamp:
1010
+ continue
1011
+ ts = parse_iso_datetime(timestamp)
1012
+ if ts < start_dt or ts > end_dt:
1013
+ continue
1014
+ for variable_id, (property_name, unit, _) in variable_defs.items():
1015
+ raw_value = props.get(property_name, {})
1016
+ if not isinstance(raw_value, dict):
1017
+ continue
1018
+ value = parse_float(raw_value.get("value"))
1019
+ if value is None:
1020
+ continue
1021
+ store.add_observation(dataset_id, source_id, entity_id, variable_id, ts, fetched_at, fetched_at, value, "1h", unit, raw_id=raw_id)
1022
+ count += 1
1023
+ return count
1024
+
1025
+
1026
+ def collect_gdelt_timeline(store: Store, fetcher: Fetcher, start_dt: datetime, end_dt: datetime, args: argparse.Namespace) -> int:
1027
+ source_id = "gdelt"
1028
+ dataset_id = "gdelt_climate_timeline"
1029
+ entity_id = "gdelt_query_climate_change"
1030
+ store.add_entity(dataset_id, source_id, entity_id, "query", "climate change", region="global", timezone_name="UTC")
1031
+ variable_id = "document_volume"
1032
+ store.add_variable(dataset_id, source_id, variable_id, "document_volume", "relative_volume", "numeric", "15min", "mean", True)
1033
+ params = {
1034
+ "query": "climate change",
1035
+ "mode": "timelinevol",
1036
+ "format": "json",
1037
+ "startdatetime": datetime_yyyymmddhhmmss(start_dt),
1038
+ "enddatetime": datetime_yyyymmddhhmmss(end_dt),
1039
+ "timelinesmooth": 0,
1040
+ }
1041
+ try:
1042
+ response, raw_id, fetched_at = fetcher.get(source_id, dataset_id, "https://api.gdeltproject.org/api/v2/doc/doc", params=params)
1043
+ except Exception:
1044
+ response, raw_id, fetched_at = fetcher.get(source_id, dataset_id, "http://api.gdeltproject.org/api/v2/doc/doc", params=params)
1045
+ payload = response.json()
1046
+ candidates: List[Dict[str, Any]] = []
1047
+ for item in payload.get("timeline", []):
1048
+ if isinstance(item, dict) and isinstance(item.get("data"), list):
1049
+ candidates.extend(item["data"])
1050
+ if not candidates and isinstance(payload.get("data"), list):
1051
+ candidates = payload["data"]
1052
+ count = 0
1053
+ for item in candidates:
1054
+ stamp = item.get("date") or item.get("datetime")
1055
+ value_raw = item.get("value") or item.get("norm") or item.get("count")
1056
+ if not stamp:
1057
+ continue
1058
+ try:
1059
+ if "-" in str(stamp):
1060
+ ts = parse_iso_datetime(str(stamp))
1061
+ else:
1062
+ ts = datetime.strptime(str(stamp)[:14], "%Y%m%d%H%M%S").replace(tzinfo=UTC)
1063
+ except Exception:
1064
+ continue
1065
+ if ts < start_dt or ts > end_dt:
1066
+ continue
1067
+ value = parse_float(value_raw)
1068
+ if value is None:
1069
+ continue
1070
+ store.add_observation(dataset_id, source_id, entity_id, variable_id, ts, fetched_at, fetched_at, value, "15min", "relative_volume", raw_id=raw_id)
1071
+ count += 1
1072
+ return count
1073
+
1074
+
1075
+ def collect_usgs_earthquake(store: Store, fetcher: Fetcher, start_dt: datetime, end_dt: datetime, args: argparse.Namespace) -> int:
1076
+ source_id = "usgs_earthquake"
1077
+ dataset_id = "usgs_earthquake_all_week_hourly"
1078
+ entity_id = "earth_global"
1079
+ store.add_entity(dataset_id, source_id, entity_id, "global_region", "Global earthquakes", region="global", timezone_name="UTC")
1080
+ variables = {
1081
+ "earthquake_count": ("events", True),
1082
+ "earthquake_avg_magnitude": ("magnitude", False),
1083
+ "earthquake_max_magnitude": ("magnitude", False),
1084
+ }
1085
+ for variable_id, (unit, is_target) in variables.items():
1086
+ store.add_variable(dataset_id, source_id, variable_id, variable_id, unit, "numeric", "1h", "aggregate", is_target)
1087
+ response, raw_id, fetched_at = fetcher.get(source_id, dataset_id, "https://earthquake.usgs.gov/earthquakes/feed/v1.0/summary/all_week.geojson")
1088
+ payload = response.json()
1089
+ bucket: Dict[datetime, List[float]] = defaultdict(list)
1090
+ for feature in payload.get("features", []):
1091
+ props = feature.get("properties", {})
1092
+ ts_ms = props.get("time")
1093
+ mag = parse_float(props.get("mag"))
1094
+ if ts_ms is None or mag is None:
1095
+ continue
1096
+ ts = floor_hour(datetime.fromtimestamp(ts_ms / 1000, tz=UTC))
1097
+ if ts < start_dt or ts > end_dt:
1098
+ continue
1099
+ bucket[ts].append(mag)
1100
+ count = 0
1101
+ for ts, mags in sorted(bucket.items()):
1102
+ values = {
1103
+ "earthquake_count": float(len(mags)),
1104
+ "earthquake_avg_magnitude": sum(mags) / len(mags),
1105
+ "earthquake_max_magnitude": max(mags),
1106
+ }
1107
+ for variable_id, value in values.items():
1108
+ store.add_observation(dataset_id, source_id, entity_id, variable_id, ts, fetched_at, fetched_at, value, "1h", variables[variable_id][0], raw_id=raw_id)
1109
+ count += 1
1110
+ return count
1111
+
1112
+
1113
+ def collect_noaa_ncei_daily(store: Store, fetcher: Fetcher, start_dt: datetime, end_dt: datetime, args: argparse.Namespace) -> int:
1114
+ source_id = "noaa_ncei"
1115
+ dataset_id = "noaa_ncei_nyc_daily_summaries"
1116
+ station_id = "USW00014732"
1117
+ entity_id = f"ncei_station_{station_id}"
1118
+ store.add_entity(
1119
+ dataset_id,
1120
+ source_id,
1121
+ entity_id,
1122
+ "station",
1123
+ "NCEI daily summaries USW00014732",
1124
+ lat=40.779,
1125
+ lon=-73.880,
1126
+ region="NY",
1127
+ country="US",
1128
+ timezone_name="UTC",
1129
+ metadata={"station": station_id},
1130
+ )
1131
+ params = {
1132
+ "dataset": "daily-summaries",
1133
+ "stations": station_id,
1134
+ "startDate": start_dt.date().isoformat(),
1135
+ "endDate": end_dt.date().isoformat(),
1136
+ "dataTypes": "TAVG,TMAX,TMIN,PRCP",
1137
+ "format": "json",
1138
+ "units": "metric",
1139
+ "includeAttributes": "false",
1140
+ }
1141
+ response, raw_id, fetched_at = fetcher.get(source_id, dataset_id, "https://www.ncei.noaa.gov/access/services/data/v1", params=params)
1142
+ rows = response.json()
1143
+ variable_defs = {
1144
+ "TAVG": ("daily_avg_temperature", "celsius", True),
1145
+ "TMAX": ("daily_max_temperature", "celsius", False),
1146
+ "TMIN": ("daily_min_temperature", "celsius", False),
1147
+ "PRCP": ("daily_precipitation", "mm", False),
1148
+ }
1149
+ for code, (variable_id, unit, is_target) in variable_defs.items():
1150
+ store.add_variable(dataset_id, source_id, variable_id, variable_id, unit, "numeric", "1d", "daily", is_target)
1151
+ count = 0
1152
+ for row in rows:
1153
+ if "DATE" not in row:
1154
+ continue
1155
+ ts = datetime.fromisoformat(row["DATE"]).replace(tzinfo=UTC)
1156
+ for code, (variable_id, unit, _) in variable_defs.items():
1157
+ value = parse_float(row.get(code))
1158
+ if value is None:
1159
+ continue
1160
+ store.add_observation(dataset_id, source_id, entity_id, variable_id, ts, fetched_at, fetched_at, value, "1d", unit, raw_id=raw_id)
1161
+ count += 1
1162
+ return count
1163
+
1164
+
1165
+ COLLECTORS: Dict[str, Callable[[Store, Fetcher, datetime, datetime, argparse.Namespace], int]] = {
1166
+ "open_meteo_weather": collect_open_meteo_weather,
1167
+ "open_meteo_air_quality": collect_open_meteo_air_quality,
1168
+ "nasa_power": collect_nasa_power,
1169
+ "usgs_water": collect_usgs_water,
1170
+ "noaa_coops": collect_noaa_coops,
1171
+ "noaa_ndbc": collect_noaa_ndbc,
1172
+ "gbfs_citibike": collect_gbfs_citibike,
1173
+ "binance": collect_binance_btcusdt,
1174
+ "coingecko": collect_coingecko_bitcoin,
1175
+ "wikimedia_pageviews": collect_wikimedia_pageviews,
1176
+ "nws_observations": collect_nws_observations,
1177
+ "gdelt": collect_gdelt_timeline,
1178
+ "usgs_earthquake": collect_usgs_earthquake,
1179
+ "noaa_ncei": collect_noaa_ncei_daily,
1180
+ }
1181
+
1182
+ DEFAULT_COLLECTORS = [
1183
+ "open_meteo_weather",
1184
+ "open_meteo_air_quality",
1185
+ "nasa_power",
1186
+ "usgs_water",
1187
+ "noaa_coops",
1188
+ "noaa_ndbc",
1189
+ "gbfs_citibike",
1190
+ "binance",
1191
+ "coingecko",
1192
+ "wikimedia_pageviews",
1193
+ "nws_observations",
1194
+ "usgs_earthquake",
1195
+ "noaa_ncei",
1196
+ ]
1197
+
1198
+
1199
+ def generate_forecast_tasks(store: Store, created_at: datetime, max_tasks_per_dataset: int = 3) -> None:
1200
+ grouped: Dict[Tuple[str, str, str], List[Dict[str, Any]]] = defaultdict(list)
1201
+ for obs in store.observations:
1202
+ grouped[(obs["dataset_id"], obs["entity_id"], obs["variable_id"])].append(obs)
1203
+
1204
+ variable_lookup = {
1205
+ (row["dataset_id"], row["variable_id"]): row
1206
+ for row in store.variables.values()
1207
+ }
1208
+ dataset_counts: Dict[str, int] = defaultdict(int)
1209
+
1210
+ for (dataset_id, entity_id, variable_id), rows in sorted(grouped.items()):
1211
+ if dataset_counts[dataset_id] >= max_tasks_per_dataset:
1212
+ continue
1213
+ var = variable_lookup.get((dataset_id, variable_id), {})
1214
+ if var.get("is_target_default") != "true":
1215
+ continue
1216
+ rows_sorted = sorted(rows, key=lambda row: row["timestamp"])
1217
+ timestamps = [parse_iso_datetime(row["timestamp"]) for row in rows_sorted]
1218
+ if not timestamps:
1219
+ continue
1220
+ freq = rows_sorted[-1]["frequency"]
1221
+ delta = frequency_delta(freq) or timedelta(hours=1)
1222
+ unique_timestamps = sorted(set(timestamps))
1223
+ if freq == "1d":
1224
+ horizon_length = 2
1225
+ input_length = min(5, max(1, len(unique_timestamps) - horizon_length))
1226
+ elif freq == "snapshot":
1227
+ input_length = 1
1228
+ horizon_length = 1
1229
+ else:
1230
+ input_length = min(96, max(1, len(unique_timestamps)))
1231
+ horizon_length = 24
1232
+
1233
+ if len(unique_timestamps) > horizon_length and freq != "snapshot":
1234
+ context_end = unique_timestamps[-horizon_length - 1]
1235
+ horizon_start = unique_timestamps[-horizon_length]
1236
+ horizon_end = unique_timestamps[-1]
1237
+ context_candidates = [ts for ts in unique_timestamps if ts <= context_end]
1238
+ context_start = context_candidates[max(0, len(context_candidates) - input_length)]
1239
+ input_length = len([ts for ts in context_candidates if ts >= context_start])
1240
+ horizon_length = len([ts for ts in unique_timestamps if horizon_start <= ts <= horizon_end])
1241
+ else:
1242
+ context_end = unique_timestamps[-1]
1243
+ context_start = unique_timestamps[max(0, len(unique_timestamps) - input_length)]
1244
+ horizon_start = context_end + delta
1245
+ horizon_end = horizon_start + delta * (horizon_length - 1)
1246
+ covariates = sorted(
1247
+ row["variable_id"]
1248
+ for (ds, vid), row in variable_lookup.items()
1249
+ if ds == dataset_id and vid != variable_id
1250
+ )
1251
+
1252
+ task_id = safe_id(f"task_{dataset_id}_{entity_id}_{variable_id}_{context_end.strftime('%Y%m%dT%H%M%SZ')}")
1253
+ store.forecast_tasks.append(
1254
+ {
1255
+ "task_id": task_id,
1256
+ "dataset_id": dataset_id,
1257
+ "target_entity_id": entity_id,
1258
+ "target_variable_id": variable_id,
1259
+ "forecast_issue_time": iso(created_at),
1260
+ "context_start": iso(context_start),
1261
+ "context_end": iso(context_end),
1262
+ "horizon_start": iso(horizon_start),
1263
+ "horizon_end": iso(horizon_end),
1264
+ "input_length": input_length,
1265
+ "horizon_length": horizon_length,
1266
+ "frequency": freq,
1267
+ "covariate_variable_ids": covariates,
1268
+ "task_split": "sample_week",
1269
+ "created_at": iso(created_at),
1270
+ }
1271
+ )
1272
+ dataset_counts[dataset_id] += 1
1273
+
1274
+
1275
+ def generate_quality_reports(store: Store, run_started: datetime, run_finished: datetime) -> None:
1276
+ grouped: Dict[Tuple[str, str, str, str], List[Dict[str, Any]]] = defaultdict(list)
1277
+ for obs in store.observations:
1278
+ grouped[(obs["dataset_id"], obs["source_id"], obs["entity_id"], obs["variable_id"])].append(obs)
1279
+ failures_by_dataset: Dict[str, int] = defaultdict(int)
1280
+ total_by_dataset: Dict[str, int] = defaultdict(int)
1281
+ for log in store.run_log:
1282
+ total_by_dataset[log["dataset_id"]] += 1
1283
+ if log["status"] != "ok":
1284
+ failures_by_dataset[log["dataset_id"]] += 1
1285
+
1286
+ for (dataset_id, source_id, entity_id, variable_id), rows in sorted(grouped.items()):
1287
+ timestamps = [parse_iso_datetime(row["timestamp"]) for row in rows]
1288
+ timestamp_strings = [row["timestamp"] for row in rows]
1289
+ duplicate_count = len(timestamp_strings) - len(set(timestamp_strings))
1290
+ latest = max(timestamps)
1291
+ earliest = min(timestamps)
1292
+ values = [parse_float(row["value"]) for row in rows if parse_float(row["value"]) is not None]
1293
+ outlier_count = 0
1294
+ if len(values) >= 5:
1295
+ mean = sum(values) / len(values)
1296
+ variance = sum((value - mean) ** 2 for value in values) / len(values)
1297
+ std = math.sqrt(variance)
1298
+ if std > 0:
1299
+ outlier_count = sum(1 for value in values if abs(value - mean) > 5 * std)
1300
+ freq = rows[-1]["frequency"]
1301
+ delta = frequency_delta(freq)
1302
+ missing_ratio = ""
1303
+ if delta is not None and len(timestamps) > 1:
1304
+ expected = int((latest - earliest) / delta) + 1
1305
+ if expected > 0:
1306
+ missing_ratio = max(0.0, min(1.0, 1.0 - len(set(timestamp_strings)) / expected))
1307
+ api_failure_rate = ""
1308
+ if total_by_dataset.get(dataset_id):
1309
+ api_failure_rate = failures_by_dataset[dataset_id] / total_by_dataset[dataset_id]
1310
+ store.quality_reports.append(
1311
+ {
1312
+ "dataset_id": dataset_id,
1313
+ "source_id": source_id,
1314
+ "entity_id": entity_id,
1315
+ "variable_id": variable_id,
1316
+ "window_start": iso(earliest),
1317
+ "window_end": iso(latest),
1318
+ "observation_count": len(rows),
1319
+ "missing_ratio": missing_ratio,
1320
+ "duplicate_count": duplicate_count,
1321
+ "freshness_delay_hours": round((run_finished - latest).total_seconds() / 3600, 3),
1322
+ "outlier_count": outlier_count,
1323
+ "api_failure_rate": api_failure_rate,
1324
+ "notes": "",
1325
+ }
1326
+ )
1327
+
1328
+
1329
+ def normalize_row(row: Dict[str, Any], fields: List[str]) -> Dict[str, Any]:
1330
+ normalized: Dict[str, Any] = {}
1331
+ for field_name in fields:
1332
+ value = row.get(field_name, "")
1333
+ if isinstance(value, (dict, list, tuple)):
1334
+ normalized[field_name] = dumps(value)
1335
+ else:
1336
+ normalized[field_name] = value
1337
+ return normalized
1338
+
1339
+
1340
+ def write_csv(path: Path, fields: List[str], rows: Iterable[Dict[str, Any]]) -> None:
1341
+ path.parent.mkdir(parents=True, exist_ok=True)
1342
+ with path.open("w", newline="", encoding="utf-8") as handle:
1343
+ writer = csv.DictWriter(handle, fieldnames=fields)
1344
+ writer.writeheader()
1345
+ for row in rows:
1346
+ writer.writerow(normalize_row(row, fields))
1347
+
1348
+
1349
+ def write_outputs(store: Store) -> None:
1350
+ normalized_dir = store.output_dir / "normalized"
1351
+ write_csv(normalized_dir / "raw_snapshots.csv", RAW_SNAPSHOT_FIELDS, store.raw_snapshots)
1352
+ write_csv(normalized_dir / "entities.csv", ENTITY_FIELDS, store.entities.values())
1353
+ write_csv(normalized_dir / "variables.csv", VARIABLE_FIELDS, store.variables.values())
1354
+ write_csv(normalized_dir / "observations.csv", OBSERVATION_FIELDS, store.observations)
1355
+ write_csv(normalized_dir / "forecast_tasks.csv", FORECAST_TASK_FIELDS, store.forecast_tasks)
1356
+ write_csv(normalized_dir / "quality_reports.csv", QUALITY_REPORT_FIELDS, store.quality_reports)
1357
+ write_csv(normalized_dir / "run_log.csv", RUN_LOG_FIELDS, store.run_log)
1358
+
1359
+ summary = {
1360
+ "generated_at": iso(utc_now()),
1361
+ "output_dir": str(store.output_dir),
1362
+ "raw_snapshots": len(store.raw_snapshots),
1363
+ "entities": len(store.entities),
1364
+ "variables": len(store.variables),
1365
+ "observations": len(store.observations),
1366
+ "forecast_tasks": len(store.forecast_tasks),
1367
+ "quality_reports": len(store.quality_reports),
1368
+ "run_log": store.run_log,
1369
+ "notes": [
1370
+ "CSV files are normalized storage artifacts for this minimal prototype.",
1371
+ "For production, keep the same schema but prefer Parquet, DuckDB, or PostgreSQL.",
1372
+ "Model-specific wide CSV files should be exported from observations + forecast_tasks.",
1373
+ ],
1374
+ }
1375
+ (store.output_dir / "summary.json").write_text(json.dumps(summary, ensure_ascii=False, indent=2), encoding="utf-8")
1376
+
1377
+
1378
+ def parse_args(argv: List[str]) -> argparse.Namespace:
1379
+ parser = argparse.ArgumentParser(description="Collect and normalize a one-week TS-Bench data sample.")
1380
+ parser.add_argument("--days", type=int, default=7, help="Number of trailing days to collect.")
1381
+ parser.add_argument("--output", default="data/sample_week", help="Output directory.")
1382
+ parser.add_argument("--timeout", type=int, default=30, help="HTTP timeout in seconds.")
1383
+ parser.add_argument("--limit-entities", type=int, default=25, help="Entity cap for high-cardinality snapshot sources such as GBFS.")
1384
+ parser.add_argument("--max-tasks-per-dataset", type=int, default=3, help="Maximum forecast tasks generated per dataset.")
1385
+ parser.add_argument("--only", nargs="*", choices=sorted(COLLECTORS.keys()), help="Run only selected collectors.")
1386
+ parser.add_argument("--list", action="store_true", help="List collectors and exit.")
1387
+ parser.add_argument("--strict", action="store_true", help="Exit non-zero if any collector fails.")
1388
+ return parser.parse_args(argv)
1389
+
1390
+
1391
+ def main(argv: List[str]) -> int:
1392
+ args = parse_args(argv)
1393
+ if args.list:
1394
+ for name in sorted(COLLECTORS):
1395
+ print(name)
1396
+ return 0
1397
+
1398
+ output_dir = Path(args.output)
1399
+ output_dir.mkdir(parents=True, exist_ok=True)
1400
+ store = Store(output_dir=output_dir)
1401
+ fetcher = Fetcher(store=store, timeout=args.timeout)
1402
+ run_started = utc_now()
1403
+ end_dt = run_started
1404
+ start_dt = end_dt - timedelta(days=args.days)
1405
+
1406
+ selected = args.only or DEFAULT_COLLECTORS
1407
+ any_failed = False
1408
+ for name in selected:
1409
+ collector = COLLECTORS[name]
1410
+ started_at = utc_now()
1411
+ before = len(store.observations)
1412
+ dataset_id = ""
1413
+ try:
1414
+ added = collector(store, fetcher, start_dt, end_dt, args)
1415
+ status = "ok"
1416
+ error = ""
1417
+ after = len(store.observations)
1418
+ observation_count = after - before if added is None else added
1419
+ matching = [row for row in store.observations[before:after]]
1420
+ dataset_id = matching[0]["dataset_id"] if matching else name
1421
+ except Exception as exc: # noqa: BLE001
1422
+ any_failed = True
1423
+ status = "failed"
1424
+ error = str(exc)
1425
+ observation_count = 0
1426
+ dataset_id = name
1427
+ finished_at = utc_now()
1428
+ store.run_log.append(
1429
+ {
1430
+ "source_id": name,
1431
+ "dataset_id": dataset_id,
1432
+ "status": status,
1433
+ "started_at": iso(started_at),
1434
+ "finished_at": iso(finished_at),
1435
+ "observation_count": observation_count,
1436
+ "error": error,
1437
+ }
1438
+ )
1439
+ print(f"{name:24s} {status:7s} observations={observation_count} {error[:100]}")
1440
+
1441
+ run_finished = utc_now()
1442
+ generate_forecast_tasks(store, created_at=run_finished, max_tasks_per_dataset=args.max_tasks_per_dataset)
1443
+ generate_quality_reports(store, run_started, run_finished)
1444
+ write_outputs(store)
1445
+ print()
1446
+ print(f"Output: {output_dir}")
1447
+ print(f"Raw snapshots: {len(store.raw_snapshots)}")
1448
+ print(f"Entities: {len(store.entities)}")
1449
+ print(f"Variables: {len(store.variables)}")
1450
+ print(f"Observations: {len(store.observations)}")
1451
+ print(f"Forecast tasks: {len(store.forecast_tasks)}")
1452
+ print(f"Quality reports: {len(store.quality_reports)}")
1453
+ return 1 if (args.strict and any_failed) else 0
1454
+
1455
+
1456
+ if __name__ == "__main__":
1457
+ raise SystemExit(main(sys.argv[1:]))
vendor/ts_bench/data/sample_week/model_inputs_latest/manifest.csv ADDED
@@ -0,0 +1,8 @@
 
 
 
 
 
 
 
 
 
1
+ task_id,dataset_id,entity_id,target_variable_id,context_rows,future_target_rows,context_csv,future_target_csv
2
+ task_binance_btcusdt_1h_crypto_btcusdt_close_20260528t020000z,binance_btcusdt_1h,crypto_btcusdt,close,96,24,data/sample_week/model_inputs_latest/task_binance_btcusdt_1h_crypto_btcusdt_close_20260528t020000z/context.csv,data/sample_week/model_inputs_latest/task_binance_btcusdt_1h_crypto_btcusdt_close_20260528t020000z/future_target.csv
3
+ task_coingecko_bitcoin_7d_coin_bitcoin_price_usd_20260528t030012z,coingecko_bitcoin_7d,coin_bitcoin,price_usd,96,24,data/sample_week/model_inputs_latest/task_coingecko_bitcoin_7d_coin_bitcoin_price_usd_20260528t030012z/context.csv,data/sample_week/model_inputs_latest/task_coingecko_bitcoin_7d_coin_bitcoin_price_usd_20260528t030012z/future_target.csv
4
+ task_nasa_power_shanghai_hourly_point_shanghai_31.2304_121.4737_t2m_20260524t230000z,nasa_power_shanghai_hourly,point_shanghai_31.2304_121.4737,T2M,69,24,data/sample_week/model_inputs_latest/task_nasa_power_shanghai_hourly_point_shanghai_31.2304_121.4737_t2m_20260524t230000z/context.csv,data/sample_week/model_inputs_latest/task_nasa_power_shanghai_hourly_point_shanghai_31.2304_121.4737_t2m_20260524t230000z/future_target.csv
5
+ task_noaa_coops_sf_water_level_coops_station_9414290_water_level_20260529t001800z,noaa_coops_sf_water_level,coops_station_9414290,water_level,96,24,data/sample_week/model_inputs_latest/task_noaa_coops_sf_water_level_coops_station_9414290_water_level_20260529t001800z/context.csv,data/sample_week/model_inputs_latest/task_noaa_coops_sf_water_level_coops_station_9414290_water_level_20260529t001800z/future_target.csv
6
+ task_nws_ksfo_observations_nws_station_ksfo_temperature_20260529t004500z,nws_ksfo_observations,nws_station_ksfo,temperature,96,24,data/sample_week/model_inputs_latest/task_nws_ksfo_observations_nws_station_ksfo_temperature_20260529t004500z/context.csv,data/sample_week/model_inputs_latest/task_nws_ksfo_observations_nws_station_ksfo_temperature_20260529t004500z/future_target.csv
7
+ task_open_meteo_shanghai_air_quality_hourly_grid_shanghai_31.2304_121.4737_pm2_5_20260528t020000z,open_meteo_shanghai_air_quality_hourly,grid_shanghai_31.2304_121.4737,pm2_5,96,24,data/sample_week/model_inputs_latest/task_open_meteo_shanghai_air_quality_hourly_grid_shanghai_31.2304_121.4737_pm2_5_20260528t020000z/context.csv,data/sample_week/model_inputs_latest/task_open_meteo_shanghai_air_quality_hourly_grid_shanghai_31.2304_121.4737_pm2_5_20260528t020000z/future_target.csv
8
+ task_usgs_potomac_iv_usgs_site_01646500_usgs_00060_20260528t235000z,usgs_potomac_iv,usgs_site_01646500,usgs_00060,96,24,data/sample_week/model_inputs_latest/task_usgs_potomac_iv_usgs_site_01646500_usgs_00060_20260528t235000z/context.csv,data/sample_week/model_inputs_latest/task_usgs_potomac_iv_usgs_site_01646500_usgs_00060_20260528t235000z/future_target.csv
vendor/ts_bench/data/sample_week/model_inputs_latest/manifest.json ADDED
@@ -0,0 +1,72 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ [
2
+ {
3
+ "task_id": "task_binance_btcusdt_1h_crypto_btcusdt_close_20260528t020000z",
4
+ "dataset_id": "binance_btcusdt_1h",
5
+ "entity_id": "crypto_btcusdt",
6
+ "target_variable_id": "close",
7
+ "context_rows": 96,
8
+ "future_target_rows": 24,
9
+ "context_csv": "data/sample_week/model_inputs_latest/task_binance_btcusdt_1h_crypto_btcusdt_close_20260528t020000z/context.csv",
10
+ "future_target_csv": "data/sample_week/model_inputs_latest/task_binance_btcusdt_1h_crypto_btcusdt_close_20260528t020000z/future_target.csv"
11
+ },
12
+ {
13
+ "task_id": "task_coingecko_bitcoin_7d_coin_bitcoin_price_usd_20260528t030012z",
14
+ "dataset_id": "coingecko_bitcoin_7d",
15
+ "entity_id": "coin_bitcoin",
16
+ "target_variable_id": "price_usd",
17
+ "context_rows": 96,
18
+ "future_target_rows": 24,
19
+ "context_csv": "data/sample_week/model_inputs_latest/task_coingecko_bitcoin_7d_coin_bitcoin_price_usd_20260528t030012z/context.csv",
20
+ "future_target_csv": "data/sample_week/model_inputs_latest/task_coingecko_bitcoin_7d_coin_bitcoin_price_usd_20260528t030012z/future_target.csv"
21
+ },
22
+ {
23
+ "task_id": "task_nasa_power_shanghai_hourly_point_shanghai_31.2304_121.4737_t2m_20260524t230000z",
24
+ "dataset_id": "nasa_power_shanghai_hourly",
25
+ "entity_id": "point_shanghai_31.2304_121.4737",
26
+ "target_variable_id": "T2M",
27
+ "context_rows": 69,
28
+ "future_target_rows": 24,
29
+ "context_csv": "data/sample_week/model_inputs_latest/task_nasa_power_shanghai_hourly_point_shanghai_31.2304_121.4737_t2m_20260524t230000z/context.csv",
30
+ "future_target_csv": "data/sample_week/model_inputs_latest/task_nasa_power_shanghai_hourly_point_shanghai_31.2304_121.4737_t2m_20260524t230000z/future_target.csv"
31
+ },
32
+ {
33
+ "task_id": "task_noaa_coops_sf_water_level_coops_station_9414290_water_level_20260529t001800z",
34
+ "dataset_id": "noaa_coops_sf_water_level",
35
+ "entity_id": "coops_station_9414290",
36
+ "target_variable_id": "water_level",
37
+ "context_rows": 96,
38
+ "future_target_rows": 24,
39
+ "context_csv": "data/sample_week/model_inputs_latest/task_noaa_coops_sf_water_level_coops_station_9414290_water_level_20260529t001800z/context.csv",
40
+ "future_target_csv": "data/sample_week/model_inputs_latest/task_noaa_coops_sf_water_level_coops_station_9414290_water_level_20260529t001800z/future_target.csv"
41
+ },
42
+ {
43
+ "task_id": "task_nws_ksfo_observations_nws_station_ksfo_temperature_20260529t004500z",
44
+ "dataset_id": "nws_ksfo_observations",
45
+ "entity_id": "nws_station_ksfo",
46
+ "target_variable_id": "temperature",
47
+ "context_rows": 96,
48
+ "future_target_rows": 24,
49
+ "context_csv": "data/sample_week/model_inputs_latest/task_nws_ksfo_observations_nws_station_ksfo_temperature_20260529t004500z/context.csv",
50
+ "future_target_csv": "data/sample_week/model_inputs_latest/task_nws_ksfo_observations_nws_station_ksfo_temperature_20260529t004500z/future_target.csv"
51
+ },
52
+ {
53
+ "task_id": "task_open_meteo_shanghai_air_quality_hourly_grid_shanghai_31.2304_121.4737_pm2_5_20260528t020000z",
54
+ "dataset_id": "open_meteo_shanghai_air_quality_hourly",
55
+ "entity_id": "grid_shanghai_31.2304_121.4737",
56
+ "target_variable_id": "pm2_5",
57
+ "context_rows": 96,
58
+ "future_target_rows": 24,
59
+ "context_csv": "data/sample_week/model_inputs_latest/task_open_meteo_shanghai_air_quality_hourly_grid_shanghai_31.2304_121.4737_pm2_5_20260528t020000z/context.csv",
60
+ "future_target_csv": "data/sample_week/model_inputs_latest/task_open_meteo_shanghai_air_quality_hourly_grid_shanghai_31.2304_121.4737_pm2_5_20260528t020000z/future_target.csv"
61
+ },
62
+ {
63
+ "task_id": "task_usgs_potomac_iv_usgs_site_01646500_usgs_00060_20260528t235000z",
64
+ "dataset_id": "usgs_potomac_iv",
65
+ "entity_id": "usgs_site_01646500",
66
+ "target_variable_id": "usgs_00060",
67
+ "context_rows": 96,
68
+ "future_target_rows": 24,
69
+ "context_csv": "data/sample_week/model_inputs_latest/task_usgs_potomac_iv_usgs_site_01646500_usgs_00060_20260528t235000z/context.csv",
70
+ "future_target_csv": "data/sample_week/model_inputs_latest/task_usgs_potomac_iv_usgs_site_01646500_usgs_00060_20260528t235000z/future_target.csv"
71
+ }
72
+ ]
vendor/ts_bench/data/sample_week/model_inputs_latest/task_binance_btcusdt_1h_crypto_btcusdt_close_20260527t230000z/context.csv ADDED
@@ -0,0 +1,97 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ timestamp,close,high,low,open,volume
2
+ 2026-05-24T00:00:00Z,76835.16,76930.65,76629.02,76752.0,392.54327
3
+ 2026-05-24T01:00:00Z,76890.04,76921.45,76754.53,76835.16,225.24128
4
+ 2026-05-24T02:00:00Z,76830.0,76968.04,76704.32,76890.04,185.87406
5
+ 2026-05-24T03:00:00Z,76750.0,77011.62,76738.5,76830.01,248.04543
6
+ 2026-05-24T04:00:00Z,76786.5,76858.26,76730.0,76750.01,163.64327
7
+ 2026-05-24T05:00:00Z,76822.23,76861.78,76651.59,76786.5,202.16815
8
+ 2026-05-24T06:00:00Z,76987.58,77065.94,76716.02,76822.23,257.8917
9
+ 2026-05-24T07:00:00Z,76799.99,77014.24,76786.2,76987.58,193.96514
10
+ 2026-05-24T08:00:00Z,76830.69,76861.77,76734.42,76799.99,226.94949
11
+ 2026-05-24T09:00:00Z,77020.53,77046.67,76821.86,76830.69,369.92973
12
+ 2026-05-24T10:00:00Z,77190.74,77543.15,76927.04,77020.54,863.69953
13
+ 2026-05-24T11:00:00Z,77320.06,77356.62,77190.73,77190.73,285.66801
14
+ 2026-05-24T12:00:00Z,77118.73,77325.72,77044.73,77320.06,271.94108
15
+ 2026-05-24T13:00:00Z,76802.23,77127.54,76688.89,77118.72,474.39799
16
+ 2026-05-24T14:00:00Z,76465.16,76840.01,76195.36,76802.23,837.87651
17
+ 2026-05-24T15:00:00Z,76371.59,76539.38,76362.78,76465.15,398.04105
18
+ 2026-05-24T16:00:00Z,76671.7,76734.3,76371.59,76371.59,359.46109
19
+ 2026-05-24T17:00:00Z,76693.34,76799.99,76566.15,76671.69,249.31214
20
+ 2026-05-24T18:00:00Z,76665.73,76788.01,76596.0,76693.34,148.14196
21
+ 2026-05-24T19:00:00Z,76773.34,76783.51,76611.17,76665.72,125.12918
22
+ 2026-05-24T20:00:00Z,76668.4,76796.18,76596.37,76773.33,560.38206
23
+ 2026-05-24T21:00:00Z,76151.14,76748.12,76108.0,76668.41,497.80071
24
+ 2026-05-24T22:00:00Z,76803.99,77012.01,76132.0,76151.14,543.11534
25
+ 2026-05-24T23:00:00Z,77064.96,77225.97,76803.99,76803.99,316.78339
26
+ 2026-05-25T00:00:00Z,77153.01,77288.0,77064.95,77064.96,281.30951
27
+ 2026-05-25T01:00:00Z,76945.27,77245.67,76914.25,77153.01,574.42489
28
+ 2026-05-25T02:00:00Z,77108.77,77485.0,76945.28,76945.28,313.96478
29
+ 2026-05-25T03:00:00Z,77026.67,77169.88,77002.3,77108.77,198.70522
30
+ 2026-05-25T04:00:00Z,77391.07,77450.0,76990.78,77026.68,251.77406
31
+ 2026-05-25T05:00:00Z,77407.0,77498.0,77324.0,77391.07,306.77798
32
+ 2026-05-25T06:00:00Z,77419.87,77650.0,77222.22,77406.99,413.85927
33
+ 2026-05-25T07:00:00Z,77401.12,77520.0,77184.62,77419.87,376.11814
34
+ 2026-05-25T08:00:00Z,77456.94,77469.64,77280.32,77401.11,352.6298
35
+ 2026-05-25T09:00:00Z,77604.56,77699.97,77425.17,77456.94,369.10396
36
+ 2026-05-25T10:00:00Z,77545.65,77681.37,77490.37,77604.56,283.53522
37
+ 2026-05-25T11:00:00Z,77357.5,77545.65,77333.33,77545.65,209.17805
38
+ 2026-05-25T12:00:00Z,77367.98,77440.46,77286.5,77357.5,261.28583
39
+ 2026-05-25T13:00:00Z,77341.05,77450.0,77286.5,77367.98,199.52985
40
+ 2026-05-25T14:00:00Z,77714.0,77773.69,77304.84,77341.05,603.07815
41
+ 2026-05-25T15:00:00Z,77628.82,77905.52,77557.0,77713.99,499.50313
42
+ 2026-05-25T16:00:00Z,77735.69,77807.29,77508.88,77628.82,342.64397
43
+ 2026-05-25T17:00:00Z,77564.01,77804.48,77550.39,77735.69,278.21782
44
+ 2026-05-25T18:00:00Z,77543.39,77658.01,77543.38,77564.0,204.7646
45
+ 2026-05-25T19:00:00Z,77441.86,77543.39,77343.61,77543.39,295.53937
46
+ 2026-05-25T20:00:00Z,77273.06,77500.0,77220.0,77441.85,314.11527
47
+ 2026-05-25T21:00:00Z,77367.03,77396.08,77273.05,77273.06,183.92616
48
+ 2026-05-25T22:00:00Z,77116.21,77449.94,77019.82,77367.03,395.72227
49
+ 2026-05-25T23:00:00Z,77322.01,77367.44,77072.36,77116.21,162.61467
50
+ 2026-05-26T00:00:00Z,76792.71,77345.0,76575.49,77322.01,686.17714
51
+ 2026-05-26T01:00:00Z,76756.98,76888.01,76590.9,76792.24,342.05131
52
+ 2026-05-26T02:00:00Z,76660.0,76810.0,76512.98,76756.98,721.43857
53
+ 2026-05-26T03:00:00Z,76664.01,76705.71,76475.59,76660.0,396.14567
54
+ 2026-05-26T04:00:00Z,76749.5,76817.28,76606.79,76664.0,392.59093
55
+ 2026-05-26T05:00:00Z,76930.01,76989.23,76704.8,76749.51,556.0432
56
+ 2026-05-26T06:00:00Z,76908.01,77059.74,76775.0,76930.0,658.30833
57
+ 2026-05-26T07:00:00Z,76730.84,76924.55,76603.73,76908.0,436.26184
58
+ 2026-05-26T08:00:00Z,76726.37,76852.01,76693.21,76730.84,246.06951
59
+ 2026-05-26T09:00:00Z,76672.19,76804.0,76649.79,76726.37,350.5237
60
+ 2026-05-26T10:00:00Z,77430.0,77621.75,76586.94,76672.2,897.13516
61
+ 2026-05-26T11:00:00Z,77183.55,77516.93,77174.94,77430.0,403.26795
62
+ 2026-05-26T12:00:00Z,77132.5,77232.52,76877.94,77183.56,402.68245
63
+ 2026-05-26T13:00:00Z,77231.56,77464.01,76691.88,77132.51,659.9668
64
+ 2026-05-26T14:00:00Z,76908.01,78080.0,76724.63,77231.56,2020.17697
65
+ 2026-05-26T15:00:00Z,76518.1,77012.0,76272.63,76908.01,1751.25838
66
+ 2026-05-26T16:00:00Z,76558.0,76710.01,76209.58,76518.1,2070.02098
67
+ 2026-05-26T17:00:00Z,75935.08,76649.91,75850.0,76558.0,1190.65802
68
+ 2026-05-26T18:00:00Z,75840.55,76273.66,75779.3,75935.09,846.30902
69
+ 2026-05-26T19:00:00Z,76052.0,76104.49,75805.23,75840.55,581.71052
70
+ 2026-05-26T20:00:00Z,76106.86,76180.0,75982.64,76052.0,324.41575
71
+ 2026-05-26T21:00:00Z,75890.0,76117.31,75850.0,76106.86,275.58323
72
+ 2026-05-26T22:00:00Z,75749.02,75998.94,75677.97,75890.0,428.12169
73
+ 2026-05-26T23:00:00Z,75930.01,75986.45,75749.02,75749.02,316.55917
74
+ 2026-05-27T00:00:00Z,76085.01,76089.7,75872.71,75930.01,780.41682
75
+ 2026-05-27T01:00:00Z,75877.44,76104.02,75824.0,76085.0,573.72878
76
+ 2026-05-27T02:00:00Z,75879.0,75890.59,75682.99,75877.44,437.25803
77
+ 2026-05-27T03:00:00Z,75683.0,75966.91,75607.62,75879.01,612.84746
78
+ 2026-05-27T04:00:00Z,75579.01,75706.77,75310.9,75683.0,817.05359
79
+ 2026-05-27T05:00:00Z,75697.99,75739.36,75579.0,75579.01,490.56114
80
+ 2026-05-27T06:00:00Z,75838.65,75900.0,75690.01,75697.99,482.86494
81
+ 2026-05-27T07:00:00Z,75977.17,75980.0,75764.0,75838.65,433.92885
82
+ 2026-05-27T08:00:00Z,75873.24,76071.41,75843.02,75977.16,308.77238
83
+ 2026-05-27T09:00:00Z,75912.6,75978.01,75710.76,75873.25,294.26595
84
+ 2026-05-27T10:00:00Z,75909.82,76043.94,75880.0,75912.61,372.61961
85
+ 2026-05-27T11:00:00Z,75769.18,76029.24,75754.1,75909.82,316.01081
86
+ 2026-05-27T12:00:00Z,75688.0,76174.15,75575.75,75769.18,621.24106
87
+ 2026-05-27T13:00:00Z,75073.4,75760.0,74866.18,75688.01,2200.23684
88
+ 2026-05-27T14:00:00Z,75031.92,75274.11,74758.33,75073.4,1567.20208
89
+ 2026-05-27T15:00:00Z,75326.01,75502.29,74983.77,75031.92,797.5234
90
+ 2026-05-27T16:00:00Z,75304.01,75408.01,75000.0,75326.02,632.49097
91
+ 2026-05-27T17:00:00Z,74860.42,75344.01,74685.58,75304.01,1103.99219
92
+ 2026-05-27T18:00:00Z,75026.67,75032.01,74663.45,74860.42,549.25309
93
+ 2026-05-27T19:00:00Z,75066.08,75105.17,74952.36,75026.67,879.80197
94
+ 2026-05-27T20:00:00Z,75274.58,75415.81,75056.25,75066.09,389.41476
95
+ 2026-05-27T21:00:00Z,74473.01,75274.58,74341.76,75274.58,653.21971
96
+ 2026-05-27T22:00:00Z,74418.13,74775.5,74260.91,74473.01,624.64107
97
+ 2026-05-27T23:00:00Z,74449.3,74572.0,74243.99,74418.12,938.42933