diff --git a/README.md b/README.md index 265a7f05e87fb9dbe72bb3c008f194b5c7184053..12d6add1b05f38a0c06f3a0c4d2d2c29b12b517b 100644 --- a/README.md +++ b/README.md @@ -13,6 +13,22 @@ python_version: 3.11.9 # TSFM Realworld Bench Leaderboard -Leaderboard for the **TSFM Realworld Benchmark** — evaluating time series foundation models on real-world multivariate forecasting tasks. +Live benchmark: every **1 hour** the Space collects TS-Bench data, runs all TSFM models via TSFM.ai, and updates the leaderboard. + +## Enable live evaluation (required) + +In **Settings → Repository secrets**, add: + +| Secret | Value | +|---|---| +| `TSFM_API_KEY` | Your key from [tsfm.ai](https://tsfm.ai/) | + +Optional variables (Settings → Variables): + +| Variable | Default | Purpose | +|---|---|---| +| `TSFM_BENCH_INTERVAL_SECONDS` | `3600` | Collect + eval cycle (1 hour) | +| `TSFM_REFRESH_SECONDS` | `60` | UI reload from disk (seconds) | + +Without `TSFM_API_KEY`, the leaderboard only reloads static CSV files and **does not** call the API. -Results are submitted via pull request to the [TSFM_Bench](https://github.com/zhouziyu02/TSFM_Bench) repository. diff --git a/app.py b/app.py index 94455eeb9efa6d0e9b4dca5d0912ffa061f34225..841b86c2d6046f51092bf1e36887b72882b97e21 100644 --- a/app.py +++ b/app.py @@ -16,6 +16,8 @@ from src.display.css_html_js import ( refresh_countdown_head_js, refresh_countdown_markup, ) +from src.benchmark_config import BENCHMARK_INTERVAL_SECONDS, DISPLAY_REFRESH_SECONDS +from src.eval_scheduler import get_scheduler, start_live_eval_scheduler from src.utils import ( get_grouped_dfs, prepare_ranks_table, @@ -23,25 +25,38 @@ from src.utils import ( ) RESULTS_PATH = os.getenv("TSFM_RESULTS_PATH", "results") -REFRESH_SECONDS = int(os.getenv("TSFM_REFRESH_SECONDS", "300")) + +start_live_eval_scheduler() def load_online_status(results_path: str = RESULTS_PATH) -> dict: status_path = Path(results_path) / "online_status.json" if not status_path.exists(): - return {"status": "unknown", "message": "No online_status.json yet. Run run_online_eval.py."} + return {"status": "unknown", "message": "No online_status.json yet."} return json.loads(status_path.read_text()) def build_status_line(status: dict) -> str: + eval_state = get_scheduler().snapshot() finished = status.get("finished_at", "n/a") - data_source = status.get("data_source", "n/a") - ts_bench_root = status.get("ts_bench_root", "") - root_hint = f" · TS-Bench: `{ts_bench_root}`" if ts_bench_root else "" + eval_status = eval_state.get("status", "unknown") + eval_message = eval_state.get("message", "") + + if eval_status == "running": + live_hint = "**Evaluating now** — collecting TS-Bench data and calling TSFM.ai…" + elif eval_status == "disabled": + live_hint = f"**Live eval off** — {eval_message}" + else: + secs = get_scheduler().seconds_until_next_run() + next_hint = f" · Next eval in ~{secs}s" if secs is not None else "" + live_hint = ( + f"**Live eval** · Last run `{finished}` · " + f"collect + eval every **{BENCHMARK_INTERVAL_SECONDS // 60} min**{next_hint}" + ) + return ( - f"**Live benchmark** · Data: `{data_source}`{root_hint} · " - f"Last eval: `{finished}` · " - f"Auto-refresh every {REFRESH_SECONDS}s" + f"{live_hint} · " + f"Display reloads every {DISPLAY_REFRESH_SECONDS}s" ) @@ -73,14 +88,14 @@ DOMAINS: list[str] = grouped_initial["domains"] INITIAL_OUTPUTS = refresh_leaderboard() demo = gr.Blocks( - css=build_custom_css(REFRESH_SECONDS), - head=refresh_countdown_head_js(REFRESH_SECONDS), + css=build_custom_css(DISPLAY_REFRESH_SECONDS), + head=refresh_countdown_head_js(DISPLAY_REFRESH_SECONDS), ) with demo: gr.HTML(TITLE) gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text") status = gr.Markdown(INITIAL_OUTPUTS[2], elem_classes="markdown-text") - gr.HTML(refresh_countdown_markup(REFRESH_SECONDS)) + gr.HTML(refresh_countdown_markup(DISPLAY_REFRESH_SECONDS)) domain_tables: list[tuple[gr.Dataframe, gr.Dataframe]] = [] with gr.Tabs(): @@ -130,6 +145,8 @@ with demo: refresh_outputs.extend([values_df, ranks_df]) demo.load(refresh_leaderboard, inputs=[], outputs=refresh_outputs) - gr.Timer(value=REFRESH_SECONDS).tick(refresh_leaderboard, inputs=[], outputs=refresh_outputs) + gr.Timer(value=DISPLAY_REFRESH_SECONDS).tick( + refresh_leaderboard, inputs=[], outputs=refresh_outputs + ) demo.queue(default_concurrency_limit=20).launch() diff --git a/configs/datasets/ts_bench.yaml b/configs/datasets/ts_bench.yaml new file mode 100644 index 0000000000000000000000000000000000000000..630a325ce823359e649e565a4812988a0749ae34 --- /dev/null +++ b/configs/datasets/ts_bench.yaml @@ -0,0 +1,29 @@ +source_type: ts_bench + +settings: + ts_bench_root: vendor/ts_bench + output_subdir: data/sample_week + collect_days: 7 + collect_timeout: 45 + limit_entities: 10 + task_limit: 20 + max_tasks_per_dataset: 1 + min_future_steps: 2 + skip_frequencies: + - snapshot + auto_refresh: true + +source_domains: + open_meteo_weather: Weather + open_meteo_air_quality: Air Quality + nasa_power: Weather + usgs_water: Hydrology + noaa_coops: Ocean + noaa_ndbc: Ocean + gbfs_citibike: Transport + binance: Finance + coingecko: Finance + wikimedia_pageviews: Web + nws_observations: Weather + usgs_earthquake: Hazards + noaa_ncei: Weather diff --git a/configs/models/online_tsfm.yaml b/configs/models/online_tsfm.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f8b68abff9075b1840ba7b60914648ced2583291 --- /dev/null +++ b/configs/models/online_tsfm.yaml @@ -0,0 +1,26 @@ +models: + - model_id: amazon/chronos-bolt-tiny + display_name: Chronos-Bolt-Tiny + model_type: zero-shot + org: Amazon / TSFM.ai + - model_id: amazon/chronos-bolt-mini + display_name: Chronos-Bolt-Mini + model_type: zero-shot + org: Amazon / TSFM.ai + - model_id: amazon/chronos-bolt-base + display_name: Chronos-Bolt-Base + model_type: zero-shot + org: Amazon / TSFM.ai + baseline: true + - model_id: amazon/chronos-2 + display_name: Chronos-2 + model_type: zero-shot + org: Amazon / TSFM.ai + - model_id: google/timesfm-2.5-200m-pytorch + display_name: TimesFM-2.5 + model_type: zero-shot + org: Google / TSFM.ai + - model_id: Salesforce/moirai-1.1-R-small + display_name: Moirai-1.1-Small + model_type: zero-shot + org: Salesforce / TSFM.ai diff --git a/requirements.txt b/requirements.txt index 325de5c60cf77240510880b9cbf8323d65e5738a..2ff0c514671fe10b6bcbf83b51f731055b0f768a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,6 +4,11 @@ starlette==0.37.2 fastapi==0.112.4 jinja2==3.1.4 uvicorn==0.30.6 -pandas -numpy -scipy +pandas>=2.0.0 +numpy~=1.26.0 +scipy~=1.11.3 +gluonts~=0.15.1 +httpx>=0.27.0 +pyyaml>=6.0 +python-dotenv==1.0.0 +requests>=2.31.0 diff --git a/results/chronos_2/all_results.csv b/results/chronos_2/all_results.csv index 1ba55f44c355306b714281668e6873455977c588..9178b2e852e566400c57e9da89f1f450313d46e5 100644 --- a/results/chronos_2/all_results.csv +++ b/results/chronos_2/all_results.csv @@ -1,11 +1,8 @@ dataset,model,eval_metrics/MSE[mean],eval_metrics/MSE[0.5],eval_metrics/MAE[0.5],eval_metrics/MASE[0.5],eval_metrics/MAPE[0.5],eval_metrics/sMAPE[0.5],eval_metrics/MSIS,eval_metrics/RMSE[mean],eval_metrics/NRMSE[mean],eval_metrics/ND[0.5],eval_metrics/mean_weighted_sum_quantile_loss,domain,num_variates -binance_btcusdt_1h_close/1h/short,Chronos-2,4380093.155741374,4380093.155741374,1850.0364583333333,2.3838636016637054,0.025076568408860828,0.024678861943730448,66.88197607794382,2092.8672092947927,0.0282057298197708,0.024933081405608456,0.021564816825717217,Finance,1 -coingecko_bitcoin_7d_price_usd/1h/short,Chronos-2,3302102.223780314,3302102.223780314,1589.05078125,1.9819221083493386,0.02156161218573524,0.021260322353283848,46.8612864887045,1817.1687383895623,0.02450082462177794,0.021425117923286136,0.017372480048838304,Finance,1 -nasa_power_shanghai_hourly_T2M/1h/short,Chronos-2,16.264854419926603,16.264854419926603,3.1164464155832925,0.9942700533878908,0.1112300523205632,0.12196958213989656,21.94742601097775,4.032970917317233,0.15713129036201698,0.12142196327823566,0.1048847735590834,Weather,1 -noaa_coops_sf_water_level_water_level/6min/short,Chronos-2,0.06003561507700905,0.06003561507700905,0.213998693972826,8.070613535689246,1.013472949369841,0.597409693051528,96.8635045441028,0.24502166246478913,0.9537009413978706,0.832949845522854,0.5562847402156419,Ocean,1 -noaa_ndbc_46013_realtime_wave_height/10min/short,Chronos-2,0.009589138295815758,0.009589138295815758,0.07559254765510559,0.5440371647347783,0.0288622829034601,0.029130192091777325,4.1629569029245,0.09792414562208729,0.0375428043484963,0.028981169136490237,0.03618823474141707,Ocean,1 -nws_ksfo_observations_temperature/1h/short,Chronos-2,0.003463902878100574,0.003463902878100574,0.02116549015045166,0.4010301182731177,0.0016484734142401373,0.0016379871537986845,9.278132686642117,0.0588549307883424,0.00453311396422428,0.0016302046009655346,0.0018722179194657846,Weather,1 -open_meteo_shanghai_air_quality_hourly_pm2_5/1h/short,Chronos-2,121.62235993895926,121.62235993895926,9.762574990590414,0.5251049426581348,0.25052453302473104,0.21224658419640266,4.581651077587976,11.028252805361294,0.2247033380106543,0.1989148441445373,0.2278692478910483,Air Quality,1 -open_meteo_shanghai_hourly_temperature_2m/1h/short,Chronos-2,0.30751845326813054,0.30751845326813054,0.4332726796468099,0.23597305034316118,0.017424389083777924,0.017404320116509994,2.091343357081799,0.5545434638223865,0.0221301007506678,0.0172905618380289,0.022378411607540734,Weather,1 -usgs_potomac_iv_usgs_00060/15min/short,Chronos-2,5269.190091451009,5269.190091451009,65.57242838541667,nan,0.0025808082601376345,0.0025768785670678665,nan,72.58918715243344,0.0028545641351112607,0.002578630642716697,0.001904930492972491,Hydrology,1 -wikimedia_time_series_hourly_pageviews/1D/short,Chronos-2,4063.1116605430143,4063.1116605430143,63.214759826660156,1.3546019962855749,0.2031273434135618,0.22642697598080944,5.089582116263253,63.74254199938228,0.20528999033617482,0.20359020878151418,0.18650881592714202,Web,1 +binance_btcusdt_1h_close/1h/short,Chronos-2,823714.9134292603,823714.9134292603,858.2594401041666,0.9497371860172586,0.011705204247515538,0.011628998753724112,11.651465649030945,907.5874136573624,0.01236475636114196,0.01169272371106779,0.008400224593419353,Finance,1 +coingecko_bitcoin_7d_price_usd/1h/short,Chronos-2,855220.5976460775,855220.5976460775,874.1985677083334,0.9460587958444222,0.011942090149853535,0.011862712920852017,11.805964342783492,924.7813782976372,0.012618568650165292,0.011928370206599736,0.008589137695491324,Finance,1 +nasa_power_shanghai_hourly_T2M/1h/short,Chronos-2,16.932999722573793,16.932999722573793,3.1624552408854165,0.9207458206411238,0.11271485949261918,0.12390656428505213,20.748803742282863,4.114972627196176,0.1603262140917773,0.12321454404213872,0.10622444551676062,Weather,1 +noaa_coops_sf_water_level_water_level/6min/short,Chronos-2,0.0002494680528126736,0.0002494680528126736,0.013614242275555929,0.853698368574301,0.010921521056233332,0.01099381527380565,7.152836038203941,0.015794557696012686,0.012375351058861507,0.010667030429280775,0.014505464409588711,Ocean,1 +nws_ksfo_observations_temperature/1h/short,Chronos-2,6.805250338098176,6.805250338098176,2.0215774377187095,2.4880953079614887,0.13046011501961127,0.11750503806375708,54.233638763427734,2.608687474209622,0.1540184486618227,0.11935512547416734,0.0937472313337025,Weather,1 +open_meteo_shanghai_air_quality_hourly_pm2_5/1h/short,Chronos-2,787.8530339560867,787.8530339560867,22.037151257197063,0.9668951191457578,0.3169416718860694,0.39892875770053077,5.485232905888252,28.06871984890096,0.4783761537649721,0.3755799236696442,0.30724907240289284,Air Quality,1 +usgs_potomac_iv_usgs_00060/15min/short,Chronos-2,12352.058849016825,12352.058849016825,90.12906901041667,nan,0.003257321805455632,0.0032514783083314887,nan,111.13981666809076,0.003992449633339587,0.0032376854606346356,0.0034832326643549696,Hydrology,1 diff --git a/results/chronos_2/config.json b/results/chronos_2/config.json index 2d7cc96fd48a2ab63426f26dc09d84edd214a748..f3b177448fb051d498bc6df0f00fddfad156fc7e 100644 --- a/results/chronos_2/config.json +++ b/results/chronos_2/config.json @@ -3,7 +3,7 @@ "model_type": "zero-shot", "model_dtype": "float32", "model_link": "https://tsfm.ai/models/amazon/chronos-2", - "code_link": "https://github.com/zhouziyu02/TSFM_Bench/blob/main/scripts/run_online_eval.py", + "code_link": "https://github.com/zhouziyu02/TSFM_Bench/blob/main/space/scripts/run_space_eval.py", "org": "Amazon / TSFM.ai", "testdata_leakage": "No", "replication_code_available": "Yes", diff --git a/results/chronos_2/online_meta.json b/results/chronos_2/online_meta.json index 3a5727a4357fed210e6fe8ab105d25bc28df4a26..571a59c452e80eec5bd7c33b956e06146ebf960e 100644 --- a/results/chronos_2/online_meta.json +++ b/results/chronos_2/online_meta.json @@ -1,67 +1,49 @@ { "model": "Chronos-2", "api_model_id": "amazon/chronos-2", - "evaluated_at": "2026-05-28T12:20:18.799459+00:00", + "evaluated_at": "2026-05-29T02:53:16.309267+00:00", "datasets": [ { - "dataset": "task_binance_btcusdt_1h_crypto_btcusdt_close_20260527t120000z", - "data_fetched_at": "2026-05-28T12:16:39.187896Z", + "dataset": "task_binance_btcusdt_1h_crypto_btcusdt_close_20260528t020000z", + "data_fetched_at": "2026-05-29T02:50:05.083862Z", "context_length": 96, "prediction_length": 24 }, { - "dataset": "task_coingecko_bitcoin_7d_coin_bitcoin_price_usd_20260527t120016z", - "data_fetched_at": "2026-05-28T12:16:39.187896Z", + "dataset": "task_coingecko_bitcoin_7d_coin_bitcoin_price_usd_20260528t030012z", + "data_fetched_at": "2026-05-29T02:50:05.083862Z", "context_length": 96, "prediction_length": 24 }, { "dataset": "task_nasa_power_shanghai_hourly_point_shanghai_31.2304_121.4737_t2m_20260524t230000z", - "data_fetched_at": "2026-05-28T12:16:39.187896Z", - "context_length": 83, + "data_fetched_at": "2026-05-29T02:50:05.083862Z", + "context_length": 69, "prediction_length": 24 }, { - "dataset": "task_noaa_coops_sf_water_level_coops_station_9414290_water_level_20260528t094200z", - "data_fetched_at": "2026-05-28T12:16:39.187896Z", + "dataset": "task_noaa_coops_sf_water_level_coops_station_9414290_water_level_20260529t001800z", + "data_fetched_at": "2026-05-29T02:50:05.083862Z", "context_length": 96, "prediction_length": 24 }, { - "dataset": "task_noaa_ndbc_46013_realtime_ndbc_station_46013_wave_height_20260528t045000z", - "data_fetched_at": "2026-05-28T12:16:39.187896Z", + "dataset": "task_nws_ksfo_observations_nws_station_ksfo_temperature_20260529t004500z", + "data_fetched_at": "2026-05-29T02:50:05.083862Z", "context_length": 96, "prediction_length": 24 }, { - "dataset": "task_nws_ksfo_observations_nws_station_ksfo_temperature_20260528t101000z", - "data_fetched_at": "2026-05-28T12:16:39.187896Z", + "dataset": "task_open_meteo_shanghai_air_quality_hourly_grid_shanghai_31.2304_121.4737_pm2_5_20260528t020000z", + "data_fetched_at": "2026-05-29T02:50:05.083862Z", "context_length": 96, "prediction_length": 24 }, { - "dataset": "task_open_meteo_shanghai_air_quality_hourly_grid_shanghai_31.2304_121.4737_pm2_5_20260527t120000z", - "data_fetched_at": "2026-05-28T12:16:39.187896Z", + "dataset": "task_usgs_potomac_iv_usgs_site_01646500_usgs_00060_20260528t235000z", + "data_fetched_at": "2026-05-29T02:50:05.083862Z", "context_length": 96, "prediction_length": 24 - }, - { - "dataset": "task_open_meteo_shanghai_hourly_grid_shanghai_31.2304_121.4737_temperature_2m_20260527t120000z", - "data_fetched_at": "2026-05-28T12:16:39.187896Z", - "context_length": 96, - "prediction_length": 24 - }, - { - "dataset": "task_usgs_potomac_iv_usgs_site_01646500_usgs_00060_20260528t095000z", - "data_fetched_at": "2026-05-28T12:16:39.187896Z", - "context_length": 96, - "prediction_length": 24 - }, - { - "dataset": "task_wikimedia_time_series_hourly_enwiki_article_time_series_pageviews_20260525t000000z", - "data_fetched_at": "2026-05-28T12:16:39.187896Z", - "context_length": 4, - "prediction_length": 2 } ] } diff --git a/results/chronos_bolt_base/all_results.csv b/results/chronos_bolt_base/all_results.csv index da67ad899e587ac32df719be976b3cd956f9893a..bce3183ccdd7be305cc3a27b4703cd96f413e7d7 100644 --- a/results/chronos_bolt_base/all_results.csv +++ b/results/chronos_bolt_base/all_results.csv @@ -1,11 +1,8 @@ dataset,model,eval_metrics/MSE[mean],eval_metrics/MSE[0.5],eval_metrics/MAE[0.5],eval_metrics/MASE[0.5],eval_metrics/MAPE[0.5],eval_metrics/sMAPE[0.5],eval_metrics/MSIS,eval_metrics/RMSE[mean],eval_metrics/NRMSE[mean],eval_metrics/ND[0.5],eval_metrics/mean_weighted_sum_quantile_loss,domain,num_variates -binance_btcusdt_1h_close/1h/short,Chronos-Bolt-Base,4627089.7309748335,4627089.7309748335,1874.4404296875,2.4153093273872144,0.025415879545412644,0.024995872396728593,52.461912288031094,2151.067114474775,0.028990094347893916,0.025261975575046464,0.019576547766102168,Finance,1 -coingecko_bitcoin_7d_price_usd/1h/short,Chronos-Bolt-Base,4946482.715515137,4946482.715515137,1928.9296875,2.4058314801622274,0.026177410818833124,0.02572777590076968,48.45609391950561,2224.0689547572792,0.02998704647293313,0.02600769371756978,0.019592513629850913,Finance,1 -nasa_power_shanghai_hourly_T2M/1h/short,Chronos-Bolt-Base,11.688720341001195,11.688720341001195,2.729196389516195,0.8707219307033777,0.09833320637961653,0.10621373079641616,8.073066633299149,3.4188770584800494,0.13320516681172243,0.10633405475219919,0.08541267410550227,Weather,1 -noaa_coops_sf_water_level_water_level/6min/short,Chronos-Bolt-Base,0.03234704147219288,0.03234704147219288,0.13382656003038088,5.047051581319789,0.6706533574633848,0.41720830815031723,22.83294145173187,0.1798528328167029,0.7000434747071089,0.5208948262007407,0.4173842468196957,Ocean,1 -noaa_ndbc_46013_realtime_wave_height/10min/short,Chronos-Bolt-Base,0.04078452699226176,0.04078452699226176,0.17054141561190286,1.2273811519446596,0.06768574491893825,0.06480219526685438,7.064554223996088,0.20195179373370706,0.07742560970848361,0.06538329192416381,0.061584868474800714,Ocean,1 -nws_ksfo_observations_temperature/1h/short,Chronos-Bolt-Base,0.0037005670213829944,0.0037005670213829944,0.01999191443125407,0.3787939590286682,0.0015589197272527502,0.0015477403017569464,13.350006060847901,0.06083228601148402,0.004685413464949044,0.0015398136615912097,0.0017142914417336535,Weather,1 -open_meteo_shanghai_air_quality_hourly_pm2_5/1h/short,Chronos-Bolt-Base,38.0265807971964,38.0265807971964,4.774677435557048,0.25681817793213635,0.1013508384060875,0.09818999823413017,3.9438365445644235,6.166569613423366,0.12564535839599894,0.0972852161289089,0.17859769229816153,Air Quality,1 -open_meteo_shanghai_hourly_temperature_2m/1h/short,Chronos-Bolt-Base,0.6043374313377777,0.6043374313377777,0.6502586205800375,0.3541499785661307,0.026396943053871125,0.026084632371007877,3.108420925181368,0.7773914273631899,0.03102326820637271,0.02594979424739109,0.027723991507018,Weather,1 -usgs_potomac_iv_usgs_00060/15min/short,Chronos-Bolt-Base,4639.297880808513,4639.297880808513,62.649576822916664,nan,0.0024655696006461968,0.0024621813211091674,nan,68.11239153640483,0.002678514495942513,0.002463689732508602,0.0015585257620341544,Hydrology,1 -wikimedia_time_series_hourly_pageviews/1D/short,Chronos-Bolt-Base,4028.8496973256115,4028.8496973256115,63.05271911621094,1.3511296953473773,0.2026653811521117,0.22577298837325582,4.94127665928432,63.473220316331926,0.20442260971443454,0.20306833853852155,0.181499555051636,Web,1 +binance_btcusdt_1h_close/1h/short,Chronos-Bolt-Base,1020807.1654103597,1020807.1654103597,978.3776041666666,1.082658167477634,0.013340172949584,0.013245844896014864,12.505715915951397,1010.3500212353933,0.013764769832701284,0.013329185181148515,0.009075649577973174,Finance,1 +coingecko_bitcoin_7d_price_usd/1h/short,Chronos-Bolt-Base,1032518.6303456625,1032518.6303456625,981.5572916666666,1.0622425427220556,0.013405463836439183,0.013309736826454506,12.85205207248118,1016.1292389975118,0.01386500297327957,0.013393271490572569,0.009228772376426409,Finance,1 +nasa_power_shanghai_hourly_T2M/1h/short,Chronos-Bolt-Base,14.357317871038353,14.357317871038353,3.029151995976766,0.8819347083000757,0.1091449078122792,0.11893372374924811,7.6038772586826555,3.7891051543917795,0.1476298724769894,0.1180208267276896,0.09493130206210582,Weather,1 +noaa_coops_sf_water_level_water_level/6min/short,Chronos-Bolt-Base,0.007210553512415767,0.007210553512415767,0.07268983622392018,4.558108585133124,0.054106267823969455,0.055993512387166634,21.160505010226164,0.08491497813940581,0.06653257944006775,0.05695393685568374,0.049410255141733854,Ocean,1 +nws_ksfo_observations_temperature/1h/short,Chronos-Bolt-Base,6.471037574445442,6.471037574445442,1.968206803003947,2.4224083729279346,0.1270358446325033,0.11470028206444982,37.253308711907806,2.543823416521957,0.15018883639982034,0.11620409169026993,0.08526762779478787,Weather,1 +open_meteo_shanghai_air_quality_hourly_pm2_5/1h/short,Chronos-Bolt-Base,964.608039366991,964.608039366991,25.43541646003723,1.115996334626862,0.3828018459905362,0.4930053251111303,8.94359384637864,31.058139663653247,0.5293249380582461,0.4334966739154405,0.35288186109406783,Air Quality,1 +usgs_potomac_iv_usgs_00060/15min/short,Chronos-Bolt-Base,31798.887013276417,31798.887013276417,151.68888346354166,nan,0.00542055609772574,0.005430840108326886,nan,178.32242431415185,0.006405834730638593,0.005449084273499476,0.0052516232677867584,Hydrology,1 diff --git a/results/chronos_bolt_base/config.json b/results/chronos_bolt_base/config.json index 2e253e8a06f844942d111d00e61b0e5c010dc521..b9d4b60d4ac48100990f9718d67320f586640f04 100644 --- a/results/chronos_bolt_base/config.json +++ b/results/chronos_bolt_base/config.json @@ -3,7 +3,7 @@ "model_type": "zero-shot", "model_dtype": "float32", "model_link": "https://tsfm.ai/models/amazon/chronos-bolt-base", - "code_link": "https://github.com/zhouziyu02/TSFM_Bench/blob/main/scripts/run_online_eval.py", + "code_link": "https://github.com/zhouziyu02/TSFM_Bench/blob/main/space/scripts/run_space_eval.py", "org": "Amazon / TSFM.ai", "testdata_leakage": "No", "replication_code_available": "Yes", diff --git a/results/chronos_bolt_base/online_meta.json b/results/chronos_bolt_base/online_meta.json index 27e37da222fed4c29377a9a3139361bf610b15c1..cda99482c61065789abfe4b6e3574e710113ccf7 100644 --- a/results/chronos_bolt_base/online_meta.json +++ b/results/chronos_bolt_base/online_meta.json @@ -1,67 +1,49 @@ { "model": "Chronos-Bolt-Base", "api_model_id": "amazon/chronos-bolt-base", - "evaluated_at": "2026-05-28T12:19:28.056546+00:00", + "evaluated_at": "2026-05-29T02:51:30.013825+00:00", "datasets": [ { - "dataset": "task_binance_btcusdt_1h_crypto_btcusdt_close_20260527t120000z", - "data_fetched_at": "2026-05-28T12:16:39.187896Z", + "dataset": "task_binance_btcusdt_1h_crypto_btcusdt_close_20260528t020000z", + "data_fetched_at": "2026-05-29T02:50:05.083862Z", "context_length": 96, "prediction_length": 24 }, { - "dataset": "task_coingecko_bitcoin_7d_coin_bitcoin_price_usd_20260527t120016z", - "data_fetched_at": "2026-05-28T12:16:39.187896Z", + "dataset": "task_coingecko_bitcoin_7d_coin_bitcoin_price_usd_20260528t030012z", + "data_fetched_at": "2026-05-29T02:50:05.083862Z", "context_length": 96, "prediction_length": 24 }, { "dataset": "task_nasa_power_shanghai_hourly_point_shanghai_31.2304_121.4737_t2m_20260524t230000z", - "data_fetched_at": "2026-05-28T12:16:39.187896Z", - "context_length": 83, + "data_fetched_at": "2026-05-29T02:50:05.083862Z", + "context_length": 69, "prediction_length": 24 }, { - "dataset": "task_noaa_coops_sf_water_level_coops_station_9414290_water_level_20260528t094200z", - "data_fetched_at": "2026-05-28T12:16:39.187896Z", + "dataset": "task_noaa_coops_sf_water_level_coops_station_9414290_water_level_20260529t001800z", + "data_fetched_at": "2026-05-29T02:50:05.083862Z", "context_length": 96, "prediction_length": 24 }, { - "dataset": "task_noaa_ndbc_46013_realtime_ndbc_station_46013_wave_height_20260528t045000z", - "data_fetched_at": "2026-05-28T12:16:39.187896Z", + "dataset": "task_nws_ksfo_observations_nws_station_ksfo_temperature_20260529t004500z", + "data_fetched_at": "2026-05-29T02:50:05.083862Z", "context_length": 96, "prediction_length": 24 }, { - "dataset": "task_nws_ksfo_observations_nws_station_ksfo_temperature_20260528t101000z", - "data_fetched_at": "2026-05-28T12:16:39.187896Z", + "dataset": "task_open_meteo_shanghai_air_quality_hourly_grid_shanghai_31.2304_121.4737_pm2_5_20260528t020000z", + "data_fetched_at": "2026-05-29T02:50:05.083862Z", "context_length": 96, "prediction_length": 24 }, { - "dataset": "task_open_meteo_shanghai_air_quality_hourly_grid_shanghai_31.2304_121.4737_pm2_5_20260527t120000z", - "data_fetched_at": "2026-05-28T12:16:39.187896Z", + "dataset": "task_usgs_potomac_iv_usgs_site_01646500_usgs_00060_20260528t235000z", + "data_fetched_at": "2026-05-29T02:50:05.083862Z", "context_length": 96, "prediction_length": 24 - }, - { - "dataset": "task_open_meteo_shanghai_hourly_grid_shanghai_31.2304_121.4737_temperature_2m_20260527t120000z", - "data_fetched_at": "2026-05-28T12:16:39.187896Z", - "context_length": 96, - "prediction_length": 24 - }, - { - "dataset": "task_usgs_potomac_iv_usgs_site_01646500_usgs_00060_20260528t095000z", - "data_fetched_at": "2026-05-28T12:16:39.187896Z", - "context_length": 96, - "prediction_length": 24 - }, - { - "dataset": "task_wikimedia_time_series_hourly_enwiki_article_time_series_pageviews_20260525t000000z", - "data_fetched_at": "2026-05-28T12:16:39.187896Z", - "context_length": 4, - "prediction_length": 2 } ] } diff --git a/results/chronos_bolt_mini/all_results.csv b/results/chronos_bolt_mini/all_results.csv index 41cc22d68c6061cc52c1d17082575b7bbecc9a41..b65cbd530c7a2f754dde9eb180fa8ce9bf6717fa 100644 --- a/results/chronos_bolt_mini/all_results.csv +++ b/results/chronos_bolt_mini/all_results.csv @@ -1,11 +1,8 @@ dataset,model,eval_metrics/MSE[mean],eval_metrics/MSE[0.5],eval_metrics/MAE[0.5],eval_metrics/MASE[0.5],eval_metrics/MAPE[0.5],eval_metrics/sMAPE[0.5],eval_metrics/MSIS,eval_metrics/RMSE[mean],eval_metrics/NRMSE[mean],eval_metrics/ND[0.5],eval_metrics/mean_weighted_sum_quantile_loss,domain,num_variates -binance_btcusdt_1h_close/1h/short,Chronos-Bolt-Mini,3679822.6813456216,3679822.6813456216,1654.4537760416667,2.1318456290822945,0.022439195282325037,0.02210438779696162,36.894830953375646,1918.2863918991923,0.02585289092712348,0.022297198789814756,0.016141385649750175,Finance,1 -coingecko_bitcoin_7d_price_usd/1h/short,Chronos-Bolt-Mini,3435441.7622146606,3435441.7622146606,1580.2054036458333,1.9708898306919778,0.02145509434972552,0.021141595171611006,22.12416891501253,1853.4944732085555,0.024990603275380642,0.02130585599630342,0.013890711488634161,Finance,1 -nasa_power_shanghai_hourly_T2M/1h/short,Chronos-Bolt-Mini,9.13818853845593,9.13818853845593,2.457893212636312,0.784165453167993,0.08935921081957772,0.09565064369787313,7.269570607721318,3.0229436876091373,0.1177789406529396,0.09576362934213734,0.07988338981433957,Weather,1 -noaa_coops_sf_water_level_water_level/6min/short,Chronos-Bolt-Mini,0.02307176451407955,0.02307176451407955,0.1410230112572511,5.318454063280177,0.6473759859576461,0.45994482734131564,19.641841542644524,0.15189392520466233,0.5912186620687938,0.5489056650822877,0.38075484208964183,Ocean,1 -noaa_ndbc_46013_realtime_wave_height/10min/short,Chronos-Bolt-Mini,0.03395013763280493,0.03395013763280493,0.15618878602981567,1.1240857326666478,0.06198749864085912,0.05956084295321493,7.4643506648433195,0.18425563121056826,0.0706411382882342,0.059880686199460895,0.0619203976492256,Ocean,1 -nws_ksfo_observations_temperature/1h/short,Chronos-Bolt-Mini,0.003293288668335966,0.003293288668335966,0.019572933514912922,0.3708553776284693,0.0015254847226304179,0.0015155839450093345,12.664317868508965,0.05738718209091614,0.004420065286277438,0.0015075429883074389,0.0017640636910896848,Weather,1 -open_meteo_shanghai_air_quality_hourly_pm2_5/1h/short,Chronos-Bolt-Mini,58.57930447065731,58.57930447065731,6.592350959777832,0.35458637460433845,0.15445332516570973,0.14256432500281468,4.4046256992472514,7.65371181000809,0.1559462430027276,0.13432075707220698,0.20333559743785112,Air Quality,1 -open_meteo_shanghai_hourly_temperature_2m/1h/short,Chronos-Bolt-Mini,0.8828099140087033,0.8828099140087033,0.7188208103179932,0.39149096453335225,0.028733217601040043,0.028834013771296452,2.773892978739956,0.9395796475066408,0.03749569442091312,0.02868589748468996,0.02969891826111223,Weather,1 -usgs_potomac_iv_usgs_00060/15min/short,Chronos-Bolt-Mini,4843.679802894592,4843.679802894592,64.029296875,nan,0.002519860464793693,0.0025163275141648713,nan,69.59655022265538,0.002736878920766392,0.0025179471161723742,0.001616091514077776,Hydrology,1 -wikimedia_time_series_hourly_pageviews/1D/short,Chronos-Bolt-Mini,4577.245442453423,4577.245442453423,67.47705841064453,1.445936965942383,0.21708162745704251,0.24360223675984644,8.007688195364816,67.65534304438506,0.21789160400768137,0.21731741839176982,0.23123530744333173,Web,1 +binance_btcusdt_1h_close/1h/short,Chronos-Bolt-Mini,851287.6321029663,851287.6321029663,882.4768880208334,0.976535855233298,0.012034191309566289,0.011955457296705658,10.656671858587634,922.6524980202278,0.012569999508968603,0.012022656496243319,0.008244497111170809,Finance,1 +coingecko_bitcoin_7d_price_usd/1h/short,Chronos-Bolt-Mini,864730.2875595093,864730.2875595093,885.3583984375,0.9581359787768308,0.012093613190106718,0.012013367698897761,11.016269186291742,929.9087522760011,0.01268853126193431,0.012080645212871425,0.0083666342836689,Finance,1 +nasa_power_shanghai_hourly_T2M/1h/short,Chronos-Bolt-Mini,9.548240780425354,9.548240780425354,2.5835858980814614,0.7522085647795073,0.09503784443039846,0.1018251998426517,3.8696742236218,3.090022779920134,0.12039245425054106,0.10066082653447453,0.08411044052817845,Weather,1 +noaa_coops_sf_water_level_water_level/6min/short,Chronos-Bolt-Mini,0.01640126373424522,0.01640126373424522,0.0986495316028595,6.185944284331548,0.07142778184577067,0.07563235656687009,25.340507814660793,0.12806741870688743,0.10034337752300845,0.07729387608529449,0.0749025769287721,Ocean,1 +nws_ksfo_observations_temperature/1h/short,Chronos-Bolt-Mini,6.4003104863786575,6.4003104863786575,1.9677637418111165,2.4218630668444514,0.1268957925752334,0.1146813703822039,35.354552146715996,2.5298834926491494,0.14936581506415642,0.11617793309585928,0.0828891098816994,Weather,1 +open_meteo_shanghai_air_quality_hourly_pm2_5/1h/short,Chronos-Bolt-Mini,1257.403550816446,1257.403550816446,29.262885729471844,1.2839291727739017,0.4375496892711623,0.5885718970399296,15.36094857308285,35.45988650315235,0.6043440602078141,0.4987283637688504,0.40799980188904916,Air Quality,1 +usgs_potomac_iv_usgs_00060/15min/short,Chronos-Bolt-Mini,68632.59780025482,68632.59780025482,211.42757161458334,nan,0.007524633240620554,0.007560872203681779,nan,261.9782391731321,0.00941098299678966,0.007595063192261637,0.006913356011138552,Hydrology,1 diff --git a/results/chronos_bolt_mini/config.json b/results/chronos_bolt_mini/config.json index e44993805fad5adad57935ddc34849451c41f753..3dc6cdda4be2be9261a58d37552685f687e4bd78 100644 --- a/results/chronos_bolt_mini/config.json +++ b/results/chronos_bolt_mini/config.json @@ -3,7 +3,7 @@ "model_type": "zero-shot", "model_dtype": "float32", "model_link": "https://tsfm.ai/models/amazon/chronos-bolt-mini", - "code_link": "https://github.com/zhouziyu02/TSFM_Bench/blob/main/scripts/run_online_eval.py", + "code_link": "https://github.com/zhouziyu02/TSFM_Bench/blob/main/space/scripts/run_space_eval.py", "org": "Amazon / TSFM.ai", "testdata_leakage": "No", "replication_code_available": "Yes", diff --git a/results/chronos_bolt_mini/online_meta.json b/results/chronos_bolt_mini/online_meta.json index 3ff45b42bd6309c2e027947db04c944324ddc1c4..108b3b99254189cd56191028d8b1be0063ea66ef 100644 --- a/results/chronos_bolt_mini/online_meta.json +++ b/results/chronos_bolt_mini/online_meta.json @@ -1,67 +1,49 @@ { "model": "Chronos-Bolt-Mini", "api_model_id": "amazon/chronos-bolt-mini", - "evaluated_at": "2026-05-28T12:18:55.358936+00:00", + "evaluated_at": "2026-05-29T02:51:10.536027+00:00", "datasets": [ { - "dataset": "task_binance_btcusdt_1h_crypto_btcusdt_close_20260527t120000z", - "data_fetched_at": "2026-05-28T12:16:39.187896Z", + "dataset": "task_binance_btcusdt_1h_crypto_btcusdt_close_20260528t020000z", + "data_fetched_at": "2026-05-29T02:50:05.083862Z", "context_length": 96, "prediction_length": 24 }, { - "dataset": "task_coingecko_bitcoin_7d_coin_bitcoin_price_usd_20260527t120016z", - "data_fetched_at": "2026-05-28T12:16:39.187896Z", + "dataset": "task_coingecko_bitcoin_7d_coin_bitcoin_price_usd_20260528t030012z", + "data_fetched_at": "2026-05-29T02:50:05.083862Z", "context_length": 96, "prediction_length": 24 }, { "dataset": "task_nasa_power_shanghai_hourly_point_shanghai_31.2304_121.4737_t2m_20260524t230000z", - "data_fetched_at": "2026-05-28T12:16:39.187896Z", - "context_length": 83, + "data_fetched_at": "2026-05-29T02:50:05.083862Z", + "context_length": 69, "prediction_length": 24 }, { - "dataset": "task_noaa_coops_sf_water_level_coops_station_9414290_water_level_20260528t094200z", - "data_fetched_at": "2026-05-28T12:16:39.187896Z", + "dataset": "task_noaa_coops_sf_water_level_coops_station_9414290_water_level_20260529t001800z", + "data_fetched_at": "2026-05-29T02:50:05.083862Z", "context_length": 96, "prediction_length": 24 }, { - "dataset": "task_noaa_ndbc_46013_realtime_ndbc_station_46013_wave_height_20260528t045000z", - "data_fetched_at": "2026-05-28T12:16:39.187896Z", + "dataset": "task_nws_ksfo_observations_nws_station_ksfo_temperature_20260529t004500z", + "data_fetched_at": "2026-05-29T02:50:05.083862Z", "context_length": 96, "prediction_length": 24 }, { - "dataset": "task_nws_ksfo_observations_nws_station_ksfo_temperature_20260528t101000z", - "data_fetched_at": "2026-05-28T12:16:39.187896Z", + "dataset": "task_open_meteo_shanghai_air_quality_hourly_grid_shanghai_31.2304_121.4737_pm2_5_20260528t020000z", + "data_fetched_at": "2026-05-29T02:50:05.083862Z", "context_length": 96, "prediction_length": 24 }, { - "dataset": "task_open_meteo_shanghai_air_quality_hourly_grid_shanghai_31.2304_121.4737_pm2_5_20260527t120000z", - "data_fetched_at": "2026-05-28T12:16:39.187896Z", + "dataset": "task_usgs_potomac_iv_usgs_site_01646500_usgs_00060_20260528t235000z", + "data_fetched_at": "2026-05-29T02:50:05.083862Z", "context_length": 96, "prediction_length": 24 - }, - { - "dataset": "task_open_meteo_shanghai_hourly_grid_shanghai_31.2304_121.4737_temperature_2m_20260527t120000z", - "data_fetched_at": "2026-05-28T12:16:39.187896Z", - "context_length": 96, - "prediction_length": 24 - }, - { - "dataset": "task_usgs_potomac_iv_usgs_site_01646500_usgs_00060_20260528t095000z", - "data_fetched_at": "2026-05-28T12:16:39.187896Z", - "context_length": 96, - "prediction_length": 24 - }, - { - "dataset": "task_wikimedia_time_series_hourly_enwiki_article_time_series_pageviews_20260525t000000z", - "data_fetched_at": "2026-05-28T12:16:39.187896Z", - "context_length": 4, - "prediction_length": 2 } ] } diff --git a/results/chronos_bolt_tiny/all_results.csv b/results/chronos_bolt_tiny/all_results.csv index 276c579057c16b0b20ee165117559a2176d60df2..7049039c0212df1abcc9ae97d9ec4dbdb9b68c9c 100644 --- a/results/chronos_bolt_tiny/all_results.csv +++ b/results/chronos_bolt_tiny/all_results.csv @@ -1,11 +1,8 @@ dataset,model,eval_metrics/MSE[mean],eval_metrics/MSE[0.5],eval_metrics/MAE[0.5],eval_metrics/MASE[0.5],eval_metrics/MAPE[0.5],eval_metrics/sMAPE[0.5],eval_metrics/MSIS,eval_metrics/RMSE[mean],eval_metrics/NRMSE[mean],eval_metrics/ND[0.5],eval_metrics/mean_weighted_sum_quantile_loss,domain,num_variates -binance_btcusdt_1h_close/1h/short,Chronos-Bolt-Tiny,4495006.537943522,4495006.537943522,1834.72265625,2.3641310092463494,0.024881456748435934,0.02447329243393806,42.788615799160766,2120.143046575755,0.028573328343718278,0.024726696135602966,0.018015038439380838,Finance,1 -coingecko_bitcoin_7d_price_usd/1h/short,Chronos-Bolt-Tiny,4406332.5283203125,4406332.5283203125,1807.1061197916667,2.2538886819785158,0.024529248640723117,0.02412824693875963,37.43977008949355,2099.1266108361147,0.028302453076857404,0.02436515067565812,0.017333996733501764,Finance,1 -nasa_power_shanghai_hourly_T2M/1h/short,Chronos-Bolt-Tiny,7.912413253015529,7.912413253015529,2.2673874696095786,0.7233865627167552,0.08212307663246772,0.0874627393188528,2.2266680874402787,2.812901216362837,0.10959533476675763,0.088341207054233,0.07361846408709673,Weather,1 -noaa_coops_sf_water_level_water_level/6min/short,Chronos-Bolt-Tiny,0.0028608058548383144,0.0028608058548383144,0.049041210363308586,1.8495096807224467,0.225792045612945,0.19658526579294686,13.967965661021097,0.05348650161338199,0.20818619230490099,0.19088372848461752,0.20820748121803842,Ocean,1 -noaa_ndbc_46013_realtime_wave_height/10min/short,Chronos-Bolt-Tiny,0.04937945652725754,0.04937945652725754,0.18734135230382284,1.348289762768264,0.07458714345199068,0.07090175042748963,7.0800555112695145,0.22221488817641705,0.0851942084062309,0.07182416237838214,0.06322130203989622,Ocean,1 -nws_ksfo_observations_temperature/1h/short,Chronos-Bolt-Tiny,0.003276212487397364,0.003276212487397364,0.02068618933359782,0.39194863412581116,0.001611035489054791,0.0016012503189502235,12.192836604810884,0.057238208282556886,0.004408591052225484,0.0015932879790811604,0.001700157311234707,Weather,1 -open_meteo_shanghai_air_quality_hourly_pm2_5/1h/short,Chronos-Bolt-Tiny,63.19570465734099,63.19570465734099,6.434090296427409,0.34607392203585513,0.14174756363352167,0.1365677615769388,4.458499596558531,7.949572608470281,0.1619744788596924,0.13109615749526107,0.20445340630382589,Air Quality,1 -open_meteo_shanghai_hourly_temperature_2m/1h/short,Chronos-Bolt-Tiny,0.5948811111434225,0.5948811111434225,0.6484154860178629,0.35314615324951076,0.026197955231929627,0.025978455964565026,2.995656525072318,0.7712853629775573,0.030779594213512538,0.025876240493323176,0.02777387188201828,Weather,1 -usgs_potomac_iv_usgs_00060/15min/short,Chronos-Bolt-Tiny,3149.2314387957254,3149.2314387957254,54.302978515625,nan,0.0021364303572376296,0.0021344827808550478,nan,56.11801349652111,0.002206836513053427,0.002135460403694904,0.0014279900766016713,Hydrology,1 -wikimedia_time_series_hourly_pageviews/1D/short,Chronos-Bolt-Tiny,4641.08992965892,4641.08992965892,67.62164306640625,1.4490352085658482,0.21732069855015004,0.24415806200991483,8.605713272094727,68.125545353112,0.21940594316622222,0.21778306945702497,0.2325466885637525,Web,1 +binance_btcusdt_1h_close/1h/short,Chronos-Bolt-Tiny,1029025.1603978475,1029025.1603978475,984.4554036458334,1.0893837703720461,0.01342267473206749,0.013327594824025202,11.50025719460482,1014.4087738174624,0.013820065318350654,0.013411987684401543,0.008981437172713323,Finance,1 +coingecko_bitcoin_7d_price_usd/1h/short,Chronos-Bolt-Tiny,1049676.785677592,1049676.785677592,991.6748046875,1.073191728111963,0.013543286102919765,0.013445979208987893,11.929663258797348,1024.537352016798,0.0139797310093777,0.013531324154281413,0.009152163608285595,Finance,1 +nasa_power_shanghai_hourly_T2M/1h/short,Chronos-Bolt-Tiny,9.325186076793065,9.325186076793065,2.57205859820048,0.7488524024372003,0.09445699084526753,0.10095493097514746,2.3972798950588126,3.0537167643370373,0.11897791117710119,0.10021170365662015,0.08079254781825497,Weather,1 +noaa_coops_sf_water_level_water_level/6min/short,Chronos-Bolt-Tiny,0.015442472707887234,0.015442472707887234,0.08998485902945201,5.64261497591828,0.06416666330862907,0.06807652745844346,21.565446339911507,0.12426774604814893,0.09736625819078952,0.07050493226238033,0.06671574848406818,Ocean,1 +nws_ksfo_observations_temperature/1h/short,Chronos-Bolt-Tiny,7.537212424866539,7.537212424866539,2.111232042312622,2.5984394366924577,0.13644075438988787,0.12216720632877787,52.66092036320614,2.745398409132368,0.16208994297460477,0.12464838626199982,0.09697055688920332,Weather,1 +open_meteo_shanghai_air_quality_hourly_pm2_5/1h/short,Chronos-Bolt-Tiny,1461.9665035634837,1461.9665035634837,32.67136557896932,1.4334785628798237,0.5069763645738149,0.7007815976613591,18.987324775681852,38.23567056510823,0.6516518430489527,0.5568192025874885,0.44639292146149856,Air Quality,1 +usgs_potomac_iv_usgs_00060/15min/short,Chronos-Bolt-Tiny,78316.96788072586,78316.96788072586,221.53458658854166,nan,0.007875539639408598,0.007920176132274283,nan,279.8516890796371,0.010053046756340802,0.00795813512666517,0.007018508513986596,Hydrology,1 diff --git a/results/chronos_bolt_tiny/config.json b/results/chronos_bolt_tiny/config.json index 67ab11eb177b8d074cb56c2dcca82b13ca851977..48073114ba76bab7f7a421684fea4efc7736035e 100644 --- a/results/chronos_bolt_tiny/config.json +++ b/results/chronos_bolt_tiny/config.json @@ -3,7 +3,7 @@ "model_type": "zero-shot", "model_dtype": "float32", "model_link": "https://tsfm.ai/models/amazon/chronos-bolt-tiny", - "code_link": "https://github.com/zhouziyu02/TSFM_Bench/blob/main/scripts/run_online_eval.py", + "code_link": "https://github.com/zhouziyu02/TSFM_Bench/blob/main/space/scripts/run_space_eval.py", "org": "Amazon / TSFM.ai", "testdata_leakage": "No", "replication_code_available": "Yes", diff --git a/results/chronos_bolt_tiny/online_meta.json b/results/chronos_bolt_tiny/online_meta.json index 644d5c32663dd67e408231d7b9440ea19c5069fb..247f8e5ea33508906ce1c9443c0c6f20e8e912da 100644 --- a/results/chronos_bolt_tiny/online_meta.json +++ b/results/chronos_bolt_tiny/online_meta.json @@ -1,67 +1,49 @@ { "model": "Chronos-Bolt-Tiny", "api_model_id": "amazon/chronos-bolt-tiny", - "evaluated_at": "2026-05-28T12:18:05.053451+00:00", + "evaluated_at": "2026-05-29T02:50:52.849106+00:00", "datasets": [ { - "dataset": "task_binance_btcusdt_1h_crypto_btcusdt_close_20260527t120000z", - "data_fetched_at": "2026-05-28T12:16:39.187896Z", + "dataset": "task_binance_btcusdt_1h_crypto_btcusdt_close_20260528t020000z", + "data_fetched_at": "2026-05-29T02:50:05.083862Z", "context_length": 96, "prediction_length": 24 }, { - "dataset": "task_coingecko_bitcoin_7d_coin_bitcoin_price_usd_20260527t120016z", - "data_fetched_at": "2026-05-28T12:16:39.187896Z", + "dataset": "task_coingecko_bitcoin_7d_coin_bitcoin_price_usd_20260528t030012z", + "data_fetched_at": "2026-05-29T02:50:05.083862Z", "context_length": 96, "prediction_length": 24 }, { "dataset": "task_nasa_power_shanghai_hourly_point_shanghai_31.2304_121.4737_t2m_20260524t230000z", - "data_fetched_at": "2026-05-28T12:16:39.187896Z", - "context_length": 83, + "data_fetched_at": "2026-05-29T02:50:05.083862Z", + "context_length": 69, "prediction_length": 24 }, { - "dataset": "task_noaa_coops_sf_water_level_coops_station_9414290_water_level_20260528t094200z", - "data_fetched_at": "2026-05-28T12:16:39.187896Z", + "dataset": "task_noaa_coops_sf_water_level_coops_station_9414290_water_level_20260529t001800z", + "data_fetched_at": "2026-05-29T02:50:05.083862Z", "context_length": 96, "prediction_length": 24 }, { - "dataset": "task_noaa_ndbc_46013_realtime_ndbc_station_46013_wave_height_20260528t045000z", - "data_fetched_at": "2026-05-28T12:16:39.187896Z", + "dataset": "task_nws_ksfo_observations_nws_station_ksfo_temperature_20260529t004500z", + "data_fetched_at": "2026-05-29T02:50:05.083862Z", "context_length": 96, "prediction_length": 24 }, { - "dataset": "task_nws_ksfo_observations_nws_station_ksfo_temperature_20260528t101000z", - "data_fetched_at": "2026-05-28T12:16:39.187896Z", + "dataset": "task_open_meteo_shanghai_air_quality_hourly_grid_shanghai_31.2304_121.4737_pm2_5_20260528t020000z", + "data_fetched_at": "2026-05-29T02:50:05.083862Z", "context_length": 96, "prediction_length": 24 }, { - "dataset": "task_open_meteo_shanghai_air_quality_hourly_grid_shanghai_31.2304_121.4737_pm2_5_20260527t120000z", - "data_fetched_at": "2026-05-28T12:16:39.187896Z", + "dataset": "task_usgs_potomac_iv_usgs_site_01646500_usgs_00060_20260528t235000z", + "data_fetched_at": "2026-05-29T02:50:05.083862Z", "context_length": 96, "prediction_length": 24 - }, - { - "dataset": "task_open_meteo_shanghai_hourly_grid_shanghai_31.2304_121.4737_temperature_2m_20260527t120000z", - "data_fetched_at": "2026-05-28T12:16:39.187896Z", - "context_length": 96, - "prediction_length": 24 - }, - { - "dataset": "task_usgs_potomac_iv_usgs_site_01646500_usgs_00060_20260528t095000z", - "data_fetched_at": "2026-05-28T12:16:39.187896Z", - "context_length": 96, - "prediction_length": 24 - }, - { - "dataset": "task_wikimedia_time_series_hourly_enwiki_article_time_series_pageviews_20260525t000000z", - "data_fetched_at": "2026-05-28T12:16:39.187896Z", - "context_length": 4, - "prediction_length": 2 } ] } diff --git a/results/dataset_properties.csv b/results/dataset_properties.csv index aa4f7ae201f56aa64f71cd7c42340b6b511a2146..dceaa4731889596b0d36f83575bbf5ac94afbf42 100644 --- a/results/dataset_properties.csv +++ b/results/dataset_properties.csv @@ -3,9 +3,6 @@ binance_btcusdt_1h_close,Finance,1h,1 coingecko_bitcoin_7d_price_usd,Finance,1h,1 nasa_power_shanghai_hourly_t2m,Weather,1h,1 noaa_coops_sf_water_level_water_level,Ocean,6min,1 -noaa_ndbc_46013_realtime_wave_height,Ocean,10min,1 nws_ksfo_observations_temperature,Weather,1h,1 open_meteo_shanghai_air_quality_hourly_pm2_5,Air Quality,1h,1 -open_meteo_shanghai_hourly_temperature_2m,Weather,1h,1 usgs_potomac_iv_usgs_00060,Hydrology,15min,1 -wikimedia_time_series_hourly_pageviews,Web,1D,1 diff --git a/results/eval_state.json b/results/eval_state.json new file mode 100644 index 0000000000000000000000000000000000000000..7f4b0fc1a2e72a3cbe699184be737e2cd3c9b982 --- /dev/null +++ b/results/eval_state.json @@ -0,0 +1,10 @@ +{ + "status": "ok", + "started_at": "2026-05-29T02:49:52.974580+00:00", + "finished_at": "2026-05-29T02:55:13.595519+00:00", + "failed_models": [], + "task_count": 7, + "model_count": 6, + "message": "Evaluation complete", + "updated_at": "2026-05-29T02:55:13.595519+00:00" +} diff --git a/results/moirai_1_1_small/all_results.csv b/results/moirai_1_1_small/all_results.csv index 9b7705496ca60b951418db08980d7d4a4f1c7229..2398704528b71c625c03f923445f71d7ab4abc93 100644 --- a/results/moirai_1_1_small/all_results.csv +++ b/results/moirai_1_1_small/all_results.csv @@ -1,11 +1,8 @@ dataset,model,eval_metrics/MSE[mean],eval_metrics/MSE[0.5],eval_metrics/MAE[0.5],eval_metrics/MASE[0.5],eval_metrics/MAPE[0.5],eval_metrics/sMAPE[0.5],eval_metrics/MSIS,eval_metrics/RMSE[mean],eval_metrics/NRMSE[mean],eval_metrics/ND[0.5],eval_metrics/mean_weighted_sum_quantile_loss,domain,num_variates -binance_btcusdt_1h_close/1h/short,Moirai-1.1-Small,8869335.108332315,7410361.406759898,2589.4026692708335,3.3365735823857965,0.035022303348886334,0.0343546582322679,33.181524624215,2978.1428958886972,0.04013665726762046,0.034897575803988286,0.022626002293161866,Finance,1 -coingecko_bitcoin_7d_price_usd/1h/short,Moirai-1.1-Small,3513908.5660069785,3177372.7269617715,1577.9192708333333,1.9680384824423314,0.021403528918522274,0.021113633378368957,50.807128803692336,1874.5422283872344,0.02527438944637319,0.021275032144936267,0.0176526858138831,Finance,1 -nasa_power_shanghai_hourly_T2M/1h/short,Moirai-1.1-Small,16.46809493020707,17.762606350991785,3.2289379835128784,1.03015932672555,0.1150119196766957,0.12686051090764325,15.835757291519093,4.058090059400736,0.15810997413863923,0.12580482285892922,0.10702413857726764,Weather,1 -noaa_coops_sf_water_level_water_level/6min/short,Moirai-1.1-Small,0.005106911324939271,0.005023066128869232,0.061788178980350494,2.3302409204783667,0.26746783672360525,0.22650807762100097,22.07225150197471,0.071462656856146,0.2781550105933736,0.24049891698571166,0.309963690852684,Ocean,1 -noaa_ndbc_46013_realtime_wave_height/10min/short,Moirai-1.1-Small,0.011788767184261436,0.009929538068242039,0.07437371214230855,0.5352652442580991,0.029046001335686772,0.02862860060252284,5.27525164074997,0.10857608937635135,0.04162661674999375,0.028513883944475068,0.039676113202017554,Ocean,1 -nws_ksfo_observations_temperature/1h/short,Moirai-1.1-Small,0.009181221596918476,0.0033276888748332567,0.017116785049438477,0.3243178534509565,0.0013366897324967708,0.001326615176037771,15.141958610055587,0.09581869127116314,0.007380130120228421,0.0013183659600123755,0.002086598079706136,Weather,1 -open_meteo_shanghai_air_quality_hourly_pm2_5/1h/short,Moirai-1.1-Small,187.9900532822021,186.3770823168015,9.616911133130392,0.5172700413546742,0.1997350052588335,0.24096720588809792,6.676616536919641,13.710946476527509,0.2793638752154284,0.1959469075568973,0.21197237663449722,Air Quality,1 -open_meteo_shanghai_hourly_temperature_2m/1h/short,Moirai-1.1-Small,0.7451148641512191,0.7336263607690702,0.7014484405517578,0.38202946077832767,0.028982091534334033,0.02845294724981401,3.3017538202957617,0.8632003615332996,0.03444763524408459,0.027992620368853683,0.026208127846003952,Weather,1 -usgs_potomac_iv_usgs_00060/15min/short,Moirai-1.1-Small,10798.155609925589,6040.659053166707,68.469970703125,nan,0.0026951572531489633,0.0026905473764017203,nan,103.91417424935632,0.004086416814655992,0.002692576268843192,0.0026380449775952625,Hydrology,1 -wikimedia_time_series_hourly_pageviews/1D/short,Moirai-1.1-Small,3089.0964928199537,2815.892555135797,52.98386764526367,1.1353685923985073,0.17051746000903956,0.18643314794028967,2.6726661137172156,55.579640992183045,0.17900045408110482,0.17064047550809555,0.13484585527060686,Web,1 +binance_btcusdt_1h_close/1h/short,Moirai-1.1-Small,952374.008026123,920160.1939131418,925.12646484375,1.0237312453261904,0.012614736997089234,0.012529663657250362,10.904908958873357,975.8965150189456,0.013295383409153661,0.012603704248102358,0.008648531284962633,Finance,1 +coingecko_bitcoin_7d_price_usd/1h/short,Moirai-1.1-Small,980882.1796239218,946688.7947781881,936.5074869791666,1.0134895871007623,0.012790573136249405,0.012702768980917836,11.206580901590142,990.3949614289855,0.013513860794401974,0.012778570474239172,0.008733789549838429,Finance,1 +nasa_power_shanghai_hourly_T2M/1h/short,Moirai-1.1-Small,22.11944576046153,23.27638368696455,3.802117625872294,1.106982912627002,0.1363775798751412,0.15233264027581442,7.944486966482941,4.703131484496423,0.18324186661696704,0.14813686012367747,0.13622131038191412,Weather,1 +noaa_coops_sf_water_level_water_level/6min/short,Moirai-1.1-Small,0.07894352857742426,0.08123021915922779,0.24160874634981155,15.1503835775314,0.17799479341783764,0.2016127114400773,229.84756618905703,0.2809689103396037,0.2201447466272617,0.18930527289948496,0.1584442261775357,Ocean,1 +nws_ksfo_observations_temperature/1h/short,Moirai-1.1-Small,7.333502640536911,7.098776238255065,2.0732465585072837,2.551688072008964,0.13368541403511328,0.12020856200648522,67.84784727830153,2.708044061779075,0.1598845202526391,0.12240570087127874,0.1052653917054816,Weather,1 +open_meteo_shanghai_air_quality_hourly_pm2_5/1h/short,Moirai-1.1-Small,973.8622120598106,1023.9657259982699,23.962568004926045,1.051374098936426,0.3256804509855051,0.4334222612374714,12.957339274599958,31.206765485384906,0.5318579730285935,0.4083948672485306,0.3651104991221374,Air Quality,1 +usgs_potomac_iv_usgs_00060/15min/short,Moirai-1.1-Small,96855.45377286275,112976.57937427361,257.6378173828125,nan,0.009145275820772208,0.009214959719790085,nan,311.2160885508054,0.011179742740935982,0.009255063040244723,0.008606347849519367,Hydrology,1 diff --git a/results/moirai_1_1_small/config.json b/results/moirai_1_1_small/config.json index b247a1e751a00ba00c8ee5d1fb4483efc762c270..52b9f26d3e58e0a8a24e99ea7726ccf0ef2f8176 100644 --- a/results/moirai_1_1_small/config.json +++ b/results/moirai_1_1_small/config.json @@ -3,7 +3,7 @@ "model_type": "zero-shot", "model_dtype": "float32", "model_link": "https://tsfm.ai/models/Salesforce/moirai-1.1-R-small", - "code_link": "https://github.com/zhouziyu02/TSFM_Bench/blob/main/scripts/run_online_eval.py", + "code_link": "https://github.com/zhouziyu02/TSFM_Bench/blob/main/space/scripts/run_space_eval.py", "org": "Salesforce / TSFM.ai", "testdata_leakage": "No", "replication_code_available": "Yes", diff --git a/results/moirai_1_1_small/online_meta.json b/results/moirai_1_1_small/online_meta.json index ff7c296352bb417e303c26ab2d78091b3883bd01..4c98616204e7dca007fdf181eb3d1609c99753bf 100644 --- a/results/moirai_1_1_small/online_meta.json +++ b/results/moirai_1_1_small/online_meta.json @@ -1,67 +1,49 @@ { "model": "Moirai-1.1-Small", "api_model_id": "Salesforce/moirai-1.1-R-small", - "evaluated_at": "2026-05-28T12:21:49.771336+00:00", + "evaluated_at": "2026-05-29T02:55:13.592899+00:00", "datasets": [ { - "dataset": "task_binance_btcusdt_1h_crypto_btcusdt_close_20260527t120000z", - "data_fetched_at": "2026-05-28T12:16:39.187896Z", + "dataset": "task_binance_btcusdt_1h_crypto_btcusdt_close_20260528t020000z", + "data_fetched_at": "2026-05-29T02:50:05.083862Z", "context_length": 96, "prediction_length": 24 }, { - "dataset": "task_coingecko_bitcoin_7d_coin_bitcoin_price_usd_20260527t120016z", - "data_fetched_at": "2026-05-28T12:16:39.187896Z", + "dataset": "task_coingecko_bitcoin_7d_coin_bitcoin_price_usd_20260528t030012z", + "data_fetched_at": "2026-05-29T02:50:05.083862Z", "context_length": 96, "prediction_length": 24 }, { "dataset": "task_nasa_power_shanghai_hourly_point_shanghai_31.2304_121.4737_t2m_20260524t230000z", - "data_fetched_at": "2026-05-28T12:16:39.187896Z", - "context_length": 83, + "data_fetched_at": "2026-05-29T02:50:05.083862Z", + "context_length": 69, "prediction_length": 24 }, { - "dataset": "task_noaa_coops_sf_water_level_coops_station_9414290_water_level_20260528t094200z", - "data_fetched_at": "2026-05-28T12:16:39.187896Z", + "dataset": "task_noaa_coops_sf_water_level_coops_station_9414290_water_level_20260529t001800z", + "data_fetched_at": "2026-05-29T02:50:05.083862Z", "context_length": 96, "prediction_length": 24 }, { - "dataset": "task_noaa_ndbc_46013_realtime_ndbc_station_46013_wave_height_20260528t045000z", - "data_fetched_at": "2026-05-28T12:16:39.187896Z", + "dataset": "task_nws_ksfo_observations_nws_station_ksfo_temperature_20260529t004500z", + "data_fetched_at": "2026-05-29T02:50:05.083862Z", "context_length": 96, "prediction_length": 24 }, { - "dataset": "task_nws_ksfo_observations_nws_station_ksfo_temperature_20260528t101000z", - "data_fetched_at": "2026-05-28T12:16:39.187896Z", + "dataset": "task_open_meteo_shanghai_air_quality_hourly_grid_shanghai_31.2304_121.4737_pm2_5_20260528t020000z", + "data_fetched_at": "2026-05-29T02:50:05.083862Z", "context_length": 96, "prediction_length": 24 }, { - "dataset": "task_open_meteo_shanghai_air_quality_hourly_grid_shanghai_31.2304_121.4737_pm2_5_20260527t120000z", - "data_fetched_at": "2026-05-28T12:16:39.187896Z", + "dataset": "task_usgs_potomac_iv_usgs_site_01646500_usgs_00060_20260528t235000z", + "data_fetched_at": "2026-05-29T02:50:05.083862Z", "context_length": 96, "prediction_length": 24 - }, - { - "dataset": "task_open_meteo_shanghai_hourly_grid_shanghai_31.2304_121.4737_temperature_2m_20260527t120000z", - "data_fetched_at": "2026-05-28T12:16:39.187896Z", - "context_length": 96, - "prediction_length": 24 - }, - { - "dataset": "task_usgs_potomac_iv_usgs_site_01646500_usgs_00060_20260528t095000z", - "data_fetched_at": "2026-05-28T12:16:39.187896Z", - "context_length": 96, - "prediction_length": 24 - }, - { - "dataset": "task_wikimedia_time_series_hourly_enwiki_article_time_series_pageviews_20260525t000000z", - "data_fetched_at": "2026-05-28T12:16:39.187896Z", - "context_length": 4, - "prediction_length": 2 } ] } diff --git a/results/online_status.json b/results/online_status.json index 5991b136d81eeaeb13becfe264520360e1682226..82b2d7ce12ea58eb6e8903ba52de179cb20e9b0c 100644 --- a/results/online_status.json +++ b/results/online_status.json @@ -1,411 +1,302 @@ { "status": "ok", - "started_at": "2026-05-28T12:16:39.312039+00:00", - "finished_at": "2026-05-28T12:21:49.773564+00:00", + "started_at": "2026-05-29T02:49:52.974580+00:00", + "finished_at": "2026-05-29T02:55:13.595519+00:00", "data_source": "ts_bench", - "data_config": "configs/datasets/ts_bench.yaml", - "ts_bench_root": "/Users/zhouziyu/Desktop/TS-Bench", - "baseline_model": "Chronos-Bolt-Base", + "data_config": "/Users/zhouziyu/Desktop/TSFM_Bench/space/configs/datasets/ts_bench.yaml", + "ts_bench_root": "/Users/zhouziyu/Desktop/TSFM_Bench/space/vendor/ts_bench", "models": { "Chronos-Bolt-Tiny": { "model": "Chronos-Bolt-Tiny", "api_model_id": "amazon/chronos-bolt-tiny", - "evaluated_at": "2026-05-28T12:18:05.053451+00:00", + "evaluated_at": "2026-05-29T02:50:52.849106+00:00", "datasets": [ { - "dataset": "task_binance_btcusdt_1h_crypto_btcusdt_close_20260527t120000z", - "data_fetched_at": "2026-05-28T12:16:39.187896Z", + "dataset": "task_binance_btcusdt_1h_crypto_btcusdt_close_20260528t020000z", + "data_fetched_at": "2026-05-29T02:50:05.083862Z", "context_length": 96, "prediction_length": 24 }, { - "dataset": "task_coingecko_bitcoin_7d_coin_bitcoin_price_usd_20260527t120016z", - "data_fetched_at": "2026-05-28T12:16:39.187896Z", + "dataset": "task_coingecko_bitcoin_7d_coin_bitcoin_price_usd_20260528t030012z", + "data_fetched_at": "2026-05-29T02:50:05.083862Z", "context_length": 96, "prediction_length": 24 }, { "dataset": "task_nasa_power_shanghai_hourly_point_shanghai_31.2304_121.4737_t2m_20260524t230000z", - "data_fetched_at": "2026-05-28T12:16:39.187896Z", - "context_length": 83, + "data_fetched_at": "2026-05-29T02:50:05.083862Z", + "context_length": 69, "prediction_length": 24 }, { - "dataset": "task_noaa_coops_sf_water_level_coops_station_9414290_water_level_20260528t094200z", - "data_fetched_at": "2026-05-28T12:16:39.187896Z", + "dataset": "task_noaa_coops_sf_water_level_coops_station_9414290_water_level_20260529t001800z", + "data_fetched_at": "2026-05-29T02:50:05.083862Z", "context_length": 96, "prediction_length": 24 }, { - "dataset": "task_noaa_ndbc_46013_realtime_ndbc_station_46013_wave_height_20260528t045000z", - "data_fetched_at": "2026-05-28T12:16:39.187896Z", + "dataset": "task_nws_ksfo_observations_nws_station_ksfo_temperature_20260529t004500z", + "data_fetched_at": "2026-05-29T02:50:05.083862Z", "context_length": 96, "prediction_length": 24 }, { - "dataset": "task_nws_ksfo_observations_nws_station_ksfo_temperature_20260528t101000z", - "data_fetched_at": "2026-05-28T12:16:39.187896Z", + "dataset": "task_open_meteo_shanghai_air_quality_hourly_grid_shanghai_31.2304_121.4737_pm2_5_20260528t020000z", + "data_fetched_at": "2026-05-29T02:50:05.083862Z", "context_length": 96, "prediction_length": 24 }, { - "dataset": "task_open_meteo_shanghai_air_quality_hourly_grid_shanghai_31.2304_121.4737_pm2_5_20260527t120000z", - "data_fetched_at": "2026-05-28T12:16:39.187896Z", + "dataset": "task_usgs_potomac_iv_usgs_site_01646500_usgs_00060_20260528t235000z", + "data_fetched_at": "2026-05-29T02:50:05.083862Z", "context_length": 96, "prediction_length": 24 - }, - { - "dataset": "task_open_meteo_shanghai_hourly_grid_shanghai_31.2304_121.4737_temperature_2m_20260527t120000z", - "data_fetched_at": "2026-05-28T12:16:39.187896Z", - "context_length": 96, - "prediction_length": 24 - }, - { - "dataset": "task_usgs_potomac_iv_usgs_site_01646500_usgs_00060_20260528t095000z", - "data_fetched_at": "2026-05-28T12:16:39.187896Z", - "context_length": 96, - "prediction_length": 24 - }, - { - "dataset": "task_wikimedia_time_series_hourly_enwiki_article_time_series_pageviews_20260525t000000z", - "data_fetched_at": "2026-05-28T12:16:39.187896Z", - "context_length": 4, - "prediction_length": 2 } ] }, "Chronos-Bolt-Mini": { "model": "Chronos-Bolt-Mini", "api_model_id": "amazon/chronos-bolt-mini", - "evaluated_at": "2026-05-28T12:18:55.358936+00:00", + "evaluated_at": "2026-05-29T02:51:10.536027+00:00", "datasets": [ { - "dataset": "task_binance_btcusdt_1h_crypto_btcusdt_close_20260527t120000z", - "data_fetched_at": "2026-05-28T12:16:39.187896Z", + "dataset": "task_binance_btcusdt_1h_crypto_btcusdt_close_20260528t020000z", + "data_fetched_at": "2026-05-29T02:50:05.083862Z", "context_length": 96, "prediction_length": 24 }, { - "dataset": "task_coingecko_bitcoin_7d_coin_bitcoin_price_usd_20260527t120016z", - "data_fetched_at": "2026-05-28T12:16:39.187896Z", + "dataset": "task_coingecko_bitcoin_7d_coin_bitcoin_price_usd_20260528t030012z", + "data_fetched_at": "2026-05-29T02:50:05.083862Z", "context_length": 96, "prediction_length": 24 }, { "dataset": "task_nasa_power_shanghai_hourly_point_shanghai_31.2304_121.4737_t2m_20260524t230000z", - "data_fetched_at": "2026-05-28T12:16:39.187896Z", - "context_length": 83, - "prediction_length": 24 - }, - { - "dataset": "task_noaa_coops_sf_water_level_coops_station_9414290_water_level_20260528t094200z", - "data_fetched_at": "2026-05-28T12:16:39.187896Z", - "context_length": 96, - "prediction_length": 24 - }, - { - "dataset": "task_noaa_ndbc_46013_realtime_ndbc_station_46013_wave_height_20260528t045000z", - "data_fetched_at": "2026-05-28T12:16:39.187896Z", - "context_length": 96, + "data_fetched_at": "2026-05-29T02:50:05.083862Z", + "context_length": 69, "prediction_length": 24 }, { - "dataset": "task_nws_ksfo_observations_nws_station_ksfo_temperature_20260528t101000z", - "data_fetched_at": "2026-05-28T12:16:39.187896Z", + "dataset": "task_noaa_coops_sf_water_level_coops_station_9414290_water_level_20260529t001800z", + "data_fetched_at": "2026-05-29T02:50:05.083862Z", "context_length": 96, "prediction_length": 24 }, { - "dataset": "task_open_meteo_shanghai_air_quality_hourly_grid_shanghai_31.2304_121.4737_pm2_5_20260527t120000z", - "data_fetched_at": "2026-05-28T12:16:39.187896Z", + "dataset": "task_nws_ksfo_observations_nws_station_ksfo_temperature_20260529t004500z", + "data_fetched_at": "2026-05-29T02:50:05.083862Z", "context_length": 96, "prediction_length": 24 }, { - "dataset": "task_open_meteo_shanghai_hourly_grid_shanghai_31.2304_121.4737_temperature_2m_20260527t120000z", - "data_fetched_at": "2026-05-28T12:16:39.187896Z", + "dataset": "task_open_meteo_shanghai_air_quality_hourly_grid_shanghai_31.2304_121.4737_pm2_5_20260528t020000z", + "data_fetched_at": "2026-05-29T02:50:05.083862Z", "context_length": 96, "prediction_length": 24 }, { - "dataset": "task_usgs_potomac_iv_usgs_site_01646500_usgs_00060_20260528t095000z", - "data_fetched_at": "2026-05-28T12:16:39.187896Z", + "dataset": "task_usgs_potomac_iv_usgs_site_01646500_usgs_00060_20260528t235000z", + "data_fetched_at": "2026-05-29T02:50:05.083862Z", "context_length": 96, "prediction_length": 24 - }, - { - "dataset": "task_wikimedia_time_series_hourly_enwiki_article_time_series_pageviews_20260525t000000z", - "data_fetched_at": "2026-05-28T12:16:39.187896Z", - "context_length": 4, - "prediction_length": 2 } ] }, "Chronos-Bolt-Base": { "model": "Chronos-Bolt-Base", "api_model_id": "amazon/chronos-bolt-base", - "evaluated_at": "2026-05-28T12:19:28.056546+00:00", + "evaluated_at": "2026-05-29T02:51:30.013825+00:00", "datasets": [ { - "dataset": "task_binance_btcusdt_1h_crypto_btcusdt_close_20260527t120000z", - "data_fetched_at": "2026-05-28T12:16:39.187896Z", + "dataset": "task_binance_btcusdt_1h_crypto_btcusdt_close_20260528t020000z", + "data_fetched_at": "2026-05-29T02:50:05.083862Z", "context_length": 96, "prediction_length": 24 }, { - "dataset": "task_coingecko_bitcoin_7d_coin_bitcoin_price_usd_20260527t120016z", - "data_fetched_at": "2026-05-28T12:16:39.187896Z", + "dataset": "task_coingecko_bitcoin_7d_coin_bitcoin_price_usd_20260528t030012z", + "data_fetched_at": "2026-05-29T02:50:05.083862Z", "context_length": 96, "prediction_length": 24 }, { "dataset": "task_nasa_power_shanghai_hourly_point_shanghai_31.2304_121.4737_t2m_20260524t230000z", - "data_fetched_at": "2026-05-28T12:16:39.187896Z", - "context_length": 83, + "data_fetched_at": "2026-05-29T02:50:05.083862Z", + "context_length": 69, "prediction_length": 24 }, { - "dataset": "task_noaa_coops_sf_water_level_coops_station_9414290_water_level_20260528t094200z", - "data_fetched_at": "2026-05-28T12:16:39.187896Z", + "dataset": "task_noaa_coops_sf_water_level_coops_station_9414290_water_level_20260529t001800z", + "data_fetched_at": "2026-05-29T02:50:05.083862Z", "context_length": 96, "prediction_length": 24 }, { - "dataset": "task_noaa_ndbc_46013_realtime_ndbc_station_46013_wave_height_20260528t045000z", - "data_fetched_at": "2026-05-28T12:16:39.187896Z", + "dataset": "task_nws_ksfo_observations_nws_station_ksfo_temperature_20260529t004500z", + "data_fetched_at": "2026-05-29T02:50:05.083862Z", "context_length": 96, "prediction_length": 24 }, { - "dataset": "task_nws_ksfo_observations_nws_station_ksfo_temperature_20260528t101000z", - "data_fetched_at": "2026-05-28T12:16:39.187896Z", + "dataset": "task_open_meteo_shanghai_air_quality_hourly_grid_shanghai_31.2304_121.4737_pm2_5_20260528t020000z", + "data_fetched_at": "2026-05-29T02:50:05.083862Z", "context_length": 96, "prediction_length": 24 }, { - "dataset": "task_open_meteo_shanghai_air_quality_hourly_grid_shanghai_31.2304_121.4737_pm2_5_20260527t120000z", - "data_fetched_at": "2026-05-28T12:16:39.187896Z", + "dataset": "task_usgs_potomac_iv_usgs_site_01646500_usgs_00060_20260528t235000z", + "data_fetched_at": "2026-05-29T02:50:05.083862Z", "context_length": 96, "prediction_length": 24 - }, - { - "dataset": "task_open_meteo_shanghai_hourly_grid_shanghai_31.2304_121.4737_temperature_2m_20260527t120000z", - "data_fetched_at": "2026-05-28T12:16:39.187896Z", - "context_length": 96, - "prediction_length": 24 - }, - { - "dataset": "task_usgs_potomac_iv_usgs_site_01646500_usgs_00060_20260528t095000z", - "data_fetched_at": "2026-05-28T12:16:39.187896Z", - "context_length": 96, - "prediction_length": 24 - }, - { - "dataset": "task_wikimedia_time_series_hourly_enwiki_article_time_series_pageviews_20260525t000000z", - "data_fetched_at": "2026-05-28T12:16:39.187896Z", - "context_length": 4, - "prediction_length": 2 } ] }, "Chronos-2": { "model": "Chronos-2", "api_model_id": "amazon/chronos-2", - "evaluated_at": "2026-05-28T12:20:18.799459+00:00", + "evaluated_at": "2026-05-29T02:53:16.309267+00:00", "datasets": [ { - "dataset": "task_binance_btcusdt_1h_crypto_btcusdt_close_20260527t120000z", - "data_fetched_at": "2026-05-28T12:16:39.187896Z", + "dataset": "task_binance_btcusdt_1h_crypto_btcusdt_close_20260528t020000z", + "data_fetched_at": "2026-05-29T02:50:05.083862Z", "context_length": 96, "prediction_length": 24 }, { - "dataset": "task_coingecko_bitcoin_7d_coin_bitcoin_price_usd_20260527t120016z", - "data_fetched_at": "2026-05-28T12:16:39.187896Z", + "dataset": "task_coingecko_bitcoin_7d_coin_bitcoin_price_usd_20260528t030012z", + "data_fetched_at": "2026-05-29T02:50:05.083862Z", "context_length": 96, "prediction_length": 24 }, { "dataset": "task_nasa_power_shanghai_hourly_point_shanghai_31.2304_121.4737_t2m_20260524t230000z", - "data_fetched_at": "2026-05-28T12:16:39.187896Z", - "context_length": 83, + "data_fetched_at": "2026-05-29T02:50:05.083862Z", + "context_length": 69, "prediction_length": 24 }, { - "dataset": "task_noaa_coops_sf_water_level_coops_station_9414290_water_level_20260528t094200z", - "data_fetched_at": "2026-05-28T12:16:39.187896Z", + "dataset": "task_noaa_coops_sf_water_level_coops_station_9414290_water_level_20260529t001800z", + "data_fetched_at": "2026-05-29T02:50:05.083862Z", "context_length": 96, "prediction_length": 24 }, { - "dataset": "task_noaa_ndbc_46013_realtime_ndbc_station_46013_wave_height_20260528t045000z", - "data_fetched_at": "2026-05-28T12:16:39.187896Z", + "dataset": "task_nws_ksfo_observations_nws_station_ksfo_temperature_20260529t004500z", + "data_fetched_at": "2026-05-29T02:50:05.083862Z", "context_length": 96, "prediction_length": 24 }, { - "dataset": "task_nws_ksfo_observations_nws_station_ksfo_temperature_20260528t101000z", - "data_fetched_at": "2026-05-28T12:16:39.187896Z", + "dataset": "task_open_meteo_shanghai_air_quality_hourly_grid_shanghai_31.2304_121.4737_pm2_5_20260528t020000z", + "data_fetched_at": "2026-05-29T02:50:05.083862Z", "context_length": 96, "prediction_length": 24 }, { - "dataset": "task_open_meteo_shanghai_air_quality_hourly_grid_shanghai_31.2304_121.4737_pm2_5_20260527t120000z", - "data_fetched_at": "2026-05-28T12:16:39.187896Z", + "dataset": "task_usgs_potomac_iv_usgs_site_01646500_usgs_00060_20260528t235000z", + "data_fetched_at": "2026-05-29T02:50:05.083862Z", "context_length": 96, "prediction_length": 24 - }, - { - "dataset": "task_open_meteo_shanghai_hourly_grid_shanghai_31.2304_121.4737_temperature_2m_20260527t120000z", - "data_fetched_at": "2026-05-28T12:16:39.187896Z", - "context_length": 96, - "prediction_length": 24 - }, - { - "dataset": "task_usgs_potomac_iv_usgs_site_01646500_usgs_00060_20260528t095000z", - "data_fetched_at": "2026-05-28T12:16:39.187896Z", - "context_length": 96, - "prediction_length": 24 - }, - { - "dataset": "task_wikimedia_time_series_hourly_enwiki_article_time_series_pageviews_20260525t000000z", - "data_fetched_at": "2026-05-28T12:16:39.187896Z", - "context_length": 4, - "prediction_length": 2 } ] }, "TimesFM-2.5": { "model": "TimesFM-2.5", "api_model_id": "google/timesfm-2.5-200m-pytorch", - "evaluated_at": "2026-05-28T12:21:19.114949+00:00", + "evaluated_at": "2026-05-29T02:54:46.508756+00:00", "datasets": [ { - "dataset": "task_binance_btcusdt_1h_crypto_btcusdt_close_20260527t120000z", - "data_fetched_at": "2026-05-28T12:16:39.187896Z", + "dataset": "task_binance_btcusdt_1h_crypto_btcusdt_close_20260528t020000z", + "data_fetched_at": "2026-05-29T02:50:05.083862Z", "context_length": 96, "prediction_length": 24 }, { - "dataset": "task_coingecko_bitcoin_7d_coin_bitcoin_price_usd_20260527t120016z", - "data_fetched_at": "2026-05-28T12:16:39.187896Z", + "dataset": "task_coingecko_bitcoin_7d_coin_bitcoin_price_usd_20260528t030012z", + "data_fetched_at": "2026-05-29T02:50:05.083862Z", "context_length": 96, "prediction_length": 24 }, { "dataset": "task_nasa_power_shanghai_hourly_point_shanghai_31.2304_121.4737_t2m_20260524t230000z", - "data_fetched_at": "2026-05-28T12:16:39.187896Z", - "context_length": 83, - "prediction_length": 24 - }, - { - "dataset": "task_noaa_coops_sf_water_level_coops_station_9414290_water_level_20260528t094200z", - "data_fetched_at": "2026-05-28T12:16:39.187896Z", - "context_length": 96, - "prediction_length": 24 - }, - { - "dataset": "task_noaa_ndbc_46013_realtime_ndbc_station_46013_wave_height_20260528t045000z", - "data_fetched_at": "2026-05-28T12:16:39.187896Z", - "context_length": 96, + "data_fetched_at": "2026-05-29T02:50:05.083862Z", + "context_length": 69, "prediction_length": 24 }, { - "dataset": "task_nws_ksfo_observations_nws_station_ksfo_temperature_20260528t101000z", - "data_fetched_at": "2026-05-28T12:16:39.187896Z", + "dataset": "task_noaa_coops_sf_water_level_coops_station_9414290_water_level_20260529t001800z", + "data_fetched_at": "2026-05-29T02:50:05.083862Z", "context_length": 96, "prediction_length": 24 }, { - "dataset": "task_open_meteo_shanghai_air_quality_hourly_grid_shanghai_31.2304_121.4737_pm2_5_20260527t120000z", - "data_fetched_at": "2026-05-28T12:16:39.187896Z", + "dataset": "task_nws_ksfo_observations_nws_station_ksfo_temperature_20260529t004500z", + "data_fetched_at": "2026-05-29T02:50:05.083862Z", "context_length": 96, "prediction_length": 24 }, { - "dataset": "task_open_meteo_shanghai_hourly_grid_shanghai_31.2304_121.4737_temperature_2m_20260527t120000z", - "data_fetched_at": "2026-05-28T12:16:39.187896Z", + "dataset": "task_open_meteo_shanghai_air_quality_hourly_grid_shanghai_31.2304_121.4737_pm2_5_20260528t020000z", + "data_fetched_at": "2026-05-29T02:50:05.083862Z", "context_length": 96, "prediction_length": 24 }, { - "dataset": "task_usgs_potomac_iv_usgs_site_01646500_usgs_00060_20260528t095000z", - "data_fetched_at": "2026-05-28T12:16:39.187896Z", + "dataset": "task_usgs_potomac_iv_usgs_site_01646500_usgs_00060_20260528t235000z", + "data_fetched_at": "2026-05-29T02:50:05.083862Z", "context_length": 96, "prediction_length": 24 - }, - { - "dataset": "task_wikimedia_time_series_hourly_enwiki_article_time_series_pageviews_20260525t000000z", - "data_fetched_at": "2026-05-28T12:16:39.187896Z", - "context_length": 4, - "prediction_length": 2 } ] }, "Moirai-1.1-Small": { "model": "Moirai-1.1-Small", "api_model_id": "Salesforce/moirai-1.1-R-small", - "evaluated_at": "2026-05-28T12:21:49.771336+00:00", + "evaluated_at": "2026-05-29T02:55:13.592899+00:00", "datasets": [ { - "dataset": "task_binance_btcusdt_1h_crypto_btcusdt_close_20260527t120000z", - "data_fetched_at": "2026-05-28T12:16:39.187896Z", + "dataset": "task_binance_btcusdt_1h_crypto_btcusdt_close_20260528t020000z", + "data_fetched_at": "2026-05-29T02:50:05.083862Z", "context_length": 96, "prediction_length": 24 }, { - "dataset": "task_coingecko_bitcoin_7d_coin_bitcoin_price_usd_20260527t120016z", - "data_fetched_at": "2026-05-28T12:16:39.187896Z", + "dataset": "task_coingecko_bitcoin_7d_coin_bitcoin_price_usd_20260528t030012z", + "data_fetched_at": "2026-05-29T02:50:05.083862Z", "context_length": 96, "prediction_length": 24 }, { "dataset": "task_nasa_power_shanghai_hourly_point_shanghai_31.2304_121.4737_t2m_20260524t230000z", - "data_fetched_at": "2026-05-28T12:16:39.187896Z", - "context_length": 83, + "data_fetched_at": "2026-05-29T02:50:05.083862Z", + "context_length": 69, "prediction_length": 24 }, { - "dataset": "task_noaa_coops_sf_water_level_coops_station_9414290_water_level_20260528t094200z", - "data_fetched_at": "2026-05-28T12:16:39.187896Z", + "dataset": "task_noaa_coops_sf_water_level_coops_station_9414290_water_level_20260529t001800z", + "data_fetched_at": "2026-05-29T02:50:05.083862Z", "context_length": 96, "prediction_length": 24 }, { - "dataset": "task_noaa_ndbc_46013_realtime_ndbc_station_46013_wave_height_20260528t045000z", - "data_fetched_at": "2026-05-28T12:16:39.187896Z", + "dataset": "task_nws_ksfo_observations_nws_station_ksfo_temperature_20260529t004500z", + "data_fetched_at": "2026-05-29T02:50:05.083862Z", "context_length": 96, "prediction_length": 24 }, { - "dataset": "task_nws_ksfo_observations_nws_station_ksfo_temperature_20260528t101000z", - "data_fetched_at": "2026-05-28T12:16:39.187896Z", + "dataset": "task_open_meteo_shanghai_air_quality_hourly_grid_shanghai_31.2304_121.4737_pm2_5_20260528t020000z", + "data_fetched_at": "2026-05-29T02:50:05.083862Z", "context_length": 96, "prediction_length": 24 }, { - "dataset": "task_open_meteo_shanghai_air_quality_hourly_grid_shanghai_31.2304_121.4737_pm2_5_20260527t120000z", - "data_fetched_at": "2026-05-28T12:16:39.187896Z", + "dataset": "task_usgs_potomac_iv_usgs_site_01646500_usgs_00060_20260528t235000z", + "data_fetched_at": "2026-05-29T02:50:05.083862Z", "context_length": 96, "prediction_length": 24 - }, - { - "dataset": "task_open_meteo_shanghai_hourly_grid_shanghai_31.2304_121.4737_temperature_2m_20260527t120000z", - "data_fetched_at": "2026-05-28T12:16:39.187896Z", - "context_length": 96, - "prediction_length": 24 - }, - { - "dataset": "task_usgs_potomac_iv_usgs_site_01646500_usgs_00060_20260528t095000z", - "data_fetched_at": "2026-05-28T12:16:39.187896Z", - "context_length": 96, - "prediction_length": 24 - }, - { - "dataset": "task_wikimedia_time_series_hourly_enwiki_article_time_series_pageviews_20260525t000000z", - "data_fetched_at": "2026-05-28T12:16:39.187896Z", - "context_length": 4, - "prediction_length": 2 } ] } diff --git a/results/timesfm_2_5/all_results.csv b/results/timesfm_2_5/all_results.csv index 7ea7bb995bdfd903aac184c42b6db92b3ec428f4..8ae5294e41c81c4548c5437e0abd4249ec467184 100644 --- a/results/timesfm_2_5/all_results.csv +++ b/results/timesfm_2_5/all_results.csv @@ -1,11 +1,8 @@ dataset,model,eval_metrics/MSE[mean],eval_metrics/MSE[0.5],eval_metrics/MAE[0.5],eval_metrics/MASE[0.5],eval_metrics/MAPE[0.5],eval_metrics/sMAPE[0.5],eval_metrics/MSIS,eval_metrics/RMSE[mean],eval_metrics/NRMSE[mean],eval_metrics/ND[0.5],eval_metrics/mean_weighted_sum_quantile_loss,domain,num_variates -binance_btcusdt_1h_close/1h/short,TimesFM-2.5,3184300.868250529,2877341.8473765054,1502.9791666666667,1.9366630929550903,0.020371679342378764,0.0201094796089761,86.07571948209704,1784.4609461264567,0.024049315263214986,0.02025576401191205,0.021796525719852494,Finance,1 -coingecko_bitcoin_7d_price_usd/1h/short,TimesFM-2.5,2978472.513104757,2691326.2771759033,1431.572265625,1.7855091583102543,0.01942591517249738,0.019179939216192834,78.84789791391036,1725.8251687539955,0.023269242362690887,0.01930184042488197,0.020704134741179717,Finance,1 -nasa_power_shanghai_hourly_T2M/1h/short,TimesFM-2.5,7.424815583824056,11.590662498969019,2.684919834136963,0.8565959528394198,0.09649247329369882,0.10426336051909445,5.473512568019115,2.7248514792230525,0.1061647697821695,0.10460896612097444,0.06960420118115515,Weather,1 -noaa_coops_sf_water_level_water_level/6min/short,TimesFM-2.5,0.014191201840556963,0.008006066655363927,0.08215828488270442,3.0984664146014134,0.37820721429781673,0.3032819543290635,207.6676558778752,0.11912683090117425,0.4636788830559884,0.31978573995484816,0.46509102471709085,Ocean,1 -noaa_ndbc_46013_realtime_wave_height/10min/short,TimesFM-2.5,0.05512604274710261,0.031698439646326904,0.15231487154960632,1.09620529318399,0.06015022141194603,0.05812306444413484,72.78888463699143,0.23478935824926694,0.09001509161876256,0.05839547933378565,0.08648648191844961,Ocean,1 -nws_ksfo_observations_temperature/1h/short,TimesFM-2.5,0.0034255919382151965,0.0033221443641195947,0.0194094975789388,0.36775869844608894,0.0015129974061875602,0.0015030149122856155,15.604036901152464,0.05852855660457719,0.0045079760301519585,0.0014949548548461198,0.0016153432104989958,Weather,1 -open_meteo_shanghai_air_quality_hourly_pm2_5/1h/short,TimesFM-2.5,128.3803764339212,199.8811043637435,9.946486552556356,0.5349970940929124,0.19267528355159025,0.23408765568966516,7.410677562806579,11.330506450901531,0.2308618296844907,0.20266208703077312,0.166486311960706,Air Quality,1 -open_meteo_shanghai_hourly_temperature_2m/1h/short,TimesFM-2.5,0.4488093384068937,0.42550756879063556,0.5227727890014648,0.28471744342993616,0.021116368406687125,0.020967622620989168,11.867516613268592,0.6699323386782382,0.026734911000282555,0.020862232169443492,0.02480197235359902,Weather,1 -usgs_potomac_iv_usgs_00060/15min/short,TimesFM-2.5,3964.9586008389792,2807.6309466362,52.009847005208336,nan,0.002045911466877187,0.0020444056954015345,nan,62.9679172344058,0.0024762084444137954,0.002045283185523513,0.002015193888024105,Hydrology,1 -wikimedia_time_series_hourly_pageviews/1D/short,TimesFM-2.5,4207.680999754579,5861.192346271244,75.43997955322266,1.6165709904262,0.24218997554725216,0.2765631520841845,22.796233912876673,64.86664011458107,0.20891027412103405,0.2429628971118282,0.177518524153541,Web,1 +binance_btcusdt_1h_close/1h/short,TimesFM-2.5,1219064.9854685466,835565.3181101481,884.322265625,0.97857792276093,0.012057590756120376,0.011980322700327062,53.58648546814461,1104.1127593993951,0.015042171211020878,0.0120477974844572,0.015083192708183289,Finance,1 +coingecko_bitcoin_7d_price_usd/1h/short,TimesFM-2.5,1230621.6017862957,849700.2018839518,888.9869791666666,0.9620628334321281,0.012141146014077656,0.012062308830606919,52.43584047207273,1109.3338549716652,0.015136772575028212,0.012130156909482296,0.015134561186691181,Finance,1 +nasa_power_shanghai_hourly_T2M/1h/short,TimesFM-2.5,7.933177144345336,11.399316419632365,2.7634520530700684,0.8045764238845047,0.10092765997658189,0.10842496844555281,9.535674024816261,2.8165896300926296,0.10973904153296879,0.10766871268224146,0.07858026268608374,Weather,1 +noaa_coops_sf_water_level_water_level/6min/short,TimesFM-2.5,0.0015351705280940802,0.0024778745030360483,0.04266408085823059,2.675305425612182,0.03174085850119802,0.03239086232536759,12.171508226109887,0.03918125225275578,0.030699292812490016,0.03342815850788563,0.021909742273566946,Ocean,1 +nws_ksfo_observations_temperature/1h/short,TimesFM-2.5,7.230681060140189,6.612742760992963,2.0171178181966147,2.4826065454727564,0.12987325798582047,0.11728228771068923,97.8729589413374,2.68899257346319,0.15875970913435808,0.11909182690459717,0.12032840390405451,Weather,1 +open_meteo_shanghai_air_quality_hourly_pm2_5/1h/short,TimesFM-2.5,1370.1968184538339,1568.2607702783378,32.57473762830099,1.429238945300226,0.48036513469849956,0.6789653743383695,33.04442361416721,37.016169689121455,0.6308678478504538,0.5551723691146495,0.45629084448376683,Air Quality,1 +usgs_potomac_iv_usgs_00060/15min/short,TimesFM-2.5,203315.88810078302,267129.3128611247,408.2639973958333,nan,0.014499470105041025,0.014667487665793741,nan,450.90563103689783,0.01619777749571254,0.01466597206630744,0.011492126628270052,Hydrology,1 diff --git a/results/timesfm_2_5/config.json b/results/timesfm_2_5/config.json index 32fc9bad9a121aa856fb3d370007f66f002b6b85..1485470102151d28157323cd39695bfb878423c3 100644 --- a/results/timesfm_2_5/config.json +++ b/results/timesfm_2_5/config.json @@ -3,7 +3,7 @@ "model_type": "zero-shot", "model_dtype": "float32", "model_link": "https://tsfm.ai/models/google/timesfm-2.5-200m-pytorch", - "code_link": "https://github.com/zhouziyu02/TSFM_Bench/blob/main/scripts/run_online_eval.py", + "code_link": "https://github.com/zhouziyu02/TSFM_Bench/blob/main/space/scripts/run_space_eval.py", "org": "Google / TSFM.ai", "testdata_leakage": "No", "replication_code_available": "Yes", diff --git a/results/timesfm_2_5/online_meta.json b/results/timesfm_2_5/online_meta.json index 40f8d3dfc3c9c116d59b61655634a961dd9403a1..3b1959a41372cc68be1f6ad4bce47b76a5ad3579 100644 --- a/results/timesfm_2_5/online_meta.json +++ b/results/timesfm_2_5/online_meta.json @@ -1,67 +1,49 @@ { "model": "TimesFM-2.5", "api_model_id": "google/timesfm-2.5-200m-pytorch", - "evaluated_at": "2026-05-28T12:21:19.114949+00:00", + "evaluated_at": "2026-05-29T02:54:46.508756+00:00", "datasets": [ { - "dataset": "task_binance_btcusdt_1h_crypto_btcusdt_close_20260527t120000z", - "data_fetched_at": "2026-05-28T12:16:39.187896Z", + "dataset": "task_binance_btcusdt_1h_crypto_btcusdt_close_20260528t020000z", + "data_fetched_at": "2026-05-29T02:50:05.083862Z", "context_length": 96, "prediction_length": 24 }, { - "dataset": "task_coingecko_bitcoin_7d_coin_bitcoin_price_usd_20260527t120016z", - "data_fetched_at": "2026-05-28T12:16:39.187896Z", + "dataset": "task_coingecko_bitcoin_7d_coin_bitcoin_price_usd_20260528t030012z", + "data_fetched_at": "2026-05-29T02:50:05.083862Z", "context_length": 96, "prediction_length": 24 }, { "dataset": "task_nasa_power_shanghai_hourly_point_shanghai_31.2304_121.4737_t2m_20260524t230000z", - "data_fetched_at": "2026-05-28T12:16:39.187896Z", - "context_length": 83, + "data_fetched_at": "2026-05-29T02:50:05.083862Z", + "context_length": 69, "prediction_length": 24 }, { - "dataset": "task_noaa_coops_sf_water_level_coops_station_9414290_water_level_20260528t094200z", - "data_fetched_at": "2026-05-28T12:16:39.187896Z", + "dataset": "task_noaa_coops_sf_water_level_coops_station_9414290_water_level_20260529t001800z", + "data_fetched_at": "2026-05-29T02:50:05.083862Z", "context_length": 96, "prediction_length": 24 }, { - "dataset": "task_noaa_ndbc_46013_realtime_ndbc_station_46013_wave_height_20260528t045000z", - "data_fetched_at": "2026-05-28T12:16:39.187896Z", + "dataset": "task_nws_ksfo_observations_nws_station_ksfo_temperature_20260529t004500z", + "data_fetched_at": "2026-05-29T02:50:05.083862Z", "context_length": 96, "prediction_length": 24 }, { - "dataset": "task_nws_ksfo_observations_nws_station_ksfo_temperature_20260528t101000z", - "data_fetched_at": "2026-05-28T12:16:39.187896Z", + "dataset": "task_open_meteo_shanghai_air_quality_hourly_grid_shanghai_31.2304_121.4737_pm2_5_20260528t020000z", + "data_fetched_at": "2026-05-29T02:50:05.083862Z", "context_length": 96, "prediction_length": 24 }, { - "dataset": "task_open_meteo_shanghai_air_quality_hourly_grid_shanghai_31.2304_121.4737_pm2_5_20260527t120000z", - "data_fetched_at": "2026-05-28T12:16:39.187896Z", + "dataset": "task_usgs_potomac_iv_usgs_site_01646500_usgs_00060_20260528t235000z", + "data_fetched_at": "2026-05-29T02:50:05.083862Z", "context_length": 96, "prediction_length": 24 - }, - { - "dataset": "task_open_meteo_shanghai_hourly_grid_shanghai_31.2304_121.4737_temperature_2m_20260527t120000z", - "data_fetched_at": "2026-05-28T12:16:39.187896Z", - "context_length": 96, - "prediction_length": 24 - }, - { - "dataset": "task_usgs_potomac_iv_usgs_site_01646500_usgs_00060_20260528t095000z", - "data_fetched_at": "2026-05-28T12:16:39.187896Z", - "context_length": 96, - "prediction_length": 24 - }, - { - "dataset": "task_wikimedia_time_series_hourly_enwiki_article_time_series_pageviews_20260525t000000z", - "data_fetched_at": "2026-05-28T12:16:39.187896Z", - "context_length": 4, - "prediction_length": 2 } ] } diff --git a/scripts/run_online_daemon.py b/scripts/run_online_daemon.py new file mode 100644 index 0000000000000000000000000000000000000000..e4c431ff8177c87e37c69fdd97310d71e0ccd0d7 --- /dev/null +++ b/scripts/run_online_daemon.py @@ -0,0 +1,63 @@ +#!/usr/bin/env python3 +"""Periodically run space/scripts/run_space_eval.py (1-hour live benchmark loop).""" + +from __future__ import annotations + +import argparse +import logging +import os +import subprocess +import sys +import time +from pathlib import Path + +logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s") +logger = logging.getLogger(__name__) + +SPACE_ROOT = Path(__file__).resolve().parents[1] +EVAL_SCRIPT = SPACE_ROOT / "scripts" / "run_space_eval.py" +DEFAULT_INTERVAL_SECONDS = int( + os.getenv( + "TSFM_BENCH_INTERVAL_SECONDS", + os.getenv("TSFM_EVAL_INTERVAL_SECONDS", "3600"), + ) +) + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--interval-minutes", + type=int, + default=max(1, DEFAULT_INTERVAL_SECONDS // 60), + help="Minutes between collect + eval cycles (default: 60)", + ) + parser.add_argument( + "--once", + action="store_true", + help="Run a single evaluation and exit", + ) + return parser.parse_args() + + +def run_eval() -> int: + cmd = [sys.executable, str(EVAL_SCRIPT)] + logger.info("Running: %s", " ".join(cmd)) + return subprocess.run(cmd, cwd=str(SPACE_ROOT), check=False).returncode + + +def main() -> None: + args = parse_args() + if args.once: + raise SystemExit(run_eval()) + + while True: + code = run_eval() + if code != 0: + logger.error("Evaluation failed with exit code %s", code) + logger.info("Sleeping %s minutes until next cycle", args.interval_minutes) + time.sleep(args.interval_minutes * 60) + + +if __name__ == "__main__": + main() diff --git a/scripts/run_space_eval.py b/scripts/run_space_eval.py new file mode 100644 index 0000000000000000000000000000000000000000..8191d189ec946ba2e365aa626a9ca8245979ff6f --- /dev/null +++ b/scripts/run_space_eval.py @@ -0,0 +1,281 @@ +#!/usr/bin/env python3 +"""Run live TS-Bench + TSFM.ai evaluation inside the HF Space (or locally).""" + +from __future__ import annotations + +import csv +import json +import logging +import os +import sys +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +import yaml + +SPACE_ROOT = Path(__file__).resolve().parents[1] +if str(SPACE_ROOT) not in sys.path: + sys.path.insert(0, str(SPACE_ROOT)) + +os.environ.setdefault("TS_BENCH_ROOT", str(SPACE_ROOT / "vendor" / "ts_bench")) + +from tsfm_bench.data.registry import load_data_source, load_dataset_properties +from tsfm_bench.data.ts_bench import TsBenchDataSource +from tsfm_bench.eval.api_predictor import DEFAULT_QUANTILES, TsfmApiConfig, TsfmApiPredictor +from tsfm_bench.eval.online_eval import run_online_eval_for_dataset + +logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s") +logger = logging.getLogger(__name__) + +DATA_CONFIG = SPACE_ROOT / "configs" / "datasets" / "ts_bench.yaml" +MODEL_CONFIG = SPACE_ROOT / "configs" / "models" / "online_tsfm.yaml" +OUTPUT_ROOT = Path(os.getenv("TSFM_RESULTS_PATH", str(SPACE_ROOT / "results"))) +EVAL_STATE_PATH = OUTPUT_ROOT / "eval_state.json" + + +def model_display_name(model_spec: dict[str, Any]) -> str: + return model_spec.get("display_name") or model_spec["model_id"] + + +def model_output_slug(model_spec: dict[str, Any]) -> str: + import re + + name = model_spec.get("display_name") or model_spec["model_id"] + return re.sub(r"[^a-zA-Z0-9]+", "_", name).strip("_").lower() + + +def load_model_specs(path: Path) -> list[dict[str, Any]]: + return yaml.safe_load(path.read_text()).get("models", []) + + +def write_model_results( + model_spec: dict[str, Any], + rows: list[list[Any]], + output_root: Path, + meta: dict[str, Any], +) -> None: + model_name = model_display_name(model_spec) + out_dir = output_root / model_output_slug(model_spec) + out_dir.mkdir(parents=True, exist_ok=True) + + csv_path = out_dir / "all_results.csv" + with csv_path.open("w", newline="") as handle: + writer = csv.writer(handle) + writer.writerow( + [ + "dataset", + "model", + "eval_metrics/MSE[mean]", + "eval_metrics/MSE[0.5]", + "eval_metrics/MAE[0.5]", + "eval_metrics/MASE[0.5]", + "eval_metrics/MAPE[0.5]", + "eval_metrics/sMAPE[0.5]", + "eval_metrics/MSIS", + "eval_metrics/RMSE[mean]", + "eval_metrics/NRMSE[mean]", + "eval_metrics/ND[0.5]", + "eval_metrics/mean_weighted_sum_quantile_loss", + "domain", + "num_variates", + ] + ) + writer.writerows(rows) + + config = { + "model": model_name, + "model_type": model_spec.get("model_type", "zero-shot"), + "model_dtype": "float32", + "model_link": model_spec.get( + "model_link", f"https://tsfm.ai/models/{model_spec['model_id']}" + ), + "code_link": "https://github.com/zhouziyu02/TSFM_Bench/blob/main/space/scripts/run_space_eval.py", + "org": model_spec.get("org", "TSFM.ai"), + "testdata_leakage": "No", + "replication_code_available": "Yes", + "api_model_id": model_spec["model_id"], + } + (out_dir / "config.json").write_text(json.dumps(config, indent=4) + "\n") + (out_dir / "online_meta.json").write_text(json.dumps(meta, indent=4) + "\n") + + +def write_dataset_properties(properties: dict[str, dict[str, Any]], output_root: Path) -> None: + rows = [ + { + "dataset": key, + "domain": values["domain"], + "frequency": values["frequency"], + "num_variates": values["num_variates"], + } + for key, values in sorted(properties.items()) + ] + csv_path = output_root / "dataset_properties.csv" + with csv_path.open("w", newline="") as handle: + writer = csv.DictWriter( + handle, + fieldnames=["dataset", "domain", "frequency", "num_variates"], + ) + writer.writeheader() + writer.writerows(rows) + + +def write_eval_state(payload: dict[str, Any]) -> None: + OUTPUT_ROOT.mkdir(parents=True, exist_ok=True) + EVAL_STATE_PATH.write_text(json.dumps(payload, indent=4) + "\n") + + +def write_run_metadata(output_root: Path, payload: dict[str, Any]) -> None: + (output_root / "online_status.json").write_text(json.dumps(payload, indent=4) + "\n") + + +def make_predictor(model_spec: dict[str, Any], pred_len: int) -> TsfmApiPredictor: + predictor = TsfmApiPredictor( + config=TsfmApiConfig(model_id=model_spec["model_id"]), + prediction_length=pred_len, + quantile_levels=DEFAULT_QUANTILES, + ) + predictor.leaderboard_name = model_display_name(model_spec) + return predictor + + +def run_evaluation() -> int: + if not os.getenv("TSFM_API_KEY"): + write_eval_state( + { + "status": "disabled", + "message": "TSFM_API_KEY not configured — set it in HF Space secrets.", + "updated_at": datetime.now(timezone.utc).isoformat(), + } + ) + logger.error("TSFM_API_KEY is missing; skipping evaluation") + return 1 + + started = datetime.now(timezone.utc).isoformat() + write_eval_state( + { + "status": "running", + "started_at": started, + "message": "Collecting TS-Bench data and calling TSFM.ai API…", + "updated_at": started, + } + ) + + try: + source = load_data_source(DATA_CONFIG) + if isinstance(source, TsBenchDataSource) and source._settings.auto_refresh: + source.refresh() + + datasets = source.list_datasets() + if not datasets: + raise RuntimeError("No TS-Bench tasks available after data collection") + + properties = load_dataset_properties(DATA_CONFIG) + model_specs = load_model_specs(MODEL_CONFIG) + OUTPUT_ROOT.mkdir(parents=True, exist_ok=True) + write_dataset_properties(properties, OUTPUT_ROOT) + + failed_models: list[str] = [] + all_model_meta: dict[str, Any] = {} + + for model_spec in model_specs: + model_name = model_display_name(model_spec) + rows: list[list[Any]] = [] + dataset_meta: list[dict[str, Any]] = [] + + try: + for ds_name in datasets: + pred_len = source.get_prediction_length(ds_name) + predictor = make_predictor(model_spec, pred_len) + try: + logger.info("Evaluating %s on %s", model_name, ds_name) + result = run_online_eval_for_dataset(source, ds_name, predictor) + except Exception: + logger.exception("Skipping %s on %s", model_name, ds_name) + continue + rows.append( + [ + result.dataset, + model_name, + result.metrics["MSE[mean]"], + result.metrics["MSE[0.5]"], + result.metrics["MAE[0.5]"], + result.metrics["MASE[0.5]"], + result.metrics["MAPE[0.5]"], + result.metrics["sMAPE[0.5]"], + result.metrics["MSIS"], + result.metrics["RMSE[mean]"], + result.metrics["NRMSE[mean]"], + result.metrics["ND[0.5]"], + result.metrics["mean_weighted_sum_quantile_loss"], + result.domain, + result.num_variates, + ] + ) + dataset_meta.append( + { + "dataset": ds_name, + "data_fetched_at": result.data_fetched_at, + "context_length": result.context_length, + "prediction_length": result.prediction_length, + } + ) + if not rows: + raise RuntimeError(f"No successful datasets for {model_name}") + model_meta = { + "model": model_name, + "api_model_id": model_spec["model_id"], + "evaluated_at": datetime.now(timezone.utc).isoformat(), + "datasets": dataset_meta, + } + write_model_results(model_spec, rows, OUTPUT_ROOT, model_meta) + all_model_meta[model_name] = model_meta + except Exception: + logger.exception("Failed evaluating %s", model_name) + failed_models.append(model_name) + + finished = datetime.now(timezone.utc).isoformat() + status = "ok" if not failed_models else "partial" + write_run_metadata( + OUTPUT_ROOT, + { + "status": status, + "started_at": started, + "finished_at": finished, + "data_source": "ts_bench", + "data_config": str(DATA_CONFIG), + "ts_bench_root": os.environ.get("TS_BENCH_ROOT", ""), + "models": all_model_meta, + "failed_models": failed_models, + }, + ) + write_eval_state( + { + "status": status, + "started_at": started, + "finished_at": finished, + "failed_models": failed_models, + "task_count": len(datasets), + "model_count": len(model_specs) - len(failed_models), + "message": "Evaluation complete", + "updated_at": finished, + } + ) + return 0 if status == "ok" else 2 + except Exception as exc: + finished = datetime.now(timezone.utc).isoformat() + logger.exception("Evaluation run failed") + write_eval_state( + { + "status": "error", + "started_at": started, + "finished_at": finished, + "message": str(exc), + "updated_at": finished, + } + ) + return 1 + + +if __name__ == "__main__": + raise SystemExit(run_evaluation()) diff --git a/src/about.py b/src/about.py index b63e00d4b884e49636300278a43fee2c0093acbe..fc73afafd853b363af76a13a00c12af234d9612e 100644 --- a/src/about.py +++ b/src/about.py @@ -3,42 +3,54 @@ TITLE = """ TSFM Realworld Bench Leaderboard """ INTRODUCTION_TEXT = """ **TSFM Realworld Bench** evaluates time series foundation models on **live TS-Bench data** with **zero-shot API inference** via [TSFM.ai](https://tsfm.ai/). -Following [GIFT-Eval](https://github.com/SalesforceAIResearch/gift-eval), each model's -**raw metric values** are aggregated with a **geometric mean** across datasets. -Per-dataset **ranks** are averaged for ranking (lower is better). + +The Space runs a **unified 1-hour loop**: collect TS-Bench data → call TSFM.ai for all models → +write new `results/` CSVs → leaderboard reloads. Set **`TSFM_API_KEY`** in Space Settings → +Repository secrets to enable it. """ LLM_BENCHMARKS_TEXT = """ -## How to participate +## How it works (live benchmark) + +1. **Background scheduler** runs every **`TSFM_BENCH_INTERVAL_SECONDS`** (default **3600 = 1 hour**): + TS-Bench collect → TSFM.ai eval → update `results/`. +2. Each run calls **TSFM.ai** for all configured models. +3. The Gradio UI reloads tables every **`TSFM_REFRESH_SECONDS`** (default 60s) from disk. + +### Required HF Space secret -1. Clone [TSFM_Bench](https://github.com/zhouziyu02/TSFM_Bench) -2. Implement your model against the streaming data interface (`tsfm_bench.data`) -3. Run evaluation and write `results//all_results.csv` -4. Open a pull request; results sync to this leaderboard +| Variable | Purpose | +|---|---| +| `TSFM_API_KEY` | TSFM.ai API key — **required** for live evaluation | + +### Optional environment variables + +| Variable | Default | Purpose | +|---|---|---| +| `TSFM_BENCH_INTERVAL_SECONDS` | `3600` | **Main knob**: seconds between collect+eval cycles (1 h) | +| `TSFM_EVAL_INTERVAL_SECONDS` | (alias) | Same as above if `TSFM_BENCH_INTERVAL_SECONDS` unset | +| `TSFM_REFRESH_SECONDS` | `60` | UI table reload interval (display only) | +| `TS_BENCH_ROOT` | `vendor/ts_bench` | Path to TS-Bench collect scripts | ## Metrics - **MASE** — Mean Absolute Scaled Error -- **CRPS** — Mean Weighted Sum Quantile Loss (probabilistic forecast quality) -- **CRPS_Rank** / **MASE_Rank** — average per-dataset rank on raw metrics (lower is better) +- **CRPS** — Mean Weighted Sum Quantile Loss +- Ranks are averaged per-dataset on raw metrics (lower is better) ## Aggregation (GIFT-Eval style) -1. Compute evaluation metrics on each dataset configuration (raw values). -2. Rank models per dataset on each metric. -3. Aggregate metric values with a **geometric mean** across datasets. -4. Aggregate ranks with an **arithmetic mean** across datasets. - -## Data interface +1. Compute evaluation metrics on each dataset (raw values). +2. Rank models per dataset. +3. Aggregate with **geometric mean** (metrics) and **arithmetic mean** (ranks). -Live series are collected from **TS-Bench** (`TsBenchDataSource`). Each evaluation -holds out the most recent forecast horizon as ground truth and calls TSFM models through -the **TSFM.ai** API for zero-shot forecasts. +## Local development ```bash export TSFM_API_KEY=your_key -python scripts/run_online_eval.py -python scripts/run_online_daemon.py --interval-minutes 60 +cd space +python scripts/run_space_eval.py # single eval run +python app.py # leaderboard + scheduler ``` """ diff --git a/src/benchmark_config.py b/src/benchmark_config.py new file mode 100644 index 0000000000000000000000000000000000000000..d67a841b1a2206628b49eee661d7fa4088ce534a --- /dev/null +++ b/src/benchmark_config.py @@ -0,0 +1,19 @@ +"""Unified timing for the live benchmark loop.""" + +from __future__ import annotations + +import os + +# One full cycle: TS-Bench collect → TSFM.ai eval → write results → leaderboard reload. +# Override with TSFM_BENCH_INTERVAL_SECONDS (preferred) or TSFM_EVAL_INTERVAL_SECONDS. +BENCHMARK_INTERVAL_SECONDS = int( + os.getenv( + "TSFM_BENCH_INTERVAL_SECONDS", + os.getenv("TSFM_EVAL_INTERVAL_SECONDS", "3600"), + ) +) + +# UI table reload (reads results/ from disk; does not call TSFM.ai). +DISPLAY_REFRESH_SECONDS = int(os.getenv("TSFM_REFRESH_SECONDS", "60")) + +BENCHMARK_INTERVAL_MINUTES = BENCHMARK_INTERVAL_SECONDS // 60 diff --git a/src/display/css_html_js.py b/src/display/css_html_js.py index 8f9f4c5147c6f4701f2234c27926f356ad90cdd0..4ef869fc74bbd0bdd6d17a734b32d813e77a465c 100644 --- a/src/display/css_html_js.py +++ b/src/display/css_html_js.py @@ -37,7 +37,7 @@ def refresh_countdown_markup(seconds: int) -> str: """Static HTML only — Gradio on HF Spaces strips