Spaces:
Running
Running
Download cref/texts.py from autogluon/fev-bench: direct link, hf CLI and curl.
- Browser
- Download file 6.35 kB
-
https://huggingface.co/spaces/autogluon/fev-bench/resolve/main/cref/texts.py
- Command line
-
hf download hf://spaces/autogluon/fev-bench/cref/texts.py
-
curl -L -o texts.py https://huggingface.co/spaces/autogluon/fev-bench/resolve/main/cref/texts.py
6.35 kB
| from __future__ import annotations | |
| PAPER_URL: str | None = None | |
| CONTACT = "auerand@amazon.de" | |
| DATA_URL = "https://doi.org/10.7910/DVN/B4OOJK" | |
| ABOUT = """ | |
| CREF (Continuous Real-Time Evaluation for Forecasting) evaluates forecasting models on | |
| *actual forecasting*. Static benchmarks score forecasts against values that are already | |
| known, and those values can leak into a model: through training data from the same period, | |
| through what a language model remembers about that time, or through an agent that simply | |
| looks the answer up. CREF takes every forecast before its forecast period begins and scores | |
| it once the actual values have been observed, which rules out all of these by design. | |
| """ | |
| SCOPE = ( | |
| "{tasks} tasks from {sources} public data sources in {domains} domains, scored continuously." | |
| ) | |
| DATA = ( | |
| "A snapshot of the data of the cycles from 2026-04-10 to 2026-11-02 is on " | |
| f"[Harvard Dataverse]({DATA_URL}) (partly outdated)." | |
| ) | |
| JOIN_TEASER = ( | |
| "Get your model scored and ranked live next to the others. Open to external teams soon." | |
| ) | |
| STEPS = [ | |
| ( | |
| ":material/target: Score each forecast", | |
| "The error $E_i$ on each item $i$, one target variable of one sample: MASE or SQL, " | |
| "scaled by the seasonal naive error on the sample's history. A missing or invalid " | |
| "forecast gets the baseline's error $B_i$.", | |
| ), | |
| ( | |
| ":material/date_range: Keep recent samples", | |
| "The last 30 days for data up to hourly, 120 days for daily and 365 days for weekly and " | |
| "monthly data, so a model that joins later is soon scored on the same samples.", | |
| ), | |
| ( | |
| ":material/balance: Weigh every task the same", | |
| "$w_i = 1 / n_{t(i)}$, with $n_{t(i)}$ the items of $i$'s task in the window. Domain, " | |
| "frequency and task boards use only their own items.", | |
| ), | |
| ] | |
| SCORE_DEFINITIONS = [ | |
| ( | |
| "Skill score", | |
| r"1 - \exp\left(\frac{\sum_i w_i \log r_i}{\sum_i w_i}\right)", | |
| r"With $r_i = \mathrm{clip}(E_i / B_i, 0.1, 10)$. 0 % is as good as the baseline.", | |
| ), | |
| ( | |
| "Win rate", | |
| r"\frac{\sum_i w_i s_i}{\sum_i w_i}", | |
| r"With $s_i$ = 1 if $E_i < B_i$, ½ if equal, else 0: the weighted share of items on which " | |
| "the model beats the baseline.", | |
| ), | |
| ( | |
| "Marginal win rate", | |
| r"\frac{1}{M - 1} \sum_{k \neq j} W_{jk}", | |
| "Model $j$'s win rate $W_{jk}$ against each other model $k$ shown, averaged.", | |
| ), | |
| ( | |
| "Coverage", | |
| None, | |
| "The share of samples, and of the score's weight, that the model forecast itself.", | |
| ), | |
| ] | |
| RELEASE = ( | |
| "A model's training data may leak a sample's actual values, directly or indirectly. Two " | |
| "modes allow both a strict and a faster comparison of models. Where a forecast doesn't " | |
| "count, the model gets the seasonal naive baseline's score. Their history follows the same rule.\n\n" | |
| "**Live** (the default): only what a model could have forecast live. A pretrained " | |
| "model counts from its checkpoint's release, LLM-based and agentic models from their first " | |
| "live day.\n\n" | |
| "**Every sample:** pretrained models count on every sample (unless " | |
| "direct leakage is reported for specific samples; those fall back to the baseline), " | |
| "LLM-based models from their training-data cutoff, agentic models from their first live day." | |
| ) | |
| PAPER = "**CREF: Forecasting Benchmarks for the Age of Agents.** NeurIPS 2026 " | |
| CITATION = """@inproceedings{auer2026cref, | |
| title={{CREF}: Forecasting Benchmarks for the Age of Agents}, | |
| author={Auer, Andreas and Ansari, Abdul Fatir and Shchur, Oleksandr and Zhang, Xiyuan and Han, Boran and Mercado, Pedro and Desai, Prateek Mutalik and Wang, Bernie and Bohlke-Schneider, Michael}, | |
| booktitle={Advances in Neural Information Processing Systems}, | |
| volume={39}, | |
| year={2026}, | |
| }""" | |
| LIVE = "Every path below is relative to `{base}`, readable over HTTPS without an account or key." | |
| FILES_NOTE = ( | |
| "`<source>` is a data source (a task can have several), `<cycle>` a cycle ID such as " | |
| "`20261031T040000`, `<date>` a day such as `2026-11-01`. There is no folder listing: " | |
| "`v1/latest.json` names the newest cycles, and the manifests name every file." | |
| ) | |
| FILES = { | |
| "Samples and scores": [ | |
| ("v1/samples/<source>/<cycle>.json", "A cycle's samples"), | |
| ("v1/forecasts/<source>/<cycle>/<model>.json", "One model's forecasts"), | |
| ("v1/ground_truth/<source>/<cycle>.json", "The actual values"), | |
| ("v1/metrics/<source>/<cycle>.json", "MASE and SQL per item"), | |
| ("v1/latest.json", "The cycles of the last 48 hours, per source"), | |
| ("v1/manifests/<date>.json", "Every file published or rewritten that day"), | |
| ("v1/backfill/<date>.json", "Files a backfill published, by cycle day"), | |
| ], | |
| "Leaderboards and changelog": [ | |
| ("v1/leaderboard/latest.json", "Live"), | |
| ("v1/leaderboard/all_samples/latest.json", "Every sample"), | |
| ("v1/leaderboard/history/<date>.json", "Each day's last, live"), | |
| ("v1/leaderboard/all_samples/history/<date>.json", "Each day's last, every sample"), | |
| ("excluded_samples.json", "Excluded samples, by changelog entry"), | |
| ("INCIDENTS.md", "The data changelog"), | |
| ("DATA_LICENSES.md", "Each source's license"), | |
| ], | |
| } | |
| CHANGELOG = ( | |
| "What changed in the benchmark's data, and which samples the leaderboard leaves out because " | |
| "of it." | |
| ) | |
| JOIN = ( | |
| "Get your model scored on data that does not exist yet, and ranked live next to the " | |
| "others: pretrained, LLM-based and agentic models alike." | |
| ) | |
| JOIN_STATUS = "The competition is not open to new teams yet; this is coming soon." | |
| JOIN_STEPS = [ | |
| ( | |
| ":material/mail: Contact us", | |
| "Tell us your team's name and about your model. We let you know when the competition " | |
| "opens.", | |
| ), | |
| ( | |
| ":material/schedule: Forecast every cycle", | |
| "Submit point and, optionally, quantile forecasts before each sample's deadline. It " | |
| "comes with the sample and is always at least an hour after the sample is published.", | |
| ), | |
| ( | |
| ":material/leaderboard: Get ranked", | |
| "Your forecasts are scored once the actual values are observed, and your model joins " | |
| "the leaderboard.", | |
| ), | |
| ] | |