fev-bench / cref /texts.py
AndiA's picture
Fetch from live bucket (#6)
b2c1aa1
Raw History Blame Contribute Delete
6.35 kB
from __future__ import annotations
PAPER_URL: str | None = None
CONTACT = "auerand@amazon.de"
DATA_URL = "https://doi.org/10.7910/DVN/B4OOJK"
ABOUT = """
CREF (Continuous Real-Time Evaluation for Forecasting) evaluates forecasting models on
*actual forecasting*. Static benchmarks score forecasts against values that are already
known, and those values can leak into a model: through training data from the same period,
through what a language model remembers about that time, or through an agent that simply
looks the answer up. CREF takes every forecast before its forecast period begins and scores
it once the actual values have been observed, which rules out all of these by design.
"""
SCOPE = (
"{tasks} tasks from {sources} public data sources in {domains} domains, scored continuously."
)
DATA = (
"A snapshot of the data of the cycles from 2026-04-10 to 2026-11-02 is on "
f"[Harvard Dataverse]({DATA_URL}) (partly outdated)."
)
JOIN_TEASER = (
"Get your model scored and ranked live next to the others. Open to external teams soon."
)
STEPS = [
(
":material/target: Score each forecast",
"The error $E_i$ on each item $i$, one target variable of one sample: MASE or SQL, "
"scaled by the seasonal naive error on the sample's history. A missing or invalid "
"forecast gets the baseline's error $B_i$.",
),
(
":material/date_range: Keep recent samples",
"The last 30 days for data up to hourly, 120 days for daily and 365 days for weekly and "
"monthly data, so a model that joins later is soon scored on the same samples.",
),
(
":material/balance: Weigh every task the same",
"$w_i = 1 / n_{t(i)}$, with $n_{t(i)}$ the items of $i$'s task in the window. Domain, "
"frequency and task boards use only their own items.",
),
]
SCORE_DEFINITIONS = [
(
"Skill score",
r"1 - \exp\left(\frac{\sum_i w_i \log r_i}{\sum_i w_i}\right)",
r"With $r_i = \mathrm{clip}(E_i / B_i, 0.1, 10)$. 0 % is as good as the baseline.",
),
(
"Win rate",
r"\frac{\sum_i w_i s_i}{\sum_i w_i}",
r"With $s_i$ = 1 if $E_i < B_i$, ½ if equal, else 0: the weighted share of items on which "
"the model beats the baseline.",
),
(
"Marginal win rate",
r"\frac{1}{M - 1} \sum_{k \neq j} W_{jk}",
"Model $j$'s win rate $W_{jk}$ against each other model $k$ shown, averaged.",
),
(
"Coverage",
None,
"The share of samples, and of the score's weight, that the model forecast itself.",
),
]
RELEASE = (
"A model's training data may leak a sample's actual values, directly or indirectly. Two "
"modes allow both a strict and a faster comparison of models. Where a forecast doesn't "
"count, the model gets the seasonal naive baseline's score. Their history follows the same rule.\n\n"
"**Live** (the default): only what a model could have forecast live. A pretrained "
"model counts from its checkpoint's release, LLM-based and agentic models from their first "
"live day.\n\n"
"**Every sample:** pretrained models count on every sample (unless "
"direct leakage is reported for specific samples; those fall back to the baseline), "
"LLM-based models from their training-data cutoff, agentic models from their first live day."
)
PAPER = "**CREF: Forecasting Benchmarks for the Age of Agents.** NeurIPS 2026 "
CITATION = """@inproceedings{auer2026cref,
title={{CREF}: Forecasting Benchmarks for the Age of Agents},
author={Auer, Andreas and Ansari, Abdul Fatir and Shchur, Oleksandr and Zhang, Xiyuan and Han, Boran and Mercado, Pedro and Desai, Prateek Mutalik and Wang, Bernie and Bohlke-Schneider, Michael},
booktitle={Advances in Neural Information Processing Systems},
volume={39},
year={2026},
}"""
LIVE = "Every path below is relative to `{base}`, readable over HTTPS without an account or key."
FILES_NOTE = (
"`<source>` is a data source (a task can have several), `<cycle>` a cycle ID such as "
"`20261031T040000`, `<date>` a day such as `2026-11-01`. There is no folder listing: "
"`v1/latest.json` names the newest cycles, and the manifests name every file."
)
FILES = {
"Samples and scores": [
("v1/samples/<source>/<cycle>.json", "A cycle's samples"),
("v1/forecasts/<source>/<cycle>/<model>.json", "One model's forecasts"),
("v1/ground_truth/<source>/<cycle>.json", "The actual values"),
("v1/metrics/<source>/<cycle>.json", "MASE and SQL per item"),
("v1/latest.json", "The cycles of the last 48 hours, per source"),
("v1/manifests/<date>.json", "Every file published or rewritten that day"),
("v1/backfill/<date>.json", "Files a backfill published, by cycle day"),
],
"Leaderboards and changelog": [
("v1/leaderboard/latest.json", "Live"),
("v1/leaderboard/all_samples/latest.json", "Every sample"),
("v1/leaderboard/history/<date>.json", "Each day's last, live"),
("v1/leaderboard/all_samples/history/<date>.json", "Each day's last, every sample"),
("excluded_samples.json", "Excluded samples, by changelog entry"),
("INCIDENTS.md", "The data changelog"),
("DATA_LICENSES.md", "Each source's license"),
],
}
CHANGELOG = (
"What changed in the benchmark's data, and which samples the leaderboard leaves out because "
"of it."
)
JOIN = (
"Get your model scored on data that does not exist yet, and ranked live next to the "
"others: pretrained, LLM-based and agentic models alike."
)
JOIN_STATUS = "The competition is not open to new teams yet; this is coming soon."
JOIN_STEPS = [
(
":material/mail: Contact us",
"Tell us your team's name and about your model. We let you know when the competition "
"opens.",
),
(
":material/schedule: Forecast every cycle",
"Submit point and, optionally, quantile forecasts before each sample's deadline. It "
"comes with the sample and is always at least an hour after the sample is published.",
),
(
":material/leaderboard: Get ranked",
"Your forecasts are scored once the actual values are observed, and your model joins "
"the leaderboard.",
),
]