File size: 6,345 Bytes
585789a
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
b2c1aa1
 
585789a
 
 
 
 
 
 
 
 
b2c1aa1
 
 
585789a
 
 
b2c1aa1
 
585789a
 
 
b2c1aa1
 
585789a
 
 
 
 
 
b2c1aa1
 
585789a
 
 
b2c1aa1
 
 
585789a
 
 
b2c1aa1
 
585789a
 
 
 
b2c1aa1
585789a
 
 
 
b2c1aa1
 
 
 
 
 
 
 
 
585789a
 
 
b2c1aa1
585789a
 
 
 
 
 
 
 
 
b2c1aa1
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
585789a
 
b2c1aa1
585789a
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
from __future__ import annotations

PAPER_URL: str | None = None
CONTACT = "auerand@amazon.de"

DATA_URL = "https://doi.org/10.7910/DVN/B4OOJK"

ABOUT = """
CREF (Continuous Real-Time Evaluation for Forecasting) evaluates forecasting models on
*actual forecasting*. Static benchmarks score forecasts against values that are already
known, and those values can leak into a model: through training data from the same period,
through what a language model remembers about that time, or through an agent that simply
looks the answer up. CREF takes every forecast before its forecast period begins and scores
it once the actual values have been observed, which rules out all of these by design.
"""

SCOPE = (
    "{tasks} tasks from {sources} public data sources in {domains} domains, scored continuously."
)

DATA = (
    "A snapshot of the data of the cycles from 2026-04-10 to 2026-11-02 is on "
    f"[Harvard Dataverse]({DATA_URL}) (partly outdated)."
)

JOIN_TEASER = (
    "Get your model scored and ranked live next to the others. Open to external teams soon."
)

STEPS = [
    (
        ":material/target: Score each forecast",
        "The error $E_i$ on each item $i$, one target variable of one sample: MASE or SQL, "
        "scaled by the seasonal naive error on the sample's history. A missing or invalid "
        "forecast gets the baseline's error $B_i$.",
    ),
    (
        ":material/date_range: Keep recent samples",
        "The last 30 days for data up to hourly, 120 days for daily and 365 days for weekly and "
        "monthly data, so a model that joins later is soon scored on the same samples.",
    ),
    (
        ":material/balance: Weigh every task the same",
        "$w_i = 1 / n_{t(i)}$, with $n_{t(i)}$ the items of $i$'s task in the window. Domain, "
        "frequency and task boards use only their own items.",
    ),
]

SCORE_DEFINITIONS = [
    (
        "Skill score",
        r"1 - \exp\left(\frac{\sum_i w_i \log r_i}{\sum_i w_i}\right)",
        r"With $r_i = \mathrm{clip}(E_i / B_i, 0.1, 10)$. 0 % is as good as the baseline.",
    ),
    (
        "Win rate",
        r"\frac{\sum_i w_i s_i}{\sum_i w_i}",
        r"With $s_i$ = 1 if $E_i < B_i$, ½ if equal, else 0: the weighted share of items on which "
        "the model beats the baseline.",
    ),
    (
        "Marginal win rate",
        r"\frac{1}{M - 1} \sum_{k \neq j} W_{jk}",
        "Model $j$'s win rate $W_{jk}$ against each other model $k$ shown, averaged.",
    ),
    (
        "Coverage",
        None,
        "The share of samples, and of the score's weight, that the model forecast itself.",
    ),
]

RELEASE = (
    "A model's training data may leak a sample's actual values, directly or indirectly. Two "
    "modes allow both a strict and a faster comparison of models. Where a forecast doesn't "
    "count, the model gets the seasonal naive baseline's score. Their history follows the same rule.\n\n"
    "**Live** (the default): only what a model could have forecast live. A pretrained "
    "model counts from its checkpoint's release, LLM-based and agentic models from their first "
    "live day.\n\n"
    "**Every sample:** pretrained models count on every sample (unless "
    "direct leakage is reported for specific samples; those fall back to the baseline), "
    "LLM-based models from their training-data cutoff, agentic models from their first live day."
)


PAPER = "**CREF: Forecasting Benchmarks for the Age of Agents.** NeurIPS 2026 "

CITATION = """@inproceedings{auer2026cref,
  title={{CREF}: Forecasting Benchmarks for the Age of Agents},
  author={Auer, Andreas and Ansari, Abdul Fatir and Shchur, Oleksandr and Zhang, Xiyuan and Han, Boran and Mercado, Pedro and Desai, Prateek Mutalik and Wang, Bernie and Bohlke-Schneider, Michael},
  booktitle={Advances in Neural Information Processing Systems},
  volume={39},
  year={2026},
}"""

LIVE = "Every path below is relative to `{base}`, readable over HTTPS without an account or key."

FILES_NOTE = (
    "`<source>` is a data source (a task can have several), `<cycle>` a cycle ID such as "
    "`20261031T040000`, `<date>` a day such as `2026-11-01`. There is no folder listing: "
    "`v1/latest.json` names the newest cycles, and the manifests name every file."
)

FILES = {
    "Samples and scores": [
        ("v1/samples/<source>/<cycle>.json", "A cycle's samples"),
        ("v1/forecasts/<source>/<cycle>/<model>.json", "One model's forecasts"),
        ("v1/ground_truth/<source>/<cycle>.json", "The actual values"),
        ("v1/metrics/<source>/<cycle>.json", "MASE and SQL per item"),
        ("v1/latest.json", "The cycles of the last 48 hours, per source"),
        ("v1/manifests/<date>.json", "Every file published or rewritten that day"),
        ("v1/backfill/<date>.json", "Files a backfill published, by cycle day"),
    ],
    "Leaderboards and changelog": [
        ("v1/leaderboard/latest.json", "Live"),
        ("v1/leaderboard/all_samples/latest.json", "Every sample"),
        ("v1/leaderboard/history/<date>.json", "Each day's last, live"),
        ("v1/leaderboard/all_samples/history/<date>.json", "Each day's last, every sample"),
        ("excluded_samples.json", "Excluded samples, by changelog entry"),
        ("INCIDENTS.md", "The data changelog"),
        ("DATA_LICENSES.md", "Each source's license"),
    ],
}

CHANGELOG = (
    "What changed in the benchmark's data, and which samples the leaderboard leaves out because "
    "of it."
)

JOIN = (
    "Get your model scored on data that does not exist yet, and ranked live next to the "
    "others: pretrained, LLM-based and agentic models alike."
)

JOIN_STATUS = "The competition is not open to new teams yet; this is coming soon."

JOIN_STEPS = [
    (
        ":material/mail: Contact us",
        "Tell us your team's name and about your model. We let you know when the competition "
        "opens.",
    ),
    (
        ":material/schedule: Forecast every cycle",
        "Submit point and, optionally, quantile forecasts before each sample's deadline. It "
        "comes with the sample and is always at least an hour after the sample is published.",
    ),
    (
        ":material/leaderboard: Get ranked",
        "Your forecasts are scored once the actual values are observed, and your model joins "
        "the leaderboard.",
    ),
]