File size: 6,213 Bytes
387a4cf
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
"""Freeze reviewer identities independently of the sealed paper outcomes."""

from __future__ import annotations

import time
from .data import canonical, digest, dump, load, read_split
from .inference import now
from .search import run_name
from .workspace import ROOT


def roster():
    """Order primary pairs before secondary controls; fork is attached to seed one."""
    return (
        [(s, a) for s in range(1, 9) for a in ("editable", "native")]
        + [(s, a) for s in range(1, 6) for a in ("uniform", "handcrafted")]
        + [(1, "fork-native")]
    )


def frozen_entry(run, endpoint=None):
    """The selected source and validation checkpoint fully identify a reviewer."""
    ep = endpoint or run["endpoints"][-1]
    workspace = next(n["workspace"] for n in run["nodes"] if n["id"] == ep["node"])
    if digest(workspace["reviewer"]) != ep["key"]:
        raise ValueError("Endpoint program identity failed verification")
    return dict(
        name=run["name"],
        seed=run["seed"],
        arm=run["arm"],
        source=workspace["reviewer"],
        key=ep["key"],
        n=ep["n"],
        compute=ep["compute"],
    )


def freeze(project, allow_incomplete=False):
    """Build the entire endpoint manifest before opening any test manuscript."""
    path = project / "results" / "freeze.json"
    if path.exists():
        return load(path)
    budget = load(project / "results" / "budget.json")
    stop_file = project / "results" / "time-stop.json"
    if allow_incomplete and time.time() < budget["deadline"] and not stop_file.exists():
        raise ValueError(
            "An incomplete cohort requires the registered deadline or timing-reserve stopping record"
        )
    entries = {
        "root": dict(
            name="root",
            seed=0,
            arm="root",
            source=ROOT["reviewer"],
            key=digest(ROOT["reviewer"]),
            n=0,
        )
    }
    runs = {}
    missing = []
    for seed, arm in roster():
        name = run_name("confirm", seed, arm)
        source = project / "results" / "runs" / (name + ".json")
        data = load(source) if source.exists() else {}
        if data.get("complete") and data["budget"] == budget["budget"]:
            runs[name] = data
            entries[name] = frozen_entry(data)
        else:
            missing.append(name)
    if missing and not allow_incomplete:
        raise ValueError("Pending confirmatory endpoints: " + ", ".join(missing))
    # Token sensitivity uses a cutoff fixed without consulting any test predictions.
    for seed in range(1, 9):
        pair = [run_name("confirm", seed, a) for a in ("editable", "native")]
        if not all(name in runs for name in pair):
            continue

        def tokens(ep):
            return ep["compute"]["input"] + ep["compute"]["output"]

        cutoff = min(tokens(runs[name]["endpoints"][-1]) for name in pair)
        for name in pair:
            eligible = [ep for ep in runs[name]["endpoints"] if tokens(ep) <= cutoff]
            entry = (
                frozen_entry(runs[name], eligible[-1])
                if eligible
                else dict(entries["root"], seed=seed, arm=runs[name]["arm"])
            )
            entries[name + "-tokens"] = dict(entry, cutoff=cutoff)
    manifest = dict(
        frozen_at=now(),
        split_sha=load(project / "results" / "split-manifest.json")["splits"]["test"][
            "sha256"
        ],
        budget=budget["budget"],
        missing=missing,
        entries=entries,
        entries_sha=digest(canonical(entries)),
    )
    dump(path, manifest)
    return manifest


def score(project, domain, split, name, entry=None):
    """Score only frozen test programs; interrupted item scoring resumes idempotently."""
    if split == "test":
        manifest = load(project / "results" / "freeze.json")
        if (
            digest(canonical(manifest["entries"])) != manifest["entries_sha"]
            or name not in manifest["entries"]
        ):
            raise ValueError("Missing or corrupt frozen endpoint")
        entry = manifest["entries"][name]
    elif split == "pilot":
        if entry is None and name == "root":
            entry = dict(
                name=name,
                seed=0,
                arm="root",
                key=digest(ROOT["reviewer"]),
                source=ROOT["reviewer"],
                n=0,
            )
        elif entry is None:
            run = load(project / "results" / "runs" / (name + ".json"))
            if not run["complete"] or run["stage"] != "pilot":
                raise ValueError(
                    "Only completed pilot searches may access the pilot evaluation"
                )
            entry = frozen_entry(run)
    else:
        raise ValueError("Unknown evaluation split")
    if digest(entry["source"]) != entry["key"]:
        raise ValueError("Frozen source hash mismatch")
    papers = read_split(project, split)
    path = project / "results" / "evaluations" / f"{split}-{name}.json"
    out = (
        load(path)
        if path.exists()
        else dict(
            split=split,
            name=name,
            seed=entry["seed"],
            arm=entry["arm"],
            key=entry["key"],
            checkpoint=entry["n"],
            rows=[],
            elapsed=0.0,
            started=now(),
            complete=False,
        )
    )
    if out["key"] != entry["key"]:
        raise ValueError("Endpoint changed during evaluation")
    done = {r["id"] for r in out["rows"]}
    for paper in papers:
        if paper["id"] in done:
            continue
        start = time.monotonic()
        result = domain.review(
            {"reviewer": entry["source"]}, paper, f"endpoint/{paper['id']}"
        )
        out["rows"].append(
            dict(
                id=paper["id"],
                label=paper["label"],
                correct=int(result["decision"] == paper["label"]),
                **result,
            )
        )
        out["elapsed"] += time.monotonic() - start
        out["compute"] = domain.broker.metrics("endpoint")
        out["complete"] = len(out["rows"]) == len(papers)
        dump(path, out)
    return out