File size: 6,213 Bytes
387a4cf | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 | """Freeze reviewer identities independently of the sealed paper outcomes."""
from __future__ import annotations
import time
from .data import canonical, digest, dump, load, read_split
from .inference import now
from .search import run_name
from .workspace import ROOT
def roster():
"""Order primary pairs before secondary controls; fork is attached to seed one."""
return (
[(s, a) for s in range(1, 9) for a in ("editable", "native")]
+ [(s, a) for s in range(1, 6) for a in ("uniform", "handcrafted")]
+ [(1, "fork-native")]
)
def frozen_entry(run, endpoint=None):
"""The selected source and validation checkpoint fully identify a reviewer."""
ep = endpoint or run["endpoints"][-1]
workspace = next(n["workspace"] for n in run["nodes"] if n["id"] == ep["node"])
if digest(workspace["reviewer"]) != ep["key"]:
raise ValueError("Endpoint program identity failed verification")
return dict(
name=run["name"],
seed=run["seed"],
arm=run["arm"],
source=workspace["reviewer"],
key=ep["key"],
n=ep["n"],
compute=ep["compute"],
)
def freeze(project, allow_incomplete=False):
"""Build the entire endpoint manifest before opening any test manuscript."""
path = project / "results" / "freeze.json"
if path.exists():
return load(path)
budget = load(project / "results" / "budget.json")
stop_file = project / "results" / "time-stop.json"
if allow_incomplete and time.time() < budget["deadline"] and not stop_file.exists():
raise ValueError(
"An incomplete cohort requires the registered deadline or timing-reserve stopping record"
)
entries = {
"root": dict(
name="root",
seed=0,
arm="root",
source=ROOT["reviewer"],
key=digest(ROOT["reviewer"]),
n=0,
)
}
runs = {}
missing = []
for seed, arm in roster():
name = run_name("confirm", seed, arm)
source = project / "results" / "runs" / (name + ".json")
data = load(source) if source.exists() else {}
if data.get("complete") and data["budget"] == budget["budget"]:
runs[name] = data
entries[name] = frozen_entry(data)
else:
missing.append(name)
if missing and not allow_incomplete:
raise ValueError("Pending confirmatory endpoints: " + ", ".join(missing))
# Token sensitivity uses a cutoff fixed without consulting any test predictions.
for seed in range(1, 9):
pair = [run_name("confirm", seed, a) for a in ("editable", "native")]
if not all(name in runs for name in pair):
continue
def tokens(ep):
return ep["compute"]["input"] + ep["compute"]["output"]
cutoff = min(tokens(runs[name]["endpoints"][-1]) for name in pair)
for name in pair:
eligible = [ep for ep in runs[name]["endpoints"] if tokens(ep) <= cutoff]
entry = (
frozen_entry(runs[name], eligible[-1])
if eligible
else dict(entries["root"], seed=seed, arm=runs[name]["arm"])
)
entries[name + "-tokens"] = dict(entry, cutoff=cutoff)
manifest = dict(
frozen_at=now(),
split_sha=load(project / "results" / "split-manifest.json")["splits"]["test"][
"sha256"
],
budget=budget["budget"],
missing=missing,
entries=entries,
entries_sha=digest(canonical(entries)),
)
dump(path, manifest)
return manifest
def score(project, domain, split, name, entry=None):
"""Score only frozen test programs; interrupted item scoring resumes idempotently."""
if split == "test":
manifest = load(project / "results" / "freeze.json")
if (
digest(canonical(manifest["entries"])) != manifest["entries_sha"]
or name not in manifest["entries"]
):
raise ValueError("Missing or corrupt frozen endpoint")
entry = manifest["entries"][name]
elif split == "pilot":
if entry is None and name == "root":
entry = dict(
name=name,
seed=0,
arm="root",
key=digest(ROOT["reviewer"]),
source=ROOT["reviewer"],
n=0,
)
elif entry is None:
run = load(project / "results" / "runs" / (name + ".json"))
if not run["complete"] or run["stage"] != "pilot":
raise ValueError(
"Only completed pilot searches may access the pilot evaluation"
)
entry = frozen_entry(run)
else:
raise ValueError("Unknown evaluation split")
if digest(entry["source"]) != entry["key"]:
raise ValueError("Frozen source hash mismatch")
papers = read_split(project, split)
path = project / "results" / "evaluations" / f"{split}-{name}.json"
out = (
load(path)
if path.exists()
else dict(
split=split,
name=name,
seed=entry["seed"],
arm=entry["arm"],
key=entry["key"],
checkpoint=entry["n"],
rows=[],
elapsed=0.0,
started=now(),
complete=False,
)
)
if out["key"] != entry["key"]:
raise ValueError("Endpoint changed during evaluation")
done = {r["id"] for r in out["rows"]}
for paper in papers:
if paper["id"] in done:
continue
start = time.monotonic()
result = domain.review(
{"reviewer": entry["source"]}, paper, f"endpoint/{paper['id']}"
)
out["rows"].append(
dict(
id=paper["id"],
label=paper["label"],
correct=int(result["decision"] == paper["label"]),
**result,
)
)
out["elapsed"] += time.monotonic() - start
out["compute"] = domain.broker.metrics("endpoint")
out["complete"] = len(out["rows"]) == len(papers)
dump(path, out)
return out
|