Deploy the results API
Browse files- README.md +1 -1
- app.py +38 -8
- tests/test_running.py +25 -0
README.md
CHANGED
|
@@ -16,7 +16,7 @@ The results API behind the leaderboard pop-up of the Steerability Challenge site
|
|
| 16 |
|
| 17 |
The index of runs is `leaderboard/runs.json` in the archive, which `challenge-leaderboard` rebuilds on every render: `{"schema", "latest", "runs": [{"name", "label", "layer", "date", "deadline", "n_ranked", "n_baselines"}, ...]}`. A run's site board is `leaderboard/<run>/site.json`, `{"schema", "run", "rows"}` with the same `run` block and one row per ranked submission and baseline.
|
| 18 |
|
| 19 |
-
The pending run is the week after the latest published one: the `run` block `{"name", "label", "layer", "pulled", "pull_finished", "n_pulled", "intake_done", "models"}` of `leaderboard/pending.json` (written by `challenge-leaderboard`, the pull, and `scripts/evaluate_submissions.py --init`; `run` is null at the schedule's end), unless the index lists that run. It has no site board. Once pulled (`pulled` true), each organization of its pull manifest has a pending file `leaderboard/<run>/pending/<org>.json`, `{"schema", "run", "org", "team", "entry"}` with the entry block of a team file, and the API adds `evaluation`, the status of the organization's submission per base model (`not_started`, `in_progress`, `completed`, or `error`) read from the run's `status.json` on every request. Apart from the `phase` below, nothing else of the status file reaches a viewer. Before the pull (`pulled` false), the pending view has no teams.
|
| 20 |
|
| 21 |
`intake_done` is true once every pulled submission has an intake report; the API reads a missing or non-boolean value as false. The API serves the block with a `phase` computed on every request from `intake_done` and the submission entries (`submission-*`) of the run's `status.json`, by the first rule that applies:
|
| 22 |
|
|
|
|
| 16 |
|
| 17 |
The index of runs is `leaderboard/runs.json` in the archive, which `challenge-leaderboard` rebuilds on every render: `{"schema", "latest", "runs": [{"name", "label", "layer", "date", "deadline", "n_ranked", "n_baselines"}, ...]}`. A run's site board is `leaderboard/<run>/site.json`, `{"schema", "run", "rows"}` with the same `run` block and one row per ranked submission and baseline.
|
| 18 |
|
| 19 |
+
The pending run is the week after the latest published one: the `run` block `{"name", "label", "layer", "pulled", "pull_finished", "n_pulled", "intake_done", "models"}` of `leaderboard/pending.json` (written by `challenge-leaderboard`, the pull, and `scripts/evaluate_submissions.py --init`; `run` is null at the schedule's end), unless the index lists that run. It has no site board. Once pulled (`pulled` true), each organization of its pull manifest has a pending file `leaderboard/<run>/pending/<org>.json`, `{"schema", "run", "org", "team", "entry"}` with the entry block of a team file, and the API adds `evaluation`, the status of the organization's submission per base model (`not_started`, `in_progress`, `completed`, or `error`) read from the run's `status.json` on every request (in a run in parts, the entries `<key>#dims`, `#guards`, and `#sides` of a model combine into one status: `not_started` until a part starts, `completed` or `error` once every part ended, `in_progress` in between). Apart from the `phase` below, nothing else of the status file reaches a viewer. Before the pull (`pulled` false), the pending view has no teams.
|
| 20 |
|
| 21 |
`intake_done` is true once every pulled submission has an intake report; the API reads a missing or non-boolean value as false. The API serves the block with a `phase` computed on every request from `intake_done` and the submission entries (`submission-*`) of the run's `status.json`, by the first rule that applies:
|
| 22 |
|
app.py
CHANGED
|
@@ -26,6 +26,10 @@ the run's `status.json` (`run_phase`, first rule that applies):
|
|
| 26 |
Both endpoints read `status.json` once per request for a pulled pending run. Only the status codes of its
|
| 27 |
submission entries enter the phase, and `/runs` gives nothing else of the file.
|
| 28 |
|
|
|
|
|
|
|
|
|
|
|
|
|
| 29 |
Settings (environment):
|
| 30 |
- `HF_TOKEN` (a Space secret): the token that reads the archive dataset. It is read from the environment only, a
|
| 31 |
missing one is logged at startup, and no other credential is ever used.
|
|
@@ -66,6 +70,7 @@ STATUS_FILE = "status.json" # `<run>/status.json`, the evaluation's record of t
|
|
| 66 |
EVALUATION_STATUSES = ("not_started", "in_progress", "completed", "error") # the status codes of its entries
|
| 67 |
ENDED_STATUSES = ("completed", "error") # the status codes of an entry whose run has ended
|
| 68 |
SUBMISSION_PREFIX = "submission-" # the key prefix of a submission's entries (the harness's `SUBMISSION_ARM_PREFIX`)
|
|
|
|
| 69 |
PULL_MANIFEST = "manifest.json" # `<run>/manifest.json`, the pull's record of the run's submissions
|
| 70 |
ORG_HASH_CHARS = 12 # the hex characters of an organization handle's SHA-256 a site board row gives (`org_hash`)
|
| 71 |
PHASES = ("upcoming", "pulled", "checked", "evaluating", "finishing") # the pending run's phases, in week order
|
|
@@ -213,14 +218,34 @@ def _status_code(entry) -> str:
|
|
| 213 |
return code if code in EVALUATION_STATUSES else "not_started"
|
| 214 |
|
| 215 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 216 |
def evaluation(status: dict | None, org: str, models: list[str]) -> dict[str, str]:
|
| 217 |
"""The evaluation's progress of one organization's submission per base model, in the order of `models`: the
|
| 218 |
-
status
|
| 219 |
-
`
|
| 220 |
-
from an entry, and no other entry is read."""
|
| 221 |
entries = status if isinstance(status, dict) else {}
|
| 222 |
-
return {model: _status_code(
|
| 223 |
-
|
| 224 |
|
| 225 |
|
| 226 |
def org_hash(org: str) -> str:
|
|
@@ -241,7 +266,8 @@ def running_teams(status: dict | None, manifest: dict | None, board: dict | None
|
|
| 241 |
Returns:
|
| 242 |
`[{"team", "org_hash", "evaluation"}]` in the manifest's order, where `team` is the manifest entry's
|
| 243 |
`team_name` (the organization handle without one, as the site board names a team), and `evaluation` maps each
|
| 244 |
-
base model ref of the team's entries, in the file's order, to its status
|
|
|
|
| 245 |
read, and no organization handle is given apart from a team named by it.
|
| 246 |
"""
|
| 247 |
entries = status if isinstance(status, dict) else {}
|
|
@@ -253,8 +279,12 @@ def running_teams(status: dict | None, manifest: dict | None, board: dict | None
|
|
| 253 |
if not isinstance(org, str) or not HANDLE.match(org):
|
| 254 |
continue
|
| 255 |
prefix = f"{SUBMISSION_PREFIX}{org}/"
|
| 256 |
-
|
| 257 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 258 |
hashed = org_hash(org)
|
| 259 |
if not codes or hashed in ranked:
|
| 260 |
continue
|
|
|
|
| 26 |
Both endpoints read `status.json` once per request for a pulled pending run. Only the status codes of its
|
| 27 |
submission entries enter the phase, and `/runs` gives nothing else of the file.
|
| 28 |
|
| 29 |
+
In a run in parts (week 4 on), a submission on a model has one entry per part, `<key>#dims`, `#guards`, and `#sides`,
|
| 30 |
+
and the status per base model (`evaluation`, `running_teams`) combines them (`model_status`): `not_started` until a
|
| 31 |
+
part starts, `completed` or `error` once every part ended, and `in_progress` in between.
|
| 32 |
+
|
| 33 |
Settings (environment):
|
| 34 |
- `HF_TOKEN` (a Space secret): the token that reads the archive dataset. It is read from the environment only, a
|
| 35 |
missing one is logged at startup, and no other credential is ever used.
|
|
|
|
| 70 |
EVALUATION_STATUSES = ("not_started", "in_progress", "completed", "error") # the status codes of its entries
|
| 71 |
ENDED_STATUSES = ("completed", "error") # the status codes of an entry whose run has ended
|
| 72 |
SUBMISSION_PREFIX = "submission-" # the key prefix of a submission's entries (the harness's `SUBMISSION_ARM_PREFIX`)
|
| 73 |
+
PART_SEPARATOR = "#" # a part's key under a run in parts is `<key>#<part>` (the harness's `PART_SEPARATOR`)
|
| 74 |
PULL_MANIFEST = "manifest.json" # `<run>/manifest.json`, the pull's record of the run's submissions
|
| 75 |
ORG_HASH_CHARS = 12 # the hex characters of an organization handle's SHA-256 a site board row gives (`org_hash`)
|
| 76 |
PHASES = ("upcoming", "pulled", "checked", "evaluating", "finishing") # the pending run's phases, in week order
|
|
|
|
| 218 |
return code if code in EVALUATION_STATUSES else "not_started"
|
| 219 |
|
| 220 |
|
| 221 |
+
def model_status(codes: list[str]) -> str:
|
| 222 |
+
"""The status of one submission on one model from the status codes of its entries, i.e., of the whole run, or of
|
| 223 |
+
each part (`<key>#<part>`) of a run in parts, by the first rule that applies:
|
| 224 |
+
|
| 225 |
+
- `not_started`: no entry, or every entry `not_started`.
|
| 226 |
+
- `error` or `completed`: every entry `completed` or `error`, `error` when any is.
|
| 227 |
+
- `in_progress`: otherwise, e.g., one part running, or one part ended and another not started.
|
| 228 |
+
"""
|
| 229 |
+
if all(code == "not_started" for code in codes):
|
| 230 |
+
return "not_started"
|
| 231 |
+
if all(code in ENDED_STATUSES for code in codes):
|
| 232 |
+
return "error" if "error" in codes else "completed"
|
| 233 |
+
return "in_progress"
|
| 234 |
+
|
| 235 |
+
|
| 236 |
+
def _model_entries(entries: dict, key: str) -> list:
|
| 237 |
+
"""The `status.json` entries of one submission on one model: the entry `key` and its parts' (`key#<part>`)."""
|
| 238 |
+
return [entry for name, entry in entries.items() if name == key or name.startswith(f"{key}{PART_SEPARATOR}")]
|
| 239 |
+
|
| 240 |
+
|
| 241 |
def evaluation(status: dict | None, org: str, models: list[str]) -> dict[str, str]:
|
| 242 |
"""The evaluation's progress of one organization's submission per base model, in the order of `models`: the
|
| 243 |
+
status (`model_status`) of the `status.json` entry `submission-<org>/<model slug>` (the ref with `/` replaced by
|
| 244 |
+
`__`) together with its parts' entries (`submission-<org>/<model slug>#<part>`), and `not_started` without the
|
| 245 |
+
file or an entry. The status code is the only field read from an entry, and no other entry is read."""
|
| 246 |
entries = status if isinstance(status, dict) else {}
|
| 247 |
+
return {model: model_status([_status_code(entry) for entry in _model_entries(
|
| 248 |
+
entries, f"{SUBMISSION_PREFIX}{org}/{model.replace('/', '__')}")]) for model in models}
|
| 249 |
|
| 250 |
|
| 251 |
def org_hash(org: str) -> str:
|
|
|
|
| 266 |
Returns:
|
| 267 |
`[{"team", "org_hash", "evaluation"}]` in the manifest's order, where `team` is the manifest entry's
|
| 268 |
`team_name` (the organization handle without one, as the site board names a team), and `evaluation` maps each
|
| 269 |
+
base model ref of the team's entries, in the file's order, to its status (`model_status`, over its parts'
|
| 270 |
+
entries in a run in parts). Nothing else of an entry is
|
| 271 |
read, and no organization handle is given apart from a team named by it.
|
| 272 |
"""
|
| 273 |
entries = status if isinstance(status, dict) else {}
|
|
|
|
| 279 |
if not isinstance(org, str) or not HANDLE.match(org):
|
| 280 |
continue
|
| 281 |
prefix = f"{SUBMISSION_PREFIX}{org}/"
|
| 282 |
+
by_model: dict[str, list[str]] = {}
|
| 283 |
+
for key, entry in entries.items():
|
| 284 |
+
if key.startswith(prefix):
|
| 285 |
+
slug = key[len(prefix):].partition(PART_SEPARATOR)[0]
|
| 286 |
+
by_model.setdefault(slug.replace("__", "/", 1), []).append(_status_code(entry))
|
| 287 |
+
codes = {model: model_status(model_codes) for model, model_codes in by_model.items()}
|
| 288 |
hashed = org_hash(org)
|
| 289 |
if not codes or hashed in ranked:
|
| 290 |
continue
|
tests/test_running.py
CHANGED
|
@@ -95,3 +95,28 @@ def test_me_lists_no_running_team_without_a_status_file(client: TestClient, arch
|
|
| 95 |
def test_the_pending_view_has_no_running_key(client: TestClient) -> None:
|
| 96 |
body = client.get("/me?run=week_03", headers=bearer("participant-token")).json()
|
| 97 |
assert body["pending"] is True and "running" not in body
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 95 |
def test_the_pending_view_has_no_running_key(client: TestClient) -> None:
|
| 96 |
body = client.get("/me?run=week_03", headers=bearer("participant-token")).json()
|
| 97 |
assert body["pending"] is True and "running" not in body
|
| 98 |
+
|
| 99 |
+
|
| 100 |
+
def test_model_status_combines_the_parts_of_a_run_in_parts() -> None:
|
| 101 |
+
status = leaderboard_api.model_status
|
| 102 |
+
assert status([]) == "not_started" and status(["not_started", "not_started"]) == "not_started"
|
| 103 |
+
assert status(["completed", "completed", "completed"]) == "completed"
|
| 104 |
+
assert status(["completed", "error", "completed"]) == "error"
|
| 105 |
+
assert status(["completed", "in_progress", "not_started"]) == "in_progress"
|
| 106 |
+
assert status(["error", "not_started"]) == "in_progress" # another part still has to run
|
| 107 |
+
assert status(["completed"]) == "completed" and status(["in_progress"]) == "in_progress"
|
| 108 |
+
|
| 109 |
+
|
| 110 |
+
def test_the_parts_of_a_model_give_one_status_per_model() -> None:
|
| 111 |
+
"""`evaluation` and `running_teams` read a run in parts by model, and a whole run's keys as before."""
|
| 112 |
+
parts = {f"submission-team-r/m__one#{part}": status_entry(code)
|
| 113 |
+
for part, code in (("dims", "completed"), ("guards", "in_progress"), ("sides", "not_started"))}
|
| 114 |
+
parts.update({f"submission-team-r/m__two#{part}": status_entry("completed") for part in ("dims", "guards", "sides")})
|
| 115 |
+
assert leaderboard_api.evaluation(parts, "team-r", list(MODELS)) == {"m/one": "in_progress", "m/two": "completed"}
|
| 116 |
+
(team,) = leaderboard_api.running_teams(parts, manifest(("team-r", "Runners")), None)
|
| 117 |
+
assert team["evaluation"] == {"m/one": "in_progress", "m/two": "completed"}
|
| 118 |
+
whole = entries("team-r", "in_progress", "completed")
|
| 119 |
+
assert leaderboard_api.evaluation(whole, "team-r", list(MODELS)) == {"m/one": "in_progress", "m/two": "completed"}
|
| 120 |
+
# another organization whose handle starts with the same letters is not read
|
| 121 |
+
other = {"submission-team-rx/m__one#dims": status_entry("error")}
|
| 122 |
+
assert leaderboard_api.evaluation(other, "team-r", list(MODELS)) == {"m/one": "not_started", "m/two": "not_started"}
|