erikjm commited on
Commit
8bbdf1a
·
verified ·
1 Parent(s): 885c230

Deploy the results API

Browse files
Files changed (3) hide show
  1. README.md +1 -1
  2. app.py +38 -8
  3. tests/test_running.py +25 -0
README.md CHANGED
@@ -16,7 +16,7 @@ The results API behind the leaderboard pop-up of the Steerability Challenge site
16
 
17
  The index of runs is `leaderboard/runs.json` in the archive, which `challenge-leaderboard` rebuilds on every render: `{"schema", "latest", "runs": [{"name", "label", "layer", "date", "deadline", "n_ranked", "n_baselines"}, ...]}`. A run's site board is `leaderboard/<run>/site.json`, `{"schema", "run", "rows"}` with the same `run` block and one row per ranked submission and baseline.
18
 
19
- The pending run is the week after the latest published one: the `run` block `{"name", "label", "layer", "pulled", "pull_finished", "n_pulled", "intake_done", "models"}` of `leaderboard/pending.json` (written by `challenge-leaderboard`, the pull, and `scripts/evaluate_submissions.py --init`; `run` is null at the schedule's end), unless the index lists that run. It has no site board. Once pulled (`pulled` true), each organization of its pull manifest has a pending file `leaderboard/<run>/pending/<org>.json`, `{"schema", "run", "org", "team", "entry"}` with the entry block of a team file, and the API adds `evaluation`, the status of the organization's submission per base model (`not_started`, `in_progress`, `completed`, or `error`) read from the run's `status.json` on every request. Apart from the `phase` below, nothing else of the status file reaches a viewer. Before the pull (`pulled` false), the pending view has no teams.
20
 
21
  `intake_done` is true once every pulled submission has an intake report; the API reads a missing or non-boolean value as false. The API serves the block with a `phase` computed on every request from `intake_done` and the submission entries (`submission-*`) of the run's `status.json`, by the first rule that applies:
22
 
 
16
 
17
  The index of runs is `leaderboard/runs.json` in the archive, which `challenge-leaderboard` rebuilds on every render: `{"schema", "latest", "runs": [{"name", "label", "layer", "date", "deadline", "n_ranked", "n_baselines"}, ...]}`. A run's site board is `leaderboard/<run>/site.json`, `{"schema", "run", "rows"}` with the same `run` block and one row per ranked submission and baseline.
18
 
19
+ The pending run is the week after the latest published one: the `run` block `{"name", "label", "layer", "pulled", "pull_finished", "n_pulled", "intake_done", "models"}` of `leaderboard/pending.json` (written by `challenge-leaderboard`, the pull, and `scripts/evaluate_submissions.py --init`; `run` is null at the schedule's end), unless the index lists that run. It has no site board. Once pulled (`pulled` true), each organization of its pull manifest has a pending file `leaderboard/<run>/pending/<org>.json`, `{"schema", "run", "org", "team", "entry"}` with the entry block of a team file, and the API adds `evaluation`, the status of the organization's submission per base model (`not_started`, `in_progress`, `completed`, or `error`) read from the run's `status.json` on every request (in a run in parts, the entries `<key>#dims`, `#guards`, and `#sides` of a model combine into one status: `not_started` until a part starts, `completed` or `error` once every part ended, `in_progress` in between). Apart from the `phase` below, nothing else of the status file reaches a viewer. Before the pull (`pulled` false), the pending view has no teams.
20
 
21
  `intake_done` is true once every pulled submission has an intake report; the API reads a missing or non-boolean value as false. The API serves the block with a `phase` computed on every request from `intake_done` and the submission entries (`submission-*`) of the run's `status.json`, by the first rule that applies:
22
 
app.py CHANGED
@@ -26,6 +26,10 @@ the run's `status.json` (`run_phase`, first rule that applies):
26
  Both endpoints read `status.json` once per request for a pulled pending run. Only the status codes of its
27
  submission entries enter the phase, and `/runs` gives nothing else of the file.
28
 
 
 
 
 
29
  Settings (environment):
30
  - `HF_TOKEN` (a Space secret): the token that reads the archive dataset. It is read from the environment only, a
31
  missing one is logged at startup, and no other credential is ever used.
@@ -66,6 +70,7 @@ STATUS_FILE = "status.json" # `<run>/status.json`, the evaluation's record of t
66
  EVALUATION_STATUSES = ("not_started", "in_progress", "completed", "error") # the status codes of its entries
67
  ENDED_STATUSES = ("completed", "error") # the status codes of an entry whose run has ended
68
  SUBMISSION_PREFIX = "submission-" # the key prefix of a submission's entries (the harness's `SUBMISSION_ARM_PREFIX`)
 
69
  PULL_MANIFEST = "manifest.json" # `<run>/manifest.json`, the pull's record of the run's submissions
70
  ORG_HASH_CHARS = 12 # the hex characters of an organization handle's SHA-256 a site board row gives (`org_hash`)
71
  PHASES = ("upcoming", "pulled", "checked", "evaluating", "finishing") # the pending run's phases, in week order
@@ -213,14 +218,34 @@ def _status_code(entry) -> str:
213
  return code if code in EVALUATION_STATUSES else "not_started"
214
 
215
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
216
  def evaluation(status: dict | None, org: str, models: list[str]) -> dict[str, str]:
217
  """The evaluation's progress of one organization's submission per base model, in the order of `models`: the
218
- status code of the `status.json` entry `submission-<org>/<model slug>` (the ref with `/` replaced by `__`), and
219
- `not_started` without the file, the entry, or a code of `EVALUATION_STATUSES`. The code is the only field read
220
- from an entry, and no other entry is read."""
221
  entries = status if isinstance(status, dict) else {}
222
- return {model: _status_code(entries.get(f"{SUBMISSION_PREFIX}{org}/{model.replace('/', '__')}"))
223
- for model in models}
224
 
225
 
226
  def org_hash(org: str) -> str:
@@ -241,7 +266,8 @@ def running_teams(status: dict | None, manifest: dict | None, board: dict | None
241
  Returns:
242
  `[{"team", "org_hash", "evaluation"}]` in the manifest's order, where `team` is the manifest entry's
243
  `team_name` (the organization handle without one, as the site board names a team), and `evaluation` maps each
244
- base model ref of the team's entries, in the file's order, to its status code. Nothing else of an entry is
 
245
  read, and no organization handle is given apart from a team named by it.
246
  """
247
  entries = status if isinstance(status, dict) else {}
@@ -253,8 +279,12 @@ def running_teams(status: dict | None, manifest: dict | None, board: dict | None
253
  if not isinstance(org, str) or not HANDLE.match(org):
254
  continue
255
  prefix = f"{SUBMISSION_PREFIX}{org}/"
256
- codes = {key[len(prefix):].replace("__", "/", 1): _status_code(entry)
257
- for key, entry in entries.items() if key.startswith(prefix)}
 
 
 
 
258
  hashed = org_hash(org)
259
  if not codes or hashed in ranked:
260
  continue
 
26
  Both endpoints read `status.json` once per request for a pulled pending run. Only the status codes of its
27
  submission entries enter the phase, and `/runs` gives nothing else of the file.
28
 
29
+ In a run in parts (week 4 on), a submission on a model has one entry per part, `<key>#dims`, `#guards`, and `#sides`,
30
+ and the status per base model (`evaluation`, `running_teams`) combines them (`model_status`): `not_started` until a
31
+ part starts, `completed` or `error` once every part ended, and `in_progress` in between.
32
+
33
  Settings (environment):
34
  - `HF_TOKEN` (a Space secret): the token that reads the archive dataset. It is read from the environment only, a
35
  missing one is logged at startup, and no other credential is ever used.
 
70
  EVALUATION_STATUSES = ("not_started", "in_progress", "completed", "error") # the status codes of its entries
71
  ENDED_STATUSES = ("completed", "error") # the status codes of an entry whose run has ended
72
  SUBMISSION_PREFIX = "submission-" # the key prefix of a submission's entries (the harness's `SUBMISSION_ARM_PREFIX`)
73
+ PART_SEPARATOR = "#" # a part's key under a run in parts is `<key>#<part>` (the harness's `PART_SEPARATOR`)
74
  PULL_MANIFEST = "manifest.json" # `<run>/manifest.json`, the pull's record of the run's submissions
75
  ORG_HASH_CHARS = 12 # the hex characters of an organization handle's SHA-256 a site board row gives (`org_hash`)
76
  PHASES = ("upcoming", "pulled", "checked", "evaluating", "finishing") # the pending run's phases, in week order
 
218
  return code if code in EVALUATION_STATUSES else "not_started"
219
 
220
 
221
+ def model_status(codes: list[str]) -> str:
222
+ """The status of one submission on one model from the status codes of its entries, i.e., of the whole run, or of
223
+ each part (`<key>#<part>`) of a run in parts, by the first rule that applies:
224
+
225
+ - `not_started`: no entry, or every entry `not_started`.
226
+ - `error` or `completed`: every entry `completed` or `error`, `error` when any is.
227
+ - `in_progress`: otherwise, e.g., one part running, or one part ended and another not started.
228
+ """
229
+ if all(code == "not_started" for code in codes):
230
+ return "not_started"
231
+ if all(code in ENDED_STATUSES for code in codes):
232
+ return "error" if "error" in codes else "completed"
233
+ return "in_progress"
234
+
235
+
236
+ def _model_entries(entries: dict, key: str) -> list:
237
+ """The `status.json` entries of one submission on one model: the entry `key` and its parts' (`key#<part>`)."""
238
+ return [entry for name, entry in entries.items() if name == key or name.startswith(f"{key}{PART_SEPARATOR}")]
239
+
240
+
241
  def evaluation(status: dict | None, org: str, models: list[str]) -> dict[str, str]:
242
  """The evaluation's progress of one organization's submission per base model, in the order of `models`: the
243
+ status (`model_status`) of the `status.json` entry `submission-<org>/<model slug>` (the ref with `/` replaced by
244
+ `__`) together with its parts' entries (`submission-<org>/<model slug>#<part>`), and `not_started` without the
245
+ file or an entry. The status code is the only field read from an entry, and no other entry is read."""
246
  entries = status if isinstance(status, dict) else {}
247
+ return {model: model_status([_status_code(entry) for entry in _model_entries(
248
+ entries, f"{SUBMISSION_PREFIX}{org}/{model.replace('/', '__')}")]) for model in models}
249
 
250
 
251
  def org_hash(org: str) -> str:
 
266
  Returns:
267
  `[{"team", "org_hash", "evaluation"}]` in the manifest's order, where `team` is the manifest entry's
268
  `team_name` (the organization handle without one, as the site board names a team), and `evaluation` maps each
269
+ base model ref of the team's entries, in the file's order, to its status (`model_status`, over its parts'
270
+ entries in a run in parts). Nothing else of an entry is
271
  read, and no organization handle is given apart from a team named by it.
272
  """
273
  entries = status if isinstance(status, dict) else {}
 
279
  if not isinstance(org, str) or not HANDLE.match(org):
280
  continue
281
  prefix = f"{SUBMISSION_PREFIX}{org}/"
282
+ by_model: dict[str, list[str]] = {}
283
+ for key, entry in entries.items():
284
+ if key.startswith(prefix):
285
+ slug = key[len(prefix):].partition(PART_SEPARATOR)[0]
286
+ by_model.setdefault(slug.replace("__", "/", 1), []).append(_status_code(entry))
287
+ codes = {model: model_status(model_codes) for model, model_codes in by_model.items()}
288
  hashed = org_hash(org)
289
  if not codes or hashed in ranked:
290
  continue
tests/test_running.py CHANGED
@@ -95,3 +95,28 @@ def test_me_lists_no_running_team_without_a_status_file(client: TestClient, arch
95
  def test_the_pending_view_has_no_running_key(client: TestClient) -> None:
96
  body = client.get("/me?run=week_03", headers=bearer("participant-token")).json()
97
  assert body["pending"] is True and "running" not in body
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
95
  def test_the_pending_view_has_no_running_key(client: TestClient) -> None:
96
  body = client.get("/me?run=week_03", headers=bearer("participant-token")).json()
97
  assert body["pending"] is True and "running" not in body
98
+
99
+
100
+ def test_model_status_combines_the_parts_of_a_run_in_parts() -> None:
101
+ status = leaderboard_api.model_status
102
+ assert status([]) == "not_started" and status(["not_started", "not_started"]) == "not_started"
103
+ assert status(["completed", "completed", "completed"]) == "completed"
104
+ assert status(["completed", "error", "completed"]) == "error"
105
+ assert status(["completed", "in_progress", "not_started"]) == "in_progress"
106
+ assert status(["error", "not_started"]) == "in_progress" # another part still has to run
107
+ assert status(["completed"]) == "completed" and status(["in_progress"]) == "in_progress"
108
+
109
+
110
+ def test_the_parts_of_a_model_give_one_status_per_model() -> None:
111
+ """`evaluation` and `running_teams` read a run in parts by model, and a whole run's keys as before."""
112
+ parts = {f"submission-team-r/m__one#{part}": status_entry(code)
113
+ for part, code in (("dims", "completed"), ("guards", "in_progress"), ("sides", "not_started"))}
114
+ parts.update({f"submission-team-r/m__two#{part}": status_entry("completed") for part in ("dims", "guards", "sides")})
115
+ assert leaderboard_api.evaluation(parts, "team-r", list(MODELS)) == {"m/one": "in_progress", "m/two": "completed"}
116
+ (team,) = leaderboard_api.running_teams(parts, manifest(("team-r", "Runners")), None)
117
+ assert team["evaluation"] == {"m/one": "in_progress", "m/two": "completed"}
118
+ whole = entries("team-r", "in_progress", "completed")
119
+ assert leaderboard_api.evaluation(whole, "team-r", list(MODELS)) == {"m/one": "in_progress", "m/two": "completed"}
120
+ # another organization whose handle starts with the same letters is not read
121
+ other = {"submission-team-rx/m__one#dims": status_entry("error")}
122
+ assert leaderboard_api.evaluation(other, "team-r", list(MODELS)) == {"m/one": "not_started", "m/two": "not_started"}