Download app.py from steerability-challenge-evals/leaderboard-api: direct link, hf CLI and curl.
- Browser
- Download file 31.8 kB
-
https://huggingface.co/spaces/steerability-challenge-evals/leaderboard-api/resolve/main/app.py
- Command line
-
hf download hf://spaces/steerability-challenge-evals/leaderboard-api/app.py
-
curl -L -o app.py https://huggingface.co/spaces/steerability-challenge-evals/leaderboard-api/resolve/main/app.py
31.8 kB
| """The results API behind the competition site's leaderboard pop-up, served from a public Hugging Face Space. | |
| The index of runs is `leaderboard/runs.json` in the archive (`RUNS`), which `challenge-leaderboard` rebuilds on every | |
| render, and the pending run (the week the site shows after the latest published one, pulled or not yet) is the | |
| `run` of `leaderboard/pending.json` (`PENDING`) when the index does not list it. `GET /healthz` gives the latest | |
| run and its date (both null without an index), `GET /runs` gives the index with the pending run under `pending`, | |
| and `GET /board/<run>` gives a run's site board (`leaderboard/<run>/site.json`), each without authentication. | |
| `GET /me` gives a signed-in viewer the files they may see for one run (`?run=<run>`, by default the index's | |
| latest): a participant gets the team file of each of their organizations, and an organizer gets every file of the | |
| run. Both also get `baselines`, the run's baseline files (`leaderboard/<run>/baselines/<arm>.json`) by arm | |
| (`baselines_payload`), the same for every signed-in viewer, so a team can read its entry against the baselines; a | |
| pending run's view has none. For the index's latest run, both also get `running`, the teams whose evaluation has not reached the run's site | |
| board yet (`running_teams`), by display name and organization hash with their status per base model, for every | |
| signed-in viewer alike. For the pending run it gives the pending files instead (`leaderboard/<run>/pending/<org>.json`), each with | |
| the evaluation's progress per base model read from the run's `status.json` (`evaluation`), or empty views while | |
| the run is not pulled (`pulled` false in its block). The server decides the viewer's handle, organizations, and | |
| role from their OAuth token alone; a request names no organization. | |
| The pending run's block, as `GET /runs` serves it under `pending` and `GET /me?run=<pending run>` under `run`, is the | |
| index's block with its `intake_done` (true once every pulled submission has an intake report, read as false when | |
| missing or not a boolean) and a `phase` computed on every request from `intake_done` and the submission entries of | |
| the run's `status.json` (`run_phase`, first rule that applies): | |
| - `upcoming`: the run is not pulled. | |
| - `finishing`: every submission entry is `completed` or `error`, and there is at least one. | |
| - `evaluating`: a submission entry is `in_progress`, `completed`, or `error`. | |
| - `checked`: `intake_done` is true. | |
| - `pulled`: otherwise. | |
| Both endpoints read `status.json` once per request for a pulled pending run. Only the status codes of its | |
| submission entries enter the phase, and `/runs` gives nothing else of the file. | |
| In a run in parts (week 4 on), a submission on a model has one entry per part, `<key>#dims`, `#guards`, and `#sides`, | |
| and the status per base model (`evaluation`, `running_teams`) combines them (`model_status`): `not_started` until a | |
| part starts, `completed` or `error` once every part ended, and `in_progress` in between. | |
| Settings (environment): | |
| - `HF_TOKEN` (a Space secret): the token that reads the archive dataset. It is read from the environment only, a | |
| missing one is logged at startup, and no other credential is ever used. | |
| - `ARCHIVE_REPO`: the private archive dataset, e.g., `steerability-challenge-evals/submissions`. | |
| - `ALLOWED_ORIGINS`: the comma-separated origins CORS allows, e.g., `https://steerability.github.io`. | |
| - `ALLOWED_ORIGIN_REGEX`: an origin pattern CORS also allows (development only), e.g., `^http://localhost:\\d+$`. | |
| - `ORGANIZER_USERS`: the comma-separated handles that get the organizer payload. | |
| - `ORGANIZER_ORG`: the organization whose members get the organizer payload. | |
| - `USERINFO_URL`: the OAuth userinfo endpoint, `https://huggingface.co/oauth/userinfo` by default. | |
| - `ARCHIVE_DIR`: a local directory read in place of the archive dataset (tests and local previews). | |
| Logs give the viewer's handle, role, run, whether it is pending, and the status code only, never a token or a | |
| file's contents. | |
| """ | |
| from __future__ import annotations | |
| import hashlib | |
| import json | |
| import logging | |
| import os | |
| import re | |
| import time | |
| from pathlib import Path | |
| import httpx | |
| from fastapi import FastAPI, Header | |
| from fastapi.middleware.cors import CORSMiddleware | |
| from fastapi.responses import JSONResponse | |
| from huggingface_hub import HfApi, hf_hub_download | |
| from huggingface_hub.errors import EntryNotFoundError, HfHubHTTPError | |
| logger = logging.getLogger("leaderboard-api") | |
| USERINFO_URL = "https://huggingface.co/oauth/userinfo" | |
| RUNS = "leaderboard/runs.json" # the index of the runs with a site board, written by `challenge-leaderboard` | |
| PENDING = "leaderboard/pending.json" # the run being evaluated, written by the render, the pull, and `--init` | |
| STATUS_FILE = "status.json" # `<run>/status.json`, the evaluation's record of the run's work, one entry per key | |
| EVALUATION_STATUSES = ("not_started", "in_progress", "completed", "error") # the status codes of its entries | |
| ENDED_STATUSES = ("completed", "error") # the status codes of an entry whose run has ended | |
| SUBMISSION_PREFIX = "submission-" # the key prefix of a submission's entries (the harness's `SUBMISSION_ARM_PREFIX`) | |
| PART_SEPARATOR = "#" # a part's key under a run in parts is `<key>#<part>` (the harness's `PART_SEPARATOR`) | |
| PULL_MANIFEST = "manifest.json" # `<run>/manifest.json`, the pull's record of the run's submissions | |
| ORG_HASH_CHARS = 12 # the hex characters of an organization handle's SHA-256 a site board row gives (`org_hash`) | |
| PHASES = ("upcoming", "pulled", "checked", "evaluating", "finishing") # the pending run's phases, in week order | |
| CACHE_SECONDS = 60 # how long a userinfo answer and a directory listing are reused | |
| USERINFO_TIMEOUT_SECONDS = 15 | |
| HANDLE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{0,95}$") # a Hugging Face user or organization handle | |
| RUN_NAME = re.compile(r"^[a-z0-9_]+$") | |
| ARM_NAME = re.compile(r"^[a-z0-9_]+$") # a baseline arm's name, the stem of its baseline file | |
| DATE = re.compile(r"^\d{4}-\d{2}-\d{2}$") | |
| def _names(value: str | None) -> list[str]: | |
| return [name.strip() for name in (value or "").split(",") if name.strip()] | |
| def settings_from_env() -> dict: | |
| """The app's settings from the environment (the module docstring lists them).""" | |
| return {"token": os.environ.get("HF_TOKEN") or None, "archive_repo": os.environ.get("ARCHIVE_REPO"), | |
| "archive_dir": os.environ.get("ARCHIVE_DIR"), "origins": _names(os.environ.get("ALLOWED_ORIGINS")), | |
| "origin_regex": os.environ.get("ALLOWED_ORIGIN_REGEX") or None, | |
| "organizer_users": set(_names(os.environ.get("ORGANIZER_USERS"))), | |
| "organizer_org": os.environ.get("ORGANIZER_ORG") or None, | |
| "userinfo_url": os.environ.get("USERINFO_URL") or USERINFO_URL} | |
| class HubArchive: | |
| """The archive dataset on the Hub, read with the configured token only (`token=False` without one, so that no | |
| cached login is used). Downloads are cached by commit, and a directory listing is reused for `CACHE_SECONDS`.""" | |
| def __init__(self, repo: str, token: str | None) -> None: | |
| self.repo = repo | |
| self.token = token if token else False | |
| self.api = HfApi(token=self.token) | |
| self._listings: dict[str, tuple[float, list[str]]] = {} | |
| def read(self, path: str) -> bytes | None: | |
| try: | |
| local = hf_hub_download(repo_id=self.repo, repo_type="dataset", filename=path, token=self.token) | |
| except EntryNotFoundError: | |
| return None | |
| return Path(local).read_bytes() | |
| def listing(self, directory: str) -> list[str]: | |
| cached = self._listings.get(directory) | |
| if cached is not None and time.monotonic() - cached[0] < CACHE_SECONDS: | |
| return cached[1] | |
| try: | |
| paths = sorted(entry.path for entry in self.api.list_repo_tree(self.repo, path_in_repo=directory, | |
| repo_type="dataset")) | |
| except EntryNotFoundError: | |
| paths = [] | |
| self._listings[directory] = (time.monotonic(), paths) | |
| return paths | |
| class DirectoryArchive: | |
| """A local copy of the archive dataset, laid out as the dataset is.""" | |
| def __init__(self, root: str | Path) -> None: | |
| self.root = Path(root) | |
| def read(self, path: str) -> bytes | None: | |
| file = self.root / path | |
| return file.read_bytes() if file.is_file() else None | |
| def listing(self, directory: str) -> list[str]: | |
| folder = self.root / directory | |
| return sorted(f"{directory}/{child.name}" for child in folder.iterdir()) if folder.is_dir() else [] | |
| class NoArchive: | |
| """The archive of an app started without `ARCHIVE_REPO` or `ARCHIVE_DIR`, which contains nothing.""" | |
| def read(self, path: str) -> bytes | None: | |
| return None | |
| def listing(self, directory: str) -> list[str]: | |
| return [] | |
| def _json(archive, path: str): | |
| data = archive.read(path) | |
| return None if data is None else json.loads(data) | |
| def _text(archive, path: str) -> str | None: | |
| data = archive.read(path) | |
| return None if data is None else data.decode("utf-8") | |
| def runs_index(archive) -> dict | None: | |
| """The index of runs (`RUNS`), or None without one or for a malformed one, i.e., one without a `runs` list, with | |
| an entry that is not a mapping whose `name` matches `RUN_NAME` and whose `date` matches `DATE`, or whose `latest` | |
| is not the name of a listed run.""" | |
| index = _json(archive, RUNS) | |
| if not isinstance(index, dict) or not isinstance(index.get("runs"), list): | |
| return None | |
| for entry in index["runs"]: | |
| if not isinstance(entry, dict) or not RUN_NAME.fullmatch(str(entry.get("name"))) \ | |
| or not DATE.fullmatch(str(entry.get("date"))): | |
| return None | |
| if index.get("latest") not in {entry["name"] for entry in index["runs"]}: | |
| return None | |
| return index | |
| def index_entry(index: dict, run: str) -> dict | None: | |
| """The index entry of one run, or None when the index does not list it.""" | |
| return next((entry for entry in index["runs"] if entry["name"] == run), None) | |
| def site_board(archive, run: str) -> dict | None: | |
| """A run's site board, `leaderboard/<run>/site.json`, or None without one.""" | |
| board = _json(archive, f"leaderboard/{run}/site.json") | |
| return board if isinstance(board, dict) else None | |
| def pending_run(archive, index: dict | None) -> dict | None: | |
| """The pending run's block, i.e., the `run` of `PENDING`, or None without one, for a malformed one (`run` not a | |
| mapping, `name` not matching `RUN_NAME`, `pulled` not a boolean, `models` not a list of strings), or when the | |
| index lists the run, since a run in the index has been published whatever the pending index says. The block is | |
| returned as the index gives it; an `intake_done` that is missing or not a boolean does not reject it.""" | |
| pending = _json(archive, PENDING) | |
| run = pending.get("run") if isinstance(pending, dict) else None | |
| if not isinstance(run, dict) or not RUN_NAME.fullmatch(str(run.get("name"))): | |
| return None | |
| if not isinstance(run.get("pulled"), bool): | |
| return None | |
| if not isinstance(run.get("models"), list) or not all(isinstance(model, str) for model in run["models"]): | |
| return None | |
| if index is not None and index_entry(index, run["name"]) is not None: | |
| return None | |
| return run | |
| def run_status(archive, run: dict) -> dict | None: | |
| """The run's `status.json` in the archive for a pulled run, or None without the file or while the run is not | |
| pulled (nothing is read then).""" | |
| return _json(archive, f"{run['name']}/{STATUS_FILE}") if run["pulled"] else None | |
| def _status_code(entry) -> str: | |
| """The status code of a `status.json` entry, `not_started` for an entry that is not a mapping or whose `status` is | |
| not one of `EVALUATION_STATUSES`.""" | |
| code = entry.get("status") if isinstance(entry, dict) else None | |
| return code if code in EVALUATION_STATUSES else "not_started" | |
| def model_status(codes: list[str]) -> str: | |
| """The status of one submission on one model from the status codes of its entries, i.e., of the whole run, or of | |
| each part (`<key>#<part>`) of a run in parts, by the first rule that applies: | |
| - `not_started`: no entry, or every entry `not_started`. | |
| - `error` or `completed`: every entry `completed` or `error`, `error` when any is. | |
| - `in_progress`: otherwise, e.g., one part running, or one part ended and another not started. | |
| """ | |
| if all(code == "not_started" for code in codes): | |
| return "not_started" | |
| if all(code in ENDED_STATUSES for code in codes): | |
| return "error" if "error" in codes else "completed" | |
| return "in_progress" | |
| def _model_entries(entries: dict, key: str) -> list: | |
| """The `status.json` entries of one submission on one model: the entry `key` and its parts' (`key#<part>`).""" | |
| return [entry for name, entry in entries.items() if name == key or name.startswith(f"{key}{PART_SEPARATOR}")] | |
| def evaluation(status: dict | None, org: str, models: list[str]) -> dict[str, str]: | |
| """The evaluation's progress of one organization's submission per base model, in the order of `models`: the | |
| status (`model_status`) of the `status.json` entry `submission-<org>/<model slug>` (the ref with `/` replaced by | |
| `__`) together with its parts' entries (`submission-<org>/<model slug>#<part>`), and `not_started` without the | |
| file or an entry. The status code is the only field read from an entry, and no other entry is read.""" | |
| entries = status if isinstance(status, dict) else {} | |
| return {model: model_status([_status_code(entry) for entry in _model_entries( | |
| entries, f"{SUBMISSION_PREFIX}{org}/{model.replace('/', '__')}")]) for model in models} | |
| def org_hash(org: str) -> str: | |
| """The first `ORG_HASH_CHARS` characters of the lowercase hex SHA-256 of an organization handle, as the harness's | |
| site board gives it (`challenge.leaderboard.org_hash`).""" | |
| return hashlib.sha256(org.encode("utf-8")).hexdigest()[:ORG_HASH_CHARS] | |
| def running_teams(status: dict | None, manifest: dict | None, board: dict | None) -> list[dict]: | |
| """The teams of a published run whose evaluation has not reached its site board, i.e., each organization of the | |
| pull manifest (`manifest`) with at least one submission entry in the run's `status.json` (`status`) and no row | |
| of the site board (`board`, matched by `org_hash`), whose entries are not all ended with an error among them. | |
| A team is listed while an entry is `not_started` or `in_progress`, and while every entry is `completed` but the | |
| board does not rank it yet (its arm is scored again and the board rendered once the last model finishes). A team | |
| whose entries all ended with at least one `error` is not listed, since it is not being evaluated any more. | |
| Returns: | |
| `[{"team", "org_hash", "evaluation"}]` in the manifest's order, where `team` is the manifest entry's | |
| `team_name` (the organization handle without one, as the site board names a team), and `evaluation` maps each | |
| base model ref of the team's entries, in the file's order, to its status (`model_status`, over its parts' | |
| entries in a run in parts). Nothing else of an entry is | |
| read, and no organization handle is given apart from a team named by it. | |
| """ | |
| entries = status if isinstance(status, dict) else {} | |
| orgs = (manifest or {}).get("orgs") if isinstance(manifest, dict) else None | |
| ranked = {row.get("org_hash") for row in (board or {}).get("rows") or [] if isinstance(row, dict)} | |
| teams = [] | |
| for pull_entry in orgs if isinstance(orgs, list) else []: | |
| org = pull_entry.get("org") if isinstance(pull_entry, dict) else None | |
| if not isinstance(org, str) or not HANDLE.match(org): | |
| continue | |
| prefix = f"{SUBMISSION_PREFIX}{org}/" | |
| by_model: dict[str, list[str]] = {} | |
| for key, entry in entries.items(): | |
| if key.startswith(prefix): | |
| slug = key[len(prefix):].partition(PART_SEPARATOR)[0] | |
| by_model.setdefault(slug.replace("__", "/", 1), []).append(_status_code(entry)) | |
| codes = {model: model_status(model_codes) for model, model_codes in by_model.items()} | |
| hashed = org_hash(org) | |
| if not codes or hashed in ranked: | |
| continue | |
| values = set(codes.values()) | |
| if values <= set(ENDED_STATUSES) and "error" in values: | |
| continue | |
| team = pull_entry.get("team_name") | |
| teams.append({"team": team if isinstance(team, str) and team.strip() else org, "org_hash": hashed, | |
| "evaluation": codes}) | |
| return teams | |
| def run_phase(run: dict, status: dict | None) -> str: | |
| """The pending run's phase, one of `PHASES`, from its block and the submission entries of its `status.json` (the | |
| keys starting with `SUBMISSION_PREFIX`), by the first rule that applies: | |
| - `upcoming`: the run is not pulled. | |
| - `finishing`: there is a submission entry, and every one is `completed` or `error`. | |
| - `evaluating`: a submission entry is `in_progress`, `completed`, or `error`. | |
| - `checked`: the block's `intake_done` is true. | |
| - `pulled`: otherwise. | |
| The status code is the only field read from an entry, an entry with no valid code counts as `not_started`, and | |
| the organizer arms' entries are not read. An `intake_done` that is missing or not a boolean counts as false. | |
| """ | |
| if run.get("pulled") is not True: | |
| return "upcoming" | |
| entries = status if isinstance(status, dict) else {} | |
| codes = [_status_code(entry) for key, entry in entries.items() if key.startswith(SUBMISSION_PREFIX)] | |
| if codes and all(code in ENDED_STATUSES for code in codes): | |
| return "finishing" | |
| if any(code != "not_started" for code in codes): | |
| return "evaluating" | |
| return "checked" if run.get("intake_done") is True else "pulled" | |
| def pending_block(run: dict, status: dict | None) -> dict: | |
| """The pending run's block as the API serves it, i.e., the index's block with `intake_done` read as a boolean | |
| (false when missing or not a boolean) and the computed `phase` (`run_phase`).""" | |
| return {**run, "intake_done": run.get("intake_done") is True, "phase": run_phase(run, status)} | |
| def viewer_role(info: dict, settings: dict) -> tuple[str, list[str]]: | |
| """`organizer` for a handle in `ORGANIZER_USERS` or a member of `ORGANIZER_ORG`, else `participant`, with the | |
| viewer's organization handles in the order the userinfo lists them.""" | |
| orgs = [org.get("preferred_username") for org in info.get("orgs") or [] if isinstance(org, dict)] | |
| orgs = [org for org in orgs if isinstance(org, str) and HANDLE.match(org)] | |
| organizer = info.get("preferred_username") in settings["organizer_users"] or ( | |
| settings["organizer_org"] is not None and settings["organizer_org"] in orgs) | |
| return ("organizer" if organizer else "participant"), orgs | |
| def run_block(archive, entry: dict) -> dict: | |
| """The `run` block of a run's site board, or the run's index entry when the board is missing (the index entry is | |
| the block `challenge-leaderboard` copied from the board).""" | |
| board = site_board(archive, entry["name"]) | |
| return board["run"] if board is not None and isinstance(board.get("run"), dict) else entry | |
| def running_payload(archive, entry: dict, latest: bool) -> list[dict]: | |
| """The `running` list of a published run's view (`running_teams`): read from the run's `status.json`, pull | |
| manifest, and site board for the index's latest run (`latest`), and empty for an earlier run, whose files are not | |
| read.""" | |
| if not latest: | |
| return [] | |
| run = entry["name"] | |
| return running_teams(_json(archive, f"{run}/{STATUS_FILE}"), _json(archive, f"{run}/{PULL_MANIFEST}"), | |
| site_board(archive, run)) | |
| def baselines_payload(archive, entry: dict) -> dict[str, dict]: | |
| """The `baselines` map of a published run's view: the baseline files under `leaderboard/<run>/baselines/` | |
| (written by `challenge-leaderboard`) by arm, i.e., by file stem. A file is kept only when it is a JSON object | |
| whose `arm` equals its stem, matches `ARM_NAME`, and names a `baseline` row of the run's site board (`team` is a | |
| baseline row's arm name), so a stray or stale file is never served. Empty without a file or a site board.""" | |
| run = entry["name"] | |
| paths = [path for path in archive.listing(f"leaderboard/{run}/baselines") if path.endswith(".json")] | |
| if not paths: | |
| return {} | |
| board = site_board(archive, run) | |
| rows = board.get("rows") if board is not None else None | |
| arms = {row.get("team") for row in rows if isinstance(row, dict) and row.get("kind") == "baseline"} \ | |
| if isinstance(rows, list) else set() | |
| baselines = {} | |
| for path in sorted(paths): | |
| stem = Path(path).stem | |
| if not ARM_NAME.fullmatch(stem) or stem not in arms: | |
| continue | |
| try: | |
| view = _json(archive, path) | |
| except ValueError: # not JSON | |
| continue | |
| if isinstance(view, dict) and view.get("arm") == stem: | |
| baselines[stem] = view | |
| return baselines | |
| def participant_payload(archive, entry: dict, orgs: list[str], *, latest: bool = False) -> dict: | |
| """The run's `run` block, the team file of each of the viewer's organizations that has one in the run, in the | |
| userinfo's order, the organizations without one under `missing`, the run's baseline files under `baselines` | |
| (`baselines_payload`), and the teams still being evaluated under `running` (`running_payload`).""" | |
| teams, missing = [], [] | |
| for org in orgs: | |
| team = _json(archive, f"leaderboard/{entry['name']}/teams/{org}.json") | |
| (missing.append(org) if team is None else teams.append(team)) | |
| return {"role": "participant", "run": run_block(archive, entry), "teams": teams, "missing": missing, | |
| "baselines": baselines_payload(archive, entry), "running": running_payload(archive, entry, latest)} | |
| def organizer_payload(archive, entry: dict, *, latest: bool = False) -> dict: | |
| """The run's `run` block, internal board (`<date>.internal.json` with the index entry's date), every team file, | |
| the run's baseline files under `baselines` (`baselines_payload`, as a participant gets them), the pull manifest, | |
| the run's progress and failures files, and the teams still being evaluated under `running` | |
| (`running_payload`).""" | |
| run = entry["name"] | |
| teams = {Path(path).stem: _json(archive, path) for path in archive.listing(f"leaderboard/{run}/teams") | |
| if path.endswith(".json")} | |
| return {"role": "organizer", "run": run_block(archive, entry), | |
| "board": _json(archive, f"leaderboard/{run}/{entry['date']}.internal.json"), "teams": teams, | |
| "baselines": baselines_payload(archive, entry), "pull": _json(archive, f"{run}/{PULL_MANIFEST}"), "progress": _text(archive, f"{run}/progress.txt"), | |
| "failures": _text(archive, f"{run}/failures.md"), "running": running_payload(archive, entry, latest)} | |
| def _with_evaluation(archive, run: dict, org: str, team: dict | None, status: dict | None) -> dict | None: | |
| """A pending file with the organization's `evaluation` block added, or None without the file.""" | |
| if not isinstance(team, dict): | |
| return None | |
| return {**team, "evaluation": evaluation(status, org, run["models"])} | |
| def pending_participant_payload(archive, run: dict, orgs: list[str], status: dict | None) -> dict: | |
| """The pending run's block (`pending_block`), the pending file of each of the viewer's organizations that has | |
| one, in the userinfo's order, each with its `evaluation` block from the run's `status.json` (`status`, read once | |
| per request), and the organizations without one under `missing`. While the run is not pulled, `teams` and | |
| `missing` are empty.""" | |
| pulled = run["pulled"] | |
| teams, missing = [], [] | |
| for org in orgs if pulled else []: | |
| team = _with_evaluation(archive, run, org, _json(archive, f"leaderboard/{run['name']}/pending/{org}.json"), | |
| status) | |
| (missing.append(org) if team is None else teams.append(team)) | |
| return {"role": "participant", "pending": True, "pulled": pulled, "run": pending_block(run, status), | |
| "teams": teams, "missing": missing} | |
| def pending_organizer_payload(archive, run: dict, status: dict | None) -> dict: | |
| """The pending run's block (`pending_block`), a null board, every pending file of the run with its `evaluation` | |
| block from the run's `status.json` (`status`, read once per request), the pull manifest, and the run's progress | |
| and failures files. While the run is not pulled, `teams` is empty and the three files are null.""" | |
| name, pulled = run["name"], run["pulled"] | |
| if not pulled: | |
| return {"role": "organizer", "pending": True, "pulled": False, "run": pending_block(run, status), | |
| "board": None, "teams": {}, "pull": None, "progress": None, "failures": None} | |
| teams = {} | |
| for path in archive.listing(f"leaderboard/{name}/pending"): | |
| if path.endswith(".json"): | |
| org = Path(path).stem | |
| teams[org] = _with_evaluation(archive, run, org, _json(archive, path), status) | |
| return {"role": "organizer", "pending": True, "pulled": True, "run": pending_block(run, status), "board": None, | |
| "teams": teams, "pull": _json(archive, f"{name}/manifest.json"), | |
| "progress": _text(archive, f"{name}/progress.txt"), "failures": _text(archive, f"{name}/failures.md")} | |
| def create_app(settings: dict | None = None) -> FastAPI: | |
| """The API with CORS for the configured origins, `GET` and the `Authorization` header only, and no cookies.""" | |
| settings = settings_from_env() if settings is None else settings | |
| if settings["archive_dir"]: | |
| archive = DirectoryArchive(settings["archive_dir"]) | |
| elif not settings["archive_repo"]: | |
| logger.error("ARCHIVE_REPO is not set: add it as a Space variable; no board can be served until then") | |
| archive = NoArchive() | |
| else: | |
| if not settings["token"]: | |
| logger.error("HF_TOKEN is not set: add the read token as a Space secret; the archive dataset cannot be " | |
| "read until then, and no other credential is used") | |
| archive = HubArchive(settings["archive_repo"], settings["token"]) | |
| app = FastAPI(docs_url=None, redoc_url=None, openapi_url=None) | |
| app.add_middleware(CORSMiddleware, allow_origins=settings["origins"], allow_origin_regex=settings["origin_regex"], | |
| allow_methods=["GET"], allow_headers=["Authorization"], allow_credentials=False) | |
| userinfo_cache: dict[str, tuple[float, dict | None]] = {} | |
| def userinfo(token: str) -> dict | None: | |
| key = hashlib.sha256(token.encode("utf-8")).hexdigest() | |
| cached = userinfo_cache.get(key) | |
| if cached is not None and time.monotonic() - cached[0] < CACHE_SECONDS: | |
| return cached[1] | |
| try: | |
| response = httpx.get(settings["userinfo_url"], headers={"Authorization": f"Bearer {token}"}, | |
| timeout=USERINFO_TIMEOUT_SECONDS) | |
| info = response.json() if response.status_code == 200 else None | |
| except (httpx.HTTPError, ValueError): | |
| info = None | |
| info = info if isinstance(info, dict) else None | |
| userinfo_cache[key] = (time.monotonic(), info) | |
| return info | |
| def unavailable(endpoint: str, exc: HfHubHTTPError) -> JSONResponse: | |
| logger.error("%s: 503; the archive could not be read (%s)", endpoint, type(exc).__name__) | |
| return JSONResponse({"error": "archive_unavailable"}, status_code=503) | |
| def healthz() -> dict: | |
| try: | |
| index = runs_index(archive) | |
| except HfHubHTTPError as exc: # e.g., no read token yet | |
| logger.error("healthz: the archive could not be read (%s)", type(exc).__name__) | |
| index = None | |
| if index is None: | |
| return {"run": None, "date": None} | |
| return {"run": index["latest"], "date": index_entry(index, index["latest"])["date"]} | |
| def runs(): | |
| try: | |
| index = runs_index(archive) | |
| except HfHubHTTPError as exc: | |
| return unavailable("runs", exc) | |
| if index is None: | |
| logger.info("runs: 503") | |
| return JSONResponse({"error": "no_board"}, status_code=503) | |
| try: | |
| pending = pending_run(archive, index) | |
| status = run_status(archive, pending) if pending is not None else None | |
| except HfHubHTTPError as exc: | |
| return unavailable("runs", exc) | |
| logger.info("runs: 200 (pending %s)", pending["name"] if pending else "-") | |
| return {**index, "pending": None if pending is None else pending_block(pending, status)} | |
| def board(run: str): | |
| try: | |
| index = runs_index(archive) if RUN_NAME.fullmatch(run) else None | |
| found = site_board(archive, run) if index is not None and index_entry(index, run) is not None else None | |
| except HfHubHTTPError as exc: | |
| return unavailable("board", exc) | |
| if found is None: | |
| logger.info("board: %s 404", run) | |
| return JSONResponse({"error": "not_found"}, status_code=404) | |
| logger.info("board: %s 200", run) | |
| return found | |
| def me(run: str | None = None, authorization: str | None = Header(default=None)): | |
| scheme, _, token = (authorization or "").partition(" ") | |
| info = userinfo(token.strip()) if scheme.lower() == "bearer" and token.strip() else None | |
| if info is None: | |
| logger.info("me: - - %s 401", run or "-") | |
| return JSONResponse({"error": "invalid_token"}, status_code=401) | |
| handle = info.get("preferred_username") | |
| role, orgs = viewer_role(info, settings) | |
| try: | |
| index = runs_index(archive) | |
| pending = pending_run(archive, index) if run is not None else None | |
| if pending is not None and run == pending["name"]: | |
| name, kind = run, "pending" | |
| status = run_status(archive, pending) | |
| payload = (pending_organizer_payload(archive, pending, status) if role == "organizer" | |
| else pending_participant_payload(archive, pending, orgs, status)) | |
| else: | |
| if index is None: | |
| logger.info("me: %s %s %s 503", handle, role, run or "-") | |
| return JSONResponse({"error": "no_board"}, status_code=503) | |
| name, kind = (index["latest"] if run is None else run), "published" | |
| entry = index_entry(index, name) if RUN_NAME.fullmatch(name) else None | |
| if entry is None: | |
| logger.info("me: %s %s %s 404", handle, role, name) | |
| return JSONResponse({"error": "not_found"}, status_code=404) | |
| latest = name == index["latest"] | |
| payload = (organizer_payload(archive, entry, latest=latest) if role == "organizer" | |
| else participant_payload(archive, entry, orgs, latest=latest)) | |
| except HfHubHTTPError as exc: # e.g., no read token yet | |
| logger.error("me: %s %s %s 503; the archive could not be read (%s)", handle, role, run or "-", | |
| type(exc).__name__) | |
| return JSONResponse({"error": "archive_unavailable"}, status_code=503) | |
| logger.info("me: %s %s %s %s 200", handle, role, name, kind) | |
| return payload | |
| return app | |
| logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s: %(message)s") | |
| app = create_app() | |