Spaces:
Sleeping
Sleeping
Download server/leaderboard.py from brianbellx/codehygiene: direct link, hf CLI and curl.
- Browser
- Download file 15.7 kB
-
https://huggingface.co/spaces/brianbellx/codehygiene/resolve/main/server/leaderboard.py
- Command line
-
hf download hf://spaces/brianbellx/codehygiene/server/leaderboard.py
-
curl -L -o leaderboard.py https://huggingface.co/spaces/brianbellx/codehygiene/resolve/main/server/leaderboard.py
15.7 kB
| """Public leaderboard of GitHub repos, each scored in full on the server. See space/TELEMETRY.md.""" | |
| from __future__ import annotations | |
| import asyncio | |
| from collections.abc import Callable, Iterable | |
| from dataclasses import asdict, dataclass | |
| from datetime import datetime, timezone | |
| import io | |
| import json | |
| import logging | |
| import re | |
| import tarfile | |
| import time | |
| from typing import Any | |
| from urllib.parse import quote | |
| import httpx | |
| from pydantic import BaseModel, ConfigDict, Field, field_validator | |
| from .config import Settings | |
| from .hygiene import SCORER_ID, SCORER_VERSION, score_file | |
| from .telemetry import ScanFile, Telemetry, scan_file_from | |
| log = logging.getLogger("space.leaderboard") | |
| API = "https://api.github.com" | |
| ARCHIVES = "https://codeload.github.com" | |
| BOARD_FILE = "leaderboard.json" | |
| BOARD_VERSION = 2 | |
| BOARD_LIMIT = 100 | |
| MIN_FILES = 5 | |
| MIN_LINES = 1_000 | |
| MAX_ARCHIVE_BYTES = 80 * 1024 * 1024 | |
| MAX_FILE_BYTES = 150_000 | |
| FRESH_SECONDS = 24 * 3600 | |
| SEED_FRESH_SECONDS = 7 * 24 * 3600 | |
| MAX_WAITING = 3 | |
| GONE_CODES = frozenset({"not_found", "too_small", "fork", "duplicate"}) | |
| RANKED_EXTENSIONS = frozenset({"py", "ts", "tsx", "js", "jsx", "mjs", "cjs", "go", "rs"}) | |
| SKIPPED_DIRS = frozenset({ | |
| "test", "tests", "__tests__", "spec", "specs", "e2e", "testdata", "fixtures", "__fixtures__", | |
| "__mocks__", "vendor", "vendored", "third_party", "thirdparty", "node_modules", "dist", | |
| "build", "out", "target", "generated", "__generated__", | |
| }) | |
| SKIPPED_SUFFIXES = (".d.ts", ".pb.go", "_pb2.py", "_test.go", "_test.py", "_generated.go") | |
| SKIPPED_INFIXES = (".test.", ".spec.", ".generated.", ".min.") | |
| GENERATED_MARKS = ( | |
| "code generated", "do not edit", "@generated", "auto-generated", "autogenerated", "file generated", | |
| ) | |
| REPO_NAME = re.compile(r"^[A-Za-z0-9][A-Za-z0-9-]{0,38}/[A-Za-z0-9._-]{1,100}$") | |
| COMMIT_SHA = re.compile(r"^[0-9a-f]{40}$") | |
| SEED_REPOS = ( | |
| "BurntSushi/ripgrep", "honojs/hono", "fastapi/fastapi", "karpathy/nanoGPT", | |
| "openai/codex", "anomalyco/opencode", "pallets/flask", "psf/requests", | |
| "astral-sh/ruff", "junegunn/fzf", "charmbracelet/bubbletea", "gin-gonic/gin", | |
| "colinhacks/zod", "vitejs/vite", "expressjs/express", "tokio-rs/axum", | |
| "tinygrad/tinygrad", "pydantic/pydantic", "sindresorhus/ky", "anthropics/claude-code", | |
| ) | |
| ScoreVerdict = float | tuple[float, dict[str, int]] | |
| Scorer = Callable[[str], ScoreVerdict] | |
| class RankError(Exception): | |
| def __init__(self, status: int, code: str, message: str, **extra: Any) -> None: | |
| super().__init__(message) | |
| self.status = status | |
| self.code = code | |
| self.message = message | |
| self.extra = extra | |
| def payload(self) -> dict[str, Any]: | |
| return {"code": self.code, "message": self.message, **self.extra} | |
| class RankBody(BaseModel): | |
| model_config = ConfigDict(extra="forbid") | |
| owner: str = Field(pattern=r"^[A-Za-z0-9][A-Za-z0-9-]{0,38}$") | |
| repo: str = Field(pattern=r"^[A-Za-z0-9._-]{1,100}$") | |
| def not_dots(cls, value: str) -> str: | |
| if not value.strip("."): | |
| raise ValueError("not a repository name") | |
| return value | |
| class Measure: | |
| sha: str | |
| files: int | |
| lines: int | |
| score: float | |
| scans: tuple[ScanFile, ...] = () | |
| class Entry: | |
| repo: str | |
| score: float | |
| files: int | |
| lines: int | |
| sha: str | |
| ranked_at: float | |
| scorer: str = "" | |
| def key(self) -> str: | |
| return self.repo.lower() | |
| def row(self) -> dict[str, Any]: | |
| return { | |
| "repo": self.repo, | |
| "score": self.score, | |
| "files": self.files, | |
| "lines": self.lines, | |
| "sha": self.sha, | |
| } | |
| def line_count(text: str) -> int: | |
| return text.count("\n") + (0 if text.endswith("\n") else 1) | |
| def ranked_path(path: str) -> bool: | |
| """Source files that count toward a repo's score: no tests, vendored, generated or minified code.""" | |
| parts = path.lower().split("/") | |
| name = parts[-1] | |
| if name.rpartition(".")[2] not in RANKED_EXTENSIONS: | |
| return False | |
| if any(part in SKIPPED_DIRS or part.startswith(".") for part in parts[:-1]): | |
| return False | |
| if name.startswith("test_") or name == "conftest.py" or name.endswith(SKIPPED_SUFFIXES): | |
| return False | |
| return not any(infix in name for infix in SKIPPED_INFIXES) | |
| def generated(text: str) -> bool: | |
| head = text[:600].lower() | |
| return any(mark in head for mark in GENERATED_MARKS) | |
| def archive_files(archive: bytes) -> tuple[str, list[tuple[str, str]]]: | |
| """Commit SHA and every ranked file of a GitHub tar.gz, read in memory; nothing touches disk.""" | |
| files = [] | |
| with tarfile.open(fileobj=io.BytesIO(archive), mode="r:gz") as tar: | |
| sha = str(tar.pax_headers.get("comment", "")) | |
| for member in tar: | |
| path = member.name.partition("/")[2] | |
| if not member.isfile() or member.size > MAX_FILE_BYTES or not ranked_path(path): | |
| continue | |
| handle = tar.extractfile(member) | |
| text = handle.read().decode("utf-8", errors="replace") if handle else "" | |
| if text.strip() and "\ufffd" not in text and not generated(text): | |
| files.append((path, text)) | |
| return (sha if COMMIT_SHA.match(sha) else ""), files | |
| def _unpack_score(verdict: ScoreVerdict) -> tuple[float, dict[str, int]]: | |
| if isinstance(verdict, tuple): | |
| return verdict | |
| return float(verdict), {} | |
| def measure(archive: bytes, scorer: Scorer) -> Measure: | |
| """Line-weighted mean file score, 0 to 100, across every ranked file in the archive.""" | |
| sha, files = archive_files(archive) | |
| total_lines = 0 | |
| weighted = 0.0 | |
| scans: list[ScanFile] = [] | |
| for path, text in files: | |
| lines = line_count(text) | |
| file_score, squiggles = _unpack_score(scorer(text)) | |
| total_lines += lines | |
| weighted += file_score * lines | |
| scans.append(scan_file_from(path, text, file_score, squiggles)) | |
| score = round(100 * weighted / total_lines, 1) if total_lines else 0.0 | |
| return Measure(sha=sha, files=len(files), lines=total_lines, score=score, scans=tuple(scans)) | |
| def qualifies(files: int, lines: int) -> bool: | |
| return files >= MIN_FILES and lines >= MIN_LINES | |
| def iso(stamp: float) -> str: | |
| return datetime.fromtimestamp(stamp, timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") | |
| def _score(value: Any) -> float | None: | |
| number = value if isinstance(value, (int, float)) and not isinstance(value, bool) else None | |
| return float(number) if number is not None and 0 <= number <= 100 else None | |
| def entry_from(raw: Any) -> Entry | None: | |
| """An entry from a saved board, or None when any field is missing or out of range.""" | |
| if not isinstance(raw, dict): | |
| return None | |
| repo, sha, files, lines = raw.get("repo"), raw.get("sha"), raw.get("files"), raw.get("lines") | |
| score, stamp = _score(raw.get("score")), raw.get("ranked_at") | |
| shaped = ( | |
| isinstance(repo, str) and REPO_NAME.match(repo) | |
| and isinstance(sha, str) and COMMIT_SHA.match(sha) | |
| and isinstance(files, int) and isinstance(lines, int) and qualifies(files, lines) | |
| and score is not None and isinstance(stamp, (int, float)) | |
| ) | |
| if not shaped: | |
| return None | |
| scorer = raw.get("scorer") | |
| return Entry( | |
| repo=repo, | |
| score=score, | |
| files=files, | |
| lines=lines, | |
| sha=sha, | |
| ranked_at=float(stamp), | |
| scorer=scorer[:64] if isinstance(scorer, str) else "", | |
| ) | |
| class Leaderboard: | |
| """Repos ranked by server-side score. Holds only public repo names, commit SHAs and numbers. | |
| Every entry records the scorer that produced it. Only entries from the running scorer are listed; | |
| older ones stay in the file until `maintain` re-ranks or drops them. | |
| """ | |
| def __init__( | |
| self, | |
| settings: Settings, | |
| transport: httpx.AsyncBaseTransport | None = None, | |
| scorer: Scorer | None = None, | |
| clock: Callable[[], float] = time.time, | |
| scorer_id: str = SCORER_ID, | |
| telemetry: Telemetry | None = None, | |
| ) -> None: | |
| self.entries: dict[str, Entry] = {} | |
| self.dirty = False | |
| self.scorer: Scorer = scorer or score_file | |
| self.scorer_id = scorer_id | |
| self.clock = clock | |
| self.telemetry = telemetry | |
| self.api_headers = {"Accept": "application/vnd.github+json", "X-GitHub-Api-Version": "2022-11-28"} | |
| if settings.github_token: | |
| self.api_headers["Authorization"] = f"Bearer {settings.github_token}" | |
| self.client = httpx.AsyncClient( | |
| transport=transport, | |
| headers={"User-Agent": "codehygiene-space"}, | |
| timeout=httpx.Timeout(60.0, connect=10.0), | |
| follow_redirects=True, | |
| ) | |
| self._turn = asyncio.Lock() | |
| self._waiting = 0 | |
| async def close(self) -> None: | |
| await self.client.aclose() | |
| def merge(self, raw: Any) -> None: | |
| rows = raw.get("entries") if isinstance(raw, dict) else None | |
| for item in rows if isinstance(rows, list) else []: | |
| entry = entry_from(item) | |
| known = self.entries.get(entry.key) if entry else None | |
| if entry and (known is None or entry.ranked_at > known.ranked_at): | |
| self.entries[entry.key] = entry | |
| def snapshot(self) -> str: | |
| rows = [asdict(entry) for entry in sorted(self.entries.values(), key=lambda entry: entry.key)] | |
| return json.dumps({"version": BOARD_VERSION, "entries": rows}, indent=2, sort_keys=True) + "\n" | |
| def current(self) -> list[Entry]: | |
| return [entry for entry in self.entries.values() if entry.scorer == self.scorer_id] | |
| def stale(self) -> list[Entry]: | |
| return [entry for entry in self.entries.values() if entry.scorer != self.scorer_id] | |
| def cleanest(self) -> list[Entry]: | |
| return sorted(self.current(), key=lambda entry: (-entry.score, entry.key)) | |
| def public(self) -> dict[str, Any]: | |
| stamps = [entry.ranked_at for entry in self.current()] | |
| return { | |
| "total": len(stamps), | |
| "updated_at": iso(max(stamps)) if stamps else None, | |
| "scorer_version": SCORER_VERSION, | |
| "cleanest": [entry.row() for entry in self.cleanest()[:BOARD_LIMIT]], | |
| } | |
| def standing(self, entry: Entry) -> dict[str, Any]: | |
| order = [ranked.key for ranked in self.cleanest()] | |
| return {**entry.row(), "rank": order.index(entry.key) + 1, "total": len(order)} | |
| def _fresh(self, entry: Entry | None, seconds: float) -> bool: | |
| return bool(entry) and entry.scorer == self.scorer_id and self.clock() - entry.ranked_at < seconds | |
| async def rank(self, owner: str, repo: str) -> dict[str, Any]: | |
| known = self.entries.get(f"{owner}/{repo}".lower()) | |
| if known and self._fresh(known, FRESH_SECONDS): | |
| return self.standing(known) | |
| if self._waiting >= MAX_WAITING: | |
| raise RankError(429, "busy", "Ranking is busy. Try again shortly.") | |
| self._waiting += 1 | |
| try: | |
| async with self._turn: | |
| entry, scans = await self._build(owner, repo) | |
| finally: | |
| self._waiting -= 1 | |
| self.entries[entry.key] = entry | |
| self.dirty = True | |
| if self.telemetry is not None and scans: | |
| self.telemetry.record_files(scans) | |
| return self.standing(entry) | |
| async def maintain(self, seeds: Iterable[str], pause: float) -> None: | |
| """Ranks seeds that are missing or old, then re-ranks entries from an older scorer.""" | |
| for slug in seeds: | |
| if not self._fresh(self.entries.get(slug.lower()), SEED_FRESH_SECONDS): | |
| await self._refresh(slug) | |
| await asyncio.sleep(pause) | |
| for entry in self.stale(): | |
| await self._refresh(entry.repo) | |
| await asyncio.sleep(pause) | |
| async def _refresh(self, slug: str) -> None: | |
| """Re-ranks one repo; an older entry is dropped when the repo can no longer be listed.""" | |
| owner, repo = slug.split("/") | |
| try: | |
| await self.rank(owner, repo) | |
| except RankError as error: | |
| log.info("refresh skipped: %s", error.code) | |
| if error.code in GONE_CODES: | |
| self._drop_stale(slug) | |
| except httpx.HTTPError as error: | |
| log.info("refresh skipped: %s", type(error).__name__) | |
| def _drop_stale(self, slug: str) -> None: | |
| known = self.entries.get(slug.lower()) | |
| if known and known.scorer != self.scorer_id: | |
| del self.entries[known.key] | |
| self.dirty = True | |
| async def _build(self, owner: str, repo: str) -> tuple[Entry, tuple[ScanFile, ...]]: | |
| meta = await self._meta(owner, repo) | |
| slug = str(meta.get("full_name") or f"{owner}/{repo}") | |
| ref = str(meta.get("default_branch") or "HEAD") | |
| current = await self._measure(slug, ref) | |
| if not qualifies(current.files, current.lines) or not current.sha: | |
| raise RankError( | |
| 422, "too_small", f"Needs {MIN_FILES}+ code files and {MIN_LINES:,}+ lines.", | |
| files=current.files, lines=current.lines, | |
| ) | |
| twin = self._twin(current.sha, slug) | |
| if twin: | |
| raise RankError(409, "duplicate", f"Already ranked as {twin.repo}.", repo=twin.repo) | |
| entry = Entry( | |
| repo=slug, | |
| score=current.score, | |
| files=current.files, | |
| lines=current.lines, | |
| sha=current.sha, | |
| ranked_at=self.clock(), | |
| scorer=self.scorer_id, | |
| ) | |
| return entry, current.scans | |
| def _twin(self, sha: str, slug: str) -> Entry | None: | |
| """An entry for the same commit under another name, which is a fork or mirror.""" | |
| others = (entry for entry in self.current() if entry.key != slug.lower()) | |
| return next((entry for entry in others if entry.sha == sha), None) | |
| async def _meta(self, owner: str, repo: str) -> dict[str, Any]: | |
| """Repo metadata, or {} when the API is rate limited; archives still download without it.""" | |
| response = await self.client.get(f"{API}/repos/{owner}/{repo}", headers=self.api_headers) | |
| if response.status_code in (403, 429): | |
| log.info("GitHub API rate limited; ranking from the archive alone") | |
| return {} | |
| if response.status_code == 404: | |
| raise RankError(404, "not_found", "Repo not found.") | |
| response.raise_for_status() | |
| meta = response.json() | |
| if meta.get("private"): | |
| raise RankError(404, "not_found", "Repo not found.") | |
| if meta.get("fork"): | |
| raise RankError(422, "fork", "Forks are not ranked.") | |
| return meta | |
| async def _measure(self, slug: str, ref: str) -> Measure: | |
| archive = await self._archive(slug, ref) | |
| return await asyncio.to_thread(measure, archive, self.scorer) | |
| async def _archive(self, slug: str, ref: str) -> bytes: | |
| chunks: list[bytes] = [] | |
| size = 0 | |
| url = f"{ARCHIVES}/{slug}/tar.gz/{quote(ref, safe='/')}" | |
| async with self.client.stream("GET", url) as response: | |
| if response.status_code == 404: | |
| raise RankError(404, "not_found", "Repo not found.") | |
| response.raise_for_status() | |
| async for chunk in response.aiter_bytes(): | |
| size += len(chunk) | |
| if size > MAX_ARCHIVE_BYTES: | |
| raise RankError(413, "too_large", "Repo is too large to rank.") | |
| chunks.append(chunk) | |
| return b"".join(chunks) | |