"""Public leaderboard of GitHub repos, each scored in full on the server. See space/TELEMETRY.md.""" from __future__ import annotations import asyncio from collections.abc import Callable, Iterable from dataclasses import asdict, dataclass from datetime import datetime, timezone import io import json import logging import re import tarfile import time from typing import Any from urllib.parse import quote import httpx from pydantic import BaseModel, ConfigDict, Field, field_validator from .config import Settings from .hygiene import SCORER_ID, SCORER_VERSION, score_file from .telemetry import ScanFile, Telemetry, scan_file_from log = logging.getLogger("space.leaderboard") API = "https://api.github.com" ARCHIVES = "https://codeload.github.com" BOARD_FILE = "leaderboard.json" BOARD_VERSION = 2 BOARD_LIMIT = 100 MIN_FILES = 5 MIN_LINES = 1_000 MAX_ARCHIVE_BYTES = 80 * 1024 * 1024 MAX_FILE_BYTES = 150_000 FRESH_SECONDS = 24 * 3600 SEED_FRESH_SECONDS = 7 * 24 * 3600 MAX_WAITING = 3 GONE_CODES = frozenset({"not_found", "too_small", "fork", "duplicate"}) RANKED_EXTENSIONS = frozenset({"py", "ts", "tsx", "js", "jsx", "mjs", "cjs", "go", "rs"}) SKIPPED_DIRS = frozenset({ "test", "tests", "__tests__", "spec", "specs", "e2e", "testdata", "fixtures", "__fixtures__", "__mocks__", "vendor", "vendored", "third_party", "thirdparty", "node_modules", "dist", "build", "out", "target", "generated", "__generated__", }) SKIPPED_SUFFIXES = (".d.ts", ".pb.go", "_pb2.py", "_test.go", "_test.py", "_generated.go") SKIPPED_INFIXES = (".test.", ".spec.", ".generated.", ".min.") GENERATED_MARKS = ( "code generated", "do not edit", "@generated", "auto-generated", "autogenerated", "file generated", ) REPO_NAME = re.compile(r"^[A-Za-z0-9][A-Za-z0-9-]{0,38}/[A-Za-z0-9._-]{1,100}$") COMMIT_SHA = re.compile(r"^[0-9a-f]{40}$") SEED_REPOS = ( "BurntSushi/ripgrep", "honojs/hono", "fastapi/fastapi", "karpathy/nanoGPT", "openai/codex", "anomalyco/opencode", "pallets/flask", "psf/requests", "astral-sh/ruff", "junegunn/fzf", "charmbracelet/bubbletea", "gin-gonic/gin", "colinhacks/zod", "vitejs/vite", "expressjs/express", "tokio-rs/axum", "tinygrad/tinygrad", "pydantic/pydantic", "sindresorhus/ky", "anthropics/claude-code", ) ScoreVerdict = float | tuple[float, dict[str, int]] Scorer = Callable[[str], ScoreVerdict] class RankError(Exception): def __init__(self, status: int, code: str, message: str, **extra: Any) -> None: super().__init__(message) self.status = status self.code = code self.message = message self.extra = extra def payload(self) -> dict[str, Any]: return {"code": self.code, "message": self.message, **self.extra} class RankBody(BaseModel): model_config = ConfigDict(extra="forbid") owner: str = Field(pattern=r"^[A-Za-z0-9][A-Za-z0-9-]{0,38}$") repo: str = Field(pattern=r"^[A-Za-z0-9._-]{1,100}$") @field_validator("repo") @classmethod def not_dots(cls, value: str) -> str: if not value.strip("."): raise ValueError("not a repository name") return value @dataclass(frozen=True) class Measure: sha: str files: int lines: int score: float scans: tuple[ScanFile, ...] = () @dataclass(frozen=True) class Entry: repo: str score: float files: int lines: int sha: str ranked_at: float scorer: str = "" @property def key(self) -> str: return self.repo.lower() def row(self) -> dict[str, Any]: return { "repo": self.repo, "score": self.score, "files": self.files, "lines": self.lines, "sha": self.sha, } def line_count(text: str) -> int: return text.count("\n") + (0 if text.endswith("\n") else 1) def ranked_path(path: str) -> bool: """Source files that count toward a repo's score: no tests, vendored, generated or minified code.""" parts = path.lower().split("/") name = parts[-1] if name.rpartition(".")[2] not in RANKED_EXTENSIONS: return False if any(part in SKIPPED_DIRS or part.startswith(".") for part in parts[:-1]): return False if name.startswith("test_") or name == "conftest.py" or name.endswith(SKIPPED_SUFFIXES): return False return not any(infix in name for infix in SKIPPED_INFIXES) def generated(text: str) -> bool: head = text[:600].lower() return any(mark in head for mark in GENERATED_MARKS) def archive_files(archive: bytes) -> tuple[str, list[tuple[str, str]]]: """Commit SHA and every ranked file of a GitHub tar.gz, read in memory; nothing touches disk.""" files = [] with tarfile.open(fileobj=io.BytesIO(archive), mode="r:gz") as tar: sha = str(tar.pax_headers.get("comment", "")) for member in tar: path = member.name.partition("/")[2] if not member.isfile() or member.size > MAX_FILE_BYTES or not ranked_path(path): continue handle = tar.extractfile(member) text = handle.read().decode("utf-8", errors="replace") if handle else "" if text.strip() and "\ufffd" not in text and not generated(text): files.append((path, text)) return (sha if COMMIT_SHA.match(sha) else ""), files def _unpack_score(verdict: ScoreVerdict) -> tuple[float, dict[str, int]]: if isinstance(verdict, tuple): return verdict return float(verdict), {} def measure(archive: bytes, scorer: Scorer) -> Measure: """Line-weighted mean file score, 0 to 100, across every ranked file in the archive.""" sha, files = archive_files(archive) total_lines = 0 weighted = 0.0 scans: list[ScanFile] = [] for path, text in files: lines = line_count(text) file_score, squiggles = _unpack_score(scorer(text)) total_lines += lines weighted += file_score * lines scans.append(scan_file_from(path, text, file_score, squiggles)) score = round(100 * weighted / total_lines, 1) if total_lines else 0.0 return Measure(sha=sha, files=len(files), lines=total_lines, score=score, scans=tuple(scans)) def qualifies(files: int, lines: int) -> bool: return files >= MIN_FILES and lines >= MIN_LINES def iso(stamp: float) -> str: return datetime.fromtimestamp(stamp, timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") def _score(value: Any) -> float | None: number = value if isinstance(value, (int, float)) and not isinstance(value, bool) else None return float(number) if number is not None and 0 <= number <= 100 else None def entry_from(raw: Any) -> Entry | None: """An entry from a saved board, or None when any field is missing or out of range.""" if not isinstance(raw, dict): return None repo, sha, files, lines = raw.get("repo"), raw.get("sha"), raw.get("files"), raw.get("lines") score, stamp = _score(raw.get("score")), raw.get("ranked_at") shaped = ( isinstance(repo, str) and REPO_NAME.match(repo) and isinstance(sha, str) and COMMIT_SHA.match(sha) and isinstance(files, int) and isinstance(lines, int) and qualifies(files, lines) and score is not None and isinstance(stamp, (int, float)) ) if not shaped: return None scorer = raw.get("scorer") return Entry( repo=repo, score=score, files=files, lines=lines, sha=sha, ranked_at=float(stamp), scorer=scorer[:64] if isinstance(scorer, str) else "", ) class Leaderboard: """Repos ranked by server-side score. Holds only public repo names, commit SHAs and numbers. Every entry records the scorer that produced it. Only entries from the running scorer are listed; older ones stay in the file until `maintain` re-ranks or drops them. """ def __init__( self, settings: Settings, transport: httpx.AsyncBaseTransport | None = None, scorer: Scorer | None = None, clock: Callable[[], float] = time.time, scorer_id: str = SCORER_ID, telemetry: Telemetry | None = None, ) -> None: self.entries: dict[str, Entry] = {} self.dirty = False self.scorer: Scorer = scorer or score_file self.scorer_id = scorer_id self.clock = clock self.telemetry = telemetry self.api_headers = {"Accept": "application/vnd.github+json", "X-GitHub-Api-Version": "2022-11-28"} if settings.github_token: self.api_headers["Authorization"] = f"Bearer {settings.github_token}" self.client = httpx.AsyncClient( transport=transport, headers={"User-Agent": "codehygiene-space"}, timeout=httpx.Timeout(60.0, connect=10.0), follow_redirects=True, ) self._turn = asyncio.Lock() self._waiting = 0 async def close(self) -> None: await self.client.aclose() def merge(self, raw: Any) -> None: rows = raw.get("entries") if isinstance(raw, dict) else None for item in rows if isinstance(rows, list) else []: entry = entry_from(item) known = self.entries.get(entry.key) if entry else None if entry and (known is None or entry.ranked_at > known.ranked_at): self.entries[entry.key] = entry def snapshot(self) -> str: rows = [asdict(entry) for entry in sorted(self.entries.values(), key=lambda entry: entry.key)] return json.dumps({"version": BOARD_VERSION, "entries": rows}, indent=2, sort_keys=True) + "\n" def current(self) -> list[Entry]: return [entry for entry in self.entries.values() if entry.scorer == self.scorer_id] def stale(self) -> list[Entry]: return [entry for entry in self.entries.values() if entry.scorer != self.scorer_id] def cleanest(self) -> list[Entry]: return sorted(self.current(), key=lambda entry: (-entry.score, entry.key)) def public(self) -> dict[str, Any]: stamps = [entry.ranked_at for entry in self.current()] return { "total": len(stamps), "updated_at": iso(max(stamps)) if stamps else None, "scorer_version": SCORER_VERSION, "cleanest": [entry.row() for entry in self.cleanest()[:BOARD_LIMIT]], } def standing(self, entry: Entry) -> dict[str, Any]: order = [ranked.key for ranked in self.cleanest()] return {**entry.row(), "rank": order.index(entry.key) + 1, "total": len(order)} def _fresh(self, entry: Entry | None, seconds: float) -> bool: return bool(entry) and entry.scorer == self.scorer_id and self.clock() - entry.ranked_at < seconds async def rank(self, owner: str, repo: str) -> dict[str, Any]: known = self.entries.get(f"{owner}/{repo}".lower()) if known and self._fresh(known, FRESH_SECONDS): return self.standing(known) if self._waiting >= MAX_WAITING: raise RankError(429, "busy", "Ranking is busy. Try again shortly.") self._waiting += 1 try: async with self._turn: entry, scans = await self._build(owner, repo) finally: self._waiting -= 1 self.entries[entry.key] = entry self.dirty = True if self.telemetry is not None and scans: self.telemetry.record_files(scans) return self.standing(entry) async def maintain(self, seeds: Iterable[str], pause: float) -> None: """Ranks seeds that are missing or old, then re-ranks entries from an older scorer.""" for slug in seeds: if not self._fresh(self.entries.get(slug.lower()), SEED_FRESH_SECONDS): await self._refresh(slug) await asyncio.sleep(pause) for entry in self.stale(): await self._refresh(entry.repo) await asyncio.sleep(pause) async def _refresh(self, slug: str) -> None: """Re-ranks one repo; an older entry is dropped when the repo can no longer be listed.""" owner, repo = slug.split("/") try: await self.rank(owner, repo) except RankError as error: log.info("refresh skipped: %s", error.code) if error.code in GONE_CODES: self._drop_stale(slug) except httpx.HTTPError as error: log.info("refresh skipped: %s", type(error).__name__) def _drop_stale(self, slug: str) -> None: known = self.entries.get(slug.lower()) if known and known.scorer != self.scorer_id: del self.entries[known.key] self.dirty = True async def _build(self, owner: str, repo: str) -> tuple[Entry, tuple[ScanFile, ...]]: meta = await self._meta(owner, repo) slug = str(meta.get("full_name") or f"{owner}/{repo}") ref = str(meta.get("default_branch") or "HEAD") current = await self._measure(slug, ref) if not qualifies(current.files, current.lines) or not current.sha: raise RankError( 422, "too_small", f"Needs {MIN_FILES}+ code files and {MIN_LINES:,}+ lines.", files=current.files, lines=current.lines, ) twin = self._twin(current.sha, slug) if twin: raise RankError(409, "duplicate", f"Already ranked as {twin.repo}.", repo=twin.repo) entry = Entry( repo=slug, score=current.score, files=current.files, lines=current.lines, sha=current.sha, ranked_at=self.clock(), scorer=self.scorer_id, ) return entry, current.scans def _twin(self, sha: str, slug: str) -> Entry | None: """An entry for the same commit under another name, which is a fork or mirror.""" others = (entry for entry in self.current() if entry.key != slug.lower()) return next((entry for entry in others if entry.sha == sha), None) async def _meta(self, owner: str, repo: str) -> dict[str, Any]: """Repo metadata, or {} when the API is rate limited; archives still download without it.""" response = await self.client.get(f"{API}/repos/{owner}/{repo}", headers=self.api_headers) if response.status_code in (403, 429): log.info("GitHub API rate limited; ranking from the archive alone") return {} if response.status_code == 404: raise RankError(404, "not_found", "Repo not found.") response.raise_for_status() meta = response.json() if meta.get("private"): raise RankError(404, "not_found", "Repo not found.") if meta.get("fork"): raise RankError(422, "fork", "Forks are not ranked.") return meta async def _measure(self, slug: str, ref: str) -> Measure: archive = await self._archive(slug, ref) return await asyncio.to_thread(measure, archive, self.scorer) async def _archive(self, slug: str, ref: str) -> bytes: chunks: list[bytes] = [] size = 0 url = f"{ARCHIVES}/{slug}/tar.gz/{quote(ref, safe='/')}" async with self.client.stream("GET", url) as response: if response.status_code == 404: raise RankError(404, "not_found", "Repo not found.") response.raise_for_status() async for chunk in response.aiter_bytes(): size += len(chunk) if size > MAX_ARCHIVE_BYTES: raise RankError(413, "too_large", "Repo is too large to rank.") chunks.append(chunk) return b"".join(chunks)