smoke_test.py

"""Check the site against the current ``data/``: ``python smoke_test.py`` (standard library only).

Checks that every subset listed in ``data/meta.json`` has its files, that the leaderboards have one
row per method with Random at 1000, that the top-level pages exist, that ``data/`` holds nothing
else, and that the version in ``index.html`` matches the newest version-history entry. For the
visual check, serve the folder (``python -m http.server 8000``) and open http://localhost:8000.
"""

from __future__ import annotations

import csv
import json
import re
import sys
from pathlib import Path

ROOT = Path(__file__).resolve().parent
DATA = ROOT / "data"
SUBSET_FILES = (
    "website_leaderboard.csv",
    "leaderboard_table.html",
    "elo_explorer.html",
)
TOP_LEVEL = {
    "meta.json",
    "subsets",
    "per_dataset.csv",
    "per_dataset_table.html",
    ".gitkeep",
}
OPTIONAL = {
    "stability.csv",
    "stability_table.html",
    "validity.csv",
    "validity_table.html",  # data generated before one table per noise level
}


def check_version(errors: list[str]) -> str:
    html = (ROOT / "index.html").read_text()
    current = re.search(r'name="selectarena-version" content="([^"]+)"', html)
    history = re.search(
        r'id="version-history">\s*<li><strong>(\d{4}/\d{2}/\d{2}) · v([^<]+)</strong>',
        html,
    )
    if not current or not history:
        errors.append("index.html: selectarena-version meta tag or version history not found")
        return "?"
    if current.group(1) != history.group(2):
        errors.append(f"index.html: version {current.group(1)} but newest history entry is v{history.group(2)}")
    return current.group(1)


def main() -> int:
    errors: list[str] = []
    version = check_version(errors)
    if not (DATA / "meta.json").exists():
        print(f"v{version}: data/ has no meta.json, the site shows 'No results published yet'.")
        return 1 if errors else 0

    meta = json.loads((DATA / "meta.json").read_text())
    for info in meta["subsets"]:
        folder = DATA / info["path"]
        for name in SUBSET_FILES:
            if not (folder / name).is_file():
                errors.append(f"missing {info['path']}/{name}")
        if (folder / "website_leaderboard.csv").is_file():
            with (folder / "website_leaderboard.csv").open() as f:
                rows = list(csv.DictReader(f))
            if len(rows) != info["n_methods"]:
                errors.append(f"{info['path']}: {len(rows)} rows for {info['n_methods']} methods")
            random = [r for r in rows if r["Method"] == "Random"]
            if random and float(random[0]["Elo"]) != 1000:
                errors.append(f"{info['path']}: Random has Elo {random[0]['Elo']}, not 1000")
            if any(r["Category"] == "Other" for r in rows):
                errors.append(f"{info['path']}: a method has no subgroup in the taxonomy ('Other')")
    if not any(s["tasks"] == s["model"] == s["budget"] == "all" for s in meta["subsets"]):
        errors.append("the default subset (all, all, all) is not published")

    listed = {info["path"] for info in meta["subsets"]}
    on_disk = {str(p.parent.relative_to(DATA)) for p in DATA.glob("subsets/*/*/*/website_leaderboard.csv")}
    if on_disk - listed:
        errors.append(f"subset folders not in meta.json (stale?): {sorted(on_disk - listed)[:5]}")
    for name in ("per_dataset_table.html",):
        if not (DATA / name).is_file():
            errors.append(f"missing data/{name}")
    if meta.get("has_stability") and not (DATA / "stability_table.html").is_file():
        errors.append("meta.json says has_stability but data/stability_table.html is missing")
    for table in meta.get("validity_tables", []):
        if not (DATA / table["path"]).is_file():
            errors.append(f"missing data/{table['path']} (validity at noise {table['noise']})")
    validity_pages = {p.name for p in DATA.glob("validity_noise_*_table.html")}
    unexpected = {p.name for p in DATA.iterdir()} - TOP_LEVEL - OPTIONAL - validity_pages
    if unexpected:
        errors.append(f"unexpected files in data/: {sorted(unexpected)}")

    for error in errors:
        print("ERROR", error)
    print(f"v{version}: {len(meta['subsets'])} subsets checked, {len(errors)} errors.")
    return 1 if errors else 0


if __name__ == "__main__":
    sys.exit(main())