scripts/check_cloud_ui.py

#!/usr/bin/env python3
"""Check a running Docker UI, with its evaluator deliberately disabled in CI."""
import json
import time


def main():
    import httpx
    from gradio_client import Client
    from check_forecast_ui import check_forecast_ui
    import subprocess
    subprocess.run(["/usr/local/bin/python", "-c",
                    "import cloud.worker; from tsfm_bench.remote_state import RemoteState"],
                   check=True, timeout=60)
    subprocess.run(["/app/.venv-tabpfn/bin/python", "-c",
                    "from tabpfn_time_series import TabPFNMode, TabPFNTSPipeline; "
                    "from tabpfn_client.service_wrapper import UserAuthenticationClient"],
                   check=True, timeout=60)

    url = "http://127.0.0.1:7860"
    with httpx.Client(trust_env=False, timeout=10) as http:
        for attempt in range(60):
            try:
                response = http.get(url + "/config")
                response.raise_for_status()
                config = response.json()
                break
            except httpx.HTTPError:
                if attempt == 59:
                    raise
                time.sleep(2)
        health = http.get(url + "/healthz")
        assert health.status_code == 503
        assert not health.json()["evaluator_enabled"]
        # Private runtime files must not be downloadable via Gradio file routes.
        for path in ("/app/.cloud-state/ui-results/online_status.json",
                     "/app/space/results/evaluation_metrics_canonical.jsonl"):
            assert http.get(url + "/file=" + path).status_code in (403, 404)
    client = Client(url, verbose=False, httpx_kwargs={"trust_env":False})
    result = client.predict(api_name="/refresh_leaderboard")
    assert "summary-grid" in result[0]
    print(json.dumps(check_forecast_ui(client, config)))


if __name__ == "__main__":
    main()