math-sight-reading / tests /test_ui.py
SoulFireMage's picture
Test: key and progress survive a server restart
0e97c8b verified
Raw History Blame Contribute Delete
6.07 kB
"""Browser test: drives the real Gradio app against a fake OpenRouter.
Run: python tests/test_ui.py (needs playwright + a Chromium)
"""
import os
import subprocess
import sys
import time
import urllib.request
sys.path.insert(0, os.path.dirname(__file__))
from fake_openrouter import Fake
from playwright.sync_api import expect, sync_playwright
ROOT = os.path.join(os.path.dirname(__file__), "..")
PORT = 7861
URL = f"http://127.0.0.1:{PORT}"
CHROMIUM = os.environ.get("CHROMIUM_PATH") or "/opt/pw-browsers/chromium-1194/chrome-linux/chrome"
fake = Fake()
env = dict(os.environ, OPENROUTER_BASE_URL=fake.url, GRADIO_SERVER_PORT=str(PORT), NO_PROXY="127.0.0.1,localhost", no_proxy="127.0.0.1,localhost")
env.pop("SPACE_ID", None)
def start_server():
proc = subprocess.Popen([sys.executable, "app.py"], cwd=ROOT, env=env, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
for _ in range(60):
try:
urllib.request.urlopen(URL, timeout=1)
return proc
except Exception:
time.sleep(0.5)
raise SystemExit("server did not start:\n" + proc.stdout.read().decode())
srv = start_server()
try:
with sync_playwright() as pw:
exe = CHROMIUM if os.path.exists(CHROMIUM) else None
browser = pw.chromium.launch(executable_path=exe)
ctx = browser.new_context()
page = ctx.new_page()
page.goto(URL)
# 1. a formula loads and KaTeX renders it
expect(page.locator("#formula .katex").first).to_be_visible(timeout=20000)
print("ok formula rendered with KaTeX")
# 2. evaluating without a key gives a clear message and no request
page.get_by_label("1. Formal sight-reading").fill("x is a vector in R^n and A is a matrix of size n by n.")
page.get_by_label("2. Concept & mechanics").fill("It measures how much A stretches x along itself.")
page.get_by_label("3. Application & metaphor").fill("PCA variance and vibration frequency of a structure.")
page.get_by_role("button", name="Evaluate").click()
expect(page.get_by_text("Add your OpenRouter API key")).to_be_visible(timeout=10000)
assert fake.requests == []
print("ok no key -> message, no request")
# 3. hints reveal progressively
page.get_by_role("button", name="Hint", exact=True).click()
expect(page.get_by_text("Hint 1 (types)")).to_be_visible()
page.get_by_role("button", name="Hint", exact=True).click()
expect(page.get_by_text("Hint 2 (mechanics)")).to_be_visible()
print("ok hint ladder")
# 4. set key + model, evaluate
page.get_by_role("tab", name="Settings").click()
page.get_by_label("OpenRouter API key").fill("sk-or-UITESTKEY")
page.get_by_role("tab", name="Drill").click()
page.get_by_role("button", name="Evaluate").click()
expect(page.get_by_text("7/10")).to_be_visible(timeout=20000)
expect(page.get_by_text("Competent").first).to_be_visible()
expect(page.get_by_text("This was:")).to_be_visible()
assert fake.requests[-1]["auth"] == "Bearer sk-or-UITESTKEY"
assert len(fake.requests) == 1
print("ok evaluation shown, key sent as bearer")
# hints used (2) are penalised: 7/10, Competent, A -> (0.7+0.5+1)/3=0.733 - 0.2 = 53%
expect(page.get_by_text("53%")).to_be_visible()
print("ok hint penalty applied")
# 5. evaluating again on the same drill is refused (no double count)
page.get_by_role("button", name="Evaluate").click()
expect(page.get_by_text("Press New formula")).to_be_visible(timeout=10000)
assert len(fake.requests) == 1
print("ok no double submission")
# 6. progress tab
page.get_by_role("tab", name="Progress").click()
expect(page.get_by_text("1 drills")).to_be_visible()
print("ok progress updated")
# 7. persistence across reload (BrowserState)
page.reload()
page.get_by_role("tab", name="Progress").click()
expect(page.get_by_text("1 drills")).to_be_visible(timeout=20000)
page.get_by_role("tab", name="Settings").click()
expect(page.get_by_label("OpenRouter API key")).to_have_value("sk-or-UITESTKEY")
print("ok progress and key persist across reload")
# 7b. persistence across a SERVER RESTART (a Space sleeping or rebuilding).
# BrowserState encrypts with a per-process random secret unless one is set,
# which silently drops the saved key and progress after a restart.
srv.terminate()
srv.wait()
srv = start_server()
page.reload()
page.get_by_role("tab", name="Progress").click()
expect(page.get_by_text("1 drills")).to_be_visible(timeout=20000)
page.get_by_role("tab", name="Settings").click()
expect(page.get_by_label("OpenRouter API key")).to_have_value("sk-or-UITESTKEY")
print("ok key and progress survive a server restart")
# 8. model list refresh via the fake server
page.get_by_role("button", name="Load current OpenRouter model list").click()
expect(page.get_by_text("2 models loaded")).to_be_visible(timeout=10000)
print("ok model list refresh")
# 9. judge error is surfaced and nothing is recorded
fake.status = 401
page.get_by_role("tab", name="Drill").click()
page.get_by_role("button", name="New formula").click()
page.get_by_label("1. Formal sight-reading").fill("Typed carefully with enough words to pass the length check.")
page.get_by_role("button", name="Evaluate").click()
expect(page.get_by_text("Judge error")).to_be_visible(timeout=20000)
expect(page.get_by_text("HTTP 401")).to_be_visible()
page.get_by_role("tab", name="Progress").click()
expect(page.get_by_text("1 drills")).to_be_visible()
print("ok judge error surfaced, nothing recorded")
browser.close()
print("ALL UI CHECKS PASSED")
finally:
srv.terminate()
fake.close()