Board review: the steps of a running review (boards found, sent, the AI busy) on the student's and the teacher's screens

mathly2/boardreview.py

@@ -111,7 +111,7 @@ class GeminiReviewer:
             return None
         return pick_model(names, self.model)
 
-    def review(self, items: list[dict]) -> dict:
+    def review(self, items: list[dict], on_event=lambda step, **info: None) -> dict:
         body = json.dumps({"contents": [{"role": "user", "parts": self._parts(items)}],
                            "generationConfig": {"temperature": 0.2, "responseMimeType": "application/json", "responseSchema": SCHEMA}}).encode()
         with self._lock:
@@ -122,7 +122,10 @@ class GeminiReviewer:
                 except urllib.error.HTTPError as exc:
                     detail = exc.read().decode("utf-8", "replace")[:500]
                     if exc.code == 429 and attempt < RETRIES:
-                        self.sleep(_retry_after(detail, 20 * (attempt + 1)))
+                        wait = _retry_after(detail, 20 * (attempt + 1))
+                        on_event("busy", seconds=round(wait))
+                        self.sleep(wait)
+                        on_event("sent")
                         continue
                     if exc.code == 404 and not getattr(self, "_switched", False):
                         other = self._another_model()
@@ -206,15 +209,27 @@ def _items(app, row: dict) -> tuple[list[dict], dict]:
 
 
 def run(app, attempt_id: str) -> None:
-    """Review one handed-in attempt and store the result on it. Never raises: a failure is stored as {"state": "failed"}."""
+    """Review one handed-in attempt and store the result on it. Never raises: a failure is stored as {"state": "failed"}.
+    While it runs, its steps are stored on the pending review (`events`: boards found, sent, the AI busy and waiting), for the screens."""
     store = app.state.store
+    started = clock.now().isoformat()
+    events: list[dict] = []
+
+    def event(step: str, **info) -> None:
+        events.append({"step": step, "at": clock.now().isoformat(), **info})
+        try:
+            store.save_attempt(attempt_id, review={"state": "pending", "at": started, "events": list(events)})
+        except Exception:                                  # the steps are a courtesy: never fail the review for them
+            log.warning("could not save a review step")
     try:
         row = store.get_attempt(attempt_id)
         items, keys = _items(app, row)
+        event("boards", count=len(items), total=len(row["questions"]))
         if not items:
             result = {"state": "done", "summary": None, "questions": {}}
         else:
-            got = app.state.reviewer.review(items)
+            event("sent")
+            got = app.state.reviewer.review(items, on_event=event)
             questions = {}
             for q in got.get("questions", []):
                 key = keys.get(q.get("n"))
@@ -271,6 +286,8 @@ def student_view(row: dict) -> dict | None:
     review = row.get("review")
     if not review:
         return None
+    if review["state"] == "pending":
+        return {"state": "pending", "at": review.get("at"), "events": review.get("events", [])}
     if review["state"] != "done":
         return {"state": review["state"]}
     return {"state": "done", "summary": review.get("summary"), "questions": review.get("questions", {})}

mathly2/boardwork.py

@@ -150,7 +150,8 @@ def _ai_state(request: Request, row: dict) -> dict:
     """The AI review as the teacher sees it: can it run here (a key is set), its state, and why it failed."""
     review = row.get("review") or {}
     return {"available": getattr(request.app.state, "reviewer", None) is not None, "state": review.get("state"),
-            "reason": review.get("reason"), "summary": review.get("summary"), "model": review.get("model")}
+            "reason": review.get("reason"), "summary": review.get("summary"), "model": review.get("model"),
+            "at": review.get("at"), "events": review.get("events", [])}
 
 
 @router.get("/teaching/work/{attempt_id}")

mathly2/static/app.css

@@ -692,8 +692,16 @@ body:has(#lesson-map) p:has(> #to-subjects) { margin-block: 0 4px; }
 .qz-review-list li p { margin: 4px 0; }
 .qz-review-list li.v-correct .tag { border-color: var(--ok, #15803d); }
 .qz-review-list li.v-wrong .tag { border-color: var(--danger); }
-.qz-review .spin { display: inline-block; width: 12px; height: 12px; border-radius: 50%; border: 2px solid var(--line); border-top-color: var(--accent); animation: spin 1s linear infinite; vertical-align: middle; }
+.qz-review .spin, .rv-steps .spin { display: inline-block; width: 12px; height: 12px; border-radius: 50%; border: 2px solid var(--line); border-top-color: var(--accent); animation: spin 1s linear infinite; vertical-align: middle; }
 @keyframes spin { to { transform: rotate(360deg); } }
 .work-ai { margin: 4px 0 8px; }
 .work-ai-panel p { margin: 4px 0; }
 .work-ai-panel button { margin-top: 6px; }
+
+/* the steps of an AI review while it runs */
+.rv-steps { margin: 6px 0; }
+.rv-steps li { display: flex; gap: 8px; align-items: center; padding: 3px 0; }
+.rv-steps li.rv-done { color: var(--muted); }
+.rv-steps li.rv-now { font-weight: 600; }
+.rv-steps .rv-ok { color: var(--ok, #15803d); }
+.rv-time { margin: 2px 0 0; font-size: .9em; }

mathly2/static/quiz.js

@@ -7,6 +7,7 @@ import { figures } from "./figures.js";
 import { esc, fmt } from "./math.js";
 import { pathHref, shuffled } from "./ui.js";
 import { hasWork, sendWork } from "./boardwork.js";
+import { stepsHtml } from "./reviewsteps.js";
 
 const LIVE_AT = new Set([300, 60, 10, 0]);        // seconds left at which the timer is announced to a screen reader
 
@@ -202,7 +203,7 @@ function reviewHtml(r) {
   const review = r.review;
   if (!review) return "";
   if (review.state === "pending") return `<section class="qz-review" id="qz-review" data-state="pending"><h4>משוב על הדרך</h4>
-    <p class="muted"><span class="spin" aria-hidden="true"></span> בודקים את הדרך שכתבתם בלוח… זה לוקח כדקה.</p></section>`;
+    <p class="muted">ה־AI עובר על הדרך שכתבתם בלוח. אפשר להמשיך לקרוא את התוצאה, המשוב יופיע כאן.</p>${stepsHtml(review)}</section>`;
   if (review.state === "failed") return `<section class="qz-review" id="qz-review" data-state="failed"><h4>משוב על הדרך</h4><p class="muted">לא הצלחנו לבדוק את הדרך הפעם. המורה יראו את הלוחות.</p></section>`;
   const rows = (r.keys || []).map((key, i) => [i + 1, review.questions[key]]).filter(([, q]) => q);
   return `<section class="qz-review" id="qz-review" data-state="done"><h4>משוב על הדרך</h4>

mathly2/static/reviewsteps.js

@@ -0,0 +1,31 @@
+// The steps of an AI review while it runs (board-review): what the server has done so far, as a short list that grows, and the time since
+// it started. Shown on the student's result and on the teacher's "בדיקת דרך" screen; both ask the server again every few seconds.
+import { esc } from "./math.js";
+
+const boards = n => n === 1 ? "לוח אחד" : `${n} לוחות`;
+
+/** One line per step; the last one is the one happening now. */
+function lines(review) {
+  const events = review.events || [];
+  if (!events.length) return [["now", "⏳", "מתחילים…"]];
+  return events.map((e, i) => {
+    const last = i === events.length - 1;
+    if (e.step === "boards") return ["done", "📋", `נמצאו ${boards(e.count)} עם דרך (מתוך ${e.total} שאלות)`];
+    if (e.step === "busy") return [last ? "now" : "done", "☕", last ? `ה־AI עמוס כרגע. מנסים שוב בעוד ${e.seconds} שניות` : "ה־AI היה עמוס, ניסינו שוב"];
+    if (e.step === "sent") return [last ? "now" : "done", last ? "🔍" : "📤", last ? "ה־AI קורא את הלוחות ובודק את הדרך, שאלה אחרי שאלה…" : "נשלח ל־AI"];
+    return ["done", "•", esc(e.step)];
+  });
+}
+
+const since = at => Math.max(0, Math.round((Date.now() - new Date(at)) / 1000));
+const elapsed = secs => secs < 60 ? `${secs} שניות` : `${Math.floor(secs / 60)}:${String(secs % 60).padStart(2, "0")} דקות`;
+
+/** The steps as HTML: a list, then "running for … (usually under a minute)". */
+export function stepsHtml(review) {
+  const at = review.at;
+  return `<ol class="plain rv-steps" aria-live="polite">${lines(review).map(([state, icon, text]) =>
+    `<li class="rv-${state}"><span aria-hidden="true">${state === "now" ? `<span class="spin"></span>` : icon}</span> ${text}${state === "done" ? ' <span class="rv-ok" aria-hidden="true">✓</span>' : ""}</li>`).join("")}</ol>
+    ${at ? `<p class="muted rv-time">כבר <span class="rv-elapsed" data-since="${esc(at)}">${elapsed(since(at))}</span> · בדרך כלל פחות מדקה</p>` : ""}`;
+}
+
+setInterval(() => document.querySelectorAll(".rv-elapsed[data-since]").forEach(el => { el.textContent = elapsed(since(el.dataset.since)); }), 1000);

mathly2/static/teacher_boardwork.js

@@ -4,6 +4,7 @@ import { api } from "./api.js";
 import { figures } from "./figures.js";
 import { esc, fmt } from "./math.js";
 import { pictureUrl } from "./media.js";
+import { stepsHtml } from "./reviewsteps.js";
 
 const KINDS = { quiz: "בוחן", exercise_test: "מבחן תרגול", simulation: "סימולציה" };
 const POINTS = [[null, "אוטומטי"], [0, "0"], [0.5, "½"], [1, "מלא"]];
@@ -24,7 +25,7 @@ function aiPanel(data) {
   const ai = data.ai || {};
   if (!ai.available) return `<div class="card work-ai-panel" id="work-ai"><p class="muted">🤖 בדיקה עם AI לא מוגדרת בשרת: חסר GEMINI_API_KEY.</p></div>`;
   const button = label => `<button type="button" id="work-ai-run">${label}</button>`;
-  const body = ai.state === "pending" ? `<p role="status">🤖 ה־AI בודק את הדרך… (עד דקה בערך)</p>`
+  const body = ai.state === "pending" ? `<p><b>🤖 בדיקה עם AI</b></p>${stepsHtml(ai)}`
     : ai.state === "failed" ? `<p class="error" role="alert">🤖 הבדיקה עם AI נכשלה. ${esc(REASONS[ai.reason] || "")}${ai.model ? ` <span dir="ltr">(${esc(ai.model)})</span>` : ""}</p>${button("לנסות שוב")}`
     : ai.state === "done" ? `<p>🤖 <b>סיכום ה־AI:</b> ${ai.summary ? fmt(ai.summary) : "לא היו לוחות לבדוק."}</p><p class="muted">הצעה בלבד: הנקודות שלכם קובעות.</p>${button("לבדוק שוב עם AI")}`
     : `<p class="muted">🤖 עוד לא נבדק עם AI.</p>${button("בדיקה עם AI")}`;
@@ -89,7 +90,10 @@ async function open(id, note = "", given = null, keep = null) {
       let now;
       try { now = await api(`/api/teaching/work/${encodeURIComponent(id)}`); } catch { return wait(); }
       if (document.getElementById("work-check")?.dataset.attempt !== data.id) return;
-      if (now.ai?.state === "pending") return wait();
+      if (now.ai?.state === "pending") {                                 // the new steps, in the box only
+        document.getElementById("work-ai")?.replaceWith(Object.assign(document.createElement("div"), { innerHTML: aiPanel(now) }).firstElementChild);
+        return wait();
+      }
       open(id, "", now, chosen);
     }, 4000); };
     wait();

mathly2/tests/test_boardreview_api.py

@@ -21,7 +21,7 @@ class FakeReviewer:
     def __init__(self, fail=None):
         self.calls, self.fail = [], fail
 
-    def review(self, items):
+    def review(self, items, on_event=lambda step, **info: None):
         self.calls.append(items)
         if self.fail:
             raise boardreview.ReviewFailure(self.fail)
@@ -174,7 +174,7 @@ def test_the_teacher_sees_the_ai_state_and_can_run_it_again_or_on_a_quiz():
     client, store, bank, test, _ = make(fake)
     attempt, _ = hand_in(client, bank, test)
     view = client.get(f"/api/teaching/work/{attempt['id']}", headers=auth(TEACHER)).json()
-    assert view["ai"] == {"available": True, "state": "failed", "reason": "rate_limited", "summary": None, "model": None}
+    assert {k: view["ai"][k] for k in ("available", "state", "reason", "summary")} == {"available": True, "state": "failed", "reason": "rate_limited", "summary": None}
     assert client.get(f"/api/teaching/work/{attempt['id']}", headers=auth(TEACHER)).json()["ai"]["state"] == "failed"   # opening does not retry
     fake.fail = None
     again = client.post(f"/api/teaching/work/{attempt['id']}/review", headers=auth(TEACHER)).json()
@@ -223,3 +223,34 @@ def test_a_model_not_found_falls_back_to_a_flash_model_the_key_has():
 def test_pick_model_prefers_stable_full_and_new():
     assert boardreview.pick_model(["gemini-3-flash-preview", "gemini-2.5-flash-lite", "gemini-2.5-flash"]) == "gemini-2.5-flash"
     assert boardreview.pick_model(["gemini-2.5-pro", "gemini-2.5-flash-preview-tts"]) is None
+
+
+def test_while_it_runs_the_review_stores_its_steps_for_the_screens():
+    seen = []
+
+    class Watching(FakeReviewer):
+        def review(self, items, on_event=lambda step, **info: None):
+            seen.append(boardreview.student_view(store.get_attempt(attempt_id)))
+            on_event("busy", seconds=8)
+            seen.append(boardreview.student_view(store.get_attempt(attempt_id)))
+            on_event("sent")
+            return super().review(items, on_event)
+    fake = Watching()
+    client, store, bank, test, _ = make(None)
+    attempt, _ = hand_in(client, bank, test, boards=2)
+    attempt_id = attempt["id"]
+    client.app.state.reviewer = fake
+    client.get(f"/api/attempts/{attempt_id}", headers=auth(KID))
+    assert [e["step"] for e in seen[0]["events"]] == ["boards", "sent"] and seen[0]["events"][0]["count"] == 2
+    assert seen[0]["state"] == "pending" and seen[0]["at"]
+    assert seen[1]["events"][-1] == {**seen[1]["events"][-1], "step": "busy", "seconds": 8}
+    assert store.get_attempt(attempt_id)["review"]["state"] == "done"
+
+
+def test_gemini_says_when_it_waits_for_the_free_tier():
+    steps = []
+    reviewer = boardreview.GeminiReviewer("key", sleep=lambda s: None)
+    replies = [http_error(429, json.dumps({"error": {"details": [{"retryDelay": "7s"}]}}).encode()), answer({"summary": "s", "questions": []})]
+    reviewer._call = lambda body: (lambda r: (_ for _ in ()).throw(r) if isinstance(r, Exception) else r)(replies.pop(0))
+    reviewer.review(items(), on_event=lambda step, **info: steps.append((step, info)))
+    assert steps == [("busy", {"seconds": 8}), ("sent", {})]

mathly2/tests/test_boardwork_browser.py

@@ -184,3 +184,47 @@ def test_the_teacher_sees_why_the_ai_failed_and_runs_it_again(browser, bank):
         assert "סיכום ה־AI" in teacher.inner_text("#work-ai") and teacher.locator(".work-q .work-ai").count() == 1
         assert teacher.get_attribute(f".work-q[data-key='{second}'] [data-points='0.5']", "aria-pressed") == "true"   # the pick stayed
         assert teacher.errors == []
+
+
+def test_while_the_ai_works_the_student_and_the_teacher_see_its_steps(browser, bank):
+    import threading
+    from datetime import timedelta
+    from mathly2 import clock
+    from mathly2.tests.test_boardreview_api import FakeReviewer
+    release = threading.Event()
+
+    class Slow(FakeReviewer):
+        def review(self, items, on_event=lambda step, **info: None):
+            on_event("busy", seconds=30)
+            release.wait(30)
+            return super().review(items, on_event)
+    lessons = [l["id"] for l in bank.lessons][:2]
+    store = InMemoryStore()
+    klass = store.create_class("setup", "A", lessons=lessons, teacher_email=TEACHER)
+    store.add_member(klass["id"], KID)
+    test = store.create_test(klass["id"], clock.school_day() + timedelta(days=9), lessons, TEACHER, created_at=clock.now() - timedelta(days=20))
+    app = create_app(Settings(teachers=frozenset({TEACHER})), bank=bank, store=store, reviewer=Slow())
+    with running(app) as url:
+        page = sign_in(browser, url, f"#/exam/{test['id']}/1/all")
+        page.wait_for_selector("#qz-q")
+        page.click("[data-board-toggle]")
+        page.wait_for_selector(".mx-board.full canvas")
+        draw_line(page)
+        page.click(".mx-board .mx-board-back")
+        page.locator(".qz-choice").first.click()
+        page.wait_for_selector(".qz-choice.picked")
+        page.click("#qz-submit")
+        page.click("#qz-yes")
+        page.wait_for_selector("#qz-review[data-state='pending'] .rv-steps li.rv-now:has-text('עמוס')", timeout=15000)
+        assert "נמצאו לוח אחד" in page.inner_text("#qz-review") and page.locator(".rv-elapsed").count() == 1
+        attempt = store.attempts_of(KID_ID, "exercise_test")[0]
+
+        teacher = teacher_page(browser, url)
+        teacher.click("#teacher-tabs [data-tab='work']")
+        teacher.click(f"[data-open='{attempt['id']}']")
+        teacher.wait_for_selector("#work-ai[data-state='pending'] .rv-steps")
+        assert "עמוס" in teacher.inner_text("#work-ai")
+        release.set()
+        page.wait_for_selector("#qz-review[data-state='done']", timeout=15000)
+        teacher.wait_for_selector("#work-ai[data-state='done']", timeout=15000)
+        assert page.errors == [] and teacher.errors == []