Serve the queue round-robin by entrant; requeue abandoned claims

There is no cap on submissions, so FIFO would let the first enthusiastic entrant own the
GPU: at ~2.5 min per ligand, ten thousand molecules is seventeen days of nobody else
being scored. Interleaving by entrant costs nothing and removes the need for a cap -
everyone's first entry is scored before anyone's second.

A worker that dies mid-job used to strand its claim forever; claims now carry a timestamp
and stale ones return to the queue.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

app.py

@@ -330,6 +330,22 @@ def models():
             "totals": {"models": len(out), "submissions": sum(x["submissions"] for x in out)}}
 
 
+# If a worker dies mid-job the entry is left marked "scoring" and would never be picked
+# up again. Anything claimed longer ago than this is treated as abandoned and requeued.
+CLAIM_TTL = int(os.environ.get("ODC_CLAIM_TTL", "3600"))
+
+
+@app.get("/api/requeue_stale")
+def requeue_stale(request: Request):
+    _auth(request)
+    now, n = int(time.time()), 0
+    for r in _latest():
+        if r.get("status") == "scoring" and now - (r.get("claimed_at") or 0) > CLAIM_TTL:
+            _append({"id": r["id"], "status": "queued"})
+            n += 1
+    return {"requeued": n}
+
+
 @app.get("/api/queue")
 def queue():
     rs = _latest()
@@ -343,14 +359,37 @@ def _auth(request: Request):
         raise HTTPException(401, "worker key required")
 
 
+def _round_robin(rows):
+    """Order queued work so that entrants take turns.
+
+    Each entrant's submissions keep their own arrival order; the queues are then read
+    across rather than down. Someone who submits a hundred molecules still gets every one
+    of them scored - just not before everyone else's first.
+    """
+    by_user = {}
+    for r in sorted(rows, key=lambda x: x.get("submitted_at") or 0):
+        who = r.get("hf_user") or r.get("display_name") or "anonymous"
+        by_user.setdefault(who, []).append(r)
+    # entrants who have been waiting longest go first within each pass
+    order = sorted(by_user, key=lambda u: by_user[u][0].get("submitted_at") or 0)
+    out, depth = [], 0
+    while any(len(by_user[u]) > depth for u in order):
+        for u in order:
+            if len(by_user[u]) > depth:
+                out.append(by_user[u][depth])
+        depth += 1
+    return out
+
+
 @app.get("/api/pending")
 def pending(request: Request, limit: int = 5):
     _auth(request)
-    out = [r for r in _latest() if r.get("status") == "queued"][:limit]
+    out = _round_robin([r for r in _latest() if r.get("status") == "queued"])[:limit]
     for r in out:
-        _append({"id": r["id"], "status": "scoring"})
+        _append({"id": r["id"], "status": "scoring", "claimed_at": int(time.time())})
     return {"items": [{"id": r["id"], "smiles": r["smiles"],
-                       "candidate_id": r["candidate_id"]} for r in out]}
+                       "candidate_id": r["candidate_id"],
+                       "entrant": r.get("hf_user") or r.get("display_name")} for r in out]}
 
 
 @app.post("/api/result")