| |
| """List and (with --delete) reclaim leaked prime tunnels. |
| |
| Why this is needed: every eval episode opens a host tunnel so the sandbox can reach the local |
| vLLM server, and the account is capped at **32 concurrent tunnels**. `PrimeTunnel.expose` releases |
| its tunnel in a `finally` that is shielded against cancellation — so a normal exit or SIGTERM |
| cleans up properly. **SIGKILL does not.** Kill -9 an eval process with 24 rollouts in flight and |
| you leak 24 tunnels; do that a few times and all 32 slots are gone. Every subsequent episode then |
| dies instantly with |
| |
| TunnelError: TunnelLimitReachedError: Maximum number of tunnels (32) reached |
| |
| which looks like a sandbox-pool problem and is not one. It is also invisible in `/resources`. |
| |
| Two rules that follow: |
| * Stop eval runs with SIGTERM, never `kill -9`, unless you clean up afterwards. |
| * Total `max_concurrent` across ALL simultaneously running eval processes must stay under 32. |
| Two arms at 24 each is 48 and cannot work, however healthy the pool looks. |
| |
| Run with no arguments to list; `--delete` removes every tunnel the token owns, so only do that |
| when no eval is running. |
| """ |
|
|
| import asyncio |
| import sys |
|
|
|
|
| async def main() -> None: |
| from prime_tunnel import TunnelClient |
|
|
| delete = "--delete" in sys.argv |
| client = TunnelClient() |
| try: |
| result = client.list_tunnels() |
| if hasattr(result, "__await__"): |
| result = await result |
| items = list(getattr(result, "items", None) or getattr(result, "tunnels", None) or result) |
| print(f"tunnels held by this token: {len(items)} / 32") |
| if not delete: |
| print("(pass --delete to reclaim them — only when no eval run is active)") |
| return |
| ids = [getattr(t, "tunnel_id", None) or getattr(t, "id", None) for t in items] |
| ids = [i for i in ids if i] |
| if not ids: |
| return |
| try: |
| res = client.bulk_delete_tunnels(ids) |
| if hasattr(res, "__await__"): |
| await res |
| print(f"deleted {len(ids)} tunnels (bulk)") |
| except Exception as exc: |
| print(f"bulk delete failed ({exc}); falling back to one at a time") |
| ok = 0 |
| for tid in ids: |
| try: |
| res = client.delete_tunnel(tid) |
| if hasattr(res, "__await__"): |
| await res |
| ok += 1 |
| except Exception as inner: |
| print(f" {tid}: {inner}") |
| print(f"deleted {ok}/{len(ids)} tunnels") |
| finally: |
| close = client.close() |
| if hasattr(close, "__await__"): |
| await close |
|
|
|
|
| if __name__ == "__main__": |
| asyncio.run(main()) |
|
|
