Deploy coordinator
Browse files
coordinator/src/swarm_coordinator/evolve_replay.py
CHANGED
|
@@ -19,6 +19,7 @@ from pathlib import Path
|
|
| 19 |
from typing import Any
|
| 20 |
|
| 21 |
from . import evolve
|
|
|
|
| 22 |
|
| 23 |
|
| 24 |
def load_recorded(path: str | Path) -> list[dict[str, Any]]:
|
|
@@ -35,7 +36,9 @@ def reference_curves(runs: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
|
| 35 |
return out
|
| 36 |
|
| 37 |
|
| 38 |
-
class EvolveReplayer:
|
|
|
|
|
|
|
| 39 |
def __init__(self, store: evolve.EvolveStore, runs: list[dict[str, Any]], *, minutes: float = 20.0,
|
| 40 |
clock: Callable[[], float] = time.time):
|
| 41 |
if minutes <= 0:
|
|
@@ -63,15 +66,3 @@ class EvolveReplayer:
|
|
| 63 |
self.store.add(user, evolve.validate(event))
|
| 64 |
n += 1
|
| 65 |
return n
|
| 66 |
-
|
| 67 |
-
def run(self, tick_s: float = 0.5) -> None:
|
| 68 |
-
while not self.done and not self._stop.wait(tick_s):
|
| 69 |
-
self.step()
|
| 70 |
-
|
| 71 |
-
def start_thread(self) -> threading.Thread:
|
| 72 |
-
t = threading.Thread(target=self.run, name="evolve-replay", daemon=True)
|
| 73 |
-
t.start()
|
| 74 |
-
return t
|
| 75 |
-
|
| 76 |
-
def stop(self) -> None:
|
| 77 |
-
self._stop.set()
|
|
|
|
| 19 |
from typing import Any
|
| 20 |
|
| 21 |
from . import evolve
|
| 22 |
+
from .replay import Ticker
|
| 23 |
|
| 24 |
|
| 25 |
def load_recorded(path: str | Path) -> list[dict[str, Any]]:
|
|
|
|
| 36 |
return out
|
| 37 |
|
| 38 |
|
| 39 |
+
class EvolveReplayer(Ticker):
|
| 40 |
+
name = "evolve-replay"
|
| 41 |
+
|
| 42 |
def __init__(self, store: evolve.EvolveStore, runs: list[dict[str, Any]], *, minutes: float = 20.0,
|
| 43 |
clock: Callable[[], float] = time.time):
|
| 44 |
if minutes <= 0:
|
|
|
|
| 66 |
self.store.add(user, evolve.validate(event))
|
| 67 |
n += 1
|
| 68 |
return n
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
coordinator/src/swarm_coordinator/loadtest.py
CHANGED
|
@@ -36,6 +36,11 @@ def percentile(xs: list[float], p: float) -> float:
|
|
| 36 |
return s[min(len(s) - 1, max(0, math.ceil(p / 100 * len(s)) - 1))]
|
| 37 |
|
| 38 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 39 |
def fake_result(cell: dict[str, Any], user: str, i: int) -> dict[str, Any]:
|
| 40 |
return {"cell_id": cell["cell_id"], "harness": cell["harness"], "harness_version": "loadtest", "task": cell["task"],
|
| 41 |
"trial": cell["trial"], "model": cell["model"], "reward": float(i % 2), "passed": i % 2 == 1,
|
|
@@ -146,8 +151,7 @@ async def run_loadtest(base_url: str, n_runners: int = 300, *, transport: httpx.
|
|
| 146 |
"runners": n_runners, "viewers": viewers, "procs": procs, "wall_s": round(wall, 2),
|
| 147 |
"requests": sum(len(v) for v in lat.values()),
|
| 148 |
"rps": round(sum(len(v) for k, v in lat.items() if k != "state") / wall, 1) if wall else None,
|
| 149 |
-
"latency_ms":
|
| 150 |
-
for k, v in sorted(lat.items())},
|
| 151 |
"errors": len(errors), "error_samples": errors[:5],
|
| 152 |
"cells_claimed": len(claims), "double_claimed": sorted(c for c, us in claims.items() if len(us) > 1),
|
| 153 |
"cells_total": final["counts"]["cells_total"], "cells_done": final["counts"]["done"],
|
|
@@ -223,8 +227,7 @@ async def run_evolve_loadtest(base_url: str, attendees: int = 500, *, interval_s
|
|
| 223 |
await asyncio.sleep(0.5)
|
| 224 |
return {"attendees": attendees, "interval_s": interval_s, "iterations": iterations, "viewers": viewers, "wall_s": round(wall, 1),
|
| 225 |
"submit_rps": round(len(lat["submit"]) / wall, 1) if wall else None,
|
| 226 |
-
"latency_ms":
|
| 227 |
-
for k, v in sorted(lat.items())},
|
| 228 |
"errors": len(errors), "error_samples": errors[:5], "rate_limited": limited, "board": board}
|
| 229 |
|
| 230 |
|
|
|
|
| 36 |
return s[min(len(s) - 1, max(0, math.ceil(p / 100 * len(s)) - 1))]
|
| 37 |
|
| 38 |
|
| 39 |
+
def latency_summary(lat: dict[str, list[float]]) -> dict[str, dict[str, float]]:
|
| 40 |
+
return {k: {"n": len(v), "p50": round(percentile(v, 50), 1), "p95": round(percentile(v, 95), 1), "max": round(max(v), 1)}
|
| 41 |
+
for k, v in sorted(lat.items()) if v}
|
| 42 |
+
|
| 43 |
+
|
| 44 |
def fake_result(cell: dict[str, Any], user: str, i: int) -> dict[str, Any]:
|
| 45 |
return {"cell_id": cell["cell_id"], "harness": cell["harness"], "harness_version": "loadtest", "task": cell["task"],
|
| 46 |
"trial": cell["trial"], "model": cell["model"], "reward": float(i % 2), "passed": i % 2 == 1,
|
|
|
|
| 151 |
"runners": n_runners, "viewers": viewers, "procs": procs, "wall_s": round(wall, 2),
|
| 152 |
"requests": sum(len(v) for v in lat.values()),
|
| 153 |
"rps": round(sum(len(v) for k, v in lat.items() if k != "state") / wall, 1) if wall else None,
|
| 154 |
+
"latency_ms": latency_summary(lat),
|
|
|
|
| 155 |
"errors": len(errors), "error_samples": errors[:5],
|
| 156 |
"cells_claimed": len(claims), "double_claimed": sorted(c for c, us in claims.items() if len(us) > 1),
|
| 157 |
"cells_total": final["counts"]["cells_total"], "cells_done": final["counts"]["done"],
|
|
|
|
| 227 |
await asyncio.sleep(0.5)
|
| 228 |
return {"attendees": attendees, "interval_s": interval_s, "iterations": iterations, "viewers": viewers, "wall_s": round(wall, 1),
|
| 229 |
"submit_rps": round(len(lat["submit"]) / wall, 1) if wall else None,
|
| 230 |
+
"latency_ms": latency_summary(lat),
|
|
|
|
| 231 |
"errors": len(errors), "error_samples": errors[:5], "rate_limited": limited, "board": board}
|
| 232 |
|
| 233 |
|
coordinator/src/swarm_coordinator/replay.py
CHANGED
|
@@ -35,7 +35,33 @@ class Event:
|
|
| 35 |
user: str
|
| 36 |
|
| 37 |
|
| 38 |
-
class
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 39 |
def __init__(self, store: Store, results: list[dict[str, Any]], *, minutes: float = 20.0, concurrency: int = 150,
|
| 40 |
clock: Callable[[], float] = time.time):
|
| 41 |
self.store, self.clock = store, clock
|
|
@@ -77,15 +103,3 @@ class Replayer:
|
|
| 77 |
except Conflict:
|
| 78 |
pass # the cell already holds a result (a replay into a live store); skip it
|
| 79 |
return n
|
| 80 |
-
|
| 81 |
-
def run(self, tick_s: float = 0.5) -> None:
|
| 82 |
-
while not self.done and not self._stop.wait(tick_s):
|
| 83 |
-
self.step()
|
| 84 |
-
|
| 85 |
-
def start_thread(self) -> threading.Thread:
|
| 86 |
-
t = threading.Thread(target=self.run, name="replay", daemon=True)
|
| 87 |
-
t.start()
|
| 88 |
-
return t
|
| 89 |
-
|
| 90 |
-
def stop(self) -> None:
|
| 91 |
-
self._stop.set()
|
|
|
|
| 35 |
user: str
|
| 36 |
|
| 37 |
|
| 38 |
+
class Ticker:
|
| 39 |
+
"""Runs `step()` every tick in a daemon thread until `done` or `stop()`. Subclasses set `_stop` and `name`."""
|
| 40 |
+
|
| 41 |
+
name = "replay"
|
| 42 |
+
_stop: threading.Event
|
| 43 |
+
|
| 44 |
+
@property
|
| 45 |
+
def done(self) -> bool:
|
| 46 |
+
raise NotImplementedError
|
| 47 |
+
|
| 48 |
+
def step(self) -> int:
|
| 49 |
+
raise NotImplementedError
|
| 50 |
+
|
| 51 |
+
def run(self, tick_s: float = 0.5) -> None:
|
| 52 |
+
while not self.done and not self._stop.wait(tick_s):
|
| 53 |
+
self.step()
|
| 54 |
+
|
| 55 |
+
def start_thread(self) -> threading.Thread:
|
| 56 |
+
t = threading.Thread(target=self.run, name=self.name, daemon=True)
|
| 57 |
+
t.start()
|
| 58 |
+
return t
|
| 59 |
+
|
| 60 |
+
def stop(self) -> None:
|
| 61 |
+
self._stop.set()
|
| 62 |
+
|
| 63 |
+
|
| 64 |
+
class Replayer(Ticker):
|
| 65 |
def __init__(self, store: Store, results: list[dict[str, Any]], *, minutes: float = 20.0, concurrency: int = 150,
|
| 66 |
clock: Callable[[], float] = time.time):
|
| 67 |
self.store, self.clock = store, clock
|
|
|
|
| 103 |
except Conflict:
|
| 104 |
pass # the cell already holds a result (a replay into a live store); skip it
|
| 105 |
return n
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
coordinator/src/swarm_coordinator/static/evolve.html
CHANGED
|
@@ -141,12 +141,13 @@ function race(s) {
|
|
| 141 |
top.forEach((r, i) => {
|
| 142 |
const [cx, cy] = [x(ends[i][0]), y(ends[i][1])], lx = W - R + 18;
|
| 143 |
g += `<circle cx="${cx}" cy="${cy}" r="8" fill="${TOP[i]}"/>`;
|
| 144 |
-
|
|
|
|
| 145 |
const name = r.user.length > 12 ? r.user.slice(0, 11) + "…" : r.user;
|
| 146 |
g += `<text x="${lx}" y="${ly[i] + 9}" font-size="${FS}" fill="${TOP[i]}">${esc(name)} ${pct(ends[i][1])}</text>`;
|
| 147 |
});
|
| 148 |
svg.innerHTML = g;
|
| 149 |
-
$("race-note").textContent = runs.length ? `${s.totals.runs} runs` : "waiting for the first seed scores";
|
| 150 |
}
|
| 151 |
|
| 152 |
function fixes(s) {
|
|
|
|
| 141 |
top.forEach((r, i) => {
|
| 142 |
const [cx, cy] = [x(ends[i][0]), y(ends[i][1])], lx = W - R + 18;
|
| 143 |
g += `<circle cx="${cx}" cy="${cy}" r="8" fill="${TOP[i]}"/>`;
|
| 144 |
+
// a dotted leader, so it never reads as the run going on to the last iteration
|
| 145 |
+
g += `<polyline fill="none" stroke="${TOP[i]}" stroke-width="2" stroke-dasharray="2 7" stroke-opacity=".8" points="${cx + 12},${cy} ${lx - 14},${cy} ${lx - 4},${ly[i]}"/>`;
|
| 146 |
const name = r.user.length > 12 ? r.user.slice(0, 11) + "…" : r.user;
|
| 147 |
g += `<text x="${lx}" y="${ly[i] + 9}" font-size="${FS}" fill="${TOP[i]}">${esc(name)} ${pct(ends[i][1])}</text>`;
|
| 148 |
});
|
| 149 |
svg.innerHTML = g;
|
| 150 |
+
$("race-note").textContent = runs.length ? `${s.totals.runs} run${s.totals.runs === 1 ? "" : "s"}` : "waiting for the first seed scores";
|
| 151 |
}
|
| 152 |
|
| 153 |
function fixes(s) {
|