joelniklaus HF Staff commited on
Commit
fa30e08
·
verified ·
1 Parent(s): a7268b6

Deploy coordinator

Browse files
coordinator/src/swarm_coordinator/evolve_replay.py CHANGED
@@ -19,6 +19,7 @@ from pathlib import Path
19
  from typing import Any
20
 
21
  from . import evolve
 
22
 
23
 
24
  def load_recorded(path: str | Path) -> list[dict[str, Any]]:
@@ -35,7 +36,9 @@ def reference_curves(runs: list[dict[str, Any]]) -> list[dict[str, Any]]:
35
  return out
36
 
37
 
38
- class EvolveReplayer:
 
 
39
  def __init__(self, store: evolve.EvolveStore, runs: list[dict[str, Any]], *, minutes: float = 20.0,
40
  clock: Callable[[], float] = time.time):
41
  if minutes <= 0:
@@ -63,15 +66,3 @@ class EvolveReplayer:
63
  self.store.add(user, evolve.validate(event))
64
  n += 1
65
  return n
66
-
67
- def run(self, tick_s: float = 0.5) -> None:
68
- while not self.done and not self._stop.wait(tick_s):
69
- self.step()
70
-
71
- def start_thread(self) -> threading.Thread:
72
- t = threading.Thread(target=self.run, name="evolve-replay", daemon=True)
73
- t.start()
74
- return t
75
-
76
- def stop(self) -> None:
77
- self._stop.set()
 
19
  from typing import Any
20
 
21
  from . import evolve
22
+ from .replay import Ticker
23
 
24
 
25
  def load_recorded(path: str | Path) -> list[dict[str, Any]]:
 
36
  return out
37
 
38
 
39
+ class EvolveReplayer(Ticker):
40
+ name = "evolve-replay"
41
+
42
  def __init__(self, store: evolve.EvolveStore, runs: list[dict[str, Any]], *, minutes: float = 20.0,
43
  clock: Callable[[], float] = time.time):
44
  if minutes <= 0:
 
66
  self.store.add(user, evolve.validate(event))
67
  n += 1
68
  return n
 
 
 
 
 
 
 
 
 
 
 
 
coordinator/src/swarm_coordinator/loadtest.py CHANGED
@@ -36,6 +36,11 @@ def percentile(xs: list[float], p: float) -> float:
36
  return s[min(len(s) - 1, max(0, math.ceil(p / 100 * len(s)) - 1))]
37
 
38
 
 
 
 
 
 
39
  def fake_result(cell: dict[str, Any], user: str, i: int) -> dict[str, Any]:
40
  return {"cell_id": cell["cell_id"], "harness": cell["harness"], "harness_version": "loadtest", "task": cell["task"],
41
  "trial": cell["trial"], "model": cell["model"], "reward": float(i % 2), "passed": i % 2 == 1,
@@ -146,8 +151,7 @@ async def run_loadtest(base_url: str, n_runners: int = 300, *, transport: httpx.
146
  "runners": n_runners, "viewers": viewers, "procs": procs, "wall_s": round(wall, 2),
147
  "requests": sum(len(v) for v in lat.values()),
148
  "rps": round(sum(len(v) for k, v in lat.items() if k != "state") / wall, 1) if wall else None,
149
- "latency_ms": {k: {"n": len(v), "p50": round(percentile(v, 50), 1), "p95": round(percentile(v, 95), 1), "max": round(max(v), 1)}
150
- for k, v in sorted(lat.items())},
151
  "errors": len(errors), "error_samples": errors[:5],
152
  "cells_claimed": len(claims), "double_claimed": sorted(c for c, us in claims.items() if len(us) > 1),
153
  "cells_total": final["counts"]["cells_total"], "cells_done": final["counts"]["done"],
@@ -223,8 +227,7 @@ async def run_evolve_loadtest(base_url: str, attendees: int = 500, *, interval_s
223
  await asyncio.sleep(0.5)
224
  return {"attendees": attendees, "interval_s": interval_s, "iterations": iterations, "viewers": viewers, "wall_s": round(wall, 1),
225
  "submit_rps": round(len(lat["submit"]) / wall, 1) if wall else None,
226
- "latency_ms": {k: {"n": len(v), "p50": round(percentile(v, 50), 1), "p95": round(percentile(v, 95), 1), "max": round(max(v), 1)}
227
- for k, v in sorted(lat.items())},
228
  "errors": len(errors), "error_samples": errors[:5], "rate_limited": limited, "board": board}
229
 
230
 
 
36
  return s[min(len(s) - 1, max(0, math.ceil(p / 100 * len(s)) - 1))]
37
 
38
 
39
+ def latency_summary(lat: dict[str, list[float]]) -> dict[str, dict[str, float]]:
40
+ return {k: {"n": len(v), "p50": round(percentile(v, 50), 1), "p95": round(percentile(v, 95), 1), "max": round(max(v), 1)}
41
+ for k, v in sorted(lat.items()) if v}
42
+
43
+
44
  def fake_result(cell: dict[str, Any], user: str, i: int) -> dict[str, Any]:
45
  return {"cell_id": cell["cell_id"], "harness": cell["harness"], "harness_version": "loadtest", "task": cell["task"],
46
  "trial": cell["trial"], "model": cell["model"], "reward": float(i % 2), "passed": i % 2 == 1,
 
151
  "runners": n_runners, "viewers": viewers, "procs": procs, "wall_s": round(wall, 2),
152
  "requests": sum(len(v) for v in lat.values()),
153
  "rps": round(sum(len(v) for k, v in lat.items() if k != "state") / wall, 1) if wall else None,
154
+ "latency_ms": latency_summary(lat),
 
155
  "errors": len(errors), "error_samples": errors[:5],
156
  "cells_claimed": len(claims), "double_claimed": sorted(c for c, us in claims.items() if len(us) > 1),
157
  "cells_total": final["counts"]["cells_total"], "cells_done": final["counts"]["done"],
 
227
  await asyncio.sleep(0.5)
228
  return {"attendees": attendees, "interval_s": interval_s, "iterations": iterations, "viewers": viewers, "wall_s": round(wall, 1),
229
  "submit_rps": round(len(lat["submit"]) / wall, 1) if wall else None,
230
+ "latency_ms": latency_summary(lat),
 
231
  "errors": len(errors), "error_samples": errors[:5], "rate_limited": limited, "board": board}
232
 
233
 
coordinator/src/swarm_coordinator/replay.py CHANGED
@@ -35,7 +35,33 @@ class Event:
35
  user: str
36
 
37
 
38
- class Replayer:
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
39
  def __init__(self, store: Store, results: list[dict[str, Any]], *, minutes: float = 20.0, concurrency: int = 150,
40
  clock: Callable[[], float] = time.time):
41
  self.store, self.clock = store, clock
@@ -77,15 +103,3 @@ class Replayer:
77
  except Conflict:
78
  pass # the cell already holds a result (a replay into a live store); skip it
79
  return n
80
-
81
- def run(self, tick_s: float = 0.5) -> None:
82
- while not self.done and not self._stop.wait(tick_s):
83
- self.step()
84
-
85
- def start_thread(self) -> threading.Thread:
86
- t = threading.Thread(target=self.run, name="replay", daemon=True)
87
- t.start()
88
- return t
89
-
90
- def stop(self) -> None:
91
- self._stop.set()
 
35
  user: str
36
 
37
 
38
+ class Ticker:
39
+ """Runs `step()` every tick in a daemon thread until `done` or `stop()`. Subclasses set `_stop` and `name`."""
40
+
41
+ name = "replay"
42
+ _stop: threading.Event
43
+
44
+ @property
45
+ def done(self) -> bool:
46
+ raise NotImplementedError
47
+
48
+ def step(self) -> int:
49
+ raise NotImplementedError
50
+
51
+ def run(self, tick_s: float = 0.5) -> None:
52
+ while not self.done and not self._stop.wait(tick_s):
53
+ self.step()
54
+
55
+ def start_thread(self) -> threading.Thread:
56
+ t = threading.Thread(target=self.run, name=self.name, daemon=True)
57
+ t.start()
58
+ return t
59
+
60
+ def stop(self) -> None:
61
+ self._stop.set()
62
+
63
+
64
+ class Replayer(Ticker):
65
  def __init__(self, store: Store, results: list[dict[str, Any]], *, minutes: float = 20.0, concurrency: int = 150,
66
  clock: Callable[[], float] = time.time):
67
  self.store, self.clock = store, clock
 
103
  except Conflict:
104
  pass # the cell already holds a result (a replay into a live store); skip it
105
  return n
 
 
 
 
 
 
 
 
 
 
 
 
coordinator/src/swarm_coordinator/static/evolve.html CHANGED
@@ -141,12 +141,13 @@ function race(s) {
141
  top.forEach((r, i) => {
142
  const [cx, cy] = [x(ends[i][0]), y(ends[i][1])], lx = W - R + 18;
143
  g += `<circle cx="${cx}" cy="${cy}" r="8" fill="${TOP[i]}"/>`;
144
- g += `<polyline fill="none" stroke="${TOP[i]}" stroke-width="2" points="${cx + 8},${cy} ${lx - 14},${cy} ${lx - 4},${ly[i]}"/>`;
 
145
  const name = r.user.length > 12 ? r.user.slice(0, 11) + "…" : r.user;
146
  g += `<text x="${lx}" y="${ly[i] + 9}" font-size="${FS}" fill="${TOP[i]}">${esc(name)} ${pct(ends[i][1])}</text>`;
147
  });
148
  svg.innerHTML = g;
149
- $("race-note").textContent = runs.length ? `${s.totals.runs} runs` : "waiting for the first seed scores";
150
  }
151
 
152
  function fixes(s) {
 
141
  top.forEach((r, i) => {
142
  const [cx, cy] = [x(ends[i][0]), y(ends[i][1])], lx = W - R + 18;
143
  g += `<circle cx="${cx}" cy="${cy}" r="8" fill="${TOP[i]}"/>`;
144
+ // a dotted leader, so it never reads as the run going on to the last iteration
145
+ g += `<polyline fill="none" stroke="${TOP[i]}" stroke-width="2" stroke-dasharray="2 7" stroke-opacity=".8" points="${cx + 12},${cy} ${lx - 14},${cy} ${lx - 4},${ly[i]}"/>`;
146
  const name = r.user.length > 12 ? r.user.slice(0, 11) + "…" : r.user;
147
  g += `<text x="${lx}" y="${ly[i] + 9}" font-size="${FS}" fill="${TOP[i]}">${esc(name)} ${pct(ends[i][1])}</text>`;
148
  });
149
  svg.innerHTML = g;
150
+ $("race-note").textContent = runs.length ? `${s.totals.runs} run${s.totals.runs === 1 ? "" : "s"}` : "waiting for the first seed scores";
151
  }
152
 
153
  function fixes(s) {