# From Space HuggingFaceSwarms/SwarmChat-Demo (bot_runner.py, itself adapted from flock's run4 eval server), # plus a trace mode: which modules proposed a reply, which won, and the bot files the winner called. """Serve a checkout of a swarm's chatbot as SwarmChat, an OpenAI-compatible endpoint for evals. python experiments/run4/eval/swarmchat_server.py --port 8000 Loading the bot takes seconds (hundreds of modules), answering takes milliseconds. So one process loads the bot and forks a worker per conversation, and every conversation starts from the same just-loaded state. The same lines get the same replies whatever the order or the concurrency of the requests, in this run or the next: the hash seed, the global `random` state (restored in each worker) and the seeds of the bot's own random.Random() instances are fixed. Replies that read the clock or seed from an object id (a few modules use id(memory)) can still change: ids move with address randomisation, which HF Jobs do not let us turn off. POST /v1/chat/completions feeds the request's user messages to the bot in order (system and assistant messages are ignored: the bot only reads user lines) and returns its last reply. There is no sandbox: the bot runs as the server's user, with an empty environment, so run it somewhere disposable such as an HF Job. """ import json import os import signal import select import socket import subprocess import sys import tempfile from pathlib import Path LOADER = r''' import io, json, os, random, resource, runpy, signal, socket, sys, time root, seed, timeout, path, workers = sys.argv[1], int(sys.argv[2]), float(sys.argv[3]), sys.argv[4], int(sys.argv[5]) sys.path.insert(0, root) status = os.fdopen(os.dup(1), "w") os.dup2(2, 1) # stray prints from the bot go to the log listener = socket.socket(socket.AF_UNIX) listener.bind(path) listener.listen(256) # A worker writes a byte here when it takes a conversation; the loader forks one per byte. spawn_r, spawn_w = os.pipe() os.write(spawn_w, bytes(workers)) def converse(): """A worker: take one connection, hold that conversation, send the result and exit.""" conn = listener.accept()[0] os.write(spawn_w, b"\0") random.setstate(loaded) # random reseeds itself from the OS in every forked child started = time.monotonic() request = json.loads(conn.makefile("rb").readline()) lines, trace = (request, False) if isinstance(request, list) else (request["lines"], bool(request.get("trace"))) limit = int(timeout) + 1 signal.alarm(limit) resource.setrlimit(resource.RLIMIT_CPU, (limit, limit + 1)) if trace: result, out = "ok", {} try: out = traced(lines) except BaseException as exc: result = f"error {type(exc).__name__}" conn.sendall((json.dumps({"replies": out.get("replies", []), "trace": out.get("trace"), "status": result, "seconds": round(time.monotonic() - started, 3)}) + "\n").encode()) os._exit(0) sys.stdin = io.StringIO("".join(line + "\n" for line in lines)) sys.stdout = buffer = io.StringIO() result = "ok" try: runpy.run_module("bot", run_name="__main__") except BaseException as exc: # the fixed __main__ catches reply errors; this is the rest result = f"error {type(exc).__name__}" conn.sendall((json.dumps({"replies": buffer.getvalue().splitlines(), "status": result, "seconds": round(time.monotonic() - started, 3)}) + "\n").encode()) os._exit(0) def traced(lines): """Replies like the fixed __main__ (one per line, whitespace-normalised, "Hello!" on errors), plus, for the last line, the bot's own decision (bot.runtime.turn: candidates and winner) and the bot files each proposing handler called. sys.monitoring records a function once and then disables that event, so tracing costs about nothing.""" import bot.runtime as rt def say(text, fn): try: return " ".join(str(fn(text)).split()) or "Hello!" except Exception: # noqa: BLE001 - the fixed __main__'s rule return "Hello!" memory = {} replies = [say(line, lambda t: rt.reply(t, memory)) for line in lines[:-1]] mon, tool, bot_dir, seen, files = sys.monitoring, 5, os.path.join(root, "bot") + os.sep, set(), {} mon.use_tool_id(tool, "pybot-trace") def start(code, offset): if code.co_filename.startswith(bot_dir): seen.add("bot/" + code.co_filename[len(bot_dir):]) return mon.DISABLE mon.register_callback(tool, mon.events.PY_START, start) def watch(name, respond): def run(text, mem): seen.clear(); mon.restart_events(); mon.set_events(tool, mon.events.PY_START) try: return respond(text, mem) finally: mon.set_events(tool, 0); files[name] = sorted(seen) return run rt._HANDLERS = [(name, priority, watch(name, respond)) for name, priority, respond in rt._HANDLERS] decision = {} def last(text): decision.update(rt.turn(text, memory)) return decision["reply"] replies.append(say(lines[-1], last)) module = decision.get("module") return {"replies": replies, "trace": {"module": module, "candidates": [c["module"] for c in decision.get("candidates", [])], "files": files.get(module, []) if module else []}} # Modules make their own random.Random(), which seeds itself from the OS: derive those seeds # from the run's seed and a counter instead, so every run starts in the same state. os_seeded, created = random.Random.seed, iter(range(1 << 62)) def derived_seed(self, a=None, version=2): os_seeded(self, f"{seed}:{next(created)}" if a is None else a, version) random.Random.seed = derived_seed random.seed = random._inst.seed random.seed(seed) started = time.monotonic() import bot.runtime loaded = random.getstate() print(json.dumps({"load_s": round(time.monotonic() - started, 2), "handlers": len(bot.runtime._HANDLERS)}), file=status, flush=True) # Keep `workers` forks waiting for a connection. The workers read the requests, never the # loader, and the kernel reaps them. signal.signal(signal.SIGCHLD, signal.SIG_IGN) while True: os.read(spawn_r, 1) if os.fork() == 0: converse() ''' class Bot: """The loader process; ask() holds one conversation in a fresh worker.""" def __init__(self, tree: Path, *, seed: int, timeout: float, workers: int, log): self.timeout = timeout self.tmp = tempfile.TemporaryDirectory(prefix="swarmchat-") self.path = os.path.join(self.tmp.name, "bot.sock") # No -I: PYTHONHASHSEED must apply. The environment holds nothing else. self.proc = subprocess.Popen( [sys.executable, "-s", "-S", "-P", "-c", LOADER, str(tree), str(seed), str(timeout), self.path, str(workers)], cwd=self.tmp.name, env={"PYTHONHASHSEED": str(seed)}, stdout=subprocess.PIPE, stderr=log, text=True, start_new_session=True) if not select.select([self.proc.stdout], [], [], 90)[0]: self.close() raise RuntimeError("The bot did not load within 90 seconds.") line = self.proc.stdout.readline() if not line: self.close() raise RuntimeError("This commit could not load its bot; try another commit.") self.ready = json.loads(line) def ask(self, lines: list[str], trace: bool = False) -> dict: with socket.socket(socket.AF_UNIX) as conn: conn.settimeout(self.timeout + 5) conn.connect(self.path) conn.sendall((json.dumps({"lines": lines, "trace": True} if trace else lines) + "\n").encode()) try: raw = conn.makefile("rb").readline() except TimeoutError: return {"replies": [], "status": "timeout"} # The worker died without replying, e.g. on its CPU or wall-clock limit. return json.loads(raw) if raw else {"replies": [], "status": "worker exited"} def alive(self) -> bool: return self.proc.poll() is None def close(self): if self.alive(): os.killpg(self.proc.pid, signal.SIGKILL) self.proc.wait() self.proc.stdout.close() self.tmp.cleanup() def user_lines(messages) -> list[str]: """User message texts as single bot input lines (the bot reads one line per turn).""" lines = [] for m in messages: if m.get("role") != "user": continue content = m.get("content") or "" if isinstance(content, list): content = " ".join(p.get("text", "") for p in content if p.get("type") == "text") lines.append(" ".join(str(content).split())) return lines