Spaces:
Running
Running
Download bot_runner.py from HuggingFaceSwarms/PyBot: direct link, hf CLI and curl.
- Browser
- Download file 8.86 kB
-
https://huggingface.co/spaces/HuggingFaceSwarms/PyBot/resolve/091df39bcce760cae781ad3e10d0316f1cbbcf99/bot_runner.py
- Command line
-
hf download hf://spaces/HuggingFaceSwarms/PyBot@091df39bcce760cae781ad3e10d0316f1cbbcf99/bot_runner.py
-
curl -L -o bot_runner.py https://huggingface.co/spaces/HuggingFaceSwarms/PyBot/resolve/091df39bcce760cae781ad3e10d0316f1cbbcf99/bot_runner.py
8.86 kB
| # From Space HuggingFaceSwarms/SwarmChat-Demo (bot_runner.py, itself adapted from flock's run4 eval server), | |
| # plus a trace mode: which modules proposed a reply, which won, and the bot files the winner called. | |
| """Serve a checkout of a swarm's chatbot as SwarmChat, an OpenAI-compatible endpoint for evals. | |
| python experiments/run4/eval/swarmchat_server.py <chatbot checkout> --port 8000 | |
| Loading the bot takes seconds (hundreds of modules), answering takes milliseconds. So one | |
| process loads the bot and forks a worker per conversation, and every conversation starts from | |
| the same just-loaded state. The same lines get the same replies whatever the order or the | |
| concurrency of the requests, in this run or the next: the hash seed, the global `random` state | |
| (restored in each worker) and the seeds of the bot's own random.Random() instances are fixed. | |
| Replies that read the clock or seed from an object id (a few modules use id(memory)) can still | |
| change: ids move with address randomisation, which HF Jobs do not let us turn off. | |
| POST /v1/chat/completions feeds the request's user messages to the bot in order (system and | |
| assistant messages are ignored: the bot only reads user lines) and returns its last reply. | |
| There is no sandbox: the bot runs as the server's user, with an empty environment, so run it | |
| somewhere disposable such as an HF Job. | |
| """ | |
| import json | |
| import os | |
| import signal | |
| import select | |
| import socket | |
| import subprocess | |
| import sys | |
| import tempfile | |
| from pathlib import Path | |
| LOADER = r''' | |
| import io, json, os, random, resource, runpy, signal, socket, sys, time | |
| root, seed, timeout, path, workers = sys.argv[1], int(sys.argv[2]), float(sys.argv[3]), sys.argv[4], int(sys.argv[5]) | |
| sys.path.insert(0, root) | |
| status = os.fdopen(os.dup(1), "w") | |
| os.dup2(2, 1) # stray prints from the bot go to the log | |
| listener = socket.socket(socket.AF_UNIX) | |
| listener.bind(path) | |
| listener.listen(256) | |
| # A worker writes a byte here when it takes a conversation; the loader forks one per byte. | |
| spawn_r, spawn_w = os.pipe() | |
| os.write(spawn_w, bytes(workers)) | |
| def converse(): | |
| """A worker: take one connection, hold that conversation, send the result and exit.""" | |
| conn = listener.accept()[0] | |
| os.write(spawn_w, b"\0") | |
| random.setstate(loaded) # random reseeds itself from the OS in every forked child | |
| started = time.monotonic() | |
| request = json.loads(conn.makefile("rb").readline()) | |
| lines, trace = (request, False) if isinstance(request, list) else (request["lines"], bool(request.get("trace"))) | |
| limit = int(timeout) + 1 | |
| signal.alarm(limit) | |
| resource.setrlimit(resource.RLIMIT_CPU, (limit, limit + 1)) | |
| if trace: | |
| result, out = "ok", {} | |
| try: | |
| out = traced(lines) | |
| except BaseException as exc: | |
| result = f"error {type(exc).__name__}" | |
| conn.sendall((json.dumps({"replies": out.get("replies", []), "trace": out.get("trace"), "status": result, | |
| "seconds": round(time.monotonic() - started, 3)}) + "\n").encode()) | |
| os._exit(0) | |
| sys.stdin = io.StringIO("".join(line + "\n" for line in lines)) | |
| sys.stdout = buffer = io.StringIO() | |
| result = "ok" | |
| try: | |
| runpy.run_module("bot", run_name="__main__") | |
| except BaseException as exc: # the fixed __main__ catches reply errors; this is the rest | |
| result = f"error {type(exc).__name__}" | |
| conn.sendall((json.dumps({"replies": buffer.getvalue().splitlines(), "status": result, | |
| "seconds": round(time.monotonic() - started, 3)}) + "\n").encode()) | |
| os._exit(0) | |
| def traced(lines): | |
| """Replies like the fixed __main__ (one per line, whitespace-normalised, "Hello!" on errors), plus, for the last | |
| line, the bot's own decision (bot.runtime.turn: candidates and winner) and the bot files each proposing handler called. | |
| sys.monitoring records a function once and then disables that event, so tracing costs about nothing.""" | |
| import bot.runtime as rt | |
| def say(text, fn): | |
| try: | |
| return " ".join(str(fn(text)).split()) or "Hello!" | |
| except Exception: # noqa: BLE001 - the fixed __main__'s rule | |
| return "Hello!" | |
| memory = {} | |
| replies = [say(line, lambda t: rt.reply(t, memory)) for line in lines[:-1]] | |
| mon, tool, bot_dir, seen, files = sys.monitoring, 5, os.path.join(root, "bot") + os.sep, set(), {} | |
| mon.use_tool_id(tool, "pybot-trace") | |
| def start(code, offset): | |
| if code.co_filename.startswith(bot_dir): | |
| seen.add("bot/" + code.co_filename[len(bot_dir):]) | |
| return mon.DISABLE | |
| mon.register_callback(tool, mon.events.PY_START, start) | |
| def watch(name, respond): | |
| def run(text, mem): | |
| seen.clear(); mon.restart_events(); mon.set_events(tool, mon.events.PY_START) | |
| try: | |
| return respond(text, mem) | |
| finally: | |
| mon.set_events(tool, 0); files[name] = sorted(seen) | |
| return run | |
| rt._HANDLERS = [(name, priority, watch(name, respond)) for name, priority, respond in rt._HANDLERS] | |
| decision = {} | |
| def last(text): | |
| decision.update(rt.turn(text, memory)) | |
| return decision["reply"] | |
| replies.append(say(lines[-1], last)) | |
| module = decision.get("module") | |
| return {"replies": replies, "trace": {"module": module, | |
| "candidates": [c["module"] for c in decision.get("candidates", [])], | |
| "files": files.get(module, []) if module else []}} | |
| # Modules make their own random.Random(), which seeds itself from the OS: derive those seeds | |
| # from the run's seed and a counter instead, so every run starts in the same state. | |
| os_seeded, created = random.Random.seed, iter(range(1 << 62)) | |
| def derived_seed(self, a=None, version=2): | |
| os_seeded(self, f"{seed}:{next(created)}" if a is None else a, version) | |
| random.Random.seed = derived_seed | |
| random.seed = random._inst.seed | |
| random.seed(seed) | |
| started = time.monotonic() | |
| import bot.runtime | |
| loaded = random.getstate() | |
| print(json.dumps({"load_s": round(time.monotonic() - started, 2), | |
| "handlers": len(bot.runtime._HANDLERS)}), file=status, flush=True) | |
| # Keep `workers` forks waiting for a connection. The workers read the requests, never the | |
| # loader, and the kernel reaps them. | |
| signal.signal(signal.SIGCHLD, signal.SIG_IGN) | |
| while True: | |
| os.read(spawn_r, 1) | |
| if os.fork() == 0: | |
| converse() | |
| ''' | |
| class Bot: | |
| """The loader process; ask() holds one conversation in a fresh worker.""" | |
| def __init__(self, tree: Path, *, seed: int, timeout: float, workers: int, log): | |
| self.timeout = timeout | |
| self.tmp = tempfile.TemporaryDirectory(prefix="swarmchat-") | |
| self.path = os.path.join(self.tmp.name, "bot.sock") | |
| # No -I: PYTHONHASHSEED must apply. The environment holds nothing else. | |
| self.proc = subprocess.Popen( | |
| [sys.executable, "-s", "-S", "-P", "-c", LOADER, str(tree), str(seed), str(timeout), | |
| self.path, str(workers)], | |
| cwd=self.tmp.name, env={"PYTHONHASHSEED": str(seed)}, stdout=subprocess.PIPE, | |
| stderr=log, text=True, start_new_session=True) | |
| if not select.select([self.proc.stdout], [], [], 90)[0]: | |
| self.close() | |
| raise RuntimeError("The bot did not load within 90 seconds.") | |
| line = self.proc.stdout.readline() | |
| if not line: | |
| self.close() | |
| raise RuntimeError("This commit could not load its bot; try another commit.") | |
| self.ready = json.loads(line) | |
| def ask(self, lines: list[str], trace: bool = False) -> dict: | |
| with socket.socket(socket.AF_UNIX) as conn: | |
| conn.settimeout(self.timeout + 5) | |
| conn.connect(self.path) | |
| conn.sendall((json.dumps({"lines": lines, "trace": True} if trace else lines) + "\n").encode()) | |
| try: | |
| raw = conn.makefile("rb").readline() | |
| except TimeoutError: | |
| return {"replies": [], "status": "timeout"} | |
| # The worker died without replying, e.g. on its CPU or wall-clock limit. | |
| return json.loads(raw) if raw else {"replies": [], "status": "worker exited"} | |
| def alive(self) -> bool: | |
| return self.proc.poll() is None | |
| def close(self): | |
| if self.alive(): | |
| os.killpg(self.proc.pid, signal.SIGKILL) | |
| self.proc.wait() | |
| self.proc.stdout.close() | |
| self.tmp.cleanup() | |
| def user_lines(messages) -> list[str]: | |
| """User message texts as single bot input lines (the bot reads one line per turn).""" | |
| lines = [] | |
| for m in messages: | |
| if m.get("role") != "user": | |
| continue | |
| content = m.get("content") or "" | |
| if isinstance(content, list): | |
| content = " ".join(p.get("text", "") for p in content if p.get("type") == "text") | |
| lines.append(" ".join(str(content).split())) | |
| return lines | |