Spaces:
Running
Running
File size: 8,863 Bytes
12ea8e3 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 | # From Space HuggingFaceSwarms/SwarmChat-Demo (bot_runner.py, itself adapted from flock's run4 eval server),
# plus a trace mode: which modules proposed a reply, which won, and the bot files the winner called.
"""Serve a checkout of a swarm's chatbot as SwarmChat, an OpenAI-compatible endpoint for evals.
python experiments/run4/eval/swarmchat_server.py <chatbot checkout> --port 8000
Loading the bot takes seconds (hundreds of modules), answering takes milliseconds. So one
process loads the bot and forks a worker per conversation, and every conversation starts from
the same just-loaded state. The same lines get the same replies whatever the order or the
concurrency of the requests, in this run or the next: the hash seed, the global `random` state
(restored in each worker) and the seeds of the bot's own random.Random() instances are fixed.
Replies that read the clock or seed from an object id (a few modules use id(memory)) can still
change: ids move with address randomisation, which HF Jobs do not let us turn off.
POST /v1/chat/completions feeds the request's user messages to the bot in order (system and
assistant messages are ignored: the bot only reads user lines) and returns its last reply.
There is no sandbox: the bot runs as the server's user, with an empty environment, so run it
somewhere disposable such as an HF Job.
"""
import json
import os
import signal
import select
import socket
import subprocess
import sys
import tempfile
from pathlib import Path
LOADER = r'''
import io, json, os, random, resource, runpy, signal, socket, sys, time
root, seed, timeout, path, workers = sys.argv[1], int(sys.argv[2]), float(sys.argv[3]), sys.argv[4], int(sys.argv[5])
sys.path.insert(0, root)
status = os.fdopen(os.dup(1), "w")
os.dup2(2, 1) # stray prints from the bot go to the log
listener = socket.socket(socket.AF_UNIX)
listener.bind(path)
listener.listen(256)
# A worker writes a byte here when it takes a conversation; the loader forks one per byte.
spawn_r, spawn_w = os.pipe()
os.write(spawn_w, bytes(workers))
def converse():
"""A worker: take one connection, hold that conversation, send the result and exit."""
conn = listener.accept()[0]
os.write(spawn_w, b"\0")
random.setstate(loaded) # random reseeds itself from the OS in every forked child
started = time.monotonic()
request = json.loads(conn.makefile("rb").readline())
lines, trace = (request, False) if isinstance(request, list) else (request["lines"], bool(request.get("trace")))
limit = int(timeout) + 1
signal.alarm(limit)
resource.setrlimit(resource.RLIMIT_CPU, (limit, limit + 1))
if trace:
result, out = "ok", {}
try:
out = traced(lines)
except BaseException as exc:
result = f"error {type(exc).__name__}"
conn.sendall((json.dumps({"replies": out.get("replies", []), "trace": out.get("trace"), "status": result,
"seconds": round(time.monotonic() - started, 3)}) + "\n").encode())
os._exit(0)
sys.stdin = io.StringIO("".join(line + "\n" for line in lines))
sys.stdout = buffer = io.StringIO()
result = "ok"
try:
runpy.run_module("bot", run_name="__main__")
except BaseException as exc: # the fixed __main__ catches reply errors; this is the rest
result = f"error {type(exc).__name__}"
conn.sendall((json.dumps({"replies": buffer.getvalue().splitlines(), "status": result,
"seconds": round(time.monotonic() - started, 3)}) + "\n").encode())
os._exit(0)
def traced(lines):
"""Replies like the fixed __main__ (one per line, whitespace-normalised, "Hello!" on errors), plus, for the last
line, the bot's own decision (bot.runtime.turn: candidates and winner) and the bot files each proposing handler called.
sys.monitoring records a function once and then disables that event, so tracing costs about nothing."""
import bot.runtime as rt
def say(text, fn):
try:
return " ".join(str(fn(text)).split()) or "Hello!"
except Exception: # noqa: BLE001 - the fixed __main__'s rule
return "Hello!"
memory = {}
replies = [say(line, lambda t: rt.reply(t, memory)) for line in lines[:-1]]
mon, tool, bot_dir, seen, files = sys.monitoring, 5, os.path.join(root, "bot") + os.sep, set(), {}
mon.use_tool_id(tool, "pybot-trace")
def start(code, offset):
if code.co_filename.startswith(bot_dir):
seen.add("bot/" + code.co_filename[len(bot_dir):])
return mon.DISABLE
mon.register_callback(tool, mon.events.PY_START, start)
def watch(name, respond):
def run(text, mem):
seen.clear(); mon.restart_events(); mon.set_events(tool, mon.events.PY_START)
try:
return respond(text, mem)
finally:
mon.set_events(tool, 0); files[name] = sorted(seen)
return run
rt._HANDLERS = [(name, priority, watch(name, respond)) for name, priority, respond in rt._HANDLERS]
decision = {}
def last(text):
decision.update(rt.turn(text, memory))
return decision["reply"]
replies.append(say(lines[-1], last))
module = decision.get("module")
return {"replies": replies, "trace": {"module": module,
"candidates": [c["module"] for c in decision.get("candidates", [])],
"files": files.get(module, []) if module else []}}
# Modules make their own random.Random(), which seeds itself from the OS: derive those seeds
# from the run's seed and a counter instead, so every run starts in the same state.
os_seeded, created = random.Random.seed, iter(range(1 << 62))
def derived_seed(self, a=None, version=2):
os_seeded(self, f"{seed}:{next(created)}" if a is None else a, version)
random.Random.seed = derived_seed
random.seed = random._inst.seed
random.seed(seed)
started = time.monotonic()
import bot.runtime
loaded = random.getstate()
print(json.dumps({"load_s": round(time.monotonic() - started, 2),
"handlers": len(bot.runtime._HANDLERS)}), file=status, flush=True)
# Keep `workers` forks waiting for a connection. The workers read the requests, never the
# loader, and the kernel reaps them.
signal.signal(signal.SIGCHLD, signal.SIG_IGN)
while True:
os.read(spawn_r, 1)
if os.fork() == 0:
converse()
'''
class Bot:
"""The loader process; ask() holds one conversation in a fresh worker."""
def __init__(self, tree: Path, *, seed: int, timeout: float, workers: int, log):
self.timeout = timeout
self.tmp = tempfile.TemporaryDirectory(prefix="swarmchat-")
self.path = os.path.join(self.tmp.name, "bot.sock")
# No -I: PYTHONHASHSEED must apply. The environment holds nothing else.
self.proc = subprocess.Popen(
[sys.executable, "-s", "-S", "-P", "-c", LOADER, str(tree), str(seed), str(timeout),
self.path, str(workers)],
cwd=self.tmp.name, env={"PYTHONHASHSEED": str(seed)}, stdout=subprocess.PIPE,
stderr=log, text=True, start_new_session=True)
if not select.select([self.proc.stdout], [], [], 90)[0]:
self.close()
raise RuntimeError("The bot did not load within 90 seconds.")
line = self.proc.stdout.readline()
if not line:
self.close()
raise RuntimeError("This commit could not load its bot; try another commit.")
self.ready = json.loads(line)
def ask(self, lines: list[str], trace: bool = False) -> dict:
with socket.socket(socket.AF_UNIX) as conn:
conn.settimeout(self.timeout + 5)
conn.connect(self.path)
conn.sendall((json.dumps({"lines": lines, "trace": True} if trace else lines) + "\n").encode())
try:
raw = conn.makefile("rb").readline()
except TimeoutError:
return {"replies": [], "status": "timeout"}
# The worker died without replying, e.g. on its CPU or wall-clock limit.
return json.loads(raw) if raw else {"replies": [], "status": "worker exited"}
def alive(self) -> bool:
return self.proc.poll() is None
def close(self):
if self.alive():
os.killpg(self.proc.pid, signal.SIGKILL)
self.proc.wait()
self.proc.stdout.close()
self.tmp.cleanup()
def user_lines(messages) -> list[str]:
"""User message texts as single bot input lines (the bot reads one line per turn)."""
lines = []
for m in messages:
if m.get("role") != "user":
continue
content = m.get("content") or ""
if isinstance(content, list):
content = " ".join(p.get("text", "") for p in content if p.get("type") == "text")
lines.append(" ".join(str(content).split()))
return lines
|