File size: 8,863 Bytes
12ea8e3
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
# From Space HuggingFaceSwarms/SwarmChat-Demo (bot_runner.py, itself adapted from flock's run4 eval server),
# plus a trace mode: which modules proposed a reply, which won, and the bot files the winner called.
"""Serve a checkout of a swarm's chatbot as SwarmChat, an OpenAI-compatible endpoint for evals.

    python experiments/run4/eval/swarmchat_server.py <chatbot checkout> --port 8000

Loading the bot takes seconds (hundreds of modules), answering takes milliseconds. So one
process loads the bot and forks a worker per conversation, and every conversation starts from
the same just-loaded state. The same lines get the same replies whatever the order or the
concurrency of the requests, in this run or the next: the hash seed, the global `random` state
(restored in each worker) and the seeds of the bot's own random.Random() instances are fixed.
Replies that read the clock or seed from an object id (a few modules use id(memory)) can still
change: ids move with address randomisation, which HF Jobs do not let us turn off.

POST /v1/chat/completions feeds the request's user messages to the bot in order (system and
assistant messages are ignored: the bot only reads user lines) and returns its last reply.
There is no sandbox: the bot runs as the server's user, with an empty environment, so run it
somewhere disposable such as an HF Job.
"""

import json
import os
import signal
import select
import socket
import subprocess
import sys
import tempfile
from pathlib import Path

LOADER = r'''
import io, json, os, random, resource, runpy, signal, socket, sys, time
root, seed, timeout, path, workers = sys.argv[1], int(sys.argv[2]), float(sys.argv[3]), sys.argv[4], int(sys.argv[5])
sys.path.insert(0, root)
status = os.fdopen(os.dup(1), "w")
os.dup2(2, 1)  # stray prints from the bot go to the log
listener = socket.socket(socket.AF_UNIX)
listener.bind(path)
listener.listen(256)
# A worker writes a byte here when it takes a conversation; the loader forks one per byte.
spawn_r, spawn_w = os.pipe()
os.write(spawn_w, bytes(workers))


def converse():
    """A worker: take one connection, hold that conversation, send the result and exit."""
    conn = listener.accept()[0]
    os.write(spawn_w, b"\0")
    random.setstate(loaded)  # random reseeds itself from the OS in every forked child
    started = time.monotonic()
    request = json.loads(conn.makefile("rb").readline())
    lines, trace = (request, False) if isinstance(request, list) else (request["lines"], bool(request.get("trace")))
    limit = int(timeout) + 1
    signal.alarm(limit)
    resource.setrlimit(resource.RLIMIT_CPU, (limit, limit + 1))
    if trace:
        result, out = "ok", {}
        try:
            out = traced(lines)
        except BaseException as exc:
            result = f"error {type(exc).__name__}"
        conn.sendall((json.dumps({"replies": out.get("replies", []), "trace": out.get("trace"), "status": result,
                                  "seconds": round(time.monotonic() - started, 3)}) + "\n").encode())
        os._exit(0)
    sys.stdin = io.StringIO("".join(line + "\n" for line in lines))
    sys.stdout = buffer = io.StringIO()
    result = "ok"
    try:
        runpy.run_module("bot", run_name="__main__")
    except BaseException as exc:  # the fixed __main__ catches reply errors; this is the rest
        result = f"error {type(exc).__name__}"
    conn.sendall((json.dumps({"replies": buffer.getvalue().splitlines(), "status": result,
                              "seconds": round(time.monotonic() - started, 3)}) + "\n").encode())
    os._exit(0)


def traced(lines):
    """Replies like the fixed __main__ (one per line, whitespace-normalised, "Hello!" on errors), plus, for the last
    line, the bot's own decision (bot.runtime.turn: candidates and winner) and the bot files each proposing handler called.
    sys.monitoring records a function once and then disables that event, so tracing costs about nothing."""
    import bot.runtime as rt
    def say(text, fn):
        try:
            return " ".join(str(fn(text)).split()) or "Hello!"
        except Exception:  # noqa: BLE001 - the fixed __main__'s rule
            return "Hello!"
    memory = {}
    replies = [say(line, lambda t: rt.reply(t, memory)) for line in lines[:-1]]
    mon, tool, bot_dir, seen, files = sys.monitoring, 5, os.path.join(root, "bot") + os.sep, set(), {}
    mon.use_tool_id(tool, "pybot-trace")
    def start(code, offset):
        if code.co_filename.startswith(bot_dir):
            seen.add("bot/" + code.co_filename[len(bot_dir):])
        return mon.DISABLE
    mon.register_callback(tool, mon.events.PY_START, start)
    def watch(name, respond):
        def run(text, mem):
            seen.clear(); mon.restart_events(); mon.set_events(tool, mon.events.PY_START)
            try:
                return respond(text, mem)
            finally:
                mon.set_events(tool, 0); files[name] = sorted(seen)
        return run
    rt._HANDLERS = [(name, priority, watch(name, respond)) for name, priority, respond in rt._HANDLERS]
    decision = {}
    def last(text):
        decision.update(rt.turn(text, memory))
        return decision["reply"]
    replies.append(say(lines[-1], last))
    module = decision.get("module")
    return {"replies": replies, "trace": {"module": module,
            "candidates": [c["module"] for c in decision.get("candidates", [])],
            "files": files.get(module, []) if module else []}}


# Modules make their own random.Random(), which seeds itself from the OS: derive those seeds
# from the run's seed and a counter instead, so every run starts in the same state.
os_seeded, created = random.Random.seed, iter(range(1 << 62))
def derived_seed(self, a=None, version=2):
    os_seeded(self, f"{seed}:{next(created)}" if a is None else a, version)
random.Random.seed = derived_seed
random.seed = random._inst.seed
random.seed(seed)
started = time.monotonic()
import bot.runtime
loaded = random.getstate()
print(json.dumps({"load_s": round(time.monotonic() - started, 2),
                  "handlers": len(bot.runtime._HANDLERS)}), file=status, flush=True)
# Keep `workers` forks waiting for a connection. The workers read the requests, never the
# loader, and the kernel reaps them.
signal.signal(signal.SIGCHLD, signal.SIG_IGN)
while True:
    os.read(spawn_r, 1)
    if os.fork() == 0:
        converse()
'''


class Bot:
    """The loader process; ask() holds one conversation in a fresh worker."""

    def __init__(self, tree: Path, *, seed: int, timeout: float, workers: int, log):
        self.timeout = timeout
        self.tmp = tempfile.TemporaryDirectory(prefix="swarmchat-")
        self.path = os.path.join(self.tmp.name, "bot.sock")
        # No -I: PYTHONHASHSEED must apply. The environment holds nothing else.
        self.proc = subprocess.Popen(
            [sys.executable, "-s", "-S", "-P", "-c", LOADER, str(tree), str(seed), str(timeout),
             self.path, str(workers)],
            cwd=self.tmp.name, env={"PYTHONHASHSEED": str(seed)}, stdout=subprocess.PIPE,
            stderr=log, text=True, start_new_session=True)
        if not select.select([self.proc.stdout], [], [], 90)[0]:
            self.close()
            raise RuntimeError("The bot did not load within 90 seconds.")
        line = self.proc.stdout.readline()
        if not line:
            self.close()
            raise RuntimeError("This commit could not load its bot; try another commit.")
        self.ready = json.loads(line)

    def ask(self, lines: list[str], trace: bool = False) -> dict:
        with socket.socket(socket.AF_UNIX) as conn:
            conn.settimeout(self.timeout + 5)
            conn.connect(self.path)
            conn.sendall((json.dumps({"lines": lines, "trace": True} if trace else lines) + "\n").encode())
            try:
                raw = conn.makefile("rb").readline()
            except TimeoutError:
                return {"replies": [], "status": "timeout"}
        # The worker died without replying, e.g. on its CPU or wall-clock limit.
        return json.loads(raw) if raw else {"replies": [], "status": "worker exited"}

    def alive(self) -> bool:
        return self.proc.poll() is None

    def close(self):
        if self.alive():
            os.killpg(self.proc.pid, signal.SIGKILL)
        self.proc.wait()
        self.proc.stdout.close()
        self.tmp.cleanup()


def user_lines(messages) -> list[str]:
    """User message texts as single bot input lines (the bot reads one line per turn)."""
    lines = []
    for m in messages:
        if m.get("role") != "user":
            continue
        content = m.get("content") or ""
        if isinstance(content, list):
            content = " ".join(p.get("text", "") for p in content if p.get("type") == "text")
        lines.append(" ".join(str(content).split()))
    return lines