File size: 10,701 Bytes
6df569c | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 | """TRIBE v2 behind one Hugging Face inference endpoint: perceptions and the navigator's decisions.
The endpoint scales to zero, so nothing of ours runs while nobody asks. The model loads once, when the endpoint wakes, and
the API (cloud/src/tribe.ts) drives it with short calls, the body being {"inputs": {...}}:
{"op": "start", "id", "kind", "seconds", "stimulus_url", "files_url"} a perception, computed in the background
{"op": "poll", "id"} its state; asking keeps the endpoint awake
{"op": "decide", "input", "effort", "vectors_url"} one step of the navigator, or its `wait`
The URLs are short-lived, signed by the API and good for one perception or for the vector cache, so this process holds no
credential of ours. The endpoint's repository holds this file, requirements.txt, encode.py and navigator.py side by side.
python3 scripts/brain/tribe/handler.py # the job queue and the vector cache, without the model
"""
import collections
import concurrent.futures
import json
import os
import pathlib
import sys
import tempfile
import threading
import time
import traceback
import urllib.error
import urllib.request
HERE = pathlib.Path(__file__).resolve().parent
sys.path[:0] = [str(HERE), str(HERE.parent)]
import navigator # noqa: E402
RESULT_FILES = ("summary.json", "rois.json", "preds.npz", "features.npz", "map.png", "stimulus.mp4")
# How long a finished job is kept, so a late poll still reads its end.
JOB_TTL_S = 3600
# A key the bucket did not hold is asked for again after this long: another replica may have read it.
ABSENT_TTL_S = 30
FETCHERS = 16
HTTP_TIMEOUT_S = 300
# One model job at a time, a perception or a batch of names: both use the same GPU.
GPU = threading.RLock()
def http(method, url, data=None, content_type="application/octet-stream"):
headers = {"user-agent": "computaco-tribe-endpoint/1", **({"content-type": content_type} if data is not None else {})}
with urllib.request.urlopen(urllib.request.Request(url, data=data, method=method, headers=headers), timeout=HTTP_TIMEOUT_S) as r:
return r.read(), dict(r.headers)
class Jobs:
"""Perceptions started and not yet read, run one at a time on one thread. `run(spec)` returns the summary or raises."""
def __init__(self, run, now=time.time):
self.run, self.now = run, now
self.cond = threading.Condition()
self.jobs = collections.OrderedDict()
threading.Thread(target=self._work, name="perceptions", daemon=True).start()
def start(self, spec):
"""Starting the same id again is the same job: the API may not have seen the first answer."""
with self.cond:
job = self.jobs.get(spec["id"])
if job is None:
job = self.jobs[spec["id"]] = {"spec": spec, "state": "queued", "at": self.now()}
self.cond.notify_all()
return self.view(job)
def poll(self, jid):
with self.cond:
for old, job in list(self.jobs.items()):
if job["state"] in ("done", "failed") and self.now() - job["at"] > JOB_TTL_S:
del self.jobs[old]
job = self.jobs.get(jid)
return self.view(job) if job else {"state": "unknown"}
@staticmethod
def view(job):
return {k: job[k] for k in ("state", "summary", "error") if k in job}
def _next(self):
return next((j for j in self.jobs.values() if j["state"] == "queued"), None)
def _work(self):
while True:
with self.cond:
while (job := self._next()) is None:
self.cond.wait()
job["state"] = "running"
try:
out = {"state": "done", "summary": self.run(job["spec"])}
except Exception as e: # noqa: BLE001 - every failure is reported to the perception
traceback.print_exc()
out = {"state": "failed", "error": str(e)[:1000]}
with self.cond:
job.update(out, at=self.now())
class CloudReader(navigator.Reader):
"""The navigator's reader with its vectors kept in the API's bucket: a replica that wakes empty uses what others read."""
def __init__(self, *args, **kwargs):
self.url, self.absent, self.sent = None, {}, set()
super().__init__(*args, **kwargs)
def fetch(self, url, keys):
"""Bring the vectors the bucket holds into the local store before a step is answered."""
self.url = url
now = time.time()
want = [k for k in dict.fromkeys(keys) if not self.has(k) and now - self.absent.get(k, 0) > ABSENT_TTL_S]
def one(key):
try:
body, _ = http("GET", f"{url}/{key}.npy")
except urllib.error.HTTPError as e:
if e.code != 404:
raise
self.absent[key] = time.time()
return
tmp = self.path(key).with_suffix(".part")
tmp.write_bytes(body)
os.replace(tmp, self.path(key))
self.sent.add(key)
with concurrent.futures.ThreadPoolExecutor(FETCHERS) as pool:
list(pool.map(one, want))
def run(self, batch):
with GPU:
super().run(batch)
for key, _ in batch:
if self.url and key not in self.sent and self.has(key):
try:
http("PUT", f"{self.url}/{key}.npy", self.path(key).read_bytes())
self.sent.add(key)
except OSError as e:
self.log(f"could not keep {key}: {e}")
def stimulus_kind(kind, name):
"""What the encoder is given: a media perception is audio or video by its file's extension."""
if kind == "media":
return "audio" if name.rsplit(".", 1)[-1] in ("wav", "mp3", "flac") else "video"
return kind
class EndpointHandler:
def __init__(self, path=""):
import encode
self.encode = encode
root = pathlib.Path(os.environ.get("TRIBE_WORK", tempfile.gettempdir())) / "tribe"
self.work = root / "jobs"
self.work.mkdir(parents=True, exist_ok=True)
cache = root / "features"
cache.mkdir(parents=True, exist_ok=True)
self.nav = navigator.Navigator(cache, root / "navigator", reader=CloudReader)
self.nav.reader.log = lambda text: print(text, flush=True)
self.jobs = Jobs(self.perceive)
def perceive(self, job):
out = self.work / job["id"]
out.mkdir(parents=True, exist_ok=True)
body, headers = http("GET", job["stimulus_url"])
name = headers.get("x-cw-file") or headers.get("X-Cw-File") or "stimulus.bin"
path = out / name
path.write_bytes(body)
with GPU:
summary = self.encode.encode_stimulus(self.nav.model, stimulus_kind(job["kind"], name), path, out / "result", float(job.get("seconds") or 20))
for f in RESULT_FILES:
p = out / "result" / f
if p.exists():
http("PUT", f"{job['files_url']}/{f}", p.read_bytes())
return summary
def __call__(self, data):
given = data.get("inputs", data)
op = given.get("op")
if op == "start":
return self.jobs.start({k: given.get(k) for k in ("id", "kind", "seconds", "stimulus_url", "files_url")})
if op == "poll":
return self.jobs.poll(given["id"])
if op == "decide":
step, effort = given["input"], given.get("effort") or "normal"
self.nav.reader.fetch(given["vectors_url"], navigator.keys_of(step, effort))
return {"decision": self.nav.decide(step, effort)}
return {"error": f"no operation {op}"}
def check():
"""The job queue and the vector cache, with stand-ins for the model and for the bucket."""
import http.server
done, gate = [], threading.Event()
def run(spec):
gate.wait(5)
if spec["id"] == "bad":
raise RuntimeError("no GPU")
done.append(spec["id"])
return {"segments": 3}
clock = [100.0]
jobs = Jobs(run, now=lambda: clock[0])
assert jobs.poll("a") == {"state": "unknown"}
assert jobs.start({"id": "a"})["state"] in ("queued", "running")
jobs.start({"id": "a"})
jobs.start({"id": "bad"})
assert jobs.poll("bad") == {"state": "queued"}, "one job runs at a time"
gate.set()
for _ in range(300):
if jobs.poll("bad")["state"] == "failed":
break
time.sleep(0.01)
assert done == ["a"], "a job started twice runs once"
assert jobs.poll("a") == {"state": "done", "summary": {"segments": 3}} and jobs.poll("bad") == {"state": "failed", "error": "no GPU"}
clock[0] += JOB_TTL_S + 1
assert jobs.poll("a") == {"state": "unknown"}, "a finished job is forgotten after its time"
bucket = {"t-held": b"held"}
gets = []
class Bucket(http.server.BaseHTTPRequestHandler):
def do_GET(self):
key = pathlib.Path(self.path).stem
gets.append(key)
self.send_response(200 if key in bucket else 404)
self.end_headers()
self.wfile.write(bucket.get(key, b""))
def do_PUT(self):
bucket[pathlib.Path(self.path).stem] = self.rfile.read(int(self.headers["content-length"]))
self.send_response(201)
self.end_headers()
def log_message(self, *_):
pass
server = http.server.ThreadingHTTPServer(("127.0.0.1", 0), Bucket)
threading.Thread(target=server.serve_forever, daemon=True).start()
url = f"http://127.0.0.1:{server.server_address[1]}/vectors"
with tempfile.TemporaryDirectory() as d:
def read_texts(items):
for key, _ in items:
reader.path(key).write_bytes(b"read")
return {}
reader = CloudReader(d, read_texts, None, background=False)
reader.fetch(url, ["t-held", "t-new"])
assert reader.has("t-held") and not reader.has("t-new"), "what the bucket holds is not read again"
reader.fetch(url, ["t-held", "t-new"])
assert gets.count("t-new") == 1 and gets.count("t-held") == 1, "a key just found absent, or held, is not asked for again"
reader.run([("t-new", ("text", "New"))])
assert bucket["t-new"] == b"read", "a vector read here is kept for the next replica"
server.shutdown()
assert stimulus_kind("media", "stimulus.flac") == "audio" and stimulus_kind("media", "stimulus.mp4") == "video" and stimulus_kind("screen", "stimulus.png") == "screen"
print(json.dumps({"ok": True}))
if __name__ == "__main__":
check()
|