computaco-tribe / handler.py
jacob-valdez's picture
ComputaCo's inference handler for TRIBE v2, with attribution
6df569c verified
Raw History Blame Contribute Delete
10.7 kB
"""TRIBE v2 behind one Hugging Face inference endpoint: perceptions and the navigator's decisions.
The endpoint scales to zero, so nothing of ours runs while nobody asks. The model loads once, when the endpoint wakes, and
the API (cloud/src/tribe.ts) drives it with short calls, the body being {"inputs": {...}}:
{"op": "start", "id", "kind", "seconds", "stimulus_url", "files_url"} a perception, computed in the background
{"op": "poll", "id"} its state; asking keeps the endpoint awake
{"op": "decide", "input", "effort", "vectors_url"} one step of the navigator, or its `wait`
The URLs are short-lived, signed by the API and good for one perception or for the vector cache, so this process holds no
credential of ours. The endpoint's repository holds this file, requirements.txt, encode.py and navigator.py side by side.
python3 scripts/brain/tribe/handler.py # the job queue and the vector cache, without the model
"""
import collections
import concurrent.futures
import json
import os
import pathlib
import sys
import tempfile
import threading
import time
import traceback
import urllib.error
import urllib.request
HERE = pathlib.Path(__file__).resolve().parent
sys.path[:0] = [str(HERE), str(HERE.parent)]
import navigator # noqa: E402
RESULT_FILES = ("summary.json", "rois.json", "preds.npz", "features.npz", "map.png", "stimulus.mp4")
# How long a finished job is kept, so a late poll still reads its end.
JOB_TTL_S = 3600
# A key the bucket did not hold is asked for again after this long: another replica may have read it.
ABSENT_TTL_S = 30
FETCHERS = 16
HTTP_TIMEOUT_S = 300
# One model job at a time, a perception or a batch of names: both use the same GPU.
GPU = threading.RLock()
def http(method, url, data=None, content_type="application/octet-stream"):
headers = {"user-agent": "computaco-tribe-endpoint/1", **({"content-type": content_type} if data is not None else {})}
with urllib.request.urlopen(urllib.request.Request(url, data=data, method=method, headers=headers), timeout=HTTP_TIMEOUT_S) as r:
return r.read(), dict(r.headers)
class Jobs:
"""Perceptions started and not yet read, run one at a time on one thread. `run(spec)` returns the summary or raises."""
def __init__(self, run, now=time.time):
self.run, self.now = run, now
self.cond = threading.Condition()
self.jobs = collections.OrderedDict()
threading.Thread(target=self._work, name="perceptions", daemon=True).start()
def start(self, spec):
"""Starting the same id again is the same job: the API may not have seen the first answer."""
with self.cond:
job = self.jobs.get(spec["id"])
if job is None:
job = self.jobs[spec["id"]] = {"spec": spec, "state": "queued", "at": self.now()}
self.cond.notify_all()
return self.view(job)
def poll(self, jid):
with self.cond:
for old, job in list(self.jobs.items()):
if job["state"] in ("done", "failed") and self.now() - job["at"] > JOB_TTL_S:
del self.jobs[old]
job = self.jobs.get(jid)
return self.view(job) if job else {"state": "unknown"}
@staticmethod
def view(job):
return {k: job[k] for k in ("state", "summary", "error") if k in job}
def _next(self):
return next((j for j in self.jobs.values() if j["state"] == "queued"), None)
def _work(self):
while True:
with self.cond:
while (job := self._next()) is None:
self.cond.wait()
job["state"] = "running"
try:
out = {"state": "done", "summary": self.run(job["spec"])}
except Exception as e: # noqa: BLE001 - every failure is reported to the perception
traceback.print_exc()
out = {"state": "failed", "error": str(e)[:1000]}
with self.cond:
job.update(out, at=self.now())
class CloudReader(navigator.Reader):
"""The navigator's reader with its vectors kept in the API's bucket: a replica that wakes empty uses what others read."""
def __init__(self, *args, **kwargs):
self.url, self.absent, self.sent = None, {}, set()
super().__init__(*args, **kwargs)
def fetch(self, url, keys):
"""Bring the vectors the bucket holds into the local store before a step is answered."""
self.url = url
now = time.time()
want = [k for k in dict.fromkeys(keys) if not self.has(k) and now - self.absent.get(k, 0) > ABSENT_TTL_S]
def one(key):
try:
body, _ = http("GET", f"{url}/{key}.npy")
except urllib.error.HTTPError as e:
if e.code != 404:
raise
self.absent[key] = time.time()
return
tmp = self.path(key).with_suffix(".part")
tmp.write_bytes(body)
os.replace(tmp, self.path(key))
self.sent.add(key)
with concurrent.futures.ThreadPoolExecutor(FETCHERS) as pool:
list(pool.map(one, want))
def run(self, batch):
with GPU:
super().run(batch)
for key, _ in batch:
if self.url and key not in self.sent and self.has(key):
try:
http("PUT", f"{self.url}/{key}.npy", self.path(key).read_bytes())
self.sent.add(key)
except OSError as e:
self.log(f"could not keep {key}: {e}")
def stimulus_kind(kind, name):
"""What the encoder is given: a media perception is audio or video by its file's extension."""
if kind == "media":
return "audio" if name.rsplit(".", 1)[-1] in ("wav", "mp3", "flac") else "video"
return kind
class EndpointHandler:
def __init__(self, path=""):
import encode
self.encode = encode
root = pathlib.Path(os.environ.get("TRIBE_WORK", tempfile.gettempdir())) / "tribe"
self.work = root / "jobs"
self.work.mkdir(parents=True, exist_ok=True)
cache = root / "features"
cache.mkdir(parents=True, exist_ok=True)
self.nav = navigator.Navigator(cache, root / "navigator", reader=CloudReader)
self.nav.reader.log = lambda text: print(text, flush=True)
self.jobs = Jobs(self.perceive)
def perceive(self, job):
out = self.work / job["id"]
out.mkdir(parents=True, exist_ok=True)
body, headers = http("GET", job["stimulus_url"])
name = headers.get("x-cw-file") or headers.get("X-Cw-File") or "stimulus.bin"
path = out / name
path.write_bytes(body)
with GPU:
summary = self.encode.encode_stimulus(self.nav.model, stimulus_kind(job["kind"], name), path, out / "result", float(job.get("seconds") or 20))
for f in RESULT_FILES:
p = out / "result" / f
if p.exists():
http("PUT", f"{job['files_url']}/{f}", p.read_bytes())
return summary
def __call__(self, data):
given = data.get("inputs", data)
op = given.get("op")
if op == "start":
return self.jobs.start({k: given.get(k) for k in ("id", "kind", "seconds", "stimulus_url", "files_url")})
if op == "poll":
return self.jobs.poll(given["id"])
if op == "decide":
step, effort = given["input"], given.get("effort") or "normal"
self.nav.reader.fetch(given["vectors_url"], navigator.keys_of(step, effort))
return {"decision": self.nav.decide(step, effort)}
return {"error": f"no operation {op}"}
def check():
"""The job queue and the vector cache, with stand-ins for the model and for the bucket."""
import http.server
done, gate = [], threading.Event()
def run(spec):
gate.wait(5)
if spec["id"] == "bad":
raise RuntimeError("no GPU")
done.append(spec["id"])
return {"segments": 3}
clock = [100.0]
jobs = Jobs(run, now=lambda: clock[0])
assert jobs.poll("a") == {"state": "unknown"}
assert jobs.start({"id": "a"})["state"] in ("queued", "running")
jobs.start({"id": "a"})
jobs.start({"id": "bad"})
assert jobs.poll("bad") == {"state": "queued"}, "one job runs at a time"
gate.set()
for _ in range(300):
if jobs.poll("bad")["state"] == "failed":
break
time.sleep(0.01)
assert done == ["a"], "a job started twice runs once"
assert jobs.poll("a") == {"state": "done", "summary": {"segments": 3}} and jobs.poll("bad") == {"state": "failed", "error": "no GPU"}
clock[0] += JOB_TTL_S + 1
assert jobs.poll("a") == {"state": "unknown"}, "a finished job is forgotten after its time"
bucket = {"t-held": b"held"}
gets = []
class Bucket(http.server.BaseHTTPRequestHandler):
def do_GET(self):
key = pathlib.Path(self.path).stem
gets.append(key)
self.send_response(200 if key in bucket else 404)
self.end_headers()
self.wfile.write(bucket.get(key, b""))
def do_PUT(self):
bucket[pathlib.Path(self.path).stem] = self.rfile.read(int(self.headers["content-length"]))
self.send_response(201)
self.end_headers()
def log_message(self, *_):
pass
server = http.server.ThreadingHTTPServer(("127.0.0.1", 0), Bucket)
threading.Thread(target=server.serve_forever, daemon=True).start()
url = f"http://127.0.0.1:{server.server_address[1]}/vectors"
with tempfile.TemporaryDirectory() as d:
def read_texts(items):
for key, _ in items:
reader.path(key).write_bytes(b"read")
return {}
reader = CloudReader(d, read_texts, None, background=False)
reader.fetch(url, ["t-held", "t-new"])
assert reader.has("t-held") and not reader.has("t-new"), "what the bucket holds is not read again"
reader.fetch(url, ["t-held", "t-new"])
assert gets.count("t-new") == 1 and gets.count("t-held") == 1, "a key just found absent, or held, is not asked for again"
reader.run([("t-new", ("text", "New"))])
assert bucket["t-new"] == b"read", "a vector read here is kept for the next replica"
server.shutdown()
assert stimulus_kind("media", "stimulus.flac") == "audio" and stimulus_kind("media", "stimulus.mp4") == "video" and stimulus_kind("screen", "stimulus.png") == "screen"
print(json.dumps({"ok": True}))
if __name__ == "__main__":
check()