Download handler.py from jacob-valdez/computaco-tribe: direct link, hf CLI and curl.
- Browser
- Download file 10.7 kB
-
https://huggingface.co/jacob-valdez/computaco-tribe/resolve/main/handler.py
- Command line
-
hf download hf://jacob-valdez/computaco-tribe/handler.py
-
curl -L -o handler.py https://huggingface.co/jacob-valdez/computaco-tribe/resolve/main/handler.py
10.7 kB
| """TRIBE v2 behind one Hugging Face inference endpoint: perceptions and the navigator's decisions. | |
| The endpoint scales to zero, so nothing of ours runs while nobody asks. The model loads once, when the endpoint wakes, and | |
| the API (cloud/src/tribe.ts) drives it with short calls, the body being {"inputs": {...}}: | |
| {"op": "start", "id", "kind", "seconds", "stimulus_url", "files_url"} a perception, computed in the background | |
| {"op": "poll", "id"} its state; asking keeps the endpoint awake | |
| {"op": "decide", "input", "effort", "vectors_url"} one step of the navigator, or its `wait` | |
| The URLs are short-lived, signed by the API and good for one perception or for the vector cache, so this process holds no | |
| credential of ours. The endpoint's repository holds this file, requirements.txt, encode.py and navigator.py side by side. | |
| python3 scripts/brain/tribe/handler.py # the job queue and the vector cache, without the model | |
| """ | |
| import collections | |
| import concurrent.futures | |
| import json | |
| import os | |
| import pathlib | |
| import sys | |
| import tempfile | |
| import threading | |
| import time | |
| import traceback | |
| import urllib.error | |
| import urllib.request | |
| HERE = pathlib.Path(__file__).resolve().parent | |
| sys.path[:0] = [str(HERE), str(HERE.parent)] | |
| import navigator # noqa: E402 | |
| RESULT_FILES = ("summary.json", "rois.json", "preds.npz", "features.npz", "map.png", "stimulus.mp4") | |
| # How long a finished job is kept, so a late poll still reads its end. | |
| JOB_TTL_S = 3600 | |
| # A key the bucket did not hold is asked for again after this long: another replica may have read it. | |
| ABSENT_TTL_S = 30 | |
| FETCHERS = 16 | |
| HTTP_TIMEOUT_S = 300 | |
| # One model job at a time, a perception or a batch of names: both use the same GPU. | |
| GPU = threading.RLock() | |
| def http(method, url, data=None, content_type="application/octet-stream"): | |
| headers = {"user-agent": "computaco-tribe-endpoint/1", **({"content-type": content_type} if data is not None else {})} | |
| with urllib.request.urlopen(urllib.request.Request(url, data=data, method=method, headers=headers), timeout=HTTP_TIMEOUT_S) as r: | |
| return r.read(), dict(r.headers) | |
| class Jobs: | |
| """Perceptions started and not yet read, run one at a time on one thread. `run(spec)` returns the summary or raises.""" | |
| def __init__(self, run, now=time.time): | |
| self.run, self.now = run, now | |
| self.cond = threading.Condition() | |
| self.jobs = collections.OrderedDict() | |
| threading.Thread(target=self._work, name="perceptions", daemon=True).start() | |
| def start(self, spec): | |
| """Starting the same id again is the same job: the API may not have seen the first answer.""" | |
| with self.cond: | |
| job = self.jobs.get(spec["id"]) | |
| if job is None: | |
| job = self.jobs[spec["id"]] = {"spec": spec, "state": "queued", "at": self.now()} | |
| self.cond.notify_all() | |
| return self.view(job) | |
| def poll(self, jid): | |
| with self.cond: | |
| for old, job in list(self.jobs.items()): | |
| if job["state"] in ("done", "failed") and self.now() - job["at"] > JOB_TTL_S: | |
| del self.jobs[old] | |
| job = self.jobs.get(jid) | |
| return self.view(job) if job else {"state": "unknown"} | |
| def view(job): | |
| return {k: job[k] for k in ("state", "summary", "error") if k in job} | |
| def _next(self): | |
| return next((j for j in self.jobs.values() if j["state"] == "queued"), None) | |
| def _work(self): | |
| while True: | |
| with self.cond: | |
| while (job := self._next()) is None: | |
| self.cond.wait() | |
| job["state"] = "running" | |
| try: | |
| out = {"state": "done", "summary": self.run(job["spec"])} | |
| except Exception as e: # noqa: BLE001 - every failure is reported to the perception | |
| traceback.print_exc() | |
| out = {"state": "failed", "error": str(e)[:1000]} | |
| with self.cond: | |
| job.update(out, at=self.now()) | |
| class CloudReader(navigator.Reader): | |
| """The navigator's reader with its vectors kept in the API's bucket: a replica that wakes empty uses what others read.""" | |
| def __init__(self, *args, **kwargs): | |
| self.url, self.absent, self.sent = None, {}, set() | |
| super().__init__(*args, **kwargs) | |
| def fetch(self, url, keys): | |
| """Bring the vectors the bucket holds into the local store before a step is answered.""" | |
| self.url = url | |
| now = time.time() | |
| want = [k for k in dict.fromkeys(keys) if not self.has(k) and now - self.absent.get(k, 0) > ABSENT_TTL_S] | |
| def one(key): | |
| try: | |
| body, _ = http("GET", f"{url}/{key}.npy") | |
| except urllib.error.HTTPError as e: | |
| if e.code != 404: | |
| raise | |
| self.absent[key] = time.time() | |
| return | |
| tmp = self.path(key).with_suffix(".part") | |
| tmp.write_bytes(body) | |
| os.replace(tmp, self.path(key)) | |
| self.sent.add(key) | |
| with concurrent.futures.ThreadPoolExecutor(FETCHERS) as pool: | |
| list(pool.map(one, want)) | |
| def run(self, batch): | |
| with GPU: | |
| super().run(batch) | |
| for key, _ in batch: | |
| if self.url and key not in self.sent and self.has(key): | |
| try: | |
| http("PUT", f"{self.url}/{key}.npy", self.path(key).read_bytes()) | |
| self.sent.add(key) | |
| except OSError as e: | |
| self.log(f"could not keep {key}: {e}") | |
| def stimulus_kind(kind, name): | |
| """What the encoder is given: a media perception is audio or video by its file's extension.""" | |
| if kind == "media": | |
| return "audio" if name.rsplit(".", 1)[-1] in ("wav", "mp3", "flac") else "video" | |
| return kind | |
| class EndpointHandler: | |
| def __init__(self, path=""): | |
| import encode | |
| self.encode = encode | |
| root = pathlib.Path(os.environ.get("TRIBE_WORK", tempfile.gettempdir())) / "tribe" | |
| self.work = root / "jobs" | |
| self.work.mkdir(parents=True, exist_ok=True) | |
| cache = root / "features" | |
| cache.mkdir(parents=True, exist_ok=True) | |
| self.nav = navigator.Navigator(cache, root / "navigator", reader=CloudReader) | |
| self.nav.reader.log = lambda text: print(text, flush=True) | |
| self.jobs = Jobs(self.perceive) | |
| def perceive(self, job): | |
| out = self.work / job["id"] | |
| out.mkdir(parents=True, exist_ok=True) | |
| body, headers = http("GET", job["stimulus_url"]) | |
| name = headers.get("x-cw-file") or headers.get("X-Cw-File") or "stimulus.bin" | |
| path = out / name | |
| path.write_bytes(body) | |
| with GPU: | |
| summary = self.encode.encode_stimulus(self.nav.model, stimulus_kind(job["kind"], name), path, out / "result", float(job.get("seconds") or 20)) | |
| for f in RESULT_FILES: | |
| p = out / "result" / f | |
| if p.exists(): | |
| http("PUT", f"{job['files_url']}/{f}", p.read_bytes()) | |
| return summary | |
| def __call__(self, data): | |
| given = data.get("inputs", data) | |
| op = given.get("op") | |
| if op == "start": | |
| return self.jobs.start({k: given.get(k) for k in ("id", "kind", "seconds", "stimulus_url", "files_url")}) | |
| if op == "poll": | |
| return self.jobs.poll(given["id"]) | |
| if op == "decide": | |
| step, effort = given["input"], given.get("effort") or "normal" | |
| self.nav.reader.fetch(given["vectors_url"], navigator.keys_of(step, effort)) | |
| return {"decision": self.nav.decide(step, effort)} | |
| return {"error": f"no operation {op}"} | |
| def check(): | |
| """The job queue and the vector cache, with stand-ins for the model and for the bucket.""" | |
| import http.server | |
| done, gate = [], threading.Event() | |
| def run(spec): | |
| gate.wait(5) | |
| if spec["id"] == "bad": | |
| raise RuntimeError("no GPU") | |
| done.append(spec["id"]) | |
| return {"segments": 3} | |
| clock = [100.0] | |
| jobs = Jobs(run, now=lambda: clock[0]) | |
| assert jobs.poll("a") == {"state": "unknown"} | |
| assert jobs.start({"id": "a"})["state"] in ("queued", "running") | |
| jobs.start({"id": "a"}) | |
| jobs.start({"id": "bad"}) | |
| assert jobs.poll("bad") == {"state": "queued"}, "one job runs at a time" | |
| gate.set() | |
| for _ in range(300): | |
| if jobs.poll("bad")["state"] == "failed": | |
| break | |
| time.sleep(0.01) | |
| assert done == ["a"], "a job started twice runs once" | |
| assert jobs.poll("a") == {"state": "done", "summary": {"segments": 3}} and jobs.poll("bad") == {"state": "failed", "error": "no GPU"} | |
| clock[0] += JOB_TTL_S + 1 | |
| assert jobs.poll("a") == {"state": "unknown"}, "a finished job is forgotten after its time" | |
| bucket = {"t-held": b"held"} | |
| gets = [] | |
| class Bucket(http.server.BaseHTTPRequestHandler): | |
| def do_GET(self): | |
| key = pathlib.Path(self.path).stem | |
| gets.append(key) | |
| self.send_response(200 if key in bucket else 404) | |
| self.end_headers() | |
| self.wfile.write(bucket.get(key, b"")) | |
| def do_PUT(self): | |
| bucket[pathlib.Path(self.path).stem] = self.rfile.read(int(self.headers["content-length"])) | |
| self.send_response(201) | |
| self.end_headers() | |
| def log_message(self, *_): | |
| pass | |
| server = http.server.ThreadingHTTPServer(("127.0.0.1", 0), Bucket) | |
| threading.Thread(target=server.serve_forever, daemon=True).start() | |
| url = f"http://127.0.0.1:{server.server_address[1]}/vectors" | |
| with tempfile.TemporaryDirectory() as d: | |
| def read_texts(items): | |
| for key, _ in items: | |
| reader.path(key).write_bytes(b"read") | |
| return {} | |
| reader = CloudReader(d, read_texts, None, background=False) | |
| reader.fetch(url, ["t-held", "t-new"]) | |
| assert reader.has("t-held") and not reader.has("t-new"), "what the bucket holds is not read again" | |
| reader.fetch(url, ["t-held", "t-new"]) | |
| assert gets.count("t-new") == 1 and gets.count("t-held") == 1, "a key just found absent, or held, is not asked for again" | |
| reader.run([("t-new", ("text", "New"))]) | |
| assert bucket["t-new"] == b"read", "a vector read here is kept for the next replica" | |
| server.shutdown() | |
| assert stimulus_kind("media", "stimulus.flac") == "audio" and stimulus_kind("media", "stimulus.mp4") == "video" and stimulus_kind("screen", "stimulus.png") == "screen" | |
| print(json.dumps({"ok": True})) | |
| if __name__ == "__main__": | |
| check() | |