code: coreai_backend downloads the models from this repo; add coreai_py.py (used by parity.py)
16c436e verified Download code/coreai_backend.py from ivanfioravanti/d1-2B-gestures-CoreAI: direct link, hf CLI and curl.
- Browser
- Download file 6.21 kB
-
https://huggingface.co/ivanfioravanti/d1-2B-gestures-CoreAI/resolve/main/code/coreai_backend.py
- Command line
-
hf download hf://ivanfioravanti/d1-2B-gestures-CoreAI/code/coreai_backend.py
-
curl -L -o coreai_backend.py https://huggingface.co/ivanfioravanti/d1-2B-gestures-CoreAI/resolve/main/code/coreai_backend.py
6.21 kB
| """The Core AI backends of decide.py: d1's exported decision function on the GPU or the Neural Engine, run by the Swift | |
| worker (swift/.build/release/d1worker), one warm process per backend, started on first use. | |
| backends = CoreAI(model, QUESTIONS) # model: the mlx-vlm d1 model (its processor and readout are reused) | |
| backends.available() # {"coreai-gpu": None or why not, "coreai-ane": ...} | |
| backends.start("coreai-gpu") # lazily, once: downloads the model the first time (3.1 GB), then specialises | |
| # (74 s on the first Neural Engine run ever), then seconds | |
| backends.decide("coreai-gpu", image) # system_one's output for a PIL frame | |
| The exported graph has the prompt for QUESTIONS baked in: edit QUESTIONS and these backends say so instead of | |
| answering other questions. | |
| The models come from https://huggingface.co/ivanfioravanti/d1-2B-gestures-CoreAI (Hugging Face cache, offline after the | |
| first download); D1_COREAI_DIR=/path/to/folder with d1_gpu_fp16.aimodel / d1_ane_fp16.aimodel uses your own export instead. | |
| """ | |
| import json, os, subprocess, tempfile, time | |
| import numpy as np | |
| from d1_common import META, build_meta, frame, load_meta, pixels, readout, uncrop_box | |
| HERE = os.path.dirname(os.path.abspath(__file__)) | |
| WORKER = os.path.join(HERE, "swift", ".build", "release", "d1worker") | |
| REPO = "ivanfioravanti/d1-2B-gestures-CoreAI" | |
| MODELS = {"coreai-gpu": ("d1_gpu_fp16.aimodel", "gpu"), "coreai-ane": ("d1_ane_fp16.aimodel", "ane")} | |
| LOCAL = os.environ.get("D1_COREAI_DIR") | |
| def model_path(name): | |
| """The .aimodel folder: from D1_COREAI_DIR, else downloaded from the Hugging Face repo on first use (then cached).""" | |
| folder = MODELS[name][0] | |
| if LOCAL: | |
| return os.path.join(LOCAL, folder) | |
| from huggingface_hub import snapshot_download | |
| return os.path.join(snapshot_download(REPO, allow_patterns=[folder + "/*"]), folder) | |
| class Unavailable(Exception): | |
| pass | |
| class Worker: | |
| def __init__(self, path, compute): | |
| self.p = subprocess.Popen([WORKER, "--model", path, "--compute", compute], stdin=subprocess.PIPE, | |
| stdout=subprocess.PIPE, text=True, bufsize=1) | |
| self.ready = self.read() # specialises (first time ever: minutes on the Neural Engine) and runs once | |
| self.pix = tempfile.NamedTemporaryFile(prefix="d1-%s-" % compute, suffix=".f32", delete=False).name | |
| def read(self): | |
| line = self.p.stdout.readline() | |
| if not line: | |
| raise RuntimeError("d1worker exited (code %s)" % self.p.poll()) | |
| out = json.loads(line) | |
| if "error" in out: | |
| raise RuntimeError("d1worker: " + out["error"]) | |
| return out | |
| def logits(self, x): | |
| x.astype(np.float32).tofile(self.pix) | |
| self.p.stdin.write(json.dumps({"pixels": self.pix}) + "\n") | |
| self.p.stdin.flush() | |
| return np.array(self.read()["logits"], dtype=np.float32).reshape(12, -1) | |
| class CoreAI: | |
| def __init__(self, model, questions): | |
| self.engine, self.questions, self.workers = model.engine, questions, {} | |
| self.meta, self.problem = None, None | |
| if not os.path.exists(META): | |
| self.problem = "coreai/prompt.json is missing" | |
| else: | |
| self.meta = load_meta() | |
| now = build_meta(self.engine, questions, write=False) | |
| if any(now[k] != self.meta[k] for k in ("ids", "slots", "union")): | |
| self.problem = "QUESTIONS changed since the Core AI export: re-export (see README, Core AI)" | |
| def available(self): | |
| out = {} | |
| for name, (folder, _) in MODELS.items(): | |
| local = LOCAL and os.path.join(LOCAL, folder) | |
| out[name] = (self.problem or (None if not local or os.path.exists(local) else "missing %s (D1_COREAI_DIR)" % local) | |
| or (None if os.path.exists(WORKER) else "missing the Swift worker: cd coreai/swift && swift build -c release")) | |
| return out | |
| def start(self, name): | |
| """The warm worker for a backend, started on first use; returns the seconds this call spent starting it.""" | |
| why = self.available().get(name, "unknown backend") | |
| if why: | |
| raise Unavailable(why) | |
| w = self.workers.get(name) | |
| if w is not None and w.p.poll() is None: | |
| return 0.0 | |
| t = time.perf_counter() | |
| try: | |
| path = model_path(name) | |
| except Exception as e: # offline before the first download, or the repo unreachable | |
| raise Unavailable("could not download %s from huggingface.co/%s: %s" % (MODELS[name][0], REPO, e)) | |
| self.workers[name] = Worker(path, MODELS[name][1]) | |
| return time.perf_counter() - t | |
| def decide(self, name, img): | |
| w = self.workers[name] | |
| try: | |
| logits = w.logits(pixels(self.engine, frame(img))) # frame(): the 640x360 the export was traced at | |
| except Exception: | |
| self.workers.pop(name, None) # a dead worker is restarted on the next request | |
| raise | |
| answers = readout(self.engine, self.questions, logits, self.meta) | |
| for a in answers.values(): # boxes come back in the crop's coordinates: map them onto the whole frame | |
| if a.get("type") == "box" and a.get("box"): | |
| a["box"] = uncrop_box(a["box"], *img.size) | |
| return {"answers": answers, "usage": {"input_tokens": len(self.meta["ids"]), "output_tokens": 0}} | |
| if __name__ == "__main__": # the crop arithmetic: python coreai_backend.py | |
| full = {"top": 0.0, "bottom": 1.0, "left": 0.0, "right": 1.0} | |
| assert uncrop_box(full, 640, 360) == full # 16:9: unchanged | |
| assert uncrop_box(full, 400, 400) == {"top": 87 / 400, "bottom": 312 / 400, "left": 0.0, "right": 1.0} # 400x225 band | |
| assert uncrop_box(full, 1000, 360) == {"top": 0.0, "bottom": 1.0, "left": 180 / 1000, "right": 820 / 1000} | |
| b = uncrop_box({"top": 0.5, "bottom": 0.5, "left": 0.25, "right": 0.75}, 640, 480) # 640x360 band at y 60..420 | |
| assert b == {"top": 240 / 480, "bottom": 240 / 480, "left": 0.25, "right": 0.75}, b | |
| print("uncrop_box ok") | |