File size: 10,701 Bytes
6df569c
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
"""TRIBE v2 behind one Hugging Face inference endpoint: perceptions and the navigator's decisions.

The endpoint scales to zero, so nothing of ours runs while nobody asks. The model loads once, when the endpoint wakes, and
the API (cloud/src/tribe.ts) drives it with short calls, the body being {"inputs": {...}}:

    {"op": "start", "id", "kind", "seconds", "stimulus_url", "files_url"}   a perception, computed in the background
    {"op": "poll", "id"}                                                    its state; asking keeps the endpoint awake
    {"op": "decide", "input", "effort", "vectors_url"}                      one step of the navigator, or its `wait`

The URLs are short-lived, signed by the API and good for one perception or for the vector cache, so this process holds no
credential of ours. The endpoint's repository holds this file, requirements.txt, encode.py and navigator.py side by side.

    python3 scripts/brain/tribe/handler.py     # the job queue and the vector cache, without the model
"""

import collections
import concurrent.futures
import json
import os
import pathlib
import sys
import tempfile
import threading
import time
import traceback
import urllib.error
import urllib.request

HERE = pathlib.Path(__file__).resolve().parent
sys.path[:0] = [str(HERE), str(HERE.parent)]

import navigator  # noqa: E402

RESULT_FILES = ("summary.json", "rois.json", "preds.npz", "features.npz", "map.png", "stimulus.mp4")
# How long a finished job is kept, so a late poll still reads its end.
JOB_TTL_S = 3600
# A key the bucket did not hold is asked for again after this long: another replica may have read it.
ABSENT_TTL_S = 30
FETCHERS = 16
HTTP_TIMEOUT_S = 300
# One model job at a time, a perception or a batch of names: both use the same GPU.
GPU = threading.RLock()


def http(method, url, data=None, content_type="application/octet-stream"):
    headers = {"user-agent": "computaco-tribe-endpoint/1", **({"content-type": content_type} if data is not None else {})}
    with urllib.request.urlopen(urllib.request.Request(url, data=data, method=method, headers=headers), timeout=HTTP_TIMEOUT_S) as r:
        return r.read(), dict(r.headers)


class Jobs:
    """Perceptions started and not yet read, run one at a time on one thread. `run(spec)` returns the summary or raises."""

    def __init__(self, run, now=time.time):
        self.run, self.now = run, now
        self.cond = threading.Condition()
        self.jobs = collections.OrderedDict()
        threading.Thread(target=self._work, name="perceptions", daemon=True).start()

    def start(self, spec):
        """Starting the same id again is the same job: the API may not have seen the first answer."""
        with self.cond:
            job = self.jobs.get(spec["id"])
            if job is None:
                job = self.jobs[spec["id"]] = {"spec": spec, "state": "queued", "at": self.now()}
                self.cond.notify_all()
            return self.view(job)

    def poll(self, jid):
        with self.cond:
            for old, job in list(self.jobs.items()):
                if job["state"] in ("done", "failed") and self.now() - job["at"] > JOB_TTL_S:
                    del self.jobs[old]
            job = self.jobs.get(jid)
            return self.view(job) if job else {"state": "unknown"}

    @staticmethod
    def view(job):
        return {k: job[k] for k in ("state", "summary", "error") if k in job}

    def _next(self):
        return next((j for j in self.jobs.values() if j["state"] == "queued"), None)

    def _work(self):
        while True:
            with self.cond:
                while (job := self._next()) is None:
                    self.cond.wait()
                job["state"] = "running"
            try:
                out = {"state": "done", "summary": self.run(job["spec"])}
            except Exception as e:  # noqa: BLE001 - every failure is reported to the perception
                traceback.print_exc()
                out = {"state": "failed", "error": str(e)[:1000]}
            with self.cond:
                job.update(out, at=self.now())


class CloudReader(navigator.Reader):
    """The navigator's reader with its vectors kept in the API's bucket: a replica that wakes empty uses what others read."""

    def __init__(self, *args, **kwargs):
        self.url, self.absent, self.sent = None, {}, set()
        super().__init__(*args, **kwargs)

    def fetch(self, url, keys):
        """Bring the vectors the bucket holds into the local store before a step is answered."""
        self.url = url
        now = time.time()
        want = [k for k in dict.fromkeys(keys) if not self.has(k) and now - self.absent.get(k, 0) > ABSENT_TTL_S]

        def one(key):
            try:
                body, _ = http("GET", f"{url}/{key}.npy")
            except urllib.error.HTTPError as e:
                if e.code != 404:
                    raise
                self.absent[key] = time.time()
                return
            tmp = self.path(key).with_suffix(".part")
            tmp.write_bytes(body)
            os.replace(tmp, self.path(key))
            self.sent.add(key)

        with concurrent.futures.ThreadPoolExecutor(FETCHERS) as pool:
            list(pool.map(one, want))

    def run(self, batch):
        with GPU:
            super().run(batch)
        for key, _ in batch:
            if self.url and key not in self.sent and self.has(key):
                try:
                    http("PUT", f"{self.url}/{key}.npy", self.path(key).read_bytes())
                    self.sent.add(key)
                except OSError as e:
                    self.log(f"could not keep {key}: {e}")


def stimulus_kind(kind, name):
    """What the encoder is given: a media perception is audio or video by its file's extension."""
    if kind == "media":
        return "audio" if name.rsplit(".", 1)[-1] in ("wav", "mp3", "flac") else "video"
    return kind


class EndpointHandler:
    def __init__(self, path=""):
        import encode
        self.encode = encode
        root = pathlib.Path(os.environ.get("TRIBE_WORK", tempfile.gettempdir())) / "tribe"
        self.work = root / "jobs"
        self.work.mkdir(parents=True, exist_ok=True)
        cache = root / "features"
        cache.mkdir(parents=True, exist_ok=True)
        self.nav = navigator.Navigator(cache, root / "navigator", reader=CloudReader)
        self.nav.reader.log = lambda text: print(text, flush=True)
        self.jobs = Jobs(self.perceive)

    def perceive(self, job):
        out = self.work / job["id"]
        out.mkdir(parents=True, exist_ok=True)
        body, headers = http("GET", job["stimulus_url"])
        name = headers.get("x-cw-file") or headers.get("X-Cw-File") or "stimulus.bin"
        path = out / name
        path.write_bytes(body)
        with GPU:
            summary = self.encode.encode_stimulus(self.nav.model, stimulus_kind(job["kind"], name), path, out / "result", float(job.get("seconds") or 20))
        for f in RESULT_FILES:
            p = out / "result" / f
            if p.exists():
                http("PUT", f"{job['files_url']}/{f}", p.read_bytes())
        return summary

    def __call__(self, data):
        given = data.get("inputs", data)
        op = given.get("op")
        if op == "start":
            return self.jobs.start({k: given.get(k) for k in ("id", "kind", "seconds", "stimulus_url", "files_url")})
        if op == "poll":
            return self.jobs.poll(given["id"])
        if op == "decide":
            step, effort = given["input"], given.get("effort") or "normal"
            self.nav.reader.fetch(given["vectors_url"], navigator.keys_of(step, effort))
            return {"decision": self.nav.decide(step, effort)}
        return {"error": f"no operation {op}"}


def check():
    """The job queue and the vector cache, with stand-ins for the model and for the bucket."""
    import http.server
    done, gate = [], threading.Event()

    def run(spec):
        gate.wait(5)
        if spec["id"] == "bad":
            raise RuntimeError("no GPU")
        done.append(spec["id"])
        return {"segments": 3}

    clock = [100.0]
    jobs = Jobs(run, now=lambda: clock[0])
    assert jobs.poll("a") == {"state": "unknown"}
    assert jobs.start({"id": "a"})["state"] in ("queued", "running")
    jobs.start({"id": "a"})
    jobs.start({"id": "bad"})
    assert jobs.poll("bad") == {"state": "queued"}, "one job runs at a time"
    gate.set()
    for _ in range(300):
        if jobs.poll("bad")["state"] == "failed":
            break
        time.sleep(0.01)
    assert done == ["a"], "a job started twice runs once"
    assert jobs.poll("a") == {"state": "done", "summary": {"segments": 3}} and jobs.poll("bad") == {"state": "failed", "error": "no GPU"}
    clock[0] += JOB_TTL_S + 1
    assert jobs.poll("a") == {"state": "unknown"}, "a finished job is forgotten after its time"

    bucket = {"t-held": b"held"}
    gets = []

    class Bucket(http.server.BaseHTTPRequestHandler):
        def do_GET(self):
            key = pathlib.Path(self.path).stem
            gets.append(key)
            self.send_response(200 if key in bucket else 404)
            self.end_headers()
            self.wfile.write(bucket.get(key, b""))

        def do_PUT(self):
            bucket[pathlib.Path(self.path).stem] = self.rfile.read(int(self.headers["content-length"]))
            self.send_response(201)
            self.end_headers()

        def log_message(self, *_):
            pass

    server = http.server.ThreadingHTTPServer(("127.0.0.1", 0), Bucket)
    threading.Thread(target=server.serve_forever, daemon=True).start()
    url = f"http://127.0.0.1:{server.server_address[1]}/vectors"
    with tempfile.TemporaryDirectory() as d:
        def read_texts(items):
            for key, _ in items:
                reader.path(key).write_bytes(b"read")
            return {}

        reader = CloudReader(d, read_texts, None, background=False)
        reader.fetch(url, ["t-held", "t-new"])
        assert reader.has("t-held") and not reader.has("t-new"), "what the bucket holds is not read again"
        reader.fetch(url, ["t-held", "t-new"])
        assert gets.count("t-new") == 1 and gets.count("t-held") == 1, "a key just found absent, or held, is not asked for again"
        reader.run([("t-new", ("text", "New"))])
        assert bucket["t-new"] == b"read", "a vector read here is kept for the next replica"
    server.shutdown()
    assert stimulus_kind("media", "stimulus.flac") == "audio" and stimulus_kind("media", "stimulus.mp4") == "video" and stimulus_kind("screen", "stimulus.png") == "screen"
    print(json.dumps({"ok": True}))


if __name__ == "__main__":
    check()