davanstrien HF Staff commited on
Commit
ddfb9e0
·
verified ·
1 Parent(s): 5f0049a

Port to FastAPI + Granian Docker Space: /api/* JSON routes, fetch() frontend, single supervised ingest worker (file lock + wait for previous worker)

Browse files
Files changed (6) hide show
  1. .dockerignore +3 -0
  2. Dockerfile +21 -0
  3. README.md +6 -4
  4. app.py +122 -17
  5. index.html +19 -9
  6. requirements.txt +5 -4
.dockerignore ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ __pycache__/
2
+ .ruff_cache/
3
+ *.pyc
Dockerfile ADDED
@@ -0,0 +1,21 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ FROM python:3.12-slim
2
+
3
+ ENV PYTHONUNBUFFERED=1 \
4
+ PIP_NO_CACHE_DIR=1 \
5
+ HF_HOME=/tmp/hf \
6
+ FIREHOSE_DIR=/data
7
+
8
+ RUN useradd -m -u 1000 user
9
+ WORKDIR /home/user/app
10
+
11
+ # CPU-only torch for the ingest worker (GLiNER2.5-Decide); gliner2 pins transformers<5 / huggingface_hub<1.0.
12
+ COPY --chown=user requirements.txt .
13
+ RUN pip install --extra-index-url https://download.pytorch.org/whl/cpu -r requirements.txt
14
+
15
+ COPY --chown=user . .
16
+ USER user
17
+ EXPOSE 7860
18
+
19
+ # One Granian worker process: the app supervises the single ingest worker (single-writer rule for the bucket).
20
+ CMD ["granian", "--interface", "asgi", "--host", "0.0.0.0", "--port", "7860", "--workers", "1", \
21
+ "--runtime-threads", "2", "app:app"]
README.md CHANGED
@@ -3,10 +3,8 @@ title: Topic Feed Lab
3
  emoji: 🧭
4
  colorFrom: indigo
5
  colorTo: green
6
- sdk: gradio
7
- sdk_version: 6.16.0
8
- python_version: "3.12"
9
- app_file: app.py
10
  pinned: false
11
  short_description: Follow and browse new Hub items by topic
12
  ---
@@ -32,6 +30,10 @@ Follow new Hugging Face Hub models, datasets and arXiv papers by topic, or brows
32
  A background worker process in this Space (cpu-upgrade, CPU only, lower priority than the page) watches the Hub for
33
  new and updated models and datasets plus HF Daily Papers, checks whether the card changed, and scores changed items
34
  with GLiNER2.5-Decide. The page loads the scores into memory and picks up new files every 20 s.
 
 
 
 
35
  - Default thresholds (`cat48-v1.1.json`) are set so each topic tags about as many items as an LLM reference labelling
36
  did on a 750-item sample, with a floor of 0.1.
37
 
 
3
  emoji: 🧭
4
  colorFrom: indigo
5
  colorTo: green
6
+ sdk: docker
7
+ app_port: 7860
 
 
8
  pinned: false
9
  short_description: Follow and browse new Hub items by topic
10
  ---
 
30
  A background worker process in this Space (cpu-upgrade, CPU only, lower priority than the page) watches the Hub for
31
  new and updated models and datasets plus HF Daily Papers, checks whether the card changed, and scores changed items
32
  with GLiNER2.5-Decide. The page loads the scores into memory and picks up new files every 20 s.
33
+ - The app is FastAPI served by Granian (Docker Space, one app process). It supervises exactly one ingest worker
34
+ (a file lock enforces this). JSON API: `GET /api/catalogue`, `/api/status`, `/api/live`, `/api/events`,
35
+ `/api/browse` (list params repeated, e.g. `?topics=robotics&topics=video`; `thresholds` is a JSON object;
36
+ docs at `/api/docs`).
37
  - Default thresholds (`cat48-v1.1.json`) are set so each topic tags about as many items as an LLM reference labelling
38
  did on a 750-item sample, with a floor of 0.1.
39
 
app.py CHANGED
@@ -1,4 +1,4 @@
1
- """Topic Feed Lab: browse and follow new Hub items by topic, from precomputed scores (gradio.Server, CPU only).
2
 
3
  Scores come from the private bucket davanstrien/hub-firehose (GLiNER2.5-Decide, 48-topic catalogue cat48-v1),
4
  mounted in the Space at FIREHOSE_DIR (default /data). The page itself runs no model; a background worker process
@@ -14,8 +14,11 @@ from datetime import datetime, timedelta, timezone
14
  import numpy as np
15
  import pandas as pd
16
  import pyarrow.parquet as pq
17
- from fastapi.responses import HTMLResponse
18
- from gradio import Server
 
 
 
19
 
20
  HERE = os.path.dirname(os.path.abspath(__file__))
21
  DATA = os.environ.get("FIREHOSE_DIR", "/data")
@@ -195,6 +198,8 @@ class Events:
195
  if changed and self.files:
196
  df = pd.concat([f[2] for f in self.files.values()], ignore_index=True)
197
  df["t_dt"] = pd.to_datetime(df["t"], utc=True, errors="coerce", format="ISO8601")
 
 
198
  df = df.dropna(subset=["t_dt"]).sort_values("t_dt").tail(20000).reset_index(drop=True)
199
  with self.lock:
200
  self.df = df
@@ -207,9 +212,7 @@ class Events:
207
 
208
 
209
  STORE = Store()
210
- STORE.refresh()
211
  EVENTS = Events()
212
- EVENTS.refresh()
213
 
214
 
215
  def refresher(obj, every):
@@ -218,10 +221,6 @@ def refresher(obj, every):
218
  obj.refresh()
219
 
220
 
221
- threading.Thread(target=refresher, args=(STORE, REFRESH_S), daemon=True).start()
222
- threading.Thread(target=refresher, args=(EVENTS, EVENTS_REFRESH_S), daemon=True).start()
223
-
224
-
225
  # ---------------------------------------------------------------- in-Space ingest worker
226
  # The worker (firehose_worker.py) runs GLiNER2.5-Decide on CPU and appends scores to the bucket mounted at DATA.
227
  # It runs as a child process at lower CPU priority so page requests stay responsive. RUN_WORKER=0 disables it.
@@ -230,6 +229,25 @@ WORKER_ARGS = ["--out", DATA, "--model-cap", "1500", "--inherit", "quantized", "
230
  "--sweep-hours", "6", "--canary-secs", "600"]
231
 
232
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
233
  def supervise_worker():
234
  import subprocess
235
  import sys
@@ -237,19 +255,41 @@ def supervise_worker():
237
  while not os.path.isdir(os.path.join(DATA, "state")):
238
  print("worker: waiting for the bucket mount at", DATA, flush=True)
239
  time.sleep(10)
 
240
  backoff = 30
241
  while True:
242
  started = time.time()
243
  proc = subprocess.Popen([sys.executable, os.path.join(HERE, "firehose_worker.py"), *WORKER_ARGS],
244
  preexec_fn=lambda: os.nice(10))
 
245
  rc = proc.wait()
246
  print("worker exited with", rc, flush=True)
247
  backoff = 30 if time.time() - started > 600 else min(600, backoff * 2)
248
  time.sleep(backoff)
249
 
250
 
251
- if RUN_WORKER:
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
252
  threading.Thread(target=supervise_worker, daemon=True).start()
 
253
 
254
 
255
  # ---------------------------------------------------------------- helpers
@@ -323,7 +363,50 @@ def status_payload(meta):
323
 
324
 
325
  # ---------------------------------------------------------------- app
326
- app = Server()
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
327
 
328
 
329
  @app.get("/", response_class=HTMLResponse)
@@ -332,21 +415,18 @@ async def homepage():
332
  return f.read()
333
 
334
 
335
- @app.api(name="catalogue")
336
  def catalogue() -> dict:
337
  """Topic catalogue: labels, notes, theme groups, broad/narrow pairs, default thresholds."""
338
  return {k: CAT[k] for k in ("catalogue_version", "labels", "notes", "groups", "pairs", "thresholds")} | {
339
  "thresholds_v1": CAT.get("thresholds_v1"), "custom_topics_enabled": CUSTOM_TOPICS_ENABLED}
340
 
341
 
342
- @app.api(name="status")
343
  def status() -> dict:
344
  """Data status: row counts, newest item, worker status (if any)."""
345
  meta, _, _ = STORE.snapshot()
346
  return status_payload(meta)
347
 
348
 
349
- @app.api(name="live")
350
  def live(topics: list, thresholds: dict, limit: int = 50, offset: int = 0, since: str = "",
351
  all_items: bool = False) -> dict:
352
  """Newest items first (by last_modified) where any of `topics` passes its threshold (all_items: every item).
@@ -377,7 +457,6 @@ def live(topics: list, thresholds: dict, limit: int = 50, offset: int = 0, since
377
  "new_since": new_since, "arrivals": arrivals, "status": status_payload(meta)}
378
 
379
 
380
- @app.api(name="events")
381
  def events(topics: list, thresholds: dict, since: str = "", limit: int = 400) -> dict:
382
  """Recent worker events, newest last, with each scored item's lane among `topics` and its top topics.
383
  Returns whether the worker is active (an event in the last 10 min), counts for the last 10 min and the status."""
@@ -420,7 +499,6 @@ def events(topics: list, thresholds: dict, since: str = "", limit: int = 400) ->
420
  SORTS = {"score", "newest", "likes", "downloads"}
421
 
422
 
423
- @app.api(name="browse")
424
  def browse(topics: list, thresholds: dict, days: int = 7, sources: list = None, q: str = "", sort: str = "score",
425
  limit: int = 30, offset: int = 0, match: str = "any") -> dict:
426
  """Items in one or more topics. days: 1, 3, 7 or 0 (all). sources: model/dataset/paper. q: text search over
@@ -471,4 +549,31 @@ def browse(topics: list, thresholds: dict, days: int = 7, sources: list = None,
471
  "summary": summary, "status": status_payload(meta)}
472
 
473
 
474
- app.launch(show_error=True)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Topic Feed Lab: browse and follow new Hub items by topic, from precomputed scores (FastAPI on Granian, Docker Space).
2
 
3
  Scores come from the private bucket davanstrien/hub-firehose (GLiNER2.5-Decide, 48-topic catalogue cat48-v1),
4
  mounted in the Space at FIREHOSE_DIR (default /data). The page itself runs no model; a background worker process
 
14
  import numpy as np
15
  import pandas as pd
16
  import pyarrow.parquet as pq
17
+ from contextlib import asynccontextmanager
18
+
19
+ from fastapi import FastAPI, Query
20
+ from fastapi.middleware.gzip import GZipMiddleware
21
+ from fastapi.responses import HTMLResponse, JSONResponse
22
 
23
  HERE = os.path.dirname(os.path.abspath(__file__))
24
  DATA = os.environ.get("FIREHOSE_DIR", "/data")
 
198
  if changed and self.files:
199
  df = pd.concat([f[2] for f in self.files.values()], ignore_index=True)
200
  df["t_dt"] = pd.to_datetime(df["t"], utc=True, errors="coerce", format="ISO8601")
201
+ # The worker may write the same event to both logs (parquet + jsonl): keep one.
202
+ df = df.drop_duplicates(["t", "source", "id", "event"])
203
  df = df.dropna(subset=["t_dt"]).sort_values("t_dt").tail(20000).reset_index(drop=True)
204
  with self.lock:
205
  self.df = df
 
212
 
213
 
214
  STORE = Store()
 
215
  EVENTS = Events()
 
216
 
217
 
218
  def refresher(obj, every):
 
221
  obj.refresh()
222
 
223
 
 
 
 
 
224
  # ---------------------------------------------------------------- in-Space ingest worker
225
  # The worker (firehose_worker.py) runs GLiNER2.5-Decide on CPU and appends scores to the bucket mounted at DATA.
226
  # It runs as a child process at lower CPU priority so page requests stay responsive. RUN_WORKER=0 disables it.
 
229
  "--sweep-hours", "6", "--canary-secs", "600"]
230
 
231
 
232
+ def wait_for_previous_worker(stale_s=150, max_wait_s=900):
233
+ """On a redeploy the old container keeps running until the new one is up. Its worker writes
234
+ state/worker_status.json every ~60 s; wait until that goes stale so two workers never write at once."""
235
+ path = os.path.join(DATA, "state", "worker_status.json")
236
+ t0 = time.time()
237
+ while time.time() - t0 < max_wait_s:
238
+ try:
239
+ with open(path, encoding="utf-8") as f:
240
+ upd = pd.to_datetime(json.load(f).get("updated_at"), utc=True)
241
+ age = (datetime.now(timezone.utc) - upd).total_seconds()
242
+ except Exception:
243
+ return
244
+ if age > stale_s:
245
+ return
246
+ print(f"worker: previous worker active {age:.0f} s ago; waiting before starting", flush=True)
247
+ time.sleep(20)
248
+ print("worker: previous worker still reporting after 15 min; starting anyway", flush=True)
249
+
250
+
251
  def supervise_worker():
252
  import subprocess
253
  import sys
 
255
  while not os.path.isdir(os.path.join(DATA, "state")):
256
  print("worker: waiting for the bucket mount at", DATA, flush=True)
257
  time.sleep(10)
258
+ wait_for_previous_worker()
259
  backoff = 30
260
  while True:
261
  started = time.time()
262
  proc = subprocess.Popen([sys.executable, os.path.join(HERE, "firehose_worker.py"), *WORKER_ARGS],
263
  preexec_fn=lambda: os.nice(10))
264
+ print(f"worker: started pid {proc.pid}", flush=True)
265
  rc = proc.wait()
266
  print("worker exited with", rc, flush=True)
267
  backoff = 30 if time.time() - started > 600 else min(600, backoff * 2)
268
  time.sleep(backoff)
269
 
270
 
271
+ _WORKER_LOCK = None # held for the life of the process that supervises the worker
272
+
273
+
274
+ def start_worker_once():
275
+ """Single-writer rule: at most one ingest worker per container. Granian runs one process (--workers 1);
276
+ the file lock also guards against a second app process (e.g. a reload) starting another worker."""
277
+ global _WORKER_LOCK
278
+ import fcntl
279
+
280
+ f = open(os.environ.get("WORKER_LOCK", "/tmp/topic-feed-worker.lock"), "w")
281
+ try:
282
+ fcntl.flock(f, fcntl.LOCK_EX | fcntl.LOCK_NB)
283
+ except OSError:
284
+ print(f"worker: another process (pid file {f.name}) supervises the worker; not starting one", flush=True)
285
+ f.close()
286
+ return False
287
+ f.write(str(os.getpid()))
288
+ f.flush()
289
+ _WORKER_LOCK = f
290
+ print(f"worker: supervisor started in pid {os.getpid()}", flush=True)
291
  threading.Thread(target=supervise_worker, daemon=True).start()
292
+ return True
293
 
294
 
295
  # ---------------------------------------------------------------- helpers
 
363
 
364
 
365
  # ---------------------------------------------------------------- app
366
+ class NPJSONResponse(JSONResponse):
367
+ """JSON that also accepts numpy scalars/arrays (value_counts, argmax results)."""
368
+
369
+ def render(self, content):
370
+ def default(o):
371
+ if isinstance(o, np.generic):
372
+ return o.item()
373
+ if isinstance(o, np.ndarray):
374
+ return o.tolist()
375
+ raise TypeError(type(o).__name__)
376
+ return json.dumps(content, default=default, separators=(",", ":"), allow_nan=False).encode()
377
+
378
+
379
+ @asynccontextmanager
380
+ async def lifespan(_app):
381
+ STORE.refresh()
382
+ EVENTS.refresh()
383
+ threading.Thread(target=refresher, args=(STORE, REFRESH_S), daemon=True).start()
384
+ threading.Thread(target=refresher, args=(EVENTS, EVENTS_REFRESH_S), daemon=True).start()
385
+ if RUN_WORKER:
386
+ start_worker_once()
387
+ yield
388
+
389
+
390
+ app = FastAPI(title="Topic Feed Lab", default_response_class=NPJSONResponse, lifespan=lifespan,
391
+ docs_url="/api/docs", openapi_url="/api/openapi.json")
392
+ app.add_middleware(GZipMiddleware, minimum_size=1000)
393
+
394
+
395
+ @app.middleware("http")
396
+ async def timing(request, call_next):
397
+ t0 = time.perf_counter()
398
+ resp = await call_next(request)
399
+ resp.headers["Server-Timing"] = f"app;dur={(time.perf_counter() - t0) * 1000:.1f}"
400
+ return resp
401
+
402
+
403
+ def _thresholds(s):
404
+ """thresholds query param: a JSON object {topic: value}."""
405
+ try:
406
+ d = json.loads(s) if s else {}
407
+ return d if isinstance(d, dict) else {}
408
+ except ValueError:
409
+ return {}
410
 
411
 
412
  @app.get("/", response_class=HTMLResponse)
 
415
  return f.read()
416
 
417
 
 
418
  def catalogue() -> dict:
419
  """Topic catalogue: labels, notes, theme groups, broad/narrow pairs, default thresholds."""
420
  return {k: CAT[k] for k in ("catalogue_version", "labels", "notes", "groups", "pairs", "thresholds")} | {
421
  "thresholds_v1": CAT.get("thresholds_v1"), "custom_topics_enabled": CUSTOM_TOPICS_ENABLED}
422
 
423
 
 
424
  def status() -> dict:
425
  """Data status: row counts, newest item, worker status (if any)."""
426
  meta, _, _ = STORE.snapshot()
427
  return status_payload(meta)
428
 
429
 
 
430
  def live(topics: list, thresholds: dict, limit: int = 50, offset: int = 0, since: str = "",
431
  all_items: bool = False) -> dict:
432
  """Newest items first (by last_modified) where any of `topics` passes its threshold (all_items: every item).
 
457
  "new_since": new_since, "arrivals": arrivals, "status": status_payload(meta)}
458
 
459
 
 
460
  def events(topics: list, thresholds: dict, since: str = "", limit: int = 400) -> dict:
461
  """Recent worker events, newest last, with each scored item's lane among `topics` and its top topics.
462
  Returns whether the worker is active (an event in the last 10 min), counts for the last 10 min and the status."""
 
499
  SORTS = {"score", "newest", "likes", "downloads"}
500
 
501
 
 
502
  def browse(topics: list, thresholds: dict, days: int = 7, sources: list = None, q: str = "", sort: str = "score",
503
  limit: int = 30, offset: int = 0, match: str = "any") -> dict:
504
  """Items in one or more topics. days: 1, 3, 7 or 0 (all). sources: model/dataset/paper. q: text search over
 
549
  "summary": summary, "status": status_payload(meta)}
550
 
551
 
552
+ # ---------------------------------------------------------------- JSON API
553
+ # Lists (topics, sources) are repeated query params: ?topics=robotics&topics=video. thresholds is a JSON object.
554
+ @app.get("/api/catalogue")
555
+ def api_catalogue():
556
+ return NPJSONResponse(catalogue(), headers={"Cache-Control": "public, max-age=3600"})
557
+
558
+
559
+ @app.get("/api/status")
560
+ def api_status():
561
+ return NPJSONResponse(status(), headers={"Cache-Control": "public, max-age=15"})
562
+
563
+
564
+ @app.get("/api/live")
565
+ def api_live(topics: list[str] = Query(default=[]), thresholds: str = "", limit: int = 50, offset: int = 0,
566
+ since: str = "", all_items: bool = False):
567
+ return live(topics, _thresholds(thresholds), limit, offset, since, all_items)
568
+
569
+
570
+ @app.get("/api/events")
571
+ def api_events(topics: list[str] = Query(default=[]), thresholds: str = "", since: str = "", limit: int = 400):
572
+ return events(topics, _thresholds(thresholds), since, limit)
573
+
574
+
575
+ @app.get("/api/browse")
576
+ def api_browse(topics: list[str] = Query(default=[]), thresholds: str = "", days: int = 7,
577
+ sources: list[str] = Query(default=[]), q: str = "", sort: str = "score", limit: int = 30,
578
+ offset: int = 0, match: str = "any"):
579
+ return browse(topics, _thresholds(thresholds), days, sources or None, q, sort, limit, offset, match)
index.html CHANGED
@@ -256,7 +256,19 @@ input[type=search]{min-width:14rem}
256
  </div>
257
 
258
  <script type="module">
259
- import { Client } from "https://cdn.jsdelivr.net/npm/@gradio/client/dist/index.min.js";
 
 
 
 
 
 
 
 
 
 
 
 
260
 
261
  const MIN_POS_FOR_MARKS = 3; // positives needed before a topic's threshold comes from the user's confirmations
262
  const PAGE = 30, POLL_MS = 20000, EVENTS_MS = 8000;
@@ -424,22 +436,20 @@ let qTimer = null;
424
  $("q").addEventListener("input", () => { clearTimeout(qTimer); qTimer = setTimeout(reload, 300); });
425
 
426
  /* ---------- data ---------- */
427
- let client = null;
428
- async function getClient(){ if (!client) client = await Client.connect(window.location.origin); return client; }
429
  let reqId = 0;
430
  async function reload(append = false, poll = false){
431
  if (!CAT || editing) return;
432
  const my = ++reqId;
433
  if (!append){ offset = 0; }
434
  try {
435
- const c = await getClient();
436
  let d;
437
  if (mode === "live"){
438
- d = (await c.predict("/live", {topics: feedTopics(), thresholds: thrMap(feedTopics()), limit: PAGE, offset, since: prevVisit || "", all_items: liveAll})).data[0];
439
  lastLive = d;
440
  } else {
441
- d = (await c.predict("/browse", {topics: browseTopics(), thresholds: thrMap(browseTopics()), days: bDays, sources: bSources,
442
- q: $("q").value, sort: bSort, limit: PAGE, offset, match: "any"})).data[0];
443
  lastBrowse = d;
444
  }
445
  if (my !== reqId) return; // a newer request superseded this one
@@ -654,7 +664,7 @@ const agoS = iso => { const s = (Date.now() - Date.parse(iso)) / 1000; return s
654
  async function pollEvents(){
655
  if (!CAT || mode !== "live" || document.hidden) return;
656
  try {
657
- const d = (await (await getClient()).predict("/events", {topics: feedTopics(), thresholds: thrMap(feedTopics()), since: lastEvT, limit: 800})).data[0];
658
  evInfo = d;
659
  const fresh = d.events.filter(e => { const k = e.t + "|" + e.source + "|" + e.id + "|" + e.event; if (evSeen.has(k)) return false; evSeen.add(k); return true; });
660
  if (fresh.length){
@@ -834,7 +844,7 @@ function flowReplay(){
834
  function renderAll(){ renderToggles(); renderFeed(); renderCal(); }
835
  async function init(){
836
  renderControls();
837
- try { CAT = (await (await getClient()).predict("/catalogue", {})).data[0]; }
838
  catch (e) {
839
  $("status").textContent = "Could not connect: " + (e?.message || e);
840
  $("feed").innerHTML = `<div class="empty">Could not reach the app. If the Space was asleep it may still be starting; reload the page in a minute.</div>`;
 
256
  </div>
257
 
258
  <script type="module">
259
+ // JSON API: GET /api/<name>. Arrays become repeated params (?topics=a&topics=b); objects are sent as JSON strings.
260
+ async function api(name, params = {}){
261
+ const q = new URLSearchParams();
262
+ for (const [k, v] of Object.entries(params)){
263
+ if (v === undefined || v === null) continue;
264
+ if (Array.isArray(v)) v.forEach(x => q.append(k, x));
265
+ else if (typeof v === "object") q.append(k, JSON.stringify(v));
266
+ else q.append(k, String(v));
267
+ }
268
+ const r = await fetch(`api/${name}?${q}`, {headers: {Accept: "application/json"}});
269
+ if (!r.ok) throw new Error(`${r.status} ${r.statusText}`);
270
+ return r.json();
271
+ }
272
 
273
  const MIN_POS_FOR_MARKS = 3; // positives needed before a topic's threshold comes from the user's confirmations
274
  const PAGE = 30, POLL_MS = 20000, EVENTS_MS = 8000;
 
436
  $("q").addEventListener("input", () => { clearTimeout(qTimer); qTimer = setTimeout(reload, 300); });
437
 
438
  /* ---------- data ---------- */
439
+
 
440
  let reqId = 0;
441
  async function reload(append = false, poll = false){
442
  if (!CAT || editing) return;
443
  const my = ++reqId;
444
  if (!append){ offset = 0; }
445
  try {
 
446
  let d;
447
  if (mode === "live"){
448
+ d = (await api("live", {topics: feedTopics(), thresholds: thrMap(feedTopics()), limit: PAGE, offset, since: prevVisit || "", all_items: liveAll}));
449
  lastLive = d;
450
  } else {
451
+ d = (await api("browse", {topics: browseTopics(), thresholds: thrMap(browseTopics()), days: bDays, sources: bSources,
452
+ q: $("q").value, sort: bSort, limit: PAGE, offset, match: "any"}));
453
  lastBrowse = d;
454
  }
455
  if (my !== reqId) return; // a newer request superseded this one
 
664
  async function pollEvents(){
665
  if (!CAT || mode !== "live" || document.hidden) return;
666
  try {
667
+ const d = await api("events", {topics: feedTopics(), thresholds: thrMap(feedTopics()), since: lastEvT, limit: 800});
668
  evInfo = d;
669
  const fresh = d.events.filter(e => { const k = e.t + "|" + e.source + "|" + e.id + "|" + e.event; if (evSeen.has(k)) return false; evSeen.add(k); return true; });
670
  if (fresh.length){
 
844
  function renderAll(){ renderToggles(); renderFeed(); renderCal(); }
845
  async function init(){
846
  renderControls();
847
+ try { CAT = await api("catalogue"); }
848
  catch (e) {
849
  $("status").textContent = "Could not connect: " + (e?.message || e);
850
  $("feed").innerHTML = `<div class="empty">Could not reach the app. If the Space was asleep it may still be starting; reload the page in a minute.</div>`;
requirements.txt CHANGED
@@ -1,9 +1,10 @@
1
- --extra-index-url https://download.pytorch.org/whl/cpu
2
- pandas
3
- pyarrow
4
- numpy
5
  torch
6
  gliner2[local]==2.0.0
7
  protobuf
8
  sentencepiece
 
 
 
 
 
9
  requests
 
 
 
 
 
1
  torch
2
  gliner2[local]==2.0.0
3
  protobuf
4
  sentencepiece
5
+ fastapi
6
+ granian
7
+ pandas
8
+ pyarrow
9
+ numpy
10
  requests