Spaces:
Running on Zero
Running on Zero
File size: 3,897 Bytes
65eca94 2422591 65eca94 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 | """Make sure the corpus and its prebuilt indices are on disk before the app loads them.
The 273k-paper corpus is ~1.6 GB, past what a free Hugging Face Space can hold
in its own repo (1 GB). It lives in a public dataset repo instead and is
downloaded once per container start, straight to the paths the config names.
Files already on disk (a laptop, a warm container) are used as they are and
nothing touches the network.
If the download fails - dataset repo unreachable, a rate limit, a half-finished
upload - the app must still come up. The config can name a smaller `fallback`
corpus bundled with the Space; resolution switches to it and says so loudly,
so a degraded deploy is visible in /health instead of being a boot failure.
"""
from __future__ import annotations
import time
from pathlib import Path
from typing import Any
def _missing(paths: dict[str, Any]) -> list[str]:
"""What is absent among the corpus files and both index caches."""
cache = Path(paths["index_cache"])
missing = [str(Path(paths[k])) for k in ("papers", "chunks") if not Path(paths[k]).exists()]
if not (cache / "bm25" / "corpus.json").exists():
missing.append(str(cache / "bm25" / "corpus.json"))
if not any(cache.glob("dense_*.npy")):
missing.append(str(cache / "dense_*.npy"))
return missing
def _complete(paths: dict[str, Any]) -> bool:
return not _missing(paths)
def _patterns(paths: dict[str, Any]) -> list[str]:
corpus_dir = Path(paths["papers"]).parent.as_posix()
return [f"{corpus_dir}/*", f"{Path(paths['index_cache']).as_posix()}/**"]
def ensure_corpus(cfg: dict[str, Any]) -> dict[str, Any]:
"""Resolve the corpus, mutating cfg["paths"] only if it has to fall back.
Returns {"source": "local" | "downloaded" | "fallback", ...} for /health.
"""
spec = cfg.get("corpus") or {}
paths = cfg["paths"]
if _complete(paths):
return {"source": "local"}
repo = spec.get("repo")
error = "no corpus.repo configured"
if repo:
try:
from huggingface_hub import snapshot_download
started = time.time()
print(f"[corpus] downloading {repo} (first start of this container) ...", flush=True)
got = snapshot_download(repo_id=repo, repo_type="dataset", local_dir=".",
allow_patterns=_patterns(paths), max_workers=8)
if _complete(paths):
took = time.time() - started
print(f"[corpus] ready in {took:.0f}s", flush=True)
return {"source": "downloaded", "repo": repo, "seconds": round(took)}
# Say exactly what is absent and where the files went: this string
# is shown in /health, the only window into a Space's boot.
# huggingface_hub does not raise for a private repo it cannot
# authenticate to: with local_dir set it quietly returns the
# (empty) folder. So "nothing arrived" almost always means no
# HF_TOKEN secret, and the message says so.
error = (f"nothing downloaded from {repo} (returned {got!r}) - it is private, so the "
f"Space needs an HF_TOKEN secret with read access; missing {_missing(paths)}")
except Exception as exc: # noqa: BLE001 - any failure must degrade, not crash
error = f"{type(exc).__name__}: {str(exc)[:160]}"
fallback = spec.get("fallback")
if fallback:
merged = dict(paths)
merged.update(fallback)
if _complete(merged):
paths.update(fallback)
print(f"[corpus] WARNING: using the bundled fallback corpus ({error})", flush=True)
return {"source": "fallback", "reason": error}
raise RuntimeError(f"corpus unavailable and no usable fallback ({error})")
|