Spaces:
Running on Zero
Running on Zero
Download src/api/corpus_fetch.py from Kagzy/ResearchOS: direct link, hf CLI and curl.
- Browser
- Download file 3.9 kB
-
https://huggingface.co/spaces/Kagzy/ResearchOS/resolve/main/src/api/corpus_fetch.py
- Command line
-
hf download hf://spaces/Kagzy/ResearchOS/src/api/corpus_fetch.py
-
curl -L -o corpus_fetch.py https://huggingface.co/spaces/Kagzy/ResearchOS/resolve/main/src/api/corpus_fetch.py
3.9 kB
| """Make sure the corpus and its prebuilt indices are on disk before the app loads them. | |
| The 273k-paper corpus is ~1.6 GB, past what a free Hugging Face Space can hold | |
| in its own repo (1 GB). It lives in a public dataset repo instead and is | |
| downloaded once per container start, straight to the paths the config names. | |
| Files already on disk (a laptop, a warm container) are used as they are and | |
| nothing touches the network. | |
| If the download fails - dataset repo unreachable, a rate limit, a half-finished | |
| upload - the app must still come up. The config can name a smaller `fallback` | |
| corpus bundled with the Space; resolution switches to it and says so loudly, | |
| so a degraded deploy is visible in /health instead of being a boot failure. | |
| """ | |
| from __future__ import annotations | |
| import time | |
| from pathlib import Path | |
| from typing import Any | |
| def _missing(paths: dict[str, Any]) -> list[str]: | |
| """What is absent among the corpus files and both index caches.""" | |
| cache = Path(paths["index_cache"]) | |
| missing = [str(Path(paths[k])) for k in ("papers", "chunks") if not Path(paths[k]).exists()] | |
| if not (cache / "bm25" / "corpus.json").exists(): | |
| missing.append(str(cache / "bm25" / "corpus.json")) | |
| if not any(cache.glob("dense_*.npy")): | |
| missing.append(str(cache / "dense_*.npy")) | |
| return missing | |
| def _complete(paths: dict[str, Any]) -> bool: | |
| return not _missing(paths) | |
| def _patterns(paths: dict[str, Any]) -> list[str]: | |
| corpus_dir = Path(paths["papers"]).parent.as_posix() | |
| return [f"{corpus_dir}/*", f"{Path(paths['index_cache']).as_posix()}/**"] | |
| def ensure_corpus(cfg: dict[str, Any]) -> dict[str, Any]: | |
| """Resolve the corpus, mutating cfg["paths"] only if it has to fall back. | |
| Returns {"source": "local" | "downloaded" | "fallback", ...} for /health. | |
| """ | |
| spec = cfg.get("corpus") or {} | |
| paths = cfg["paths"] | |
| if _complete(paths): | |
| return {"source": "local"} | |
| repo = spec.get("repo") | |
| error = "no corpus.repo configured" | |
| if repo: | |
| try: | |
| from huggingface_hub import snapshot_download | |
| started = time.time() | |
| print(f"[corpus] downloading {repo} (first start of this container) ...", flush=True) | |
| got = snapshot_download(repo_id=repo, repo_type="dataset", local_dir=".", | |
| allow_patterns=_patterns(paths), max_workers=8) | |
| if _complete(paths): | |
| took = time.time() - started | |
| print(f"[corpus] ready in {took:.0f}s", flush=True) | |
| return {"source": "downloaded", "repo": repo, "seconds": round(took)} | |
| # Say exactly what is absent and where the files went: this string | |
| # is shown in /health, the only window into a Space's boot. | |
| # huggingface_hub does not raise for a private repo it cannot | |
| # authenticate to: with local_dir set it quietly returns the | |
| # (empty) folder. So "nothing arrived" almost always means no | |
| # HF_TOKEN secret, and the message says so. | |
| error = (f"nothing downloaded from {repo} (returned {got!r}) - it is private, so the " | |
| f"Space needs an HF_TOKEN secret with read access; missing {_missing(paths)}") | |
| except Exception as exc: # noqa: BLE001 - any failure must degrade, not crash | |
| error = f"{type(exc).__name__}: {str(exc)[:160]}" | |
| fallback = spec.get("fallback") | |
| if fallback: | |
| merged = dict(paths) | |
| merged.update(fallback) | |
| if _complete(merged): | |
| paths.update(fallback) | |
| print(f"[corpus] WARNING: using the bundled fallback corpus ({error})", flush=True) | |
| return {"source": "fallback", "reason": error} | |
| raise RuntimeError(f"corpus unavailable and no usable fallback ({error})") | |