File size: 3,897 Bytes
65eca94
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2422591
 
 
 
 
 
65eca94
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
"""Make sure the corpus and its prebuilt indices are on disk before the app loads them.



The 273k-paper corpus is ~1.6 GB, past what a free Hugging Face Space can hold

in its own repo (1 GB). It lives in a public dataset repo instead and is

downloaded once per container start, straight to the paths the config names.

Files already on disk (a laptop, a warm container) are used as they are and

nothing touches the network.



If the download fails - dataset repo unreachable, a rate limit, a half-finished

upload - the app must still come up. The config can name a smaller `fallback`

corpus bundled with the Space; resolution switches to it and says so loudly,

so a degraded deploy is visible in /health instead of being a boot failure.

"""

from __future__ import annotations

import time
from pathlib import Path
from typing import Any


def _missing(paths: dict[str, Any]) -> list[str]:
    """What is absent among the corpus files and both index caches."""
    cache = Path(paths["index_cache"])
    missing = [str(Path(paths[k])) for k in ("papers", "chunks") if not Path(paths[k]).exists()]
    if not (cache / "bm25" / "corpus.json").exists():
        missing.append(str(cache / "bm25" / "corpus.json"))
    if not any(cache.glob("dense_*.npy")):
        missing.append(str(cache / "dense_*.npy"))
    return missing


def _complete(paths: dict[str, Any]) -> bool:
    return not _missing(paths)


def _patterns(paths: dict[str, Any]) -> list[str]:
    corpus_dir = Path(paths["papers"]).parent.as_posix()
    return [f"{corpus_dir}/*", f"{Path(paths['index_cache']).as_posix()}/**"]


def ensure_corpus(cfg: dict[str, Any]) -> dict[str, Any]:
    """Resolve the corpus, mutating cfg["paths"] only if it has to fall back.



    Returns {"source": "local" | "downloaded" | "fallback", ...} for /health.

    """
    spec = cfg.get("corpus") or {}
    paths = cfg["paths"]
    if _complete(paths):
        return {"source": "local"}

    repo = spec.get("repo")
    error = "no corpus.repo configured"
    if repo:
        try:
            from huggingface_hub import snapshot_download

            started = time.time()
            print(f"[corpus] downloading {repo} (first start of this container) ...", flush=True)
            got = snapshot_download(repo_id=repo, repo_type="dataset", local_dir=".",
                                    allow_patterns=_patterns(paths), max_workers=8)
            if _complete(paths):
                took = time.time() - started
                print(f"[corpus] ready in {took:.0f}s", flush=True)
                return {"source": "downloaded", "repo": repo, "seconds": round(took)}
            # Say exactly what is absent and where the files went: this string
            # is shown in /health, the only window into a Space's boot.
            # huggingface_hub does not raise for a private repo it cannot
            # authenticate to: with local_dir set it quietly returns the
            # (empty) folder. So "nothing arrived" almost always means no
            # HF_TOKEN secret, and the message says so.
            error = (f"nothing downloaded from {repo} (returned {got!r}) - it is private, so the "
                     f"Space needs an HF_TOKEN secret with read access; missing {_missing(paths)}")
        except Exception as exc:  # noqa: BLE001 - any failure must degrade, not crash
            error = f"{type(exc).__name__}: {str(exc)[:160]}"

    fallback = spec.get("fallback")
    if fallback:
        merged = dict(paths)
        merged.update(fallback)
        if _complete(merged):
            paths.update(fallback)
            print(f"[corpus] WARNING: using the bundled fallback corpus ({error})", flush=True)
            return {"source": "fallback", "reason": error}
    raise RuntimeError(f"corpus unavailable and no usable fallback ({error})")