File size: 6,647 Bytes
e8c5cc2
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
e5c059f
 
 
e8c5cc2
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
"""Assemble a deploy/ folder ready to push to Hugging Face Spaces.



    python scripts/prepare_deploy.py



Two things happen here that matter.



**The papers file gets slimmed.** It carries `full_text` for every paper with a

PDF - 82 MB on disk and more once parsed into memory. The chunks file already

holds that text, split into the passages retrieval actually uses, so the copy in

the papers file is dead weight at serving time. Only citation lookup reads these

records, and only for metadata.



**The prebuilt indices come along.** Building BM25 and FAISS over 40k chunks

takes a few minutes; on a Space that would happen on every cold start, and free

Spaces sleep. Shipping `cache/ml/` turns a three-minute wake into seconds.

"""

from __future__ import annotations

import argparse
import json
import shutil
import sys
from pathlib import Path

sys.path.insert(0, str(Path(__file__).resolve().parents[1]))

if hasattr(sys.stdout, "reconfigure"):
    sys.stdout.reconfigure(encoding="utf-8", errors="replace")

from src.config import load_config

# Fields the serving path never reads. `full_text` is the big one.
DROP_FIELDS = ("full_text", "metadata")


def slim_papers(source: Path, target: Path) -> tuple[int, int, int]:
    """Copy papers without their full text. Returns (count, before, after)."""
    target.parent.mkdir(parents=True, exist_ok=True)
    kept = 0
    with open(source, encoding="utf-8") as src, open(target, "w", encoding="utf-8") as dst:
        for line in src:
            if not line.strip():
                continue
            record = json.loads(line)
            for field in DROP_FIELDS:
                record.pop(field, None)
            # `full_text` is not optional in the dataclass, so keep the key with
            # a null value rather than removing it - read_papers() constructs
            # Paper(**record) and would fail on a missing field.
            record["full_text"] = None
            record["metadata"] = {}
            dst.write(json.dumps(record, ensure_ascii=False) + "\n")
            kept += 1
    return kept, source.stat().st_size, target.stat().st_size


def copy_tree(source: Path, target: Path) -> int:
    if not source.exists():
        return 0
    if target.exists():
        shutil.rmtree(target)
    shutil.copytree(source, target)
    return sum(f.stat().st_size for f in target.rglob("*") if f.is_file())


def main() -> None:
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument("--config", default="configs/ml.yaml")
    parser.add_argument("--out", default="deploy")
    parser.add_argument("--skip-index", action="store_true",
                        help="omit prebuilt indices; the Space rebuilds them on first boot")
    args = parser.parse_args()

    cfg = load_config(args.config)
    out = Path(args.out)
    out.mkdir(parents=True, exist_ok=True)
    mb = lambda n: f"{n / 1e6:.0f} MB"  # noqa: E731

    # Entrypoints this script used to write. A `Dockerfile` left over from the
    # Docker-Space era sitting beside a `sdk: gradio` README is ambiguous at
    # best, so drop anything we no longer emit rather than letting deploy/
    # accumulate across rebuilds.
    for stale in ("Dockerfile", "Dockerfile.space", "Procfile"):
        (out / stale).unlink(missing_ok=True)

    papers_src = Path(cfg["paths"]["papers"])
    chunks_src = Path(cfg["paths"]["chunks"])
    for path in (papers_src, chunks_src):
        if not path.exists():
            raise SystemExit(f"{path} is missing. Run scripts/ingest_arxiv.py first.")

    count, before, after = slim_papers(papers_src, out / "data" / papers_src.name)
    print(f"[deploy] papers   {count:,} records  {mb(before)} -> {mb(after)}")

    shutil.copy(chunks_src, out / "data" / chunks_src.name)
    chunk_bytes = (out / "data" / chunks_src.name).stat().st_size
    print(f"[deploy] chunks   {mb(chunk_bytes)}")

    index_bytes = 0
    if args.skip_index:
        print("[deploy] indices  skipped - the Space will build them on first boot (~3 min)")
    else:
        index_bytes = copy_tree(Path(cfg["paths"]["index_cache"]), out / "cache" / "ml")
        print(f"[deploy] indices  {mb(index_bytes)}" if index_bytes
              else "[deploy] indices  none found; run a query locally to build them first")

    # --- everything else the Space needs, so deploy/ is pushable as-is ------
    for name in ("src", "configs", "scripts"):
        copy_tree(Path(name), out / name)
    # The Gradio SDK runs `app.py` from the repo root, so it moves up a level.
    shutil.copy("spaces/app.py", out / "app.py")
    # Serving dependencies only - not the repo's training/eval superset. See
    # the header of spaces/requirements.txt for why that distinction matters.
    shutil.copy("spaces/requirements.txt", out / "requirements.txt")
    shutil.copy("spaces/README.md", out / "README.md")

    # Spaces rejects files over 10 MB outside LFS; data and indices are far over.
    (out / ".gitattributes").write_text(
        "data/*.jsonl filter=lfs diff=lfs merge=lfs -text\n"
        "cache/ml/** filter=lfs diff=lfs merge=lfs -text\n",
        encoding="utf-8",
    )
    # A stray .env copied into a public Space would publish the Groq key.
    (out / ".gitignore").write_text(".env\n__pycache__/\n*.pyc\nresults/\n", encoding="utf-8")
    for junk in list(out.rglob("__pycache__")):
        shutil.rmtree(junk, ignore_errors=True)
    for stray in out.rglob(".env"):
        stray.unlink(missing_ok=True)

    total = sum(f.stat().st_size for f in out.rglob("*") if f.is_file())
    print(f"\n[deploy] {out}/ is a complete Gradio Space repo - {mb(total)}")
    print("[deploy] app.py, README.md, requirements.txt, src/, configs/,")
    print("[deploy] scripts/, data/, cache/, .gitattributes, .gitignore")
    print("\nCreate the Space first (SDK Gradio, hardware ZeroGPU - the only free")
    print("option for a personal account), then push:")
    print(f"  cd {out}")
    print("  git init -b main && git lfs install")
    print('  git add -A && git commit -m "ResearchOS"')
    print("  git remote add origin https://huggingface.co/spaces/USER/SPACE")
    # --force because the web form seeds the Space with its own README, so a
    # fresh local history shares no ancestor and is rejected as non-fast-forward.
    print("  git push -u origin main --force")
    print("\nThen set GROQ_API_KEY and RESEARCHOS_ALLOWED_ORIGINS in Space settings.")
    print("Full sequence in DEPLOY.md.")


if __name__ == "__main__":
    main()