Spaces:
Running on Zero
Running on Zero
File size: 6,647 Bytes
e8c5cc2 e5c059f e8c5cc2 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 | """Assemble a deploy/ folder ready to push to Hugging Face Spaces.
python scripts/prepare_deploy.py
Two things happen here that matter.
**The papers file gets slimmed.** It carries `full_text` for every paper with a
PDF - 82 MB on disk and more once parsed into memory. The chunks file already
holds that text, split into the passages retrieval actually uses, so the copy in
the papers file is dead weight at serving time. Only citation lookup reads these
records, and only for metadata.
**The prebuilt indices come along.** Building BM25 and FAISS over 40k chunks
takes a few minutes; on a Space that would happen on every cold start, and free
Spaces sleep. Shipping `cache/ml/` turns a three-minute wake into seconds.
"""
from __future__ import annotations
import argparse
import json
import shutil
import sys
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
if hasattr(sys.stdout, "reconfigure"):
sys.stdout.reconfigure(encoding="utf-8", errors="replace")
from src.config import load_config
# Fields the serving path never reads. `full_text` is the big one.
DROP_FIELDS = ("full_text", "metadata")
def slim_papers(source: Path, target: Path) -> tuple[int, int, int]:
"""Copy papers without their full text. Returns (count, before, after)."""
target.parent.mkdir(parents=True, exist_ok=True)
kept = 0
with open(source, encoding="utf-8") as src, open(target, "w", encoding="utf-8") as dst:
for line in src:
if not line.strip():
continue
record = json.loads(line)
for field in DROP_FIELDS:
record.pop(field, None)
# `full_text` is not optional in the dataclass, so keep the key with
# a null value rather than removing it - read_papers() constructs
# Paper(**record) and would fail on a missing field.
record["full_text"] = None
record["metadata"] = {}
dst.write(json.dumps(record, ensure_ascii=False) + "\n")
kept += 1
return kept, source.stat().st_size, target.stat().st_size
def copy_tree(source: Path, target: Path) -> int:
if not source.exists():
return 0
if target.exists():
shutil.rmtree(target)
shutil.copytree(source, target)
return sum(f.stat().st_size for f in target.rglob("*") if f.is_file())
def main() -> None:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--config", default="configs/ml.yaml")
parser.add_argument("--out", default="deploy")
parser.add_argument("--skip-index", action="store_true",
help="omit prebuilt indices; the Space rebuilds them on first boot")
args = parser.parse_args()
cfg = load_config(args.config)
out = Path(args.out)
out.mkdir(parents=True, exist_ok=True)
mb = lambda n: f"{n / 1e6:.0f} MB" # noqa: E731
# Entrypoints this script used to write. A `Dockerfile` left over from the
# Docker-Space era sitting beside a `sdk: gradio` README is ambiguous at
# best, so drop anything we no longer emit rather than letting deploy/
# accumulate across rebuilds.
for stale in ("Dockerfile", "Dockerfile.space", "Procfile"):
(out / stale).unlink(missing_ok=True)
papers_src = Path(cfg["paths"]["papers"])
chunks_src = Path(cfg["paths"]["chunks"])
for path in (papers_src, chunks_src):
if not path.exists():
raise SystemExit(f"{path} is missing. Run scripts/ingest_arxiv.py first.")
count, before, after = slim_papers(papers_src, out / "data" / papers_src.name)
print(f"[deploy] papers {count:,} records {mb(before)} -> {mb(after)}")
shutil.copy(chunks_src, out / "data" / chunks_src.name)
chunk_bytes = (out / "data" / chunks_src.name).stat().st_size
print(f"[deploy] chunks {mb(chunk_bytes)}")
index_bytes = 0
if args.skip_index:
print("[deploy] indices skipped - the Space will build them on first boot (~3 min)")
else:
index_bytes = copy_tree(Path(cfg["paths"]["index_cache"]), out / "cache" / "ml")
print(f"[deploy] indices {mb(index_bytes)}" if index_bytes
else "[deploy] indices none found; run a query locally to build them first")
# --- everything else the Space needs, so deploy/ is pushable as-is ------
for name in ("src", "configs", "scripts"):
copy_tree(Path(name), out / name)
# The Gradio SDK runs `app.py` from the repo root, so it moves up a level.
shutil.copy("spaces/app.py", out / "app.py")
# Serving dependencies only - not the repo's training/eval superset. See
# the header of spaces/requirements.txt for why that distinction matters.
shutil.copy("spaces/requirements.txt", out / "requirements.txt")
shutil.copy("spaces/README.md", out / "README.md")
# Spaces rejects files over 10 MB outside LFS; data and indices are far over.
(out / ".gitattributes").write_text(
"data/*.jsonl filter=lfs diff=lfs merge=lfs -text\n"
"cache/ml/** filter=lfs diff=lfs merge=lfs -text\n",
encoding="utf-8",
)
# A stray .env copied into a public Space would publish the Groq key.
(out / ".gitignore").write_text(".env\n__pycache__/\n*.pyc\nresults/\n", encoding="utf-8")
for junk in list(out.rglob("__pycache__")):
shutil.rmtree(junk, ignore_errors=True)
for stray in out.rglob(".env"):
stray.unlink(missing_ok=True)
total = sum(f.stat().st_size for f in out.rglob("*") if f.is_file())
print(f"\n[deploy] {out}/ is a complete Gradio Space repo - {mb(total)}")
print("[deploy] app.py, README.md, requirements.txt, src/, configs/,")
print("[deploy] scripts/, data/, cache/, .gitattributes, .gitignore")
print("\nCreate the Space first (SDK Gradio, hardware ZeroGPU - the only free")
print("option for a personal account), then push:")
print(f" cd {out}")
print(" git init -b main && git lfs install")
print(' git add -A && git commit -m "ResearchOS"')
print(" git remote add origin https://huggingface.co/spaces/USER/SPACE")
# --force because the web form seeds the Space with its own README, so a
# fresh local history shares no ancestor and is rejected as non-fast-forward.
print(" git push -u origin main --force")
print("\nThen set GROQ_API_KEY and RESEARCHOS_ALLOWED_ORIGINS in Space settings.")
print("Full sequence in DEPLOY.md.")
if __name__ == "__main__":
main()
|