Spaces:
Running on Zero
Running on Zero
Download scripts/prepare_deploy.py from Kagzy/ResearchOS: direct link, hf CLI and curl.
- Browser
- Download file 6.65 kB
-
https://huggingface.co/spaces/Kagzy/ResearchOS/resolve/main/scripts/prepare_deploy.py
- Command line
-
hf download hf://spaces/Kagzy/ResearchOS/scripts/prepare_deploy.py
-
curl -L -o prepare_deploy.py https://huggingface.co/spaces/Kagzy/ResearchOS/resolve/main/scripts/prepare_deploy.py
6.65 kB
| """Assemble a deploy/ folder ready to push to Hugging Face Spaces. | |
| python scripts/prepare_deploy.py | |
| Two things happen here that matter. | |
| **The papers file gets slimmed.** It carries `full_text` for every paper with a | |
| PDF - 82 MB on disk and more once parsed into memory. The chunks file already | |
| holds that text, split into the passages retrieval actually uses, so the copy in | |
| the papers file is dead weight at serving time. Only citation lookup reads these | |
| records, and only for metadata. | |
| **The prebuilt indices come along.** Building BM25 and FAISS over 40k chunks | |
| takes a few minutes; on a Space that would happen on every cold start, and free | |
| Spaces sleep. Shipping `cache/ml/` turns a three-minute wake into seconds. | |
| """ | |
| from __future__ import annotations | |
| import argparse | |
| import json | |
| import shutil | |
| import sys | |
| from pathlib import Path | |
| sys.path.insert(0, str(Path(__file__).resolve().parents[1])) | |
| if hasattr(sys.stdout, "reconfigure"): | |
| sys.stdout.reconfigure(encoding="utf-8", errors="replace") | |
| from src.config import load_config | |
| # Fields the serving path never reads. `full_text` is the big one. | |
| DROP_FIELDS = ("full_text", "metadata") | |
| def slim_papers(source: Path, target: Path) -> tuple[int, int, int]: | |
| """Copy papers without their full text. Returns (count, before, after).""" | |
| target.parent.mkdir(parents=True, exist_ok=True) | |
| kept = 0 | |
| with open(source, encoding="utf-8") as src, open(target, "w", encoding="utf-8") as dst: | |
| for line in src: | |
| if not line.strip(): | |
| continue | |
| record = json.loads(line) | |
| for field in DROP_FIELDS: | |
| record.pop(field, None) | |
| # `full_text` is not optional in the dataclass, so keep the key with | |
| # a null value rather than removing it - read_papers() constructs | |
| # Paper(**record) and would fail on a missing field. | |
| record["full_text"] = None | |
| record["metadata"] = {} | |
| dst.write(json.dumps(record, ensure_ascii=False) + "\n") | |
| kept += 1 | |
| return kept, source.stat().st_size, target.stat().st_size | |
| def copy_tree(source: Path, target: Path) -> int: | |
| if not source.exists(): | |
| return 0 | |
| if target.exists(): | |
| shutil.rmtree(target) | |
| shutil.copytree(source, target) | |
| return sum(f.stat().st_size for f in target.rglob("*") if f.is_file()) | |
| def main() -> None: | |
| parser = argparse.ArgumentParser(description=__doc__) | |
| parser.add_argument("--config", default="configs/ml.yaml") | |
| parser.add_argument("--out", default="deploy") | |
| parser.add_argument("--skip-index", action="store_true", | |
| help="omit prebuilt indices; the Space rebuilds them on first boot") | |
| args = parser.parse_args() | |
| cfg = load_config(args.config) | |
| out = Path(args.out) | |
| out.mkdir(parents=True, exist_ok=True) | |
| mb = lambda n: f"{n / 1e6:.0f} MB" # noqa: E731 | |
| # Entrypoints this script used to write. A `Dockerfile` left over from the | |
| # Docker-Space era sitting beside a `sdk: gradio` README is ambiguous at | |
| # best, so drop anything we no longer emit rather than letting deploy/ | |
| # accumulate across rebuilds. | |
| for stale in ("Dockerfile", "Dockerfile.space", "Procfile"): | |
| (out / stale).unlink(missing_ok=True) | |
| papers_src = Path(cfg["paths"]["papers"]) | |
| chunks_src = Path(cfg["paths"]["chunks"]) | |
| for path in (papers_src, chunks_src): | |
| if not path.exists(): | |
| raise SystemExit(f"{path} is missing. Run scripts/ingest_arxiv.py first.") | |
| count, before, after = slim_papers(papers_src, out / "data" / papers_src.name) | |
| print(f"[deploy] papers {count:,} records {mb(before)} -> {mb(after)}") | |
| shutil.copy(chunks_src, out / "data" / chunks_src.name) | |
| chunk_bytes = (out / "data" / chunks_src.name).stat().st_size | |
| print(f"[deploy] chunks {mb(chunk_bytes)}") | |
| index_bytes = 0 | |
| if args.skip_index: | |
| print("[deploy] indices skipped - the Space will build them on first boot (~3 min)") | |
| else: | |
| index_bytes = copy_tree(Path(cfg["paths"]["index_cache"]), out / "cache" / "ml") | |
| print(f"[deploy] indices {mb(index_bytes)}" if index_bytes | |
| else "[deploy] indices none found; run a query locally to build them first") | |
| # --- everything else the Space needs, so deploy/ is pushable as-is ------ | |
| for name in ("src", "configs", "scripts"): | |
| copy_tree(Path(name), out / name) | |
| # The Gradio SDK runs `app.py` from the repo root, so it moves up a level. | |
| shutil.copy("spaces/app.py", out / "app.py") | |
| # Serving dependencies only - not the repo's training/eval superset. See | |
| # the header of spaces/requirements.txt for why that distinction matters. | |
| shutil.copy("spaces/requirements.txt", out / "requirements.txt") | |
| shutil.copy("spaces/README.md", out / "README.md") | |
| # Spaces rejects files over 10 MB outside LFS; data and indices are far over. | |
| (out / ".gitattributes").write_text( | |
| "data/*.jsonl filter=lfs diff=lfs merge=lfs -text\n" | |
| "cache/ml/** filter=lfs diff=lfs merge=lfs -text\n", | |
| encoding="utf-8", | |
| ) | |
| # A stray .env copied into a public Space would publish the Groq key. | |
| (out / ".gitignore").write_text(".env\n__pycache__/\n*.pyc\nresults/\n", encoding="utf-8") | |
| for junk in list(out.rglob("__pycache__")): | |
| shutil.rmtree(junk, ignore_errors=True) | |
| for stray in out.rglob(".env"): | |
| stray.unlink(missing_ok=True) | |
| total = sum(f.stat().st_size for f in out.rglob("*") if f.is_file()) | |
| print(f"\n[deploy] {out}/ is a complete Gradio Space repo - {mb(total)}") | |
| print("[deploy] app.py, README.md, requirements.txt, src/, configs/,") | |
| print("[deploy] scripts/, data/, cache/, .gitattributes, .gitignore") | |
| print("\nCreate the Space first (SDK Gradio, hardware ZeroGPU - the only free") | |
| print("option for a personal account), then push:") | |
| print(f" cd {out}") | |
| print(" git init -b main && git lfs install") | |
| print(' git add -A && git commit -m "ResearchOS"') | |
| print(" git remote add origin https://huggingface.co/spaces/USER/SPACE") | |
| # --force because the web form seeds the Space with its own README, so a | |
| # fresh local history shares no ancestor and is rejected as non-fast-forward. | |
| print(" git push -u origin main --force") | |
| print("\nThen set GROQ_API_KEY and RESEARCHOS_ALLOWED_ORIGINS in Space settings.") | |
| print("Full sequence in DEPLOY.md.") | |
| if __name__ == "__main__": | |
| main() | |