chore(deploy): add upload_data.py + HF deploy runbook for runtime data (#25)

Runtime data (ChromaDB index, graph pickle, trials.jsonl) is gitignored and served
to the Space from the candle-fire-data dataset repo at startup. There was no tooling
to push it, so the dataset could silently drift from the ingested data — that's how
trials.jsonl went missing and broke trial search on the Space.

- scripts/upload_data.py: uploads chroma/, graph/als_graph.pkl, trials/trials.jsonl to
the dataset repo (mirrors app._ensure_data). Supports --dry-run and --only.
- config.HF_DATASET_REPO: single constant for the dataset repo id, now shared by both
the uploader and app._ensure_data so upload/download targets can't drift.
- README: "Deploy to Hugging Face" section documenting the two-repo model and the
required order — upload data first, then git push hf main (the Space cold-starts with
no persistent storage and re-pulls data on every rebuild).

Co-authored-by: Claude Opus 4.8 <noreply@anthropic.com>

README.md

@@ -76,6 +76,39 @@ uv run python app.py
 uv run python main.py "What is the evidence for tofersen targeting SOD1?"
 ```
 
+## Deploy to Hugging Face
+
+Deployment uses **two repos**, because runtime data is kept separate from code:
+
+| Repo | Holds | Shipped by |
+|---|---|---|
+| Space `KevinIsCoding/candle-fire` (git remote `hf`) | **Code** + small committed data (`landscape.json`, seeds, tool schemas) | `git push hf main` |
+| Dataset `KevinIsCoding/candle-fire-data` | **Large runtime data** — ChromaDB index, graph pickle, `trials.jsonl` (all gitignored) | `scripts/upload_data.py` |
+
+The large data is gitignored (see `.gitignore`) and never travels with the code. On startup the Space pulls it from the dataset repo via `app._ensure_data()`. The Space has **no persistent storage**, so every restart is a clean cold start that re-downloads this data — which means the dataset repo must always hold the current files.
+
+**Deploy in this order (data first, then code):**
+
+```bash
+# 0. Merge the approved PR into main and sync locally
+git checkout main && git pull origin main
+
+# 1. Push runtime data to the dataset repo (REQUIRED before the code push)
+#    Needs a HF token with write access: `huggingface-cli login` or HF_TOKEN env var.
+uv run python scripts/upload_data.py            # uploads chroma/, graph, trials.jsonl
+uv run python scripts/upload_data.py --dry-run  # preview targets + sizes, no writes
+uv run python scripts/upload_data.py --only trials   # push just one target
+
+# 2. Deploy the code (triggers the Space rebuild)
+git push hf main
+```
+
+> **Why data first:** the Space cold-starts on rebuild and immediately fetches data from the
+> dataset repo. If you push code before uploading data, the Space can boot against stale or
+> missing files — e.g. a missing `trials.jsonl` silently disables trial search. Always run
+> `upload_data.py` before `git push hf main`. The upload/download target is a single constant
+> (`config.HF_DATASET_REPO`) shared by the script and the app, so they can't drift.
+
 ## Architecture
 
 ```

app.py

@@ -11,7 +11,7 @@ from dotenv import load_dotenv
 load_dotenv()
 
 from agents.research_agent import stream_research_agent
-from config import CHROMA_COLLECTION, CHROMA_DIR, GRAPH_PICKLE_PATH, TRIALS_PATH
+from config import CHROMA_COLLECTION, CHROMA_DIR, GRAPH_PICKLE_PATH, HF_DATASET_REPO, TRIALS_PATH
 from logging_config import get_logger
 from rag.indexer import load_collection
 
@@ -45,7 +45,7 @@ def _load_collection():
         return None
 
 
-_HF_DATASET = "KevinIsCoding/candle-fire-data"
+_HF_DATASET = HF_DATASET_REPO
 
 
 def _ensure_data() -> None:

config.py

@@ -23,6 +23,11 @@ GRAPH_JSON_PATH = DATA_DIR / "graph" / "als_graph.json"
 CHROMA_DIR = DATA_DIR / "chroma"
 CHROMA_COLLECTION = "als_papers"
 
+# Runtime data lives OUTSIDE git (see .gitignore) and is served from a HF dataset repo:
+# scripts/upload_data.py pushes it there, app._ensure_data() pulls it on Space startup.
+# Both sides read this one constant so the upload/download targets can never drift.
+HF_DATASET_REPO = "KevinIsCoding/candle-fire-data"
+
 # Experimental therapy landscape (offline-built, committed to git — small)
 LANDSCAPE_PATH = DATA_DIR / "landscape" / "landscape.json"
 LANDSCAPE_PROGRESS_PATH = DATA_DIR / "landscape" / ".progress.json"

scripts/upload_data.py

@@ -0,0 +1,113 @@
+"""Upload runtime data to the Hugging Face dataset repo (deploy prerequisite).
+
+Runtime data (ChromaDB index, the graph pickle, trials.jsonl) is intentionally kept
+OUT of git — it is large and pipeline-generated (see .gitignore). The HF Space fetches
+it at startup via app._ensure_data(). This script is the other half of that contract:
+it pushes the current local data to the dataset repo so the Space has fresh data to pull.
+
+Run it BEFORE `git push hf main` on every deploy — otherwise the Space serves whatever
+data the dataset repo last had (or, if a file was never uploaded, none — which silently
+breaks features like trial search).
+
+What it uploads (mirrors _ensure_data exactly):
+  data/chroma/            → chroma/            (the ChromaDB store)
+  data/graph/als_graph.pkl → graph/als_graph.pkl
+  data/trials/trials.jsonl → trials/trials.jsonl
+
+Usage:
+  uv run python scripts/upload_data.py            # upload everything
+  uv run python scripts/upload_data.py --dry-run  # list what would upload, no writes
+  uv run python scripts/upload_data.py --only trials   # one target (chroma|graph|trials)
+
+Auth: needs a Hugging Face token with write access to the dataset repo. Provide it via
+`huggingface-cli login`, or the HF_TOKEN / HUGGING_FACE_HUB_TOKEN env var.
+"""
+from __future__ import annotations
+
+import argparse
+import sys
+from pathlib import Path
+
+# Allow running as `python scripts/upload_data.py` (repo root on sys.path).
+sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
+
+from config import CHROMA_DIR, GRAPH_PICKLE_PATH, HF_DATASET_REPO, TRIALS_PATH  # noqa: E402
+
+
+def _human_size(n: int) -> str:
+    for unit in ("B", "KB", "MB", "GB"):
+        if n < 1024 or unit == "GB":
+            return f"{n:.1f}{unit}" if unit != "B" else f"{n}B"
+        n /= 1024
+    return f"{n:.1f}GB"
+
+
+def _dir_size(path: Path) -> int:
+    return sum(f.stat().st_size for f in path.rglob("*") if f.is_file())
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description="Upload runtime data to the HF dataset repo.")
+    parser.add_argument("--dry-run", action="store_true", help="Show what would upload; write nothing.")
+    parser.add_argument("--only", choices=["chroma", "graph", "trials"],
+                        help="Upload just one target (default: all).")
+    parser.add_argument("--repo", default=HF_DATASET_REPO, help="Override the dataset repo id.")
+    args = parser.parse_args()
+
+    # (key, kind, local_path, path_in_repo)
+    targets = [
+        ("chroma", "folder", CHROMA_DIR, "chroma"),
+        ("graph", "file", GRAPH_PICKLE_PATH, "graph/als_graph.pkl"),
+        ("trials", "file", TRIALS_PATH, "trials/trials.jsonl"),
+    ]
+    if args.only:
+        targets = [t for t in targets if t[0] == args.only]
+
+    # Verify every selected target exists locally before touching the network.
+    missing = [str(p) for _, _, p, _ in targets if not p.exists()]
+    if missing:
+        print("ERROR — local data missing (run the offline pipeline first):", file=sys.stderr)
+        for m in missing:
+            print(f"  - {m}", file=sys.stderr)
+        return 1
+
+    print(f"Dataset repo: {args.repo}")
+    for key, kind, local, in_repo in targets:
+        size = _dir_size(local) if kind == "folder" else local.stat().st_size
+        print(f"  {key:7} {kind:6} {local}  ({_human_size(size)})  →  {in_repo}")
+
+    if args.dry_run:
+        print("\n--dry-run: nothing uploaded.")
+        return 0
+
+    from huggingface_hub import HfApi
+    api = HfApi()
+    try:
+        who = api.whoami().get("name")
+    except Exception:
+        print("ERROR — not authenticated. Run `huggingface-cli login` or set HF_TOKEN.", file=sys.stderr)
+        return 1
+    print(f"Authenticated as: {who}\n")
+
+    for key, kind, local, in_repo in targets:
+        print(f"Uploading {key} ...")
+        if kind == "folder":
+            api.upload_folder(
+                folder_path=str(local), path_in_repo=in_repo,
+                repo_id=args.repo, repo_type="dataset",
+                commit_message=f"Update {in_repo} (deploy)",
+            )
+        else:
+            api.upload_file(
+                path_or_fileobj=str(local), path_in_repo=in_repo,
+                repo_id=args.repo, repo_type="dataset",
+                commit_message=f"Update {in_repo} (deploy)",
+            )
+        print(f"  done: {in_repo}")
+
+    print("\nAll runtime data uploaded. Now deploy the code: git push hf main")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())