scripts/deploy_hf.py

#!/usr/bin/env python3
"""
Deploy AgAdvisor to a Hugging Face Streamlit Space.

Creates (or updates) a private Space, sets the API keys as Space secrets (read
from your local .env — they never leave your machine except into HF's secret
store), uploads the app + the committed offline index, and configures runtime
env vars so it serves from the on-disk index (single process, no Docker).

Run it yourself from the repo root (your terminal, your keys, your call):

    export HF_TOKEN=hf_xxx            # a write token: https://huggingface.co/settings/tokens
    python scripts/deploy_hf.py                      # -> private Space tirtho149/agadvisor
    python scripts/deploy_hf.py --repo you/agadvisor --public   # options

After it finishes, open the Space URL — HF builds and launches it automatically.
"""

import argparse
import os
import re
import sys
import tempfile
from pathlib import Path

from dotenv import load_dotenv
from huggingface_hub import HfApi

ROOT = Path(__file__).resolve().parents[1]

# HF Space config written as the Space's README.md.
# NOTE: HF retired the Streamlit SDK (create_repo accepts only gradio|docker|static), AND
# Gradio/Docker on free cpu-basic now requires PRO. The one free path for a real app is
# Gradio on ZeroGPU, so the Space runs app.py (Gradio) rather than the Streamlit UI.
README = """---
title: AgAdvisor
emoji: 🌿
colorFrom: green
colorTo: gray
sdk: gradio
sdk_version: 5.50.0
app_file: app.py
python_version: "3.12"
pinned: false
short_description: CDMS pesticide-label AI assistant
license: other
license_name: iowa-state-university-proprietary
---

# 🌿 AgAdvisor
Conversational assistant over CDMS pesticide labels (offline preprocessed index),
with weather / soil / agronomic tools and page-level citations.

Copyright © 2026 Iowa State University. All rights reserved.

Pesticide labels are legally binding. This tool is a research prototype and is not a
substitute for reading the product label. Always verify against the label of record.
"""

LICENSE = """Copyright © 2026 Iowa State University. All rights reserved.

This software and its accompanying data were developed at Iowa State University.
No license or other right is granted to any third party to use, copy, modify, or
distribute this software, in whole or in part, without prior written permission.

Licensing terms are subject to Iowa State University policy; contact the Iowa State
University Research Foundation (ISURF) regarding use or redistribution.
"""

# spaCy model isn't on PyPI as a normal dep -> install it from the release wheel.
SPACY_MODEL_LINE = (
    "en_core_web_sm @ https://github.com/explosion/spacy-models/releases/download/"
    "en_core_web_sm-3.8.0/en_core_web_sm-3.8.0-py3-none-any.whl\n"
)

# Never upload these to the Space. The Space repo is the publication boundary — the
# upload is the working tree (not git history), so this list is the ONLY thing standing
# between a local file and the public internet. Add, don't remove.
IGNORE = [
    ".git*", "**/.git/**",
    ".env", ".env.*", "*.bak", "*.bak.*",
    "config/api_keys.json",                       # potential local secrets
    "**/__pycache__/**", "*.pyc",
    "data/*.corrupted_*", "data/*.prebuild.*", "data/cdms_metadata_recovered.db",
    "data/qdrant_local/.lock", "data/qdrant_local/**/*.lock",
    "README.md", "requirements.txt",              # replaced with HF-specific versions
    "env/**", "scratchpad/**",

    # Accounts DB: usernames + bcrypt hashes. Empty today, but a redeploy after real
    # signups would publish the user table.
    "data/app/**",

    # Unpublished work: docs/ holds the CHI 2027 draft, which is anonymized for review.
    "docs/**",

    # Real personal email addresses; untracked AND not gitignored.
    "email_to_joshua.txt",

    # Third-party CDMS pesticide labels (manufacturer copyright, cdms.net terms). Not
    # read at serve time: offline mode nulls the PDF loaders (cdms_label_tool.py:44-49)
    # and answers come from the Qdrant store + cdms_metadata.db, with citation URLs from
    # the Qdrant `pdf_url` payload.
    "data/pdfs/**",

    # Internal team docs: RUNBOOK carries the live EC2 IP, the private repo name and
    # collaborator names; the ISA reports are internal feedback correspondence.
    "RUNBOOK.md",
    "AgAdvisor_ISA_Fixes_Report.md", "AgAdvisor_ISA_Fixes_Report.pdf",
    "AgAdvisor_ISA_Fixes_Visual.md", "CHANGELOG_ISA_FEEDBACK.md", "FIXES.md",
    "roundup_chunks_detailed.txt",                # debug dump of indexed chunks

    # Local runtime noise.
    "logs/**", "*.log", "evaluation_reports/**",
    ".DS_Store", "**/.DS_Store", ".pytest_cache/**", ".ruff_cache/**",
]


def dry_run(quiet: bool = False):
    """List what upload_folder would actually publish, using HF's own matcher.

    The Space repo is public-facing and the upload is the working tree, so 'what
    exactly ships' is worth being able to answer without pushing. Exits non-zero if
    any sensitive path or credential-shaped string is in the upload set.
    """
    from huggingface_hub.utils import filter_repo_objects

    all_files = [str(p.relative_to(ROOT)) for p in ROOT.rglob("*") if p.is_file()]
    kept = sorted(filter_repo_objects(all_files, ignore_patterns=IGNORE))
    excluded = sorted(set(all_files) - set(kept))

    total = sum((ROOT / f).stat().st_size for f in kept)
    print(f"WOULD UPLOAD — {len(kept)} files, {total / 1e6:.1f} MB\n")
    if not quiet:
        for f in kept:
            print(f"  + {f}")

        print(f"\nEXCLUDED — {len(excluded)} files")
        for pat in ("data/app", "docs/", "email_to_joshua", "data/pdfs", ".env"):
            hits = [f for f in excluded if f.startswith(pat) or f == pat]
            print(f"  {'✅' if hits or pat == '.env' else '⚠️ '} {pat}: {len(hits)} excluded")

    leaked = [f for f in kept if f.startswith(("data/app", "docs/", "data/pdfs"))
              or f in ("email_to_joshua.txt", ".env", "RUNBOOK.md", ".DS_Store")]
    if leaked:
        print("\n❌ SENSITIVE FILES WOULD BE PUBLISHED:")
        for f in leaked:
            print(f"     {f}")
        sys.exit(1)
    print("\n✅ No sensitive paths in the upload set.")

    hits = scan_for_secrets(kept)
    if hits:
        print("\n❌ CREDENTIAL-SHAPED STRINGS IN THE UPLOAD SET:")
        for f, pat, line in hits:
            print(f"     {f}: {pat} -> {line[:60]}")
        sys.exit(1)
    print("✅ No live credentials in the upload set.")


# Live-credential shapes. The Space is public and the upload is the working tree, so
# a key pasted into any tracked file would be published — check rather than assume.
SECRET_PATTERNS = {
    "OpenAI key": re.compile(r"sk-[A-Za-z0-9_\-]{20,}"),
    "Tavily key": re.compile(r"tvly-[A-Za-z0-9_\-]{10,}"),
    "HF token": re.compile(r"hf_[A-Za-z0-9]{30,}"),
    "AWS key": re.compile(r"AKIA[0-9A-Z]{16}"),
}


def scan_for_secrets(files):
    hits = []
    for f in files:
        p = ROOT / f
        try:
            text = p.read_text(errors="ignore")
        except Exception:
            continue  # binary (PDF/sqlite) — skip
        for name, pat in SECRET_PATTERNS.items():
            for line in text.splitlines():
                if pat.search(line) and "your_" not in line and "example" not in line.lower():
                    hits.append((f, name, line.strip()))
    return hits


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("--repo", default="tirtho149/agadvisor", help="HF repo id (user/space)")
    ap.add_argument("--public", action="store_true", help="make the Space public (default: private)")
    ap.add_argument("--dry-run", action="store_true",
                    help="print exactly which files WOULD be uploaded, then exit. Touches nothing.")
    args = ap.parse_args()

    if args.dry_run:
        dry_run()
        return

    # Load .env first so HF_TOKEN can live there rather than being passed on the command
    # line (shell history / process list are not places for a write-scoped token).
    load_dotenv(ROOT / ".env")

    token = os.environ.get("HF_TOKEN")
    if not token:
        sys.exit("❌ No HF_TOKEN. Put it in .env (HF_TOKEN=hf_...) or export it.")

    api = HfApi(token=token)

    # HF now gates ALL dynamic Spaces behind PRO: hosting a Gradio/Docker Space on
    # even cpu-basic returns 402 for a free account (only Static Spaces are free).
    # Given PRO is required regardless, cpu-basic (2 vCPU / 16 GB, always-on) is the
    # right tier for this CPU-only app — ZeroGPU's GPU-quota model is a poor fit here.
    print(f"→ creating Space {args.repo} (private={not args.public}, hardware=cpu-basic) …")
    api.create_repo(args.repo, repo_type="space", space_sdk="gradio",
                    space_hardware="cpu-basic",
                    private=not args.public, exist_ok=True)

    # Secrets from .env (values read locally; not printed).
    # SESSION_SECRET + HF_DATA_TOKEN power the accounts/persistent-history feature.
    for k in ("OPENAI_API_KEY", "TAVILY_API_KEY", "OPENWEATHER_API_KEY",
              "SESSION_SECRET", "HF_DATA_TOKEN"):
        v = os.getenv(k)
        if v:
            api.add_space_secret(args.repo, k, v)
            print(f"   secret set: {k}")
        else:
            print(f"   ⚠️  {k} not in .env — set it in the HF Space UI")

    # Ensure the private Dataset that durably stores accounts/history exists.
    data_repo = os.getenv("HF_DATA_REPO")
    if data_repo:
        api.create_repo(data_repo, repo_type="dataset", private=True, exist_ok=True)
        api.add_space_variable(args.repo, "HF_DATA_REPO", data_repo)
        print(f"   dataset ready + var set: HF_DATA_REPO={data_repo}")
    else:
        print("   ⚠️  HF_DATA_REPO not set — user accounts/history will NOT persist "
              "across restarts. Set it in .env (e.g. <user>/agadvisor-userdata).")

    # Non-secret runtime vars: serve from the committed on-disk index, single process.
    for k, v in (("CDMS_OFFLINE_INDEX", "1"), ("QDRANT_FORCE_LOCAL", "1")):
        api.add_space_variable(args.repo, k, v)
        print(f"   var set: {k}={v}")

    # HF-specific README + requirements + LICENSE (staged, then uploaded).
    with tempfile.TemporaryDirectory() as td:
        (Path(td) / "README.md").write_text(README)
        (Path(td) / "LICENSE").write_text(LICENSE)
        reqs = (ROOT / "requirements.txt").read_text()
        (Path(td) / "requirements.txt").write_text(reqs.rstrip() + "\n\n# spaCy model\n" + SPACY_MODEL_LINE)
        for name in ("README.md", "LICENSE", "requirements.txt"):
            api.upload_file(path_or_fileobj=str(Path(td) / name),
                            path_in_repo=name, repo_id=args.repo, repo_type="space")

    # The upload is the working tree and the Space may be public, so gate the push on
    # the same checks --dry-run runs. Exits non-zero (before uploading) if either fails.
    print("→ pre-flight: checking the upload set for secrets / sensitive paths …")
    dry_run(quiet=True)

    print("→ uploading app + committed index (excludes .env/secrets/junk) …")
    api.upload_folder(repo_id=args.repo, repo_type="space", folder_path=str(ROOT),
                      ignore_patterns=IGNORE, commit_message="Deploy AgAdvisor")

    print(f"\n✅ Done. Space building at: https://huggingface.co/spaces/{args.repo}")
    print("   Watch the build logs there; first build is slow (torch/spacy).")


if __name__ == "__main__":
    main()