scripts/deploy_hf.sh

#!/usr/bin/env bash
# Deploy Learning Focused MCP as a Hugging Face Docker Space + private corpus dataset.
# Daily ingest is an HF Job (08:00 Pacific / 15:00 UTC during PDT), not Cloud Run.
# Does not deploy to GCP project inferpoker. Does not print secret values.
set -euo pipefail

ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
cd "${ROOT}"

HF_USER="${HF_USER:-LeoWalker}"
SPACE_ID="${HF_SPACE_ID:-${HF_USER}/learningfocused-mcp}"
CORPUS_REPO="${HF_CORPUS_REPO:-${HF_USER}/learningfocused-corpus}"
SPACE_HOST="$(echo "${SPACE_ID}" | tr '[:upper:]' '[:lower:]' | tr '/' '-').hf.space"
SPACE_URL="https://${SPACE_HOST}"
SYNC_CORPUS="${SYNC_CORPUS:-1}"
SCHEDULE_JOB="${SCHEDULE_JOB:-1}"
# 08:00 America/Los_Angeles during PDT. HF Jobs cron is UTC (no TZ flag).
JOB_CRON="${HF_INGEST_CRON:-0 15 * * *}"
JOB_TIMEOUT="${HF_INGEST_TIMEOUT:-4h}"
JOB_FLAVOR="${HF_INGEST_FLAVOR:-cpu-basic}"
SECRETS_FILE="$(mktemp "${TMPDIR:-/tmp}/lf-hf-secrets.XXXXXX")"
trap 'rm -f "${SECRETS_FILE}"' EXIT
chmod 600 "${SECRETS_FILE}"

if ! hf auth whoami >/dev/null 2>&1; then
  echo "Hugging Face CLI is not logged in. Run: hf auth login" >&2
  echo "Need a write token for ${HF_USER} (Spaces + datasets + Jobs)." >&2
  exit 2
fi

echo ">> Hugging Face account OK"
echo ">> Space:  ${SPACE_ID}"
echo ">> Corpus: ${CORPUS_REPO}"
echo ">> URL:    ${SPACE_URL}/mcp"

echo ">> Creating private corpus dataset ${CORPUS_REPO}"
hf repos create "${CORPUS_REPO}" --type dataset --private --exist-ok

echo ">> Creating Docker Space ${SPACE_ID} (cpu-basic, sleep enabled)"
hf repos create "${SPACE_ID}" \
  --type space \
  --space-sdk docker \
  --public \
  --exist-ok \
  --flavor cpu-basic

if [[ "${SYNC_CORPUS}" == "1" ]]; then
  echo ">> Syncing chroma + derived artifacts to ${CORPUS_REPO} (not podcast_downloads)"
  PREFIXES=(
    chroma_db
    transcripts
    segmented_transcripts
    combined_summaries
    substack_articles
    article_summaries
    youtube_videos
    youtube_summaries
  )
  for prefix in "${PREFIXES[@]}"; do
    dest="${ROOT}/${prefix}"
    mkdir -p "${dest}"
    if [[ "${prefix}" == "chroma_db" && -f "${dest}/chroma.sqlite3" ]]; then
      echo "   local ${prefix}/ (chroma.sqlite3 present)"
      continue
    fi
    if [[ "${prefix}" != "chroma_db" && -n "$(ls -A "${dest}" 2>/dev/null || true)" ]]; then
      echo "   local ${prefix}/"
      continue
    fi
    echo "   gcs gs://inferpoker-learningfocused/${prefix}/ (one-time copy off InferPoker)"
    gcloud storage rsync "gs://inferpoker-learningfocused/${prefix}" "${dest}" \
      --recursive --checksums-only --project=inferpoker
  done
  if [[ ! -f "${ROOT}/chroma_db/chroma.sqlite3" ]]; then
    echo "chroma.sqlite3 missing after sync" >&2
    exit 1
  fi
  echo ">> Uploading corpus snapshot to Hugging Face"
  HF_CORPUS_REPO="${CORPUS_REPO}" \
    CHROMA_DIR="${ROOT}/chroma_db" \
    uv run python -c "from src.hf_corpus import upload_artifacts; upload_artifacts()"
fi

echo ">> Uploading Space sources"
hf upload "${SPACE_ID}" "${ROOT}/spaces/README.md" README.md --repo-type space --commit-message "Space README"
hf upload "${SPACE_ID}" "${ROOT}/Dockerfile" Dockerfile --repo-type space
hf upload "${SPACE_ID}" "${ROOT}/pyproject.toml" pyproject.toml --repo-type space
hf upload "${SPACE_ID}" "${ROOT}/uv.lock" uv.lock --repo-type space
hf upload "${SPACE_ID}" "${ROOT}/.dockerignore" .dockerignore --repo-type space
hf upload "${SPACE_ID}" "${ROOT}/src" src --repo-type space --commit-message "MCP source"
hf upload "${SPACE_ID}" "${ROOT}/scripts" scripts --repo-type space --commit-message "entrypoints"

echo ">> Setting Space variables (not secrets)"
SPACE_ID="${SPACE_ID}" CORPUS_REPO="${CORPUS_REPO}" uv run python - <<'PY'
import os
from huggingface_hub import HfApi

api = HfApi()
space = os.environ["SPACE_ID"]
corpus = os.environ["CORPUS_REPO"]
variables = {
    "MCP_HOST": "0.0.0.0",
    "HYDRATE_CHROMA": "1",
    "HF_CORPUS_REPO": corpus,
    "HF_SPACE_ID": space,
    "CHROMA_DIR": "/home/user/data/chroma_db",
    "MCP_RATE_LIMIT_PER_MINUTE": "60",
    "MCP_SNIPPET_LIMIT": "32768",
    "PORT": "7860",
}
for key, value in variables.items():
    api.add_space_variable(space, key, value)
print("space variables set:", ",".join(variables))
PY

echo ">> Copying API keys into Space secrets (values not printed)"
uv run python "${ROOT}/scripts/set_hf_secrets.py" --space "${SPACE_ID}" --write-job-secrets-file "${SECRETS_FILE}"

if [[ "${SCHEDULE_JOB}" == "1" ]]; then
  echo ">> Removing previous learningfocused ingest schedules if any"
  uv run python - <<'PY'
from huggingface_hub import delete_scheduled_job, list_scheduled_jobs

try:
    jobs = list(list_scheduled_jobs())
except Exception as exc:  # noqa: BLE001
    print(f"WARNING: could not list scheduled jobs: {exc}")
    jobs = []
for job in jobs:
    labels = getattr(job, "labels", None) or {}
    if labels.get("app") == "learningfocused" and labels.get("role") == "daily-ingest":
        print(f"deleting scheduled job {job.id}")
        delete_scheduled_job(job.id)
PY
  echo ">> Scheduling daily ingest job ${JOB_CRON} UTC flavor=${JOB_FLAVOR}"
  hf jobs scheduled run "${JOB_CRON}" \
    "hf.co/spaces/${SPACE_ID}" \
    python -m src.pipeline.daily_ingest \
    --flavor "${JOB_FLAVOR}" \
    --timeout "${JOB_TIMEOUT}" \
    --env "HF_CORPUS_REPO=${CORPUS_REPO}" \
    --env "HF_SPACE_ID=${SPACE_ID}" \
    --env "HYDRATE_CHROMA=0" \
    --env "CHROMA_DIR=/home/user/app/chroma_db" \
    --env "DAILY_ART19_DOWNLOAD_LIMIT=5" \
    --env "MCP_HOST=0.0.0.0" \
    --secrets-file "${SECRETS_FILE}" \
    --secrets HF_TOKEN \
    --label app=learningfocused \
    --label role=daily-ingest
fi

echo
echo "Space:   https://huggingface.co/spaces/${SPACE_ID}"
echo "MCP:     ${SPACE_URL}/mcp"
echo "Health:  ${SPACE_URL}/health"
echo "Corpus:  https://huggingface.co/datasets/${CORPUS_REPO}"
echo
echo "Wait for the Space to reach Running, then smoke /health and /mcp."
echo "Cursor mcp.json url: ${SPACE_URL}/mcp"