#!/usr/bin/env bash # Deploy Learning Focused MCP as a Hugging Face Docker Space + private corpus dataset. # Daily ingest is an HF Job (08:00 Pacific / 15:00 UTC during PDT), not Cloud Run. # Does not deploy to GCP project inferpoker. Does not print secret values. set -euo pipefail ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" cd "${ROOT}" HF_USER="${HF_USER:-LeoWalker}" SPACE_ID="${HF_SPACE_ID:-${HF_USER}/learningfocused-mcp}" CORPUS_REPO="${HF_CORPUS_REPO:-${HF_USER}/learningfocused-corpus}" SPACE_HOST="$(echo "${SPACE_ID}" | tr '[:upper:]' '[:lower:]' | tr '/' '-').hf.space" SPACE_URL="https://${SPACE_HOST}" SYNC_CORPUS="${SYNC_CORPUS:-1}" SCHEDULE_JOB="${SCHEDULE_JOB:-1}" # 08:00 America/Los_Angeles during PDT. HF Jobs cron is UTC (no TZ flag). JOB_CRON="${HF_INGEST_CRON:-0 15 * * *}" JOB_TIMEOUT="${HF_INGEST_TIMEOUT:-4h}" JOB_FLAVOR="${HF_INGEST_FLAVOR:-cpu-basic}" SECRETS_FILE="$(mktemp "${TMPDIR:-/tmp}/lf-hf-secrets.XXXXXX")" trap 'rm -f "${SECRETS_FILE}"' EXIT chmod 600 "${SECRETS_FILE}" if ! hf auth whoami >/dev/null 2>&1; then echo "Hugging Face CLI is not logged in. Run: hf auth login" >&2 echo "Need a write token for ${HF_USER} (Spaces + datasets + Jobs)." >&2 exit 2 fi echo ">> Hugging Face account OK" echo ">> Space: ${SPACE_ID}" echo ">> Corpus: ${CORPUS_REPO}" echo ">> URL: ${SPACE_URL}/mcp" echo ">> Creating private corpus dataset ${CORPUS_REPO}" hf repos create "${CORPUS_REPO}" --type dataset --private --exist-ok echo ">> Creating Docker Space ${SPACE_ID} (cpu-basic, sleep enabled)" hf repos create "${SPACE_ID}" \ --type space \ --space-sdk docker \ --public \ --exist-ok \ --flavor cpu-basic if [[ "${SYNC_CORPUS}" == "1" ]]; then echo ">> Syncing chroma + derived artifacts to ${CORPUS_REPO} (not podcast_downloads)" PREFIXES=( chroma_db transcripts segmented_transcripts combined_summaries substack_articles article_summaries youtube_videos youtube_summaries ) for prefix in "${PREFIXES[@]}"; do dest="${ROOT}/${prefix}" mkdir -p "${dest}" if [[ "${prefix}" == "chroma_db" && -f "${dest}/chroma.sqlite3" ]]; then echo " local ${prefix}/ (chroma.sqlite3 present)" continue fi if [[ "${prefix}" != "chroma_db" && -n "$(ls -A "${dest}" 2>/dev/null || true)" ]]; then echo " local ${prefix}/" continue fi echo " gcs gs://inferpoker-learningfocused/${prefix}/ (one-time copy off InferPoker)" gcloud storage rsync "gs://inferpoker-learningfocused/${prefix}" "${dest}" \ --recursive --checksums-only --project=inferpoker done if [[ ! -f "${ROOT}/chroma_db/chroma.sqlite3" ]]; then echo "chroma.sqlite3 missing after sync" >&2 exit 1 fi echo ">> Uploading corpus snapshot to Hugging Face" HF_CORPUS_REPO="${CORPUS_REPO}" \ CHROMA_DIR="${ROOT}/chroma_db" \ uv run python -c "from src.hf_corpus import upload_artifacts; upload_artifacts()" fi echo ">> Uploading Space sources" hf upload "${SPACE_ID}" "${ROOT}/spaces/README.md" README.md --repo-type space --commit-message "Space README" hf upload "${SPACE_ID}" "${ROOT}/Dockerfile" Dockerfile --repo-type space hf upload "${SPACE_ID}" "${ROOT}/pyproject.toml" pyproject.toml --repo-type space hf upload "${SPACE_ID}" "${ROOT}/uv.lock" uv.lock --repo-type space hf upload "${SPACE_ID}" "${ROOT}/.dockerignore" .dockerignore --repo-type space hf upload "${SPACE_ID}" "${ROOT}/src" src --repo-type space --commit-message "MCP source" hf upload "${SPACE_ID}" "${ROOT}/scripts" scripts --repo-type space --commit-message "entrypoints" echo ">> Setting Space variables (not secrets)" SPACE_ID="${SPACE_ID}" CORPUS_REPO="${CORPUS_REPO}" uv run python - <<'PY' import os from huggingface_hub import HfApi api = HfApi() space = os.environ["SPACE_ID"] corpus = os.environ["CORPUS_REPO"] variables = { "MCP_HOST": "0.0.0.0", "HYDRATE_CHROMA": "1", "HF_CORPUS_REPO": corpus, "HF_SPACE_ID": space, "CHROMA_DIR": "/home/user/data/chroma_db", "MCP_RATE_LIMIT_PER_MINUTE": "60", "MCP_SNIPPET_LIMIT": "32768", "PORT": "7860", } for key, value in variables.items(): api.add_space_variable(space, key, value) print("space variables set:", ",".join(variables)) PY echo ">> Copying API keys into Space secrets (values not printed)" uv run python "${ROOT}/scripts/set_hf_secrets.py" --space "${SPACE_ID}" --write-job-secrets-file "${SECRETS_FILE}" if [[ "${SCHEDULE_JOB}" == "1" ]]; then echo ">> Removing previous learningfocused ingest schedules if any" uv run python - <<'PY' from huggingface_hub import delete_scheduled_job, list_scheduled_jobs try: jobs = list(list_scheduled_jobs()) except Exception as exc: # noqa: BLE001 print(f"WARNING: could not list scheduled jobs: {exc}") jobs = [] for job in jobs: labels = getattr(job, "labels", None) or {} if labels.get("app") == "learningfocused" and labels.get("role") == "daily-ingest": print(f"deleting scheduled job {job.id}") delete_scheduled_job(job.id) PY echo ">> Scheduling daily ingest job ${JOB_CRON} UTC flavor=${JOB_FLAVOR}" hf jobs scheduled run "${JOB_CRON}" \ "hf.co/spaces/${SPACE_ID}" \ python -m src.pipeline.daily_ingest \ --flavor "${JOB_FLAVOR}" \ --timeout "${JOB_TIMEOUT}" \ --env "HF_CORPUS_REPO=${CORPUS_REPO}" \ --env "HF_SPACE_ID=${SPACE_ID}" \ --env "HYDRATE_CHROMA=0" \ --env "CHROMA_DIR=/home/user/app/chroma_db" \ --env "DAILY_ART19_DOWNLOAD_LIMIT=5" \ --env "MCP_HOST=0.0.0.0" \ --secrets-file "${SECRETS_FILE}" \ --secrets HF_TOKEN \ --label app=learningfocused \ --label role=daily-ingest fi echo echo "Space: https://huggingface.co/spaces/${SPACE_ID}" echo "MCP: ${SPACE_URL}/mcp" echo "Health: ${SPACE_URL}/health" echo "Corpus: https://huggingface.co/datasets/${CORPUS_REPO}" echo echo "Wait for the Space to reach Running, then smoke /health and /mcp." echo "Cursor mcp.json url: ${SPACE_URL}/mcp"