Upload 6 files

.github/workflows/ci-deploy.yml

@@ -0,0 +1,80 @@
+name: Test, backup, deploy and verify
+
+on:
+  push:
+    branches: [main]
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+concurrency:
+  group: vectorization-hf-space
+  cancel-in-progress: false
+
+jobs:
+  unit-tests:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
+          cache: pip
+      - name: Install CPU-only dependencies
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install torch==2.6.0 --index-url https://download.pytorch.org/whl/cpu
+          python -m pip install -r requirements-dev.txt
+      - name: Test validation and authorization
+        run: pytest -q tests/test_app.py --junitxml=unit-results.xml
+      - uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: unit-test-results
+          path: unit-results.xml
+          retention-days: 90
+
+  backup-and-deploy:
+    needs: unit-tests
+    if: github.ref == 'refs/heads/main' && vars.HF_DEPLOY_ENABLED == 'true'
+    runs-on: ubuntu-latest
+    timeout-minutes: 70
+    env:
+      HF_TOKEN: ${{ secrets.HF_TOKEN }}
+      API_TEST_KEY: ${{ secrets.API_TEST_KEY }}
+      SPACE_URL: https://smartdataalex-jupyter.hf.space
+      EXPECTED_SHA: ${{ github.sha }}
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
+      - run: python -m pip install 'huggingface-hub==0.30.2' 'httpx==0.28.1'
+      - name: Verify deployment credentials exist
+        run: |
+          test -n "$HF_TOKEN" || { echo 'Missing HF_TOKEN GitHub secret'; exit 1; }
+          test -n "$API_TEST_KEY" || { echo 'Missing API_TEST_KEY GitHub secret'; exit 1; }
+      - name: Backup existing Space before any modification
+        run: python scripts/backup_space.py
+      - name: Upload original Space snapshot
+        uses: actions/upload-artifact@v4
+        with:
+          name: space-backup-${{ github.run_id }}
+          path: ${{ runner.temp }}/hf-space-backup/
+          retention-days: 90
+          if-no-files-found: error
+      - name: Set release identifier
+        run: echo "$GITHUB_SHA" > VERSION
+      - name: Deploy backed-up revision to Hugging Face Docker Space
+        run: python scripts/deploy_space.py
+      - name: Smoke-test actual published API (must match this commit)
+        run: python scripts/smoke_live.py
+      - name: Upload deployment test results
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: live-api-results-${{ github.run_id }}
+          path: smoke-result.json
+          if-no-files-found: ignore
+          retention-days: 90

scripts/backup_space.py

@@ -0,0 +1,40 @@
+"""Fail-closed predeployment backup of the existing Hugging Face Space."""
+
+import json
+import os
+import tarfile
+from pathlib import Path
+
+from huggingface_hub import HfApi, snapshot_download
+
+SPACE = "SmartDataAlex/Jupyter"
+ROOT = Path(os.environ.get("RUNNER_TEMP", "/tmp")) / "hf-space-backup"
+TOKEN = os.environ.get("HF_TOKEN")
+if not TOKEN:
+    raise RuntimeError("HF_TOKEN is required to read and back up the existing Space")
+
+api = HfApi(token=TOKEN)
+info = api.repo_info(repo_id=SPACE, repo_type="space")
+revision = info.sha
+if not revision:
+    raise RuntimeError("Cannot determine the current Space revision; refusing to deploy")
+
+ROOT.mkdir(parents=True, exist_ok=True)
+snapshot_path = Path(snapshot_download(
+    repo_id=SPACE,
+    repo_type="space",
+    revision=revision,
+    token=TOKEN,
+    local_dir=ROOT / "snapshot",
+))
+files = [p for p in snapshot_path.rglob("*") if p.is_file() and ".cache" not in p.parts]
+if not files:
+    raise RuntimeError("Space snapshot is empty; refusing to overwrite it")
+archive = ROOT / f"SmartDataAlex-Jupyter-{revision}.tar.gz"
+with tarfile.open(archive, "w:gz") as output:
+    for path in files:
+        output.add(path, arcname=path.relative_to(snapshot_path))
+(ROOT / "backup-manifest.json").write_text(json.dumps({
+    "space": SPACE, "revision": revision, "files": len(files), "archive": archive.name
+}, indent=2), encoding="utf-8")
+print(f"Space backed up: revision={revision}, files={len(files)}, archive={archive}")

scripts/deploy_space.py

@@ -0,0 +1,53 @@
+"""Atomically replace Space content only if its predeployment backup is current."""
+
+from __future__ import annotations
+
+import json
+import os
+from pathlib import Path
+
+from huggingface_hub import HfApi
+
+SPACE = "SmartDataAlex/Jupyter"
+ROOT = Path(os.environ.get("RUNNER_TEMP", "/tmp")) / "hf-space-backup"
+TOKEN = os.environ.get("HF_TOKEN")
+
+
+def main() -> None:
+    if not TOKEN:
+        raise RuntimeError("Missing HF_TOKEN")
+    manifest_path = ROOT / "backup-manifest.json"
+    if not manifest_path.is_file():
+        raise RuntimeError("No completed backup manifest; refusing to deploy")
+    backup = json.loads(manifest_path.read_text(encoding="utf-8"))
+    revision = backup.get("revision")
+    if backup.get("space") != SPACE or not revision:
+        raise RuntimeError("Backup is for a different Space or lacks a revision")
+    archive = ROOT / backup.get("archive", "")
+    if not archive.is_file() or archive.stat().st_size == 0:
+        raise RuntimeError("Backup archive missing; refusing to deploy")
+
+    api = HfApi(token=TOKEN)
+    current = api.repo_info(repo_id=SPACE, repo_type="space").sha
+    if current != revision:
+        raise RuntimeError("Space changed after backup; refusing to overwrite newer content")
+
+    # Upload_folder handles deletions and additions within the same commit.
+    # parent_commit prevents accidentally overwriting a concurrent Space update.
+    result = api.upload_folder(
+        repo_id=SPACE,
+        repo_type="space",
+        folder_path=".",
+        allow_patterns=[
+            "README.md", "Dockerfile", ".dockerignore", "VERSION",
+            "requirements.txt", "app/**", "scripts/download_model.py",
+        ],
+        delete_patterns="*",
+        parent_commit=revision,
+        commit_message=f"Deploy vectorization service from GitHub {os.environ.get('GITHUB_SHA', 'unknown')}",
+    )
+    print(f"Deployed {SPACE}: {result.commit_url}")
+
+
+if __name__ == "__main__":
+    main()

scripts/download_model.py

@@ -0,0 +1,19 @@
+"""Cache an immutable, minimal E5 model snapshot in the Docker image."""
+
+import os
+
+from huggingface_hub import snapshot_download
+
+snapshot_download(
+    repo_id="intfloat/multilingual-e5-small",
+    revision="fd1525a9fd15316a2d503bf26ab031a61d056e98",
+    local_dir=os.getenv("E5_MODEL_DIR", "/opt/models/multilingual-e5-small"),
+    allow_patterns=[
+        "config.json",
+        "model.safetensors",
+        "sentencepiece.bpe.model",
+        "special_tokens_map.json",
+        "tokenizer.json",
+        "tokenizer_config.json",
+    ],
+)

scripts/restore_space.py

@@ -0,0 +1,55 @@
+"""Explicit/manual rollback of a previously backed-up Hugging Face Space.
+
+Use only with a trusted backup archive produced by backup_space.py.
+"""
+
+from __future__ import annotations
+
+import argparse
+import os
+import tarfile
+import tempfile
+from pathlib import Path
+
+from huggingface_hub import HfApi
+
+SPACE = "SmartDataAlex/Jupyter"
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description="Restore existing Space files from a backup archive")
+    parser.add_argument("archive", type=Path)
+    parser.add_argument("--confirm-space", required=True, help=f"Must be exactly {SPACE}")
+    args = parser.parse_args()
+    if args.confirm_space != SPACE:
+        raise SystemExit(f"Refusing to restore: --confirm-space must equal {SPACE}")
+    token = os.getenv("HF_TOKEN")
+    if not token:
+        raise SystemExit("HF_TOKEN environment variable is required")
+    if not args.archive.is_file():
+        raise SystemExit("Backup archive not found")
+
+    with tempfile.TemporaryDirectory(prefix="space-rollback-") as temp:
+        dest = Path(temp)
+        with tarfile.open(args.archive, "r:gz") as backup:
+            members = backup.getmembers()
+            if not members or len(members) > 10000:
+                raise SystemExit("Invalid backup archive")
+            for member in members:
+                if (not member.isfile() or member.name.startswith("/")
+                    or ".." in Path(member.name).parts):
+                    raise SystemExit("Unsafe or unsupported backup entry")
+            backup.extractall(path=dest, filter="data")
+
+        HfApi(token=token).upload_folder(
+            repo_id=SPACE,
+            repo_type="space",
+            folder_path=str(dest),
+            delete_patterns="*",
+            commit_message=f"Restore Space from backup {args.archive.name}",
+        )
+        print(f"Restored {SPACE} from {args.archive.name}")
+
+
+if __name__ == "__main__":
+    main()

scripts/smoke_live.py

@@ -0,0 +1,109 @@
+"""Test the deployed HTTPS service, not local code. Store JSON report for AI review."""
+
+from __future__ import annotations
+
+import json
+import math
+import os
+import sys
+import time
+from datetime import datetime, timezone
+from pathlib import Path
+
+import httpx
+
+SPACE_URL = os.getenv("SPACE_URL", "https://smartdataalex-jupyter.hf.space").rstrip("/")
+KEY = os.environ.get("API_TEST_KEY", "")
+EXPECTED_SHA = os.environ.get("EXPECTED_SHA", "")
+MAX_WAIT_SECONDS = int(os.getenv("LIVE_WAIT_SECONDS", "2400"))
+REPORT = Path(os.getenv("SMOKE_REPORT", "smoke-result.json"))
+TESTS = []
+
+
+def check(name: str, condition: bool, details: str = "") -> None:
+    TESTS.append({"name": name, "passed": bool(condition), "details": details})
+    if not condition:
+        raise AssertionError(f"{name} failed: {details}")
+
+
+def cosine(a: list[float], b: list[float]) -> float:
+    dot = sum(x * y for x, y in zip(a, b))
+    na = math.sqrt(sum(x * x for x in a))
+    nb = math.sqrt(sum(y * y for y in b))
+    return dot / (na * nb)
+
+
+def run() -> None:
+    if not KEY:
+        raise ValueError("API_TEST_KEY is required for real authenticated tests")
+    headers = {"X-API-Key": KEY}
+    with httpx.Client(timeout=90.0) as client:
+        deadline = time.monotonic() + MAX_WAIT_SECONDS
+        while time.monotonic() < deadline:
+            try:
+                health = client.get(f"{SPACE_URL}/health", timeout=20)
+                if health.status_code == 200 and health.json().get("status") == "ok":
+                    version = health.json().get("version", "")
+                    if not EXPECTED_SHA or version == EXPECTED_SHA:
+                        break
+            except (httpx.HTTPError, ValueError):
+                pass
+            time.sleep(15)
+        else:
+            raise TimeoutError("Live service did not reach the expected release version")
+        check("public_health", health.status_code == 200)
+        check("expected_deployment", not EXPECTED_SHA or health.json().get("version") == EXPECTED_SHA)
+
+        unauthorized = client.post(f"{SPACE_URL}/embed", json={"text": "hello", "type": "query"})
+        check("missing_key_rejected", unauthorized.status_code == 401, str(unauthorized.status_code))
+        forbidden = client.post(f"{SPACE_URL}/embed", json={"text": "hello", "type": "query"}, headers={"X-API-Key": "wrong-key"})
+        check("wrong_key_rejected", forbidden.status_code == 401, str(forbidden.status_code))
+        check("model_metadata_protected", client.get(f"{SPACE_URL}/model").status_code == 401)
+
+        model_response = client.get(f"{SPACE_URL}/model", headers=headers)
+        check("model_metadata_authorized", model_response.status_code == 200)
+        info = model_response.json()
+        check("model_and_dimensions", info.get("embedding_model") == "intfloat/multilingual-e5-small" and info.get("dimensions") == 384)
+        check("no_extra_normalization", info.get("normalization") == "none")
+
+        def embed(text: str, kind: str = "query") -> dict:
+            response = client.post(f"{SPACE_URL}/embed", headers=headers, json={"text": text, "type": kind})
+            check(f"embed_{len(TESTS)}", response.status_code == 200, str(response.status_code))
+            return response.json()
+
+        sample = embed("How to find similar documents?")
+        sample_again = embed("How to find similar documents?")
+        check("response_echo", sample.get("text") == "How to find similar documents?" and sample.get("type") == "query")
+        check("token_count", isinstance(sample.get("tokens_count"), int) and sample["tokens_count"] > 0)
+        vector = sample["embedding"]
+        check("dimension_consistent", len(vector) == sample.get("dimensions") == 384)
+        check("vector_numeric", all(isinstance(x, (int, float)) and not isinstance(x, bool) and math.isfinite(x) for x in vector))
+        check("reproducibility", max(abs(a - b) for a, b in zip(vector, sample_again["embedding"])) < 1e-5)
+        check("passage_mode", embed("A short sentence.", "passage")["type"] == "passage")
+        check("unicode", embed("Проверка текста на русском языке")["dimensions"] == 384)
+
+        related = embed("How to search for similar documents?")
+        unrelated = embed("The strawberry cake is delicious.")
+        related_score = cosine(vector, related["embedding"])
+        unrelated_score = cosine(vector, unrelated["embedding"])
+        check("semantic_sanity", related_score > unrelated_score, f"related={related_score:.4f} unrelated={unrelated_score:.4f}")
+
+        for invalid in [{"text": "", "type": "query"}, {"text": "ok", "type": "invalid"}, {"type": "query"}]:
+            response = client.post(f"{SPACE_URL}/embed", headers=headers, json=invalid)
+            check("invalid_input_rejected", response.status_code == 422, str(response.status_code))
+
+
+if __name__ == "__main__":
+    started = datetime.now(timezone.utc).isoformat()
+    error = None
+    try:
+        run()
+    except Exception as exc:
+        error = f"{type(exc).__name__}: {exc}"
+    report = {
+        "started_at": started, "space_url": SPACE_URL, "expected_sha": EXPECTED_SHA or None,
+        "passed": error is None, "error": error, "checks": TESTS,
+    }
+    REPORT.write_text(json.dumps(report, indent=2, ensure_ascii=False), encoding="utf-8")
+    print(json.dumps(report, ensure_ascii=False, indent=2))
+    sys.exit(1 if error is not None else 0)