Upload app.py

metadata_search/app.py

@@ -10,6 +10,58 @@ from dotenv import load_dotenv
 
 load_dotenv(override=True)
 
+# ---------------------------------------------------------------------------
+# Hugging Face Dataset bootstrap
+# Download only the search-critical metadata + Qdrant index at startup.
+# The large PDFs/Summary/Figures assets are intentionally NOT downloaded here.
+# ---------------------------------------------------------------------------
+def _bootstrap_search_data() -> None:
+    if os.getenv("WATERSCOPE_DATA_BOOTSTRAP", "1").lower() not in {"1", "true", "yes"}:
+        return
+
+    from huggingface_hub import snapshot_download
+
+    repo_id = os.getenv(
+        "WATERSCOPE_DATASET_REPO",
+        "EnvironmentalAI/WaterScopeAI-Data",
+    ).strip()
+    data_root = Path(
+        os.getenv("WATERSCOPE_DATA_ROOT", "/tmp/waterscope-data")
+    ).resolve()
+    data_root.mkdir(parents=True, exist_ok=True)
+
+    print(f"[bootstrap] Loading search data from {repo_id}")
+    snapshot_download(
+        repo_id=repo_id,
+        repo_type="dataset",
+        local_dir=str(data_root),
+        allow_patterns=[
+            "metadata/papers.csv",
+            "metadata_search_data/qdrant/**",
+            "metadata_search_data/qdrant_manifest.json",
+        ],
+    )
+
+    paper_csv = data_root / "metadata" / "papers.csv"
+    qdrant_path = data_root / "metadata_search_data" / "qdrant"
+    qdrant_manifest = data_root / "metadata_search_data" / "qdrant_manifest.json"
+
+    if not paper_csv.is_file():
+        raise RuntimeError(f"Dataset bootstrap did not produce papers.csv: {paper_csv}")
+    if not qdrant_path.is_dir():
+        raise RuntimeError(f"Dataset bootstrap did not produce Qdrant directory: {qdrant_path}")
+
+    # Set these before importing metadata_search.config/search modules.
+    os.environ["PAPER_CSV"] = str(paper_csv)
+    os.environ["METADATA_QDRANT_PATH"] = str(qdrant_path)
+    os.environ["METADATA_QDRANT_MANIFEST"] = str(qdrant_manifest)
+
+    print(f"[bootstrap] PAPER_CSV={paper_csv}")
+    print(f"[bootstrap] METADATA_QDRANT_PATH={qdrant_path}")
+
+
+_bootstrap_search_data()
+
 from flask import Flask, jsonify, request, send_from_directory
 
 from metadata_search.chat_api import chat_payload, get_papers_payload