Upload 2 files

metadata_search/research_app.py

@@ -0,0 +1,143 @@
+#!/usr/bin/env python3
+"""Standalone research-only API for configurable WaterScope-AI search.
+
+Run this file instead of production metadata_search/app.py when conducting
+retrieval experiments. No production source file is modified.
+"""
+
+from __future__ import annotations
+
+import os
+from pathlib import Path
+from time import monotonic
+
+from dotenv import load_dotenv
+
+load_dotenv(override=True)
+
+
+def _bootstrap_search_data() -> None:
+    """Mirror production search-data bootstrap without importing production app.py."""
+    if os.getenv("WATERSCOPE_DATA_BOOTSTRAP", "1").lower() not in {"1", "true", "yes"}:
+        return
+
+    from huggingface_hub import HfApi, hf_hub_download
+    from huggingface_hub.hf_api import RepoFile
+
+    repo_id = os.getenv("WATERSCOPE_DATASET_REPO", "EnvironmentalAI/WaterScopeAI-Data").strip()
+    data_root = Path(os.getenv("WATERSCOPE_DATA_ROOT", "/tmp/waterscope-data")).resolve()
+    data_root.mkdir(parents=True, exist_ok=True)
+    started = monotonic()
+
+    def log(message: str) -> None:
+        print(f"[research-bootstrap +{monotonic() - started:.1f}s] {message}", flush=True)
+
+    def download(filename: str) -> None:
+        log(f"Downloading {filename}")
+        hf_hub_download(
+            repo_id=repo_id,
+            repo_type="dataset",
+            filename=filename,
+            local_dir=str(data_root),
+        )
+
+    api = HfApi()
+    index_prefix = "metadata_search_data/qdrant/"
+    download("metadata/papers.csv")
+    download("metadata_search_data/qdrant_manifest.json")
+
+    index_files: list[str] = []
+    for entry in api.list_repo_tree(
+        repo_id=repo_id,
+        repo_type="dataset",
+        path_in_repo=index_prefix.rstrip("/"),
+        recursive=True,
+        expand=False,
+    ):
+        if isinstance(entry, RepoFile) and entry.path.startswith(index_prefix):
+            index_files.append(entry.path)
+
+    if not index_files:
+        raise RuntimeError(f"No Qdrant files found under {index_prefix}")
+    for filename in sorted(set(index_files)):
+        download(filename)
+
+    paper_csv = data_root / "metadata" / "papers.csv"
+    qdrant_path = data_root / "metadata_search_data" / "qdrant"
+    qdrant_manifest = data_root / "metadata_search_data" / "qdrant_manifest.json"
+
+    if not paper_csv.is_file() or not qdrant_path.is_dir():
+        raise RuntimeError("Research bootstrap did not produce required search data.")
+
+    os.environ["PAPER_CSV"] = str(paper_csv)
+    os.environ["METADATA_QDRANT_PATH"] = str(qdrant_path)
+    os.environ["METADATA_QDRANT_MANIFEST"] = str(qdrant_manifest)
+    log("Research search data ready")
+
+
+_bootstrap_search_data()
+
+from flask import Flask, jsonify, request  # noqa: E402
+from metadata_search.config import PAPER_CSV  # noqa: E402
+from metadata_search.research_search import research_search  # noqa: E402
+
+if not Path(PAPER_CSV).is_file():
+    raise SystemExit(f"Paper dataset not found: {PAPER_CSV}")
+
+app = Flask(__name__)
+
+
+def _float_arg(name: str, default: float) -> float:
+    raw = request.args.get(name)
+    return default if raw in {None, ""} else float(raw)
+
+
+def _int_arg(name: str, default: int | None = None) -> int | None:
+    raw = request.args.get(name)
+    if raw in {None, ""}:
+        return default
+    return int(raw)
+
+
+@app.get("/")
+def home():
+    return jsonify(
+        {
+            "service": "WaterScopeAI Research Search",
+            "status": "running",
+            "endpoint": "/api/research/search",
+            "example": "/api/research/search?q=PFAS&top_k=100&bm25_weight=0.5&semantic_weight=0.5&rrf_k=60",
+        }
+    )
+
+
+@app.get("/api/research/health")
+def health():
+    return jsonify({"status": "ok", "paper_csv": str(PAPER_CSV)})
+
+
+@app.get("/api/research/search")
+def api_research_search():
+    query = request.args.get("q", "").strip()
+    if not query:
+        return jsonify({"error": "q is required"}), 400
+    try:
+        payload = research_search(
+            query,
+            top_k=int(request.args.get("top_k", "100")),
+            bm25_weight=_float_arg("bm25_weight", 0.5),
+            semantic_weight=_float_arg("semantic_weight", 0.5),
+            rrf_k=int(request.args.get("rrf_k", "60")),
+            year_from=_int_arg("year_from"),
+            year_to=_int_arg("year_to"),
+        )
+        return jsonify(payload)
+    except (TypeError, ValueError) as exc:
+        return jsonify({"error": str(exc)}), 400
+    except Exception as exc:
+        return jsonify({"error": f"{type(exc).__name__}: {exc}"}), 500
+
+
+if __name__ == "__main__":
+    port = int(os.getenv("RESEARCH_SEARCH_PORT", os.getenv("PORT", "7860")))
+    app.run(host=os.getenv("HOST", "0.0.0.0"), port=port, debug=False, use_reloader=False)

metadata_search/research_search.py

@@ -0,0 +1,169 @@
+"""Research-only configurable metadata search for WaterScope-AI.
+
+This module intentionally does not modify production search_service.py.
+It reuses the production corpus, BM25 index, semantic vector store, embedder,
+boolean-query parser, and metadata enrichment, while exposing experimental
+weighted-RRF parameters for reproducible retrieval studies.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+from backend.paper_linking import enrich_metadata_row, format_authors
+from metadata_search.boolean_query import matches_boolean_query, parse_boolean_query, plan_summary
+from metadata_search.hybrid_search import weighted_rrf_scores
+from metadata_search.search_service import (
+    get_bm25_index,
+    get_embedder,
+    get_rows,
+    get_vector_store,
+    metadata_auto_rebuild_enabled,
+    ensure_metadata_index,
+)
+
+
+def _validate_weights(bm25_weight: float, semantic_weight: float) -> tuple[float, float]:
+    if bm25_weight < 0 or semantic_weight < 0:
+        raise ValueError("Weights must be non-negative.")
+    total = bm25_weight + semantic_weight
+    if total <= 0:
+        raise ValueError("bm25_weight + semantic_weight must be > 0.")
+    return bm25_weight / total, semantic_weight / total
+
+
+def research_search(
+    query: str,
+    *,
+    top_k: int = 100,
+    bm25_weight: float = 0.5,
+    semantic_weight: float = 0.5,
+    rrf_k: int = 60,
+    year_from: int | None = None,
+    year_to: int | None = None,
+) -> dict[str, Any]:
+    """Run the production-equivalent search with configurable weighted RRF.
+
+    Boolean/exact syntax and metadata matching use the same production modules.
+    The returned result list is ranked *after* configurable fusion, so BM25-only
+    and semantic-only settings are not biased by the production 0.5/0.5 ranking.
+    """
+    query = (query or "").strip()
+    if not query:
+        raise ValueError("query is required")
+    if top_k < 1 or top_k > 500:
+        raise ValueError("top_k must be between 1 and 500")
+    if rrf_k < 1:
+        raise ValueError("rrf_k must be >= 1")
+
+    bm25_weight, semantic_weight = _validate_weights(bm25_weight, semantic_weight)
+
+    rows = get_rows()
+    parsed = parse_boolean_query(query)
+    ranking_query = parsed.ranking_text or query
+
+    bm25_scores = get_bm25_index().score(ranking_query)
+
+    # Semantic scores are needed whenever semantic_weight > 0.
+    if semantic_weight > 0:
+        store = get_vector_store()
+        if store.points_count() == 0:
+            if metadata_auto_rebuild_enabled():
+                ensure_metadata_index(force=True)
+            else:
+                raise RuntimeError(
+                    "Prebuilt Qdrant index is unavailable and METADATA_AUTO_REBUILD=0."
+                )
+        query_vector = get_embedder().embed_query(ranking_query)
+        semantic_scores = store.semantic_scores(query_vector, len(rows))
+    else:
+        semantic_scores = [0.0] * len(rows)
+
+    # For BM25-only, semantic ranks remain zero. For all other settings, use
+    # the same weighted-RRF implementation as production.
+    if semantic_weight == 0:
+        semantic_scores = [0.0] * len(rows)
+
+    final_scores, bm25_ranks, bm25_rrf, semantic_ranks, semantic_rrf = weighted_rrf_scores(
+        bm25_scores,
+        semantic_scores,
+        bm25_weight=bm25_weight,
+        semantic_weight=semantic_weight,
+        rrf_k=rrf_k,
+    )
+
+    ranked_indices = sorted(
+        range(len(final_scores)),
+        key=lambda idx: (final_scores[idx], bm25_scores[idx], semantic_scores[idx]),
+        reverse=True,
+    )
+
+    results: list[dict[str, Any]] = []
+    for row_index in ranked_indices:
+        row = rows[row_index]
+        enriched = enrich_metadata_row(row, row_index)
+
+        match_row = {
+            "title": enriched["title"],
+            "abstract": enriched["abstract"],
+            "full_abstract": enriched["abstract"],
+            "keywords": enriched["keywords"],
+        }
+        if not matches_boolean_query(match_row, parsed):
+            continue
+
+        if year_from is not None or year_to is not None:
+            try:
+                publication_year = int(str(enriched.get("year") or "").strip())
+            except (TypeError, ValueError):
+                continue
+            if year_from is not None and publication_year < year_from:
+                continue
+            if year_to is not None and publication_year > year_to:
+                continue
+
+        results.append(
+            {
+                "rank": len(results) + 1,
+                "title": enriched["title"],
+                "authors": enriched["authors"],
+                "authors_display": format_authors(enriched["authors"]),
+                "year": enriched["year"],
+                "journal": (row.get("SO") or "").strip(),
+                "publication_type": (row.get("DT") or "").strip(),
+                "doi": enriched["doi"],
+                "keywords": enriched["keywords"],
+                "abstract": enriched["abstract"],
+                "document_id": enriched.get("document_id"),
+                "metadata_row_index": enriched.get("metadata_row_index"),
+                "txt_file": enriched.get("txt_file", ""),
+                "pdf_file": (row.get("PDF_file") or row.get("pdf_file") or "").strip(),
+                "final_score": final_scores[row_index],
+                "rrf_score": final_scores[row_index],
+                "bm25_score": bm25_scores[row_index],
+                "semantic_score": semantic_scores[row_index],
+                "bm25_rank": bm25_ranks[row_index],
+                "semantic_rank": semantic_ranks[row_index],
+                "bm25_rrf_score": bm25_rrf[row_index],
+                "semantic_rrf_score": semantic_rrf[row_index],
+                "search_query_parsed": plan_summary(parsed),
+            }
+        )
+        if len(results) >= top_k:
+            break
+
+    return {
+        "query": query,
+        "ranking_query": ranking_query,
+        "query_parsed": plan_summary(parsed),
+        "parameters": {
+            "top_k": top_k,
+            "bm25_weight": bm25_weight,
+            "semantic_weight": semantic_weight,
+            "rrf_k": rrf_k,
+            "year_from": year_from,
+            "year_to": year_to,
+        },
+        "result_count": len(results),
+        "results": results,
+    }