Upload app.py

metadata_search/app.py

@@ -20,7 +20,8 @@ def _bootstrap_search_data() -> None:
     if os.getenv("WATERSCOPE_DATA_BOOTSTRAP", "1").lower() not in {"1", "true", "yes"}:
         return
 
-    from huggingface_hub import HfApi, hf_hub_download
+    from huggingface_hub import HfApi, RepoFile, hf_hub_download
+    from time import monotonic
 
     repo_id = os.getenv(
         "WATERSCOPE_DATASET_REPO",
@@ -31,32 +32,55 @@ def _bootstrap_search_data() -> None:
     ).resolve()
     data_root.mkdir(parents=True, exist_ok=True)
 
-    print(f"[bootstrap] Inspecting dataset {repo_id}")
+    started = monotonic()
+
+    def log(message: str) -> None:
+        print(f"[bootstrap pid={os.getpid()} +{monotonic() - started:.1f}s] {message}", flush=True)
+
     api = HfApi()
-    repo_files = api.list_repo_files(repo_id=repo_id, repo_type="dataset")
-
-    wanted = [
-        path for path in repo_files
-        if path == "metadata/papers.csv"
-        or path == "metadata_search_data/qdrant_manifest.json"
-        or path.startswith("metadata_search_data/qdrant/")
-    ]
-
-    print(f"[bootstrap] Found {len(wanted)} search-data files")
-    if not wanted:
-        raise RuntimeError(
-            "No search-data files found in Dataset. Expected "
-            "metadata/papers.csv and metadata_search_data/qdrant/..."
-        )
+    index_prefix = "metadata_search_data/qdrant/"
+    log(f"Starting search-data bootstrap for {repo_id}")
 
-    for filename in wanted:
-        print(f"[bootstrap] Downloading {filename}")
-        hf_hub_download(
+    def download(filename: str) -> None:
+        log(f"Downloading {filename}")
+        try:
+            hf_hub_download(
+                repo_id=repo_id,
+                repo_type="dataset",
+                filename=filename,
+                local_dir=str(data_root),
+            )
+        except Exception as exc:
+            log(f"Download failed for {filename}: {type(exc).__name__}: {exc}")
+            raise
+        log(f"Ready: {filename}")
+
+    # Known metadata paths do not require a repository-wide file listing.
+    download("metadata/papers.csv")
+    download("metadata_search_data/qdrant_manifest.json")
+
+    log(f"Listing only {index_prefix}")
+    try:
+        index_files = []
+        for entry in api.list_repo_tree(
             repo_id=repo_id,
             repo_type="dataset",
-            filename=filename,
-            local_dir=str(data_root),
-        )
+            path_in_repo=index_prefix.rstrip("/"),
+            recursive=True,
+            expand=False,
+        ):
+            if isinstance(entry, RepoFile) and entry.path.startswith(index_prefix):
+                index_files.append(entry.path)
+                log(f"Found index file: {entry.path}")
+    except Exception as exc:
+        log(f"Qdrant listing failed: {type(exc).__name__}: {exc}")
+        raise
+
+    if not index_files:
+        raise RuntimeError(f"No Qdrant files found in Dataset under {index_prefix}")
+    log(f"Found {len(index_files)} Qdrant files; media/PDF/summary folders were not listed")
+    for filename in sorted(set(index_files)):
+        download(filename)
 
     paper_csv = data_root / "metadata" / "papers.csv"
     qdrant_path = data_root / "metadata_search_data" / "qdrant"
@@ -75,6 +99,7 @@ def _bootstrap_search_data() -> None:
     print(f"[bootstrap] PAPER_CSV={paper_csv}")
     print(f"[bootstrap] METADATA_QDRANT_PATH={qdrant_path}")
     print(f"[bootstrap] METADATA_QDRANT_MANIFEST={qdrant_manifest}")
+    log("Search-data bootstrap complete")
 
 
 _bootstrap_search_data()