Upload 3 files

metadata_search/app.py

@@ -273,12 +273,9 @@ def paper_pdf_file(filename: str):
     )
 
 
-@app.get("/figures/<path:paper_key>/<path:filename>")
+@app.get("/figures/<path:paper_key>/<filename>")
 def paper_figure_file(paper_key: str, filename: str):
-    safe_paper_key = Path(paper_key).name
-    safe_name = Path(filename).name
-
-    target = resolve_figure_asset(safe_paper_key, safe_name)
+    target = resolve_figure_asset(paper_key, filename)
     if target is None:
         return jsonify({"error": "Figure file not found."}), 404
 

metadata_search/config.py

@@ -55,7 +55,7 @@ METADATA_SEARCH_PORT = int(
 PAPER_FIGURES_DIR = Path(
     os.getenv(
         "PAPER_FIGURES_DIR",
-        str(PROJECT_ROOT / "results_metadata (2025-2026)"),
+        str(PROJECT_ROOT / "Media"),
     )
 )
 

metadata_search/paper_figures.py

@@ -6,7 +6,7 @@ import json
 import logging
 import os
 import unicodedata
-from pathlib import Path
+from pathlib import Path, PurePosixPath, PureWindowsPath
 from urllib.parse import quote
 
 from metadata_search.config import PAPER_FIGURES_DIR
@@ -23,7 +23,7 @@ HF_DATASET_REPO = os.getenv(
     "WATERSCOPE_DATASET_REPO",
     "EnvironmentalAI/WaterScopeAI-Data",
 ).strip()
-HF_FIGURES_PREFIX = os.getenv("WATERSCOPE_FIGURES_PREFIX", "Figures").strip() or "Figures"
+HF_FIGURES_PREFIX = os.getenv("WATERSCOPE_FIGURES_PREFIX", "Media").strip() or "Media"
 HF_FIGURES_CACHE = Path(
     os.getenv("WATERSCOPE_FIGURES_CACHE", "/tmp/waterscope-figure-cache")
 ).expanduser().resolve()
@@ -66,10 +66,37 @@ def _hf_download(repo_file: str) -> Path | None:
     return path if path.is_file() else None
 
 
+def _media_folder(folder: str) -> str | None:
+    """Preserve relative year/study paths without allowing path traversal."""
+    if not folder or "\\" in folder or ":" in folder:
+        return None
+    path = PurePosixPath(folder)
+    if path.is_absolute() or any(part in {"", ".", ".."} for part in folder.split("/")):
+        return None
+    return path.as_posix()
+
+
+def _batch_folders(path: Path) -> list[str]:
+    """The overall manifest supplies study folders; yearly reports are not scanned."""
+    batch = json.loads(path.read_text(encoding="utf-8-sig"))
+    if not isinstance(batch.get("papers"), list):
+        raise ValueError(f"Missing papers list in media batch manifest: {path}")
+    folders = []
+    for record in batch["papers"]:
+        folder = _media_folder(str(record.get("output_directory") or ""))
+        if folder is None:
+            raise ValueError("Invalid media output_directory in batch manifest")
+        if folder not in folders:
+            folders.append(folder)
+    return folders
+
+
 def resolve_figure_asset(folder: str, filename: str) -> Path | None:
     """Resolve one figure asset locally, then fetch only that asset if needed."""
-    safe_folder = Path(folder).name
-    safe_name = Path(filename).name
+    safe_folder = _media_folder(folder)
+    if safe_folder is None or not filename or "/" in filename or "\\" in filename or filename in {".", ".."}:
+        return None
+    safe_name = filename
 
     root = figures_dir()
     if root is not None:
@@ -133,10 +160,13 @@ def _load_index() -> tuple[dict[str, str], dict[str, dict]]:
     # 1) Existing local behavior.
     root = figures_dir()
     if root is not None:
-        for entry in sorted(root.iterdir()):
-            if not entry.is_dir():
-                continue
-            manifest_path = entry / "manifest.json"
+        batch_path = root / "batch_manifest.json"
+        folders = (_batch_folders(batch_path) if batch_path.is_file() else
+                   [path.parent.relative_to(root).as_posix()
+                    for path in sorted(root.rglob("manifest.json"))
+                    if path.parent != root])
+        for folder in folders:
+            manifest_path = root / folder / "manifest.json"
             if not manifest_path.is_file():
                 continue
             try:
@@ -145,34 +175,36 @@ def _load_index() -> tuple[dict[str, str], dict[str, dict]]:
                 logger.warning("Skipping figure manifest %s: %s", manifest_path, exc)
                 continue
 
-            folder = entry.name
             manifests[folder] = manifest
-            _register_stem(stem_index, folder, folder)
+            _register_stem(stem_index, PurePosixPath(folder).name, folder)
 
             source_pdf = (manifest.get("source_pdf") or "").strip()
             if source_pdf:
-                _register_stem(stem_index, Path(source_pdf).name, folder)
+                _register_stem(stem_index, PureWindowsPath(source_pdf).name, folder)
 
     # 2) Cloud fallback. List repository paths once, then download only
     #    manifest.json files. Image assets remain on-demand.
     if not manifests and HF_FIGURES_ON_DEMAND:
         try:
-            repo_files = _hf_api().list_repo_files(
-                repo_id=HF_DATASET_REPO,
-                repo_type="dataset",
-            )
             suffix = "/manifest.json"
             prefix = f"{HF_FIGURES_PREFIX}/"
-            manifest_files = [
-                path for path in repo_files
-                if path.startswith(prefix) and path.endswith(suffix)
-            ]
+            batch_path = _hf_download(f"{prefix}batch_manifest.json")
+            if batch_path is not None:
+                manifest_files = [f"{prefix}{folder}{suffix}"
+                                  for folder in _batch_folders(batch_path)]
+            else:
+                # Support data without an overall manifest; only study manifests.
+                repo_files = _hf_api().list_repo_files(
+                    repo_id=HF_DATASET_REPO, repo_type="dataset",
+                )
+                manifest_files = [path for path in repo_files
+                                  if path.startswith(prefix) and path.endswith(suffix)]
 
             for repo_file in manifest_files:
                 relative = repo_file[len(prefix):]
                 folder = relative[:-len(suffix)]
-                # The current WaterScope dataset uses one immediate paper folder.
-                if not folder or "/" in folder:
+                # Keep the year component when resolving study manifests.
+                if _media_folder(folder) is None:
                     continue
 
                 manifest_path = _hf_download(repo_file)
@@ -185,11 +217,11 @@ def _load_index() -> tuple[dict[str, str], dict[str, dict]]:
                     continue
 
                 manifests[folder] = manifest
-                _register_stem(stem_index, folder, folder)
+                _register_stem(stem_index, PurePosixPath(folder).name, folder)
 
                 source_pdf = (manifest.get("source_pdf") or "").strip()
                 if source_pdf:
-                    _register_stem(stem_index, Path(source_pdf).name, folder)
+                    _register_stem(stem_index, PureWindowsPath(source_pdf).name, folder)
 
             logger.info(
                 "Indexed figures for %s papers from HF Dataset %s",
@@ -219,11 +251,11 @@ def _lookup_folder(*, txt_file: str = "", pdf_file: str = "", paper_key: str = "
             return paper_key
         target = normalize_stem(paper_key)
         for folder in manifests:
-            if normalize_stem(folder) == target:
+            if normalize_stem(folder) == target or normalize_stem(PurePosixPath(folder).name) == target:
                 return folder
 
     for candidate in (txt_file, pdf_file):
-        stem = normalize_stem(candidate)
+        stem = normalize_stem(PureWindowsPath(candidate).name)
         if not stem:
             continue
         if stem in stem_index:
@@ -262,7 +294,7 @@ def resolve_figure_folder(
 
 
 def figure_image_url(folder: str, filename: str) -> str:
-    return f"/figures/{quote(folder, safe='')}/{quote(filename, safe='')}"
+    return f"/figures/{quote(folder, safe='/')}/{quote(filename, safe='')}"
 
 
 def figures_payload_for_folder(folder: str) -> dict | None:
@@ -293,7 +325,7 @@ def figures_payload_for_folder(folder: str) -> dict | None:
     source_pdf = (manifest.get("source_pdf") or "").strip()
     return {
         "paper_key": folder,
-        "source_pdf": Path(source_pdf).name if source_pdf else folder,
+        "source_pdf": PureWindowsPath(source_pdf).name if source_pdf else folder,
         "item_count": len(items),
         "items": items,
     }