Spaces:
Sleeping
Sleeping
Upload summaries.py
Browse files- local_rag/summaries.py +20 -64
local_rag/summaries.py
CHANGED
|
@@ -9,8 +9,8 @@ from huggingface_hub import hf_hub_download
|
|
| 9 |
from local_rag.config import LocalRagConfig
|
| 10 |
|
| 11 |
TITLE_PATTERN = re.compile(r"\*\*Title:\*\*\s*(.+)", re.MULTILINE)
|
| 12 |
-
_FILENAME_TITLE = re.compile(r"^.+?\
|
| 13 |
-
_FILENAME_YEAR = re.compile(r"^.+?\
|
| 14 |
|
| 15 |
HF_DATASET_REPO = os.getenv("WATERSCOPE_DATASET_REPO", "EnvironmentalAI/WaterScopeAI-Data")
|
| 16 |
HF_SUMMARY_PREFIX = os.getenv("WATERSCOPE_SUMMARY_PREFIX", "Summary")
|
|
@@ -29,14 +29,24 @@ def _year_from_filename(file_name: str) -> str | None:
|
|
| 29 |
|
| 30 |
def _download_cloud_summary(file_name: str) -> Path | None:
|
| 31 |
"""Download one summary from the HF Dataset and cache it locally."""
|
|
|
|
|
|
|
| 32 |
if not HF_SUMMARY_ON_DEMAND:
|
|
|
|
| 33 |
return None
|
| 34 |
|
| 35 |
year = _year_from_filename(file_name)
|
|
|
|
|
|
|
| 36 |
if not year:
|
|
|
|
| 37 |
return None
|
| 38 |
|
| 39 |
repo_file = f"{HF_SUMMARY_PREFIX}/{year}/{Path(file_name).name}"
|
|
|
|
|
|
|
|
|
|
|
|
|
| 40 |
try:
|
| 41 |
downloaded = hf_hub_download(
|
| 42 |
repo_id=HF_DATASET_REPO,
|
|
@@ -44,72 +54,18 @@ def _download_cloud_summary(file_name: str) -> Path | None:
|
|
| 44 |
filename=repo_file,
|
| 45 |
local_dir=str(HF_SUMMARY_CACHE),
|
| 46 |
)
|
| 47 |
-
except Exception:
|
|
|
|
|
|
|
|
|
|
|
|
|
| 48 |
return None
|
| 49 |
|
| 50 |
path = Path(downloaded)
|
| 51 |
-
|
| 52 |
-
|
| 53 |
-
|
| 54 |
-
def summary_search_dirs() -> list[Path]:
|
| 55 |
-
"""Directories to search for summary .txt files (first match wins)."""
|
| 56 |
-
dirs: list[Path] = []
|
| 57 |
-
for raw in (os.getenv("PAPER_SUMMARIES_DIRS") or "").split(":"):
|
| 58 |
-
raw = raw.strip()
|
| 59 |
-
if raw:
|
| 60 |
-
dirs.append(Path(raw).expanduser().resolve())
|
| 61 |
-
|
| 62 |
-
override = os.getenv("PAPER_SUMMARIES_DIR")
|
| 63 |
-
if override:
|
| 64 |
-
dirs.append(Path(override).expanduser().resolve())
|
| 65 |
-
|
| 66 |
-
project_root = Path(__file__).resolve().parents[1]
|
| 67 |
-
dirs.append(project_root / "paper_summaries")
|
| 68 |
-
|
| 69 |
-
unique: list[Path] = []
|
| 70 |
-
seen: set[str] = set()
|
| 71 |
-
for path in dirs:
|
| 72 |
-
key = str(path)
|
| 73 |
-
if key in seen:
|
| 74 |
-
continue
|
| 75 |
-
seen.add(key)
|
| 76 |
-
unique.append(path)
|
| 77 |
-
return unique
|
| 78 |
|
| 79 |
-
|
| 80 |
-
def summaries_dir() -> Path:
|
| 81 |
-
for path in summary_search_dirs():
|
| 82 |
-
if path.exists():
|
| 83 |
-
return path
|
| 84 |
-
return Path(__file__).resolve().parents[1] / "paper_summaries"
|
| 85 |
-
|
| 86 |
-
|
| 87 |
-
def _manifest_path(config: LocalRagConfig | None = None) -> Path:
|
| 88 |
-
cfg = config or LocalRagConfig.from_env()
|
| 89 |
-
return cfg.manifest_path
|
| 90 |
-
|
| 91 |
-
|
| 92 |
-
def load_manifest(config: LocalRagConfig | None = None) -> dict:
|
| 93 |
-
path = _manifest_path(config)
|
| 94 |
-
if not path.exists():
|
| 95 |
-
raise FileNotFoundError(f"Manifest not found: {path}. Rebuild local_rag_data/manifest.json first.")
|
| 96 |
-
return json.loads(path.read_text(encoding="utf-8"))
|
| 97 |
-
|
| 98 |
-
|
| 99 |
-
def title_from_filename(file_name: str) -> str:
|
| 100 |
-
stem = Path(file_name).stem
|
| 101 |
-
match = _FILENAME_TITLE.match(stem)
|
| 102 |
-
if match:
|
| 103 |
-
return match.group(1).strip()
|
| 104 |
-
return stem
|
| 105 |
-
|
| 106 |
-
|
| 107 |
-
def extract_title(file_name: str, summary: str | None = None) -> str:
|
| 108 |
-
if summary:
|
| 109 |
-
match = TITLE_PATTERN.search(summary)
|
| 110 |
-
if match:
|
| 111 |
-
return match.group(1).strip()
|
| 112 |
-
return title_from_filename(file_name)
|
| 113 |
|
| 114 |
|
| 115 |
def summary_path(file_name: str) -> Path | None:
|
|
|
|
| 9 |
from local_rag.config import LocalRagConfig
|
| 10 |
|
| 11 |
TITLE_PATTERN = re.compile(r"\*\*Title:\*\*\s*(.+)", re.MULTILINE)
|
| 12 |
+
_FILENAME_TITLE = re.compile(r"^.+?\s-\s\d{4}\s-\s(.+)$")
|
| 13 |
+
_FILENAME_YEAR = re.compile(r"^.+?\s-\s(\d{4})\s-\s")
|
| 14 |
|
| 15 |
HF_DATASET_REPO = os.getenv("WATERSCOPE_DATASET_REPO", "EnvironmentalAI/WaterScopeAI-Data")
|
| 16 |
HF_SUMMARY_PREFIX = os.getenv("WATERSCOPE_SUMMARY_PREFIX", "Summary")
|
|
|
|
| 29 |
|
| 30 |
def _download_cloud_summary(file_name: str) -> Path | None:
|
| 31 |
"""Download one summary from the HF Dataset and cache it locally."""
|
| 32 |
+
print(f"[summary] requested file = {file_name!r}", flush=True)
|
| 33 |
+
|
| 34 |
if not HF_SUMMARY_ON_DEMAND:
|
| 35 |
+
print("[summary] on-demand download disabled", flush=True)
|
| 36 |
return None
|
| 37 |
|
| 38 |
year = _year_from_filename(file_name)
|
| 39 |
+
print(f"[summary] detected year = {year!r}", flush=True)
|
| 40 |
+
|
| 41 |
if not year:
|
| 42 |
+
print("[summary] could not detect year from filename; download skipped", flush=True)
|
| 43 |
return None
|
| 44 |
|
| 45 |
repo_file = f"{HF_SUMMARY_PREFIX}/{year}/{Path(file_name).name}"
|
| 46 |
+
print(f"[summary] dataset repo = {HF_DATASET_REPO}", flush=True)
|
| 47 |
+
print(f"[summary] repo file = {repo_file!r}", flush=True)
|
| 48 |
+
print(f"[summary] cache dir = {HF_SUMMARY_CACHE}", flush=True)
|
| 49 |
+
|
| 50 |
try:
|
| 51 |
downloaded = hf_hub_download(
|
| 52 |
repo_id=HF_DATASET_REPO,
|
|
|
|
| 54 |
filename=repo_file,
|
| 55 |
local_dir=str(HF_SUMMARY_CACHE),
|
| 56 |
)
|
| 57 |
+
except Exception as exc:
|
| 58 |
+
print(
|
| 59 |
+
f"[summary] download ERROR: {type(exc).__name__}: {exc}",
|
| 60 |
+
flush=True,
|
| 61 |
+
)
|
| 62 |
return None
|
| 63 |
|
| 64 |
path = Path(downloaded)
|
| 65 |
+
print(f"[summary] downloaded path = {path}", flush=True)
|
| 66 |
+
print(f"[summary] downloaded exists = {path.is_file()}", flush=True)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 67 |
|
| 68 |
+
return path if path.is_file() else None
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 69 |
|
| 70 |
|
| 71 |
def summary_path(file_name: str) -> Path | None:
|