starfriend commited on
Commit
6ce619c
·
verified ·
1 Parent(s): 543dc84

Upload summaries.py

Browse files
Files changed (1) hide show
  1. local_rag/summaries.py +20 -64
local_rag/summaries.py CHANGED
@@ -9,8 +9,8 @@ from huggingface_hub import hf_hub_download
9
  from local_rag.config import LocalRagConfig
10
 
11
  TITLE_PATTERN = re.compile(r"\*\*Title:\*\*\s*(.+)", re.MULTILINE)
12
- _FILENAME_TITLE = re.compile(r"^.+?\\s-\\s\\d{4}\\s-\\s(.+)$")
13
- _FILENAME_YEAR = re.compile(r"^.+?\\s-\\s(\\d{4})\\s-\\s")
14
 
15
  HF_DATASET_REPO = os.getenv("WATERSCOPE_DATASET_REPO", "EnvironmentalAI/WaterScopeAI-Data")
16
  HF_SUMMARY_PREFIX = os.getenv("WATERSCOPE_SUMMARY_PREFIX", "Summary")
@@ -29,14 +29,24 @@ def _year_from_filename(file_name: str) -> str | None:
29
 
30
  def _download_cloud_summary(file_name: str) -> Path | None:
31
  """Download one summary from the HF Dataset and cache it locally."""
 
 
32
  if not HF_SUMMARY_ON_DEMAND:
 
33
  return None
34
 
35
  year = _year_from_filename(file_name)
 
 
36
  if not year:
 
37
  return None
38
 
39
  repo_file = f"{HF_SUMMARY_PREFIX}/{year}/{Path(file_name).name}"
 
 
 
 
40
  try:
41
  downloaded = hf_hub_download(
42
  repo_id=HF_DATASET_REPO,
@@ -44,72 +54,18 @@ def _download_cloud_summary(file_name: str) -> Path | None:
44
  filename=repo_file,
45
  local_dir=str(HF_SUMMARY_CACHE),
46
  )
47
- except Exception:
 
 
 
 
48
  return None
49
 
50
  path = Path(downloaded)
51
- return path if path.is_file() else None
52
-
53
-
54
- def summary_search_dirs() -> list[Path]:
55
- """Directories to search for summary .txt files (first match wins)."""
56
- dirs: list[Path] = []
57
- for raw in (os.getenv("PAPER_SUMMARIES_DIRS") or "").split(":"):
58
- raw = raw.strip()
59
- if raw:
60
- dirs.append(Path(raw).expanduser().resolve())
61
-
62
- override = os.getenv("PAPER_SUMMARIES_DIR")
63
- if override:
64
- dirs.append(Path(override).expanduser().resolve())
65
-
66
- project_root = Path(__file__).resolve().parents[1]
67
- dirs.append(project_root / "paper_summaries")
68
-
69
- unique: list[Path] = []
70
- seen: set[str] = set()
71
- for path in dirs:
72
- key = str(path)
73
- if key in seen:
74
- continue
75
- seen.add(key)
76
- unique.append(path)
77
- return unique
78
 
79
-
80
- def summaries_dir() -> Path:
81
- for path in summary_search_dirs():
82
- if path.exists():
83
- return path
84
- return Path(__file__).resolve().parents[1] / "paper_summaries"
85
-
86
-
87
- def _manifest_path(config: LocalRagConfig | None = None) -> Path:
88
- cfg = config or LocalRagConfig.from_env()
89
- return cfg.manifest_path
90
-
91
-
92
- def load_manifest(config: LocalRagConfig | None = None) -> dict:
93
- path = _manifest_path(config)
94
- if not path.exists():
95
- raise FileNotFoundError(f"Manifest not found: {path}. Rebuild local_rag_data/manifest.json first.")
96
- return json.loads(path.read_text(encoding="utf-8"))
97
-
98
-
99
- def title_from_filename(file_name: str) -> str:
100
- stem = Path(file_name).stem
101
- match = _FILENAME_TITLE.match(stem)
102
- if match:
103
- return match.group(1).strip()
104
- return stem
105
-
106
-
107
- def extract_title(file_name: str, summary: str | None = None) -> str:
108
- if summary:
109
- match = TITLE_PATTERN.search(summary)
110
- if match:
111
- return match.group(1).strip()
112
- return title_from_filename(file_name)
113
 
114
 
115
  def summary_path(file_name: str) -> Path | None:
 
9
  from local_rag.config import LocalRagConfig
10
 
11
  TITLE_PATTERN = re.compile(r"\*\*Title:\*\*\s*(.+)", re.MULTILINE)
12
+ _FILENAME_TITLE = re.compile(r"^.+?\s-\s\d{4}\s-\s(.+)$")
13
+ _FILENAME_YEAR = re.compile(r"^.+?\s-\s(\d{4})\s-\s")
14
 
15
  HF_DATASET_REPO = os.getenv("WATERSCOPE_DATASET_REPO", "EnvironmentalAI/WaterScopeAI-Data")
16
  HF_SUMMARY_PREFIX = os.getenv("WATERSCOPE_SUMMARY_PREFIX", "Summary")
 
29
 
30
  def _download_cloud_summary(file_name: str) -> Path | None:
31
  """Download one summary from the HF Dataset and cache it locally."""
32
+ print(f"[summary] requested file = {file_name!r}", flush=True)
33
+
34
  if not HF_SUMMARY_ON_DEMAND:
35
+ print("[summary] on-demand download disabled", flush=True)
36
  return None
37
 
38
  year = _year_from_filename(file_name)
39
+ print(f"[summary] detected year = {year!r}", flush=True)
40
+
41
  if not year:
42
+ print("[summary] could not detect year from filename; download skipped", flush=True)
43
  return None
44
 
45
  repo_file = f"{HF_SUMMARY_PREFIX}/{year}/{Path(file_name).name}"
46
+ print(f"[summary] dataset repo = {HF_DATASET_REPO}", flush=True)
47
+ print(f"[summary] repo file = {repo_file!r}", flush=True)
48
+ print(f"[summary] cache dir = {HF_SUMMARY_CACHE}", flush=True)
49
+
50
  try:
51
  downloaded = hf_hub_download(
52
  repo_id=HF_DATASET_REPO,
 
54
  filename=repo_file,
55
  local_dir=str(HF_SUMMARY_CACHE),
56
  )
57
+ except Exception as exc:
58
+ print(
59
+ f"[summary] download ERROR: {type(exc).__name__}: {exc}",
60
+ flush=True,
61
+ )
62
  return None
63
 
64
  path = Path(downloaded)
65
+ print(f"[summary] downloaded path = {path}", flush=True)
66
+ print(f"[summary] downloaded exists = {path.is_file()}", flush=True)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
67
 
68
+ return path if path.is_file() else None
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
69
 
70
 
71
  def summary_path(file_name: str) -> Path | None: