UWaterlooRFRChatBot / ingest /source_links.py
Anna Zhou
Add source links for CS61 CS63 CS64
be8bd08
Raw History Blame Contribute Delete
6.96 kB
"""Helpers for attaching source-paper links to extracted case-study chunks."""
from __future__ import annotations
import json
import re
from pathlib import Path
REPO_ROOT = Path(__file__).resolve().parent.parent
DATA_RAW = REPO_ROOT / "data" / "raw"
TRACKED_OVERRIDES = REPO_ROOT / "ingest" / "source_link_overrides.json"
_DOI_RE = re.compile(r"\b10\.\d{4,9}/[-._;()/:A-Z0-9]+", re.I)
_URL_RE = re.compile(r"https?://[^\s)>\]]+")
_KNOWN_BAD_DOIS = {
"10.1016/j.ijdrr.2020.315193",
"10.1016/j.ijdrr.2025.007988",
"10.1080/17477891.2017.1316763",
}
def _find_case_folder(case_id: str) -> Path | None:
for p in DATA_RAW.iterdir():
if p.is_dir() and re.match(rf"^{re.escape(case_id)}($|-)", p.name):
return p
return None
def _clean_doi(raw: str) -> str:
return raw.strip().rstrip(".,;:")
def _is_generic_title(title: str) -> bool:
return (
not title
or title.startswith("DOI ")
or title.startswith("Location:")
or title.startswith("URL:")
or title in {"Source", "Source URL"}
)
def _title_from_source_line(line: str, doi: str | None = None) -> str:
line = re.sub(r"\s+", " ", line).strip()
line = re.sub(r"^(PRIMARY|SUPPLEMENTAL|GREY LITERATURE)\s*:\s*", "", line, flags=re.I)
if doi:
line = re.sub(rf"\bDOI:\s*{re.escape(doi)}\b.*$", "", line, flags=re.I).strip()
line = line.rstrip(" .;-")
line = re.sub(r"\s+(URL|Local source|Local PDF|Covers|Note):\s+.*$", "", line, flags=re.I)
return line[:320]
def _dedupe_links(links: list[dict]) -> list[dict]:
by_key: dict[str, dict] = {}
order: list[str] = []
for link in links:
doi = _clean_doi(link.get("doi", "")) if link.get("doi") else ""
url = (link.get("url") or "").strip()
pdf_url = (link.get("pdf_url") or "").strip()
if doi and not url:
url = f"https://doi.org/{doi}"
key = (doi or url or pdf_url).lower()
if not key:
continue
candidate = {
"title": (link.get("title") or "Source").strip(),
"doi": doi,
"url": url,
"pdf_url": pdf_url,
}
candidate = {k: v for k, v in candidate.items() if v}
existing = by_key.get(key)
if existing is None:
by_key[key] = candidate
order.append(key)
continue
if _is_generic_title(existing.get("title", "")) and not _is_generic_title(candidate.get("title", "")):
existing["title"] = candidate["title"]
for field in ("doi", "url", "pdf_url"):
if not existing.get(field) and candidate.get(field):
existing[field] = candidate[field]
return [by_key[key] for key in order]
def _load_override_links(case_id: str) -> list[dict]:
links: list[dict] = []
for overrides_path in (TRACKED_OVERRIDES, DATA_RAW / "source_link_overrides.json"):
if not overrides_path.exists():
continue
try:
overrides = json.loads(overrides_path.read_text(encoding="utf-8"))
except json.JSONDecodeError:
continue
case_links = overrides.get(case_id, [])
if isinstance(case_links, list):
links.extend(
link
for link in case_links
if (link.get("doi") or "").lower() not in _KNOWN_BAD_DOIS
)
return links
def _add_sources_txt(folder: Path, links: list[dict]) -> None:
sources_txt = folder / "sources.txt"
if not sources_txt.exists():
return
last_title = ""
last_meaningful_title = ""
for line in sources_txt.read_text(encoding="utf-8", errors="ignore").splitlines():
stripped = line.strip()
if not stripped:
continue
is_heading = bool(re.match(r"^(PRIMARY SOURCE|SUPPLEMENTAL SOURCES|GREY LITERATURE|NONE)\s*:?$", stripped, re.I))
is_case_info = bool(re.match(r"^(Case ID|Location):", stripped, re.I))
has_identifier = bool(_DOI_RE.search(stripped) or _URL_RE.search(stripped))
if not is_heading and not is_case_info:
title = _title_from_source_line(stripped)
if title and not title.startswith(("Local source:", "Local PDF:", "Covers:", "Note:")):
last_title = title
if not has_identifier:
last_meaningful_title = title
for doi in _DOI_RE.findall(stripped):
doi = _clean_doi(doi)
if doi.lower() in _KNOWN_BAD_DOIS:
continue
links.append({
"title": _title_from_source_line(stripped, doi) or last_title or f"DOI {doi}",
"doi": doi,
"url": f"https://doi.org/{doi}",
})
for url in _URL_RE.findall(stripped):
title = _title_from_source_line(stripped)
if title.startswith(("URL:", "https://", "http://")) or is_case_info:
title = last_meaningful_title or last_title
links.append({
"title": title or "Source URL",
"url": url.rstrip(".,;"),
})
def _add_sources_json(folder: Path, links: list[dict]) -> None:
sources_json = folder / "sources.json"
if not sources_json.exists():
return
try:
catalog = json.loads(sources_json.read_text(encoding="utf-8"))
except json.JSONDecodeError:
catalog = {}
for source in catalog.get("sources", []):
status = str(source.get("status", "")).lower()
if "rejected" in status:
continue
doi = _clean_doi(str(source.get("doi", ""))) if source.get("doi") else ""
links.append({
"title": source.get("apa") or source.get("title") or source.get("seed_citation") or "Source",
"doi": doi,
"url": source.get("url") or (f"https://doi.org/{doi}" if doi else ""),
"pdf_url": source.get("pdf_url") or "",
})
def _add_dois_txt(folder: Path, links: list[dict]) -> None:
dois_txt = folder / "dois.txt"
if not dois_txt.exists():
return
for doi in _DOI_RE.findall(dois_txt.read_text(encoding="utf-8", errors="ignore")):
doi = _clean_doi(doi)
if doi.lower() in _KNOWN_BAD_DOIS:
continue
links.append({"title": f"DOI {doi}", "doi": doi, "url": f"https://doi.org/{doi}"})
def load_source_links(case_id: str) -> list[dict]:
"""Return article/DOI/PDF links known for a case study.
The source catalog files live in data/raw and are not all shipped to the
app, so ingestion embeds a compact copy of these links into data/extracted.
"""
folder = _find_case_folder(case_id)
if not folder:
return []
links: list[dict] = []
links.extend(_load_override_links(case_id))
_add_sources_txt(folder, links)
_add_sources_json(folder, links)
_add_dois_txt(folder, links)
return _dedupe_links(links)