Spaces:
Sleeping
Sleeping
File size: 6,961 Bytes
0c2e8f6 b6c1096 0c2e8f6 b6c1096 be8bd08 b6c1096 0c2e8f6 b6c1096 0c2e8f6 b6c1096 0c2e8f6 b6c1096 0c2e8f6 b6c1096 0c2e8f6 b6c1096 0c2e8f6 b6c1096 0c2e8f6 b6c1096 0c2e8f6 b6c1096 0c2e8f6 b6c1096 0c2e8f6 b6c1096 0c2e8f6 b6c1096 0c2e8f6 b6c1096 0c2e8f6 b6c1096 0c2e8f6 b6c1096 0c2e8f6 b6c1096 0c2e8f6 b6c1096 0c2e8f6 b6c1096 0c2e8f6 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 | """Helpers for attaching source-paper links to extracted case-study chunks."""
from __future__ import annotations
import json
import re
from pathlib import Path
REPO_ROOT = Path(__file__).resolve().parent.parent
DATA_RAW = REPO_ROOT / "data" / "raw"
TRACKED_OVERRIDES = REPO_ROOT / "ingest" / "source_link_overrides.json"
_DOI_RE = re.compile(r"\b10\.\d{4,9}/[-._;()/:A-Z0-9]+", re.I)
_URL_RE = re.compile(r"https?://[^\s)>\]]+")
_KNOWN_BAD_DOIS = {
"10.1016/j.ijdrr.2020.315193",
"10.1016/j.ijdrr.2025.007988",
"10.1080/17477891.2017.1316763",
}
def _find_case_folder(case_id: str) -> Path | None:
for p in DATA_RAW.iterdir():
if p.is_dir() and re.match(rf"^{re.escape(case_id)}($|-)", p.name):
return p
return None
def _clean_doi(raw: str) -> str:
return raw.strip().rstrip(".,;:")
def _is_generic_title(title: str) -> bool:
return (
not title
or title.startswith("DOI ")
or title.startswith("Location:")
or title.startswith("URL:")
or title in {"Source", "Source URL"}
)
def _title_from_source_line(line: str, doi: str | None = None) -> str:
line = re.sub(r"\s+", " ", line).strip()
line = re.sub(r"^(PRIMARY|SUPPLEMENTAL|GREY LITERATURE)\s*:\s*", "", line, flags=re.I)
if doi:
line = re.sub(rf"\bDOI:\s*{re.escape(doi)}\b.*$", "", line, flags=re.I).strip()
line = line.rstrip(" .;-")
line = re.sub(r"\s+(URL|Local source|Local PDF|Covers|Note):\s+.*$", "", line, flags=re.I)
return line[:320]
def _dedupe_links(links: list[dict]) -> list[dict]:
by_key: dict[str, dict] = {}
order: list[str] = []
for link in links:
doi = _clean_doi(link.get("doi", "")) if link.get("doi") else ""
url = (link.get("url") or "").strip()
pdf_url = (link.get("pdf_url") or "").strip()
if doi and not url:
url = f"https://doi.org/{doi}"
key = (doi or url or pdf_url).lower()
if not key:
continue
candidate = {
"title": (link.get("title") or "Source").strip(),
"doi": doi,
"url": url,
"pdf_url": pdf_url,
}
candidate = {k: v for k, v in candidate.items() if v}
existing = by_key.get(key)
if existing is None:
by_key[key] = candidate
order.append(key)
continue
if _is_generic_title(existing.get("title", "")) and not _is_generic_title(candidate.get("title", "")):
existing["title"] = candidate["title"]
for field in ("doi", "url", "pdf_url"):
if not existing.get(field) and candidate.get(field):
existing[field] = candidate[field]
return [by_key[key] for key in order]
def _load_override_links(case_id: str) -> list[dict]:
links: list[dict] = []
for overrides_path in (TRACKED_OVERRIDES, DATA_RAW / "source_link_overrides.json"):
if not overrides_path.exists():
continue
try:
overrides = json.loads(overrides_path.read_text(encoding="utf-8"))
except json.JSONDecodeError:
continue
case_links = overrides.get(case_id, [])
if isinstance(case_links, list):
links.extend(
link
for link in case_links
if (link.get("doi") or "").lower() not in _KNOWN_BAD_DOIS
)
return links
def _add_sources_txt(folder: Path, links: list[dict]) -> None:
sources_txt = folder / "sources.txt"
if not sources_txt.exists():
return
last_title = ""
last_meaningful_title = ""
for line in sources_txt.read_text(encoding="utf-8", errors="ignore").splitlines():
stripped = line.strip()
if not stripped:
continue
is_heading = bool(re.match(r"^(PRIMARY SOURCE|SUPPLEMENTAL SOURCES|GREY LITERATURE|NONE)\s*:?$", stripped, re.I))
is_case_info = bool(re.match(r"^(Case ID|Location):", stripped, re.I))
has_identifier = bool(_DOI_RE.search(stripped) or _URL_RE.search(stripped))
if not is_heading and not is_case_info:
title = _title_from_source_line(stripped)
if title and not title.startswith(("Local source:", "Local PDF:", "Covers:", "Note:")):
last_title = title
if not has_identifier:
last_meaningful_title = title
for doi in _DOI_RE.findall(stripped):
doi = _clean_doi(doi)
if doi.lower() in _KNOWN_BAD_DOIS:
continue
links.append({
"title": _title_from_source_line(stripped, doi) or last_title or f"DOI {doi}",
"doi": doi,
"url": f"https://doi.org/{doi}",
})
for url in _URL_RE.findall(stripped):
title = _title_from_source_line(stripped)
if title.startswith(("URL:", "https://", "http://")) or is_case_info:
title = last_meaningful_title or last_title
links.append({
"title": title or "Source URL",
"url": url.rstrip(".,;"),
})
def _add_sources_json(folder: Path, links: list[dict]) -> None:
sources_json = folder / "sources.json"
if not sources_json.exists():
return
try:
catalog = json.loads(sources_json.read_text(encoding="utf-8"))
except json.JSONDecodeError:
catalog = {}
for source in catalog.get("sources", []):
status = str(source.get("status", "")).lower()
if "rejected" in status:
continue
doi = _clean_doi(str(source.get("doi", ""))) if source.get("doi") else ""
links.append({
"title": source.get("apa") or source.get("title") or source.get("seed_citation") or "Source",
"doi": doi,
"url": source.get("url") or (f"https://doi.org/{doi}" if doi else ""),
"pdf_url": source.get("pdf_url") or "",
})
def _add_dois_txt(folder: Path, links: list[dict]) -> None:
dois_txt = folder / "dois.txt"
if not dois_txt.exists():
return
for doi in _DOI_RE.findall(dois_txt.read_text(encoding="utf-8", errors="ignore")):
doi = _clean_doi(doi)
if doi.lower() in _KNOWN_BAD_DOIS:
continue
links.append({"title": f"DOI {doi}", "doi": doi, "url": f"https://doi.org/{doi}"})
def load_source_links(case_id: str) -> list[dict]:
"""Return article/DOI/PDF links known for a case study.
The source catalog files live in data/raw and are not all shipped to the
app, so ingestion embeds a compact copy of these links into data/extracted.
"""
folder = _find_case_folder(case_id)
if not folder:
return []
links: list[dict] = []
links.extend(_load_override_links(case_id))
_add_sources_txt(folder, links)
_add_sources_json(folder, links)
_add_dois_txt(folder, links)
return _dedupe_links(links)
|