UWaterlooRFRChatBot / ingest /create_summary_chunks.py
Anna Zhou
Add deterministic geographic metadata queries
31a06e1
Raw History Blame Contribute Delete
5.46 kB
"""
create_summary_chunks.py β€” Extract Case Study Overview sections as summary chunks.
Reads each CS*_Ver2.md, pulls the CASE STUDY OVERVIEW section (the rich intro block),
and writes data/extracted/case_summaries.json β€” one chunk per case study.
These chunks are indexed alongside the section-level chunks so summary/comparison
queries have a concise, self-contained chunk for every case study.
Usage:
python3 ingest/create_summary_chunks.py
"""
from __future__ import annotations
import json
import re
import sys
from pathlib import Path
REPO_ROOT = Path(__file__).resolve().parent.parent
DATA_RAW = REPO_ROOT / "data" / "raw"
EXTRACTED_DIR = REPO_ROOT / "data" / "extracted"
OUT_FILE = EXTRACTED_DIR / "case_summaries.json"
if str(REPO_ROOT) not in sys.path:
sys.path.insert(0, str(REPO_ROOT))
from ingest.source_links import load_source_links
from ingest.geo_metadata import geographic_metadata
# Match overview section β€” handles #, ##, and variations in heading text
_OVERVIEW_RE = re.compile(
r"#{1,2}\s+CASE STUDY OVERVIEW\s*\n(.*?)(?=\n#{1,2}\s+(?:Is there|Date,|What is|Was this|Who or|Other notes|CODING|General Info)|\n---\n#{1,2}\s+Is there|\Z)",
re.IGNORECASE | re.DOTALL,
)
# Fallback: grab everything before the first Q&A question
_PREAMBLE_RE = re.compile(
r"^(.*?)(?=\n#{1,3}\s+(?:Is there any evidence|Date,\s+Number|What is the occurrence|General Information|Case Identification|CODING QUESTIONS))",
re.IGNORECASE | re.DOTALL,
)
_H2_RE = re.compile(r"^##\s+(.+)$", re.MULTILINE)
_FALLBACK_SECTION_LIMIT = 6000
def _find_case_folder(case_id: str) -> Path | None:
for p in DATA_RAW.iterdir():
if p.is_dir() and re.match(rf"^{re.escape(case_id)}($|-)", p.name):
return p
return None
def _load_case_meta(case_id: str) -> dict:
folder = _find_case_folder(case_id)
if folder:
meta_path = folder / "case_meta.json"
if meta_path.exists():
return json.loads(meta_path.read_text())
return {"case_id": case_id, "name": case_id, "location": "Unknown", "country": "Unknown"}
def _all_case_ids() -> list[str]:
ids = []
for p in DATA_RAW.iterdir():
if p.is_dir():
m = re.match(r"^(CS\d+)", p.name)
if m:
ids.append(m.group(1))
return sorted(set(ids), key=lambda x: int(x[2:]))
def _fallback_opening_sections(text: str) -> str:
"""Use opening memo sections when a Ver2 lacks a Case Study Overview heading."""
matches = list(_H2_RE.finditer(text))
if not matches:
return ""
pieces = []
for i, match in enumerate(matches):
heading = match.group(1).strip()
if not re.search(r"case identification|general information", heading, re.I):
continue
start = match.end()
end = matches[i + 1].start() if i + 1 < len(matches) else len(text)
body = text[start:end].strip()
if body:
pieces.append(f"## {heading}\n\n{body}")
if len("\n\n".join(pieces)) >= _FALLBACK_SECTION_LIMIT:
break
return "\n\n".join(pieces)[:_FALLBACK_SECTION_LIMIT].strip()
def build_summary_chunks() -> list[dict]:
chunks = []
for case_id in _all_case_ids():
folder = _find_case_folder(case_id)
if not folder:
continue
ver2_path = folder / "pipeline_output" / f"{case_id}_Ver2.md"
if not ver2_path.exists():
continue
text = ver2_path.read_text(encoding="utf-8")
m = _OVERVIEW_RE.search(text)
if m:
overview_text = m.group(1).strip()
else:
# Fallback: grab preamble before Q&A questions
m2 = _PREAMBLE_RE.match(text)
overview_text = m2.group(1).strip() if m2 else ""
if len(overview_text) < 100:
overview_text = _fallback_opening_sections(text)
if not overview_text:
print(f" [{case_id}] No CASE STUDY OVERVIEW section found β€” skipping.")
continue
if len(overview_text) < 100:
print(f" [{case_id}] Overview too short β€” skipping.")
continue
meta = _load_case_meta(case_id)
geo = geographic_metadata(meta.get("location", ""), meta.get("country", ""))
full_text = (
f"Case study: {meta.get('name', case_id)} "
f"({meta.get('location', '')}, {meta.get('country', '')})\n"
f"Section: Case Study Overview (Summary)\n\n"
f"{overview_text}"
)
chunks.append({
"case_id": case_id,
"chunk_index": 999,
"section": "overview_summary",
"location": meta.get("location", ""),
"country": meta.get("country", ""),
**geo,
"source": f"Pipeline Ver2 β€” {meta.get('name', case_id)}",
"source_links": load_source_links(case_id),
"text": full_text,
})
print(f" [{case_id}] Extracted overview ({len(overview_text)} chars)")
return chunks
def main():
EXTRACTED_DIR.mkdir(parents=True, exist_ok=True)
print("Extracting Case Study Overview sections...\n")
chunks = build_summary_chunks()
OUT_FILE.write_text(json.dumps(chunks, indent=2, ensure_ascii=False))
print(f"\nWrote {len(chunks)} summary chunks β†’ {OUT_FILE.relative_to(REPO_ROOT)}")
print("Next: python3 ingest/embed_and_index.py")
if __name__ == "__main__":
main()