Spaces:
Sleeping
Sleeping
Download ingest/create_summary_chunks.py from UWRFR/UWaterlooRFRChatBot: direct link, hf CLI and curl.
- Browser
- Download file 5.46 kB
-
https://huggingface.co/spaces/UWRFR/UWaterlooRFRChatBot/resolve/main/ingest/create_summary_chunks.py
- Command line
-
hf download hf://spaces/UWRFR/UWaterlooRFRChatBot/ingest/create_summary_chunks.py
-
curl -L -o create_summary_chunks.py https://huggingface.co/spaces/UWRFR/UWaterlooRFRChatBot/resolve/main/ingest/create_summary_chunks.py
5.46 kB
| """ | |
| create_summary_chunks.py β Extract Case Study Overview sections as summary chunks. | |
| Reads each CS*_Ver2.md, pulls the CASE STUDY OVERVIEW section (the rich intro block), | |
| and writes data/extracted/case_summaries.json β one chunk per case study. | |
| These chunks are indexed alongside the section-level chunks so summary/comparison | |
| queries have a concise, self-contained chunk for every case study. | |
| Usage: | |
| python3 ingest/create_summary_chunks.py | |
| """ | |
| from __future__ import annotations | |
| import json | |
| import re | |
| import sys | |
| from pathlib import Path | |
| REPO_ROOT = Path(__file__).resolve().parent.parent | |
| DATA_RAW = REPO_ROOT / "data" / "raw" | |
| EXTRACTED_DIR = REPO_ROOT / "data" / "extracted" | |
| OUT_FILE = EXTRACTED_DIR / "case_summaries.json" | |
| if str(REPO_ROOT) not in sys.path: | |
| sys.path.insert(0, str(REPO_ROOT)) | |
| from ingest.source_links import load_source_links | |
| from ingest.geo_metadata import geographic_metadata | |
| # Match overview section β handles #, ##, and variations in heading text | |
| _OVERVIEW_RE = re.compile( | |
| r"#{1,2}\s+CASE STUDY OVERVIEW\s*\n(.*?)(?=\n#{1,2}\s+(?:Is there|Date,|What is|Was this|Who or|Other notes|CODING|General Info)|\n---\n#{1,2}\s+Is there|\Z)", | |
| re.IGNORECASE | re.DOTALL, | |
| ) | |
| # Fallback: grab everything before the first Q&A question | |
| _PREAMBLE_RE = re.compile( | |
| r"^(.*?)(?=\n#{1,3}\s+(?:Is there any evidence|Date,\s+Number|What is the occurrence|General Information|Case Identification|CODING QUESTIONS))", | |
| re.IGNORECASE | re.DOTALL, | |
| ) | |
| _H2_RE = re.compile(r"^##\s+(.+)$", re.MULTILINE) | |
| _FALLBACK_SECTION_LIMIT = 6000 | |
| def _find_case_folder(case_id: str) -> Path | None: | |
| for p in DATA_RAW.iterdir(): | |
| if p.is_dir() and re.match(rf"^{re.escape(case_id)}($|-)", p.name): | |
| return p | |
| return None | |
| def _load_case_meta(case_id: str) -> dict: | |
| folder = _find_case_folder(case_id) | |
| if folder: | |
| meta_path = folder / "case_meta.json" | |
| if meta_path.exists(): | |
| return json.loads(meta_path.read_text()) | |
| return {"case_id": case_id, "name": case_id, "location": "Unknown", "country": "Unknown"} | |
| def _all_case_ids() -> list[str]: | |
| ids = [] | |
| for p in DATA_RAW.iterdir(): | |
| if p.is_dir(): | |
| m = re.match(r"^(CS\d+)", p.name) | |
| if m: | |
| ids.append(m.group(1)) | |
| return sorted(set(ids), key=lambda x: int(x[2:])) | |
| def _fallback_opening_sections(text: str) -> str: | |
| """Use opening memo sections when a Ver2 lacks a Case Study Overview heading.""" | |
| matches = list(_H2_RE.finditer(text)) | |
| if not matches: | |
| return "" | |
| pieces = [] | |
| for i, match in enumerate(matches): | |
| heading = match.group(1).strip() | |
| if not re.search(r"case identification|general information", heading, re.I): | |
| continue | |
| start = match.end() | |
| end = matches[i + 1].start() if i + 1 < len(matches) else len(text) | |
| body = text[start:end].strip() | |
| if body: | |
| pieces.append(f"## {heading}\n\n{body}") | |
| if len("\n\n".join(pieces)) >= _FALLBACK_SECTION_LIMIT: | |
| break | |
| return "\n\n".join(pieces)[:_FALLBACK_SECTION_LIMIT].strip() | |
| def build_summary_chunks() -> list[dict]: | |
| chunks = [] | |
| for case_id in _all_case_ids(): | |
| folder = _find_case_folder(case_id) | |
| if not folder: | |
| continue | |
| ver2_path = folder / "pipeline_output" / f"{case_id}_Ver2.md" | |
| if not ver2_path.exists(): | |
| continue | |
| text = ver2_path.read_text(encoding="utf-8") | |
| m = _OVERVIEW_RE.search(text) | |
| if m: | |
| overview_text = m.group(1).strip() | |
| else: | |
| # Fallback: grab preamble before Q&A questions | |
| m2 = _PREAMBLE_RE.match(text) | |
| overview_text = m2.group(1).strip() if m2 else "" | |
| if len(overview_text) < 100: | |
| overview_text = _fallback_opening_sections(text) | |
| if not overview_text: | |
| print(f" [{case_id}] No CASE STUDY OVERVIEW section found β skipping.") | |
| continue | |
| if len(overview_text) < 100: | |
| print(f" [{case_id}] Overview too short β skipping.") | |
| continue | |
| meta = _load_case_meta(case_id) | |
| geo = geographic_metadata(meta.get("location", ""), meta.get("country", "")) | |
| full_text = ( | |
| f"Case study: {meta.get('name', case_id)} " | |
| f"({meta.get('location', '')}, {meta.get('country', '')})\n" | |
| f"Section: Case Study Overview (Summary)\n\n" | |
| f"{overview_text}" | |
| ) | |
| chunks.append({ | |
| "case_id": case_id, | |
| "chunk_index": 999, | |
| "section": "overview_summary", | |
| "location": meta.get("location", ""), | |
| "country": meta.get("country", ""), | |
| **geo, | |
| "source": f"Pipeline Ver2 β {meta.get('name', case_id)}", | |
| "source_links": load_source_links(case_id), | |
| "text": full_text, | |
| }) | |
| print(f" [{case_id}] Extracted overview ({len(overview_text)} chars)") | |
| return chunks | |
| def main(): | |
| EXTRACTED_DIR.mkdir(parents=True, exist_ok=True) | |
| print("Extracting Case Study Overview sections...\n") | |
| chunks = build_summary_chunks() | |
| OUT_FILE.write_text(json.dumps(chunks, indent=2, ensure_ascii=False)) | |
| print(f"\nWrote {len(chunks)} summary chunks β {OUT_FILE.relative_to(REPO_ROOT)}") | |
| print("Next: python3 ingest/embed_and_index.py") | |
| if __name__ == "__main__": | |
| main() | |