Spaces:
Sleeping
Sleeping
File size: 5,461 Bytes
51b3964 0c2e8f6 51b3964 0c2e8f6 31a06e1 0c2e8f6 51b3964 318732d 51b3964 318732d 51b3964 318732d 51b3964 31a06e1 51b3964 31a06e1 51b3964 0c2e8f6 51b3964 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 | """
create_summary_chunks.py β Extract Case Study Overview sections as summary chunks.
Reads each CS*_Ver2.md, pulls the CASE STUDY OVERVIEW section (the rich intro block),
and writes data/extracted/case_summaries.json β one chunk per case study.
These chunks are indexed alongside the section-level chunks so summary/comparison
queries have a concise, self-contained chunk for every case study.
Usage:
python3 ingest/create_summary_chunks.py
"""
from __future__ import annotations
import json
import re
import sys
from pathlib import Path
REPO_ROOT = Path(__file__).resolve().parent.parent
DATA_RAW = REPO_ROOT / "data" / "raw"
EXTRACTED_DIR = REPO_ROOT / "data" / "extracted"
OUT_FILE = EXTRACTED_DIR / "case_summaries.json"
if str(REPO_ROOT) not in sys.path:
sys.path.insert(0, str(REPO_ROOT))
from ingest.source_links import load_source_links
from ingest.geo_metadata import geographic_metadata
# Match overview section β handles #, ##, and variations in heading text
_OVERVIEW_RE = re.compile(
r"#{1,2}\s+CASE STUDY OVERVIEW\s*\n(.*?)(?=\n#{1,2}\s+(?:Is there|Date,|What is|Was this|Who or|Other notes|CODING|General Info)|\n---\n#{1,2}\s+Is there|\Z)",
re.IGNORECASE | re.DOTALL,
)
# Fallback: grab everything before the first Q&A question
_PREAMBLE_RE = re.compile(
r"^(.*?)(?=\n#{1,3}\s+(?:Is there any evidence|Date,\s+Number|What is the occurrence|General Information|Case Identification|CODING QUESTIONS))",
re.IGNORECASE | re.DOTALL,
)
_H2_RE = re.compile(r"^##\s+(.+)$", re.MULTILINE)
_FALLBACK_SECTION_LIMIT = 6000
def _find_case_folder(case_id: str) -> Path | None:
for p in DATA_RAW.iterdir():
if p.is_dir() and re.match(rf"^{re.escape(case_id)}($|-)", p.name):
return p
return None
def _load_case_meta(case_id: str) -> dict:
folder = _find_case_folder(case_id)
if folder:
meta_path = folder / "case_meta.json"
if meta_path.exists():
return json.loads(meta_path.read_text())
return {"case_id": case_id, "name": case_id, "location": "Unknown", "country": "Unknown"}
def _all_case_ids() -> list[str]:
ids = []
for p in DATA_RAW.iterdir():
if p.is_dir():
m = re.match(r"^(CS\d+)", p.name)
if m:
ids.append(m.group(1))
return sorted(set(ids), key=lambda x: int(x[2:]))
def _fallback_opening_sections(text: str) -> str:
"""Use opening memo sections when a Ver2 lacks a Case Study Overview heading."""
matches = list(_H2_RE.finditer(text))
if not matches:
return ""
pieces = []
for i, match in enumerate(matches):
heading = match.group(1).strip()
if not re.search(r"case identification|general information", heading, re.I):
continue
start = match.end()
end = matches[i + 1].start() if i + 1 < len(matches) else len(text)
body = text[start:end].strip()
if body:
pieces.append(f"## {heading}\n\n{body}")
if len("\n\n".join(pieces)) >= _FALLBACK_SECTION_LIMIT:
break
return "\n\n".join(pieces)[:_FALLBACK_SECTION_LIMIT].strip()
def build_summary_chunks() -> list[dict]:
chunks = []
for case_id in _all_case_ids():
folder = _find_case_folder(case_id)
if not folder:
continue
ver2_path = folder / "pipeline_output" / f"{case_id}_Ver2.md"
if not ver2_path.exists():
continue
text = ver2_path.read_text(encoding="utf-8")
m = _OVERVIEW_RE.search(text)
if m:
overview_text = m.group(1).strip()
else:
# Fallback: grab preamble before Q&A questions
m2 = _PREAMBLE_RE.match(text)
overview_text = m2.group(1).strip() if m2 else ""
if len(overview_text) < 100:
overview_text = _fallback_opening_sections(text)
if not overview_text:
print(f" [{case_id}] No CASE STUDY OVERVIEW section found β skipping.")
continue
if len(overview_text) < 100:
print(f" [{case_id}] Overview too short β skipping.")
continue
meta = _load_case_meta(case_id)
geo = geographic_metadata(meta.get("location", ""), meta.get("country", ""))
full_text = (
f"Case study: {meta.get('name', case_id)} "
f"({meta.get('location', '')}, {meta.get('country', '')})\n"
f"Section: Case Study Overview (Summary)\n\n"
f"{overview_text}"
)
chunks.append({
"case_id": case_id,
"chunk_index": 999,
"section": "overview_summary",
"location": meta.get("location", ""),
"country": meta.get("country", ""),
**geo,
"source": f"Pipeline Ver2 β {meta.get('name', case_id)}",
"source_links": load_source_links(case_id),
"text": full_text,
})
print(f" [{case_id}] Extracted overview ({len(overview_text)} chars)")
return chunks
def main():
EXTRACTED_DIR.mkdir(parents=True, exist_ok=True)
print("Extracting Case Study Overview sections...\n")
chunks = build_summary_chunks()
OUT_FILE.write_text(json.dumps(chunks, indent=2, ensure_ascii=False))
print(f"\nWrote {len(chunks)} summary chunks β {OUT_FILE.relative_to(REPO_ROOT)}")
print("Next: python3 ingest/embed_and_index.py")
if __name__ == "__main__":
main()
|