File size: 5,461 Bytes
51b3964
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
0c2e8f6
51b3964
 
 
 
 
 
 
0c2e8f6
 
 
 
31a06e1
0c2e8f6
51b3964
 
 
 
 
 
 
 
 
 
 
 
318732d
 
 
51b3964
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
318732d
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
51b3964
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
318732d
 
51b3964
 
 
 
 
 
 
 
 
31a06e1
51b3964
 
 
 
 
 
 
 
 
 
 
 
 
31a06e1
51b3964
0c2e8f6
51b3964
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
"""
create_summary_chunks.py β€” Extract Case Study Overview sections as summary chunks.

Reads each CS*_Ver2.md, pulls the CASE STUDY OVERVIEW section (the rich intro block),
and writes data/extracted/case_summaries.json β€” one chunk per case study.

These chunks are indexed alongside the section-level chunks so summary/comparison
queries have a concise, self-contained chunk for every case study.

Usage:
    python3 ingest/create_summary_chunks.py
"""

from __future__ import annotations

import json
import re
import sys
from pathlib import Path

REPO_ROOT = Path(__file__).resolve().parent.parent
DATA_RAW = REPO_ROOT / "data" / "raw"
EXTRACTED_DIR = REPO_ROOT / "data" / "extracted"
OUT_FILE = EXTRACTED_DIR / "case_summaries.json"

if str(REPO_ROOT) not in sys.path:
    sys.path.insert(0, str(REPO_ROOT))

from ingest.source_links import load_source_links
from ingest.geo_metadata import geographic_metadata

# Match overview section β€” handles #, ##, and variations in heading text
_OVERVIEW_RE = re.compile(
    r"#{1,2}\s+CASE STUDY OVERVIEW\s*\n(.*?)(?=\n#{1,2}\s+(?:Is there|Date,|What is|Was this|Who or|Other notes|CODING|General Info)|\n---\n#{1,2}\s+Is there|\Z)",
    re.IGNORECASE | re.DOTALL,
)

# Fallback: grab everything before the first Q&A question
_PREAMBLE_RE = re.compile(
    r"^(.*?)(?=\n#{1,3}\s+(?:Is there any evidence|Date,\s+Number|What is the occurrence|General Information|Case Identification|CODING QUESTIONS))",
    re.IGNORECASE | re.DOTALL,
)

_H2_RE = re.compile(r"^##\s+(.+)$", re.MULTILINE)
_FALLBACK_SECTION_LIMIT = 6000


def _find_case_folder(case_id: str) -> Path | None:
    for p in DATA_RAW.iterdir():
        if p.is_dir() and re.match(rf"^{re.escape(case_id)}($|-)", p.name):
            return p
    return None


def _load_case_meta(case_id: str) -> dict:
    folder = _find_case_folder(case_id)
    if folder:
        meta_path = folder / "case_meta.json"
        if meta_path.exists():
            return json.loads(meta_path.read_text())
    return {"case_id": case_id, "name": case_id, "location": "Unknown", "country": "Unknown"}


def _all_case_ids() -> list[str]:
    ids = []
    for p in DATA_RAW.iterdir():
        if p.is_dir():
            m = re.match(r"^(CS\d+)", p.name)
            if m:
                ids.append(m.group(1))
    return sorted(set(ids), key=lambda x: int(x[2:]))


def _fallback_opening_sections(text: str) -> str:
    """Use opening memo sections when a Ver2 lacks a Case Study Overview heading."""
    matches = list(_H2_RE.finditer(text))
    if not matches:
        return ""

    pieces = []
    for i, match in enumerate(matches):
        heading = match.group(1).strip()
        if not re.search(r"case identification|general information", heading, re.I):
            continue
        start = match.end()
        end = matches[i + 1].start() if i + 1 < len(matches) else len(text)
        body = text[start:end].strip()
        if body:
            pieces.append(f"## {heading}\n\n{body}")
        if len("\n\n".join(pieces)) >= _FALLBACK_SECTION_LIMIT:
            break

    return "\n\n".join(pieces)[:_FALLBACK_SECTION_LIMIT].strip()


def build_summary_chunks() -> list[dict]:
    chunks = []
    for case_id in _all_case_ids():
        folder = _find_case_folder(case_id)
        if not folder:
            continue
        ver2_path = folder / "pipeline_output" / f"{case_id}_Ver2.md"
        if not ver2_path.exists():
            continue

        text = ver2_path.read_text(encoding="utf-8")
        m = _OVERVIEW_RE.search(text)
        if m:
            overview_text = m.group(1).strip()
        else:
            # Fallback: grab preamble before Q&A questions
            m2 = _PREAMBLE_RE.match(text)
            overview_text = m2.group(1).strip() if m2 else ""
            if len(overview_text) < 100:
                overview_text = _fallback_opening_sections(text)

        if not overview_text:
            print(f"  [{case_id}] No CASE STUDY OVERVIEW section found β€” skipping.")
            continue
        if len(overview_text) < 100:
            print(f"  [{case_id}] Overview too short β€” skipping.")
            continue

        meta = _load_case_meta(case_id)
        geo = geographic_metadata(meta.get("location", ""), meta.get("country", ""))
        full_text = (
            f"Case study: {meta.get('name', case_id)} "
            f"({meta.get('location', '')}, {meta.get('country', '')})\n"
            f"Section: Case Study Overview (Summary)\n\n"
            f"{overview_text}"
        )

        chunks.append({
            "case_id": case_id,
            "chunk_index": 999,
            "section": "overview_summary",
            "location": meta.get("location", ""),
            "country": meta.get("country", ""),
            **geo,
            "source": f"Pipeline Ver2 β€” {meta.get('name', case_id)}",
            "source_links": load_source_links(case_id),
            "text": full_text,
        })
        print(f"  [{case_id}] Extracted overview ({len(overview_text)} chars)")

    return chunks


def main():
    EXTRACTED_DIR.mkdir(parents=True, exist_ok=True)
    print("Extracting Case Study Overview sections...\n")
    chunks = build_summary_chunks()
    OUT_FILE.write_text(json.dumps(chunks, indent=2, ensure_ascii=False))
    print(f"\nWrote {len(chunks)} summary chunks β†’ {OUT_FILE.relative_to(REPO_ROOT)}")
    print("Next: python3 ingest/embed_and_index.py")


if __name__ == "__main__":
    main()