File size: 15,005 Bytes
66227f0
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
"""Multi-format structure-aware document morphing and table-header-preserving chunker."""

from __future__ import annotations

import csv
import hashlib
import io
import json
import re
from dataclasses import asdict, dataclass, field
from html.parser import HTMLParser
from pathlib import Path
from typing import Any


@dataclass(frozen=True)
class DocChunk:
    """A structure-preserving document chunk with section breadcrumbs and neighbor links."""

    chunk_id: str
    doc_id: str
    chunk_index: int
    section_path: tuple[str, ...]
    content: str
    kind: str = "prose"  # "prose" | "table" | "code"
    namespace: str = "default"
    metadata: dict[str, Any] = field(default_factory=dict)

    @property
    def breadcrumb_str(self) -> str:
        """Human-readable section breadcrumb."""
        return " > ".join(self.section_path) if self.section_path else self.doc_id

    def to_dict(self) -> dict[str, Any]:
        """Serialize chunk to a JSON-compatible dictionary."""
        data = asdict(self)
        data["section_path"] = list(self.section_path)
        return data

    @classmethod
    def from_dict(cls, data: dict[str, Any]) -> DocChunk:
        """Deserialize chunk from a dictionary."""
        return cls(
            chunk_id=str(data["chunk_id"]),
            doc_id=str(data["doc_id"]),
            chunk_index=int(data.get("chunk_index", 0)),
            section_path=tuple(data.get("section_path") or ()),
            content=str(data["content"]),
            kind=str(data.get("kind", "prose")),
            namespace=str(data.get("namespace", "default")),
            metadata=dict(data.get("metadata") or {}),
        )


class _HTMLToMarkdownParser(HTMLParser):
    """Lightweight HTML-to-Markdown converter preserving headings, tables, code blocks, and lists."""

    _IGNORED_TAGS = frozenset({"script", "style", "noscript", "svg", "head"})

    def __init__(self) -> None:
        super().__init__(convert_charrefs=True)
        self._out: list[str] = []
        self._ignore_depth = 0
        self._in_pre = False
        self._in_table = False
        self._table_rows: list[list[str]] = []
        self._current_row: list[str] = []
        self._current_cell: list[str] = []
        self._in_cell = False

    def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
        t = tag.lower()
        if t in self._IGNORED_TAGS:
            self._ignore_depth += 1
            return
        if self._ignore_depth > 0:
            return

        if t in {"h1", "h2", "h3", "h4", "h5", "h6"}:
            level = int(t[1])
            self._out.append(f"\n\n{'#' * level} ")
        elif t == "p" or t == "div" or t == "section" or t == "article":
            self._out.append("\n\n")
        elif t == "br":
            self._out.append("\n")
        elif t == "li":
            self._out.append("\n- ")
        elif t == "pre":
            self._in_pre = True
            self._out.append("\n\n```\n")
        elif t == "code" and not self._in_pre:
            self._out.append("`")
        elif t == "table":
            self._in_table = True
            self._table_rows = []
        elif t == "tr" and self._in_table:
            self._current_row = []
        elif t in {"th", "td"} and self._in_table:
            self._in_cell = True
            self._current_cell = []

    def handle_endtag(self, tag: str) -> None:
        t = tag.lower()
        if t in self._IGNORED_TAGS:
            self._ignore_depth = max(0, self._ignore_depth - 1)
            return
        if self._ignore_depth > 0:
            return

        if t in {"h1", "h2", "h3", "h4", "h5", "h6"}:
            self._out.append("\n\n")
        elif t == "pre":
            self._in_pre = False
            self._out.append("\n```\n\n")
        elif t == "code" and not self._in_pre:
            self._out.append("`")
        elif t in {"th", "td"} and self._in_table and self._in_cell:
            cell_text = " ".join("".join(self._current_cell).split()).replace("|", "\\|")
            self._current_row.append(cell_text)
            self._in_cell = False
        elif t == "tr" and self._in_table:
            if any(self._current_row):
                self._table_rows.append(self._current_row)
            self._current_row = []
        elif t == "table" and self._in_table:
            self._in_table = False
            if self._table_rows:
                cols = max(len(r) for r in self._table_rows)
                padded = [r + [""] * (cols - len(r)) for r in self._table_rows]
                header = "| " + " | ".join(padded[0]) + " |"
                sep = "| " + " | ".join(["---"] * cols) + " |"
                body = ["| " + " | ".join(r) + " |" for r in padded[1:]]
                self._out.append("\n\n" + "\n".join([header, sep, *body]) + "\n\n")
            self._table_rows = []

    def handle_data(self, data: str) -> None:
        if self._ignore_depth > 0:
            return
        if self._in_cell:
            self._current_cell.append(data)
        elif self._in_pre:
            self._out.append(data)
        else:
            collapsed = re.sub(r"[ \t\r\n]+", " ", data)
            self._out.append(collapsed)

    def get_markdown(self) -> str:
        # Flush any trailing unclosed table or cell
        if self._in_table:
            if self._in_cell:
                cell_text = " ".join("".join(self._current_cell).split()).replace("|", "\\|")
                self._current_row.append(cell_text)
                self._in_cell = False
            if any(self._current_row):
                self._table_rows.append(self._current_row)
                self._current_row = []
            if self._table_rows:
                cols = max(len(r) for r in self._table_rows)
                padded = [r + [""] * (cols - len(r)) for r in self._table_rows]
                header = "| " + " | ".join(padded[0]) + " |"
                sep = "| " + " | ".join(["---"] * cols) + " |"
                body = ["| " + " | ".join(r) + " |" for r in padded[1:]]
                self._out.append("\n\n" + "\n".join([header, sep, *body]) + "\n\n")
            self._in_table = False
        if self._in_pre:
            self._out.append("\n```\n\n")
            self._in_pre = False

        raw = "".join(self._out)
        return re.sub(r"\n{3,}", "\n\n", raw).strip()


def html_to_markdown(html_text: str) -> str:
    """Convert raw HTML into structured Markdown preserving headings, tables, and code blocks."""
    parser = _HTMLToMarkdownParser()
    parser.feed(html_text)
    parser.close()
    return parser.get_markdown()


def csv_to_markdown_table(csv_text: str, delimiter: str = ",") -> str:
    """Convert CSV/TSV text into a normalized Markdown table."""
    reader = csv.reader(io.StringIO(csv_text.strip()), delimiter=delimiter)
    rows = [[cell.strip().replace("|", "\\|") for cell in row] for row in reader if any(c.strip() for c in row)]
    if not rows:
        return ""
    cols = max(len(r) for r in rows)
    padded = [r + [""] * (cols - len(r)) for r in rows]
    header = "| " + " | ".join(padded[0]) + " |"
    sep = "| " + " | ".join(["---"] * cols) + " |"
    body = ["| " + " | ".join(r) + " |" for r in padded[1:]]
    return "\n".join([header, sep, *body])


_HEADING_RE = re.compile(r"^(#{1,6})\s+(.+?)\s*$")
_FENCE_RE = re.compile(r"^```")
_TABLE_ROW_RE = re.compile(r"^\|.*\|$")
_TABLE_SEP_RE = re.compile(r"^\|\s*:?-{2,}:?\s*(\|\s*:?-{2,}:?\s*)*\|?$")


class DocumentMorpher:
    """Structure-aware document chunker that preserves code blocks and repeats table headers across splits."""

    SUPPORTED_EXTENSIONS = frozenset({".md", ".markdown", ".txt", ".html", ".htm", ".csv", ".tsv", ".json"})

    def __init__(self, max_chars: int = 1000) -> None:
        if max_chars < 80:
            raise ValueError("max_chars must be >= 80")
        self.max_chars = max_chars

    def morph_text(
        self,
        doc_id: str,
        text: str,
        *,
        fmt: str = "markdown",
        namespace: str = "default",
        metadata: dict[str, Any] | None = None,
    ) -> list[DocChunk]:
        """Normalize text from markdown/html/csv/json/txt and split into structure-preserving chunks."""
        fmt_lower = fmt.lower().lstrip(".")
        if fmt_lower in {"html", "htm"}:
            normalized = html_to_markdown(text)
        elif fmt_lower == "csv":
            normalized = csv_to_markdown_table(text, delimiter=",")
        elif fmt_lower == "tsv":
            normalized = csv_to_markdown_table(text, delimiter="\t")
        elif fmt_lower == "json":
            try:
                parsed = json.loads(text)
                normalized = f"```json\n{json.dumps(parsed, ensure_ascii=False, indent=2)}\n```"
            except json.JSONDecodeError:
                normalized = text
        else:
            normalized = text

        return self._chunk_markdown(doc_id, normalized, namespace=namespace, base_metadata=metadata)

    def morph_file(
        self,
        path: str | Path,
        *,
        root_dir: str | Path | None = None,
        namespace: str = "default",
    ) -> list[DocChunk]:
        """Parse and chunk a single file on disk."""
        file_path = Path(path).resolve()
        rel_id = str(file_path.relative_to(Path(root_dir).resolve())) if root_dir else file_path.name
        ext = file_path.suffix.lower()
        raw_text = file_path.read_text(encoding="utf-8", errors="replace")
        return self.morph_text(
            doc_id=rel_id.replace("\\", "/"),
            text=raw_text,
            fmt=ext,
            namespace=namespace,
            metadata={"source_path": str(file_path)},
        )

    def morph_directory(
        self,
        directory: str | Path,
        *,
        namespace: str = "default",
    ) -> list[DocChunk]:
        """Recursively parse and chunk all supported documents in a directory."""
        root = Path(directory).resolve()
        if root.is_file():
            return self.morph_file(root, namespace=namespace)

        all_chunks: list[DocChunk] = []
        for path in sorted(root.rglob("*")):
            if any(part.startswith(".") for part in path.relative_to(root).parts):
                continue
            if path.is_file() and path.suffix.lower() in self.SUPPORTED_EXTENSIONS:
                all_chunks.extend(self.morph_file(path, root_dir=root, namespace=namespace))
        return all_chunks

    def _chunk_markdown(
        self,
        doc_id: str,
        text: str,
        *,
        namespace: str,
        base_metadata: dict[str, Any] | None,
    ) -> list[DocChunk]:
        meta = dict(base_metadata or {})
        heading_stack: list[tuple[int, str]] = []
        chunks: list[DocChunk] = []
        prose_buf: list[str] = []

        lines = text.splitlines()
        idx = 0

        def current_section() -> tuple[str, ...]:
            return tuple(title for _, title in heading_stack)

        def emit(content: str, kind: str) -> None:
            cleaned = content.strip()
            if not cleaned:
                return
            sec = current_section()
            chunk_idx = len(chunks)
            digest = hashlib.sha256(
                f"{namespace}|{doc_id}|{chunk_idx}|{'/'.join(sec)}|{kind}|{cleaned}".encode("utf-8")
            ).hexdigest()[:12]
            chunks.append(
                DocChunk(
                    chunk_id=f"{doc_id}#{chunk_idx}:{digest}",
                    doc_id=doc_id,
                    chunk_index=chunk_idx,
                    section_path=sec,
                    content=cleaned,
                    kind=kind,
                    namespace=namespace,
                    metadata=dict(meta),
                )
            )

        def flush_prose() -> None:
            if not prose_buf:
                return
            joined = "\n".join(prose_buf).strip()
            prose_buf.clear()
            if not joined:
                return
            while len(joined) > self.max_chars:
                split_at = joined.rfind("\n\n", 0, self.max_chars)
                if split_at <= 0:
                    split_at = joined.rfind("\n", 0, self.max_chars)
                if split_at <= 0:
                    split_at = self.max_chars
                emit(joined[:split_at], "prose")
                joined = joined[split_at:].lstrip()
            if joined:
                emit(joined, "prose")

        def emit_table(table_lines: list[str]) -> None:
            full_table = "\n".join(table_lines).strip()
            if len(full_table) <= self.max_chars or len(table_lines) <= 2:
                emit(full_table, "table")
                return

            has_header = len(table_lines) >= 2 and bool(_TABLE_SEP_RE.match(table_lines[1].strip()))
            header_prefix = table_lines[:2] if has_header else []
            data_rows = table_lines[2:] if has_header else table_lines

            current_batch = list(header_prefix)
            current_len = sum(len(r) + 1 for r in current_batch)

            for row in data_rows:
                row_len = len(row) + 1
                if len(current_batch) > len(header_prefix) and (current_len + row_len > self.max_chars):
                    emit("\n".join(current_batch), "table")
                    current_batch = list(header_prefix)
                    current_len = sum(len(r) + 1 for r in current_batch)
                current_batch.append(row)
                current_len += row_len

            if len(current_batch) > len(header_prefix):
                emit("\n".join(current_batch), "table")

        while idx < len(lines):
            line = lines[idx]
            heading_match = _HEADING_RE.match(line)
            if heading_match:
                flush_prose()
                level = len(heading_match.group(1))
                title = heading_match.group(2).strip()
                while heading_stack and heading_stack[-1][0] >= level:
                    heading_stack.pop()
                heading_stack.append((level, title))
                idx += 1
                continue

            if _FENCE_RE.match(line.strip()):
                flush_prose()
                code_lines = [line]
                idx += 1
                while idx < len(lines):
                    code_lines.append(lines[idx])
                    if _FENCE_RE.match(lines[idx].strip()):
                        idx += 1
                        break
                    idx += 1
                emit("\n".join(code_lines), "code")
                continue

            if _TABLE_ROW_RE.match(line.strip()):
                flush_prose()
                table_lines = [line]
                idx += 1
                while idx < len(lines) and _TABLE_ROW_RE.match(lines[idx].strip()):
                    table_lines.append(lines[idx])
                    idx += 1
                emit_table(table_lines)
                continue

            prose_buf.append(line)
            idx += 1

        flush_prose()
        return chunks