Spaces:
Running
Running
Shoufa Zhang
Deploy DocMorph & SeekMorph interactive Studio and 5-layer production benchmark
66227f0 Download docmorph/parser.py from zhangshoufa/DocMorph: direct link, hf CLI and curl.
- Browser
- Download file 15 kB
-
https://huggingface.co/spaces/zhangshoufa/DocMorph/resolve/main/docmorph/parser.py
- Command line
-
hf download hf://spaces/zhangshoufa/DocMorph/docmorph/parser.py
-
curl -L -o parser.py https://huggingface.co/spaces/zhangshoufa/DocMorph/resolve/main/docmorph/parser.py
15 kB
| """Multi-format structure-aware document morphing and table-header-preserving chunker.""" | |
| from __future__ import annotations | |
| import csv | |
| import hashlib | |
| import io | |
| import json | |
| import re | |
| from dataclasses import asdict, dataclass, field | |
| from html.parser import HTMLParser | |
| from pathlib import Path | |
| from typing import Any | |
| class DocChunk: | |
| """A structure-preserving document chunk with section breadcrumbs and neighbor links.""" | |
| chunk_id: str | |
| doc_id: str | |
| chunk_index: int | |
| section_path: tuple[str, ...] | |
| content: str | |
| kind: str = "prose" # "prose" | "table" | "code" | |
| namespace: str = "default" | |
| metadata: dict[str, Any] = field(default_factory=dict) | |
| def breadcrumb_str(self) -> str: | |
| """Human-readable section breadcrumb.""" | |
| return " > ".join(self.section_path) if self.section_path else self.doc_id | |
| def to_dict(self) -> dict[str, Any]: | |
| """Serialize chunk to a JSON-compatible dictionary.""" | |
| data = asdict(self) | |
| data["section_path"] = list(self.section_path) | |
| return data | |
| def from_dict(cls, data: dict[str, Any]) -> DocChunk: | |
| """Deserialize chunk from a dictionary.""" | |
| return cls( | |
| chunk_id=str(data["chunk_id"]), | |
| doc_id=str(data["doc_id"]), | |
| chunk_index=int(data.get("chunk_index", 0)), | |
| section_path=tuple(data.get("section_path") or ()), | |
| content=str(data["content"]), | |
| kind=str(data.get("kind", "prose")), | |
| namespace=str(data.get("namespace", "default")), | |
| metadata=dict(data.get("metadata") or {}), | |
| ) | |
| class _HTMLToMarkdownParser(HTMLParser): | |
| """Lightweight HTML-to-Markdown converter preserving headings, tables, code blocks, and lists.""" | |
| _IGNORED_TAGS = frozenset({"script", "style", "noscript", "svg", "head"}) | |
| def __init__(self) -> None: | |
| super().__init__(convert_charrefs=True) | |
| self._out: list[str] = [] | |
| self._ignore_depth = 0 | |
| self._in_pre = False | |
| self._in_table = False | |
| self._table_rows: list[list[str]] = [] | |
| self._current_row: list[str] = [] | |
| self._current_cell: list[str] = [] | |
| self._in_cell = False | |
| def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None: | |
| t = tag.lower() | |
| if t in self._IGNORED_TAGS: | |
| self._ignore_depth += 1 | |
| return | |
| if self._ignore_depth > 0: | |
| return | |
| if t in {"h1", "h2", "h3", "h4", "h5", "h6"}: | |
| level = int(t[1]) | |
| self._out.append(f"\n\n{'#' * level} ") | |
| elif t == "p" or t == "div" or t == "section" or t == "article": | |
| self._out.append("\n\n") | |
| elif t == "br": | |
| self._out.append("\n") | |
| elif t == "li": | |
| self._out.append("\n- ") | |
| elif t == "pre": | |
| self._in_pre = True | |
| self._out.append("\n\n```\n") | |
| elif t == "code" and not self._in_pre: | |
| self._out.append("`") | |
| elif t == "table": | |
| self._in_table = True | |
| self._table_rows = [] | |
| elif t == "tr" and self._in_table: | |
| self._current_row = [] | |
| elif t in {"th", "td"} and self._in_table: | |
| self._in_cell = True | |
| self._current_cell = [] | |
| def handle_endtag(self, tag: str) -> None: | |
| t = tag.lower() | |
| if t in self._IGNORED_TAGS: | |
| self._ignore_depth = max(0, self._ignore_depth - 1) | |
| return | |
| if self._ignore_depth > 0: | |
| return | |
| if t in {"h1", "h2", "h3", "h4", "h5", "h6"}: | |
| self._out.append("\n\n") | |
| elif t == "pre": | |
| self._in_pre = False | |
| self._out.append("\n```\n\n") | |
| elif t == "code" and not self._in_pre: | |
| self._out.append("`") | |
| elif t in {"th", "td"} and self._in_table and self._in_cell: | |
| cell_text = " ".join("".join(self._current_cell).split()).replace("|", "\\|") | |
| self._current_row.append(cell_text) | |
| self._in_cell = False | |
| elif t == "tr" and self._in_table: | |
| if any(self._current_row): | |
| self._table_rows.append(self._current_row) | |
| self._current_row = [] | |
| elif t == "table" and self._in_table: | |
| self._in_table = False | |
| if self._table_rows: | |
| cols = max(len(r) for r in self._table_rows) | |
| padded = [r + [""] * (cols - len(r)) for r in self._table_rows] | |
| header = "| " + " | ".join(padded[0]) + " |" | |
| sep = "| " + " | ".join(["---"] * cols) + " |" | |
| body = ["| " + " | ".join(r) + " |" for r in padded[1:]] | |
| self._out.append("\n\n" + "\n".join([header, sep, *body]) + "\n\n") | |
| self._table_rows = [] | |
| def handle_data(self, data: str) -> None: | |
| if self._ignore_depth > 0: | |
| return | |
| if self._in_cell: | |
| self._current_cell.append(data) | |
| elif self._in_pre: | |
| self._out.append(data) | |
| else: | |
| collapsed = re.sub(r"[ \t\r\n]+", " ", data) | |
| self._out.append(collapsed) | |
| def get_markdown(self) -> str: | |
| # Flush any trailing unclosed table or cell | |
| if self._in_table: | |
| if self._in_cell: | |
| cell_text = " ".join("".join(self._current_cell).split()).replace("|", "\\|") | |
| self._current_row.append(cell_text) | |
| self._in_cell = False | |
| if any(self._current_row): | |
| self._table_rows.append(self._current_row) | |
| self._current_row = [] | |
| if self._table_rows: | |
| cols = max(len(r) for r in self._table_rows) | |
| padded = [r + [""] * (cols - len(r)) for r in self._table_rows] | |
| header = "| " + " | ".join(padded[0]) + " |" | |
| sep = "| " + " | ".join(["---"] * cols) + " |" | |
| body = ["| " + " | ".join(r) + " |" for r in padded[1:]] | |
| self._out.append("\n\n" + "\n".join([header, sep, *body]) + "\n\n") | |
| self._in_table = False | |
| if self._in_pre: | |
| self._out.append("\n```\n\n") | |
| self._in_pre = False | |
| raw = "".join(self._out) | |
| return re.sub(r"\n{3,}", "\n\n", raw).strip() | |
| def html_to_markdown(html_text: str) -> str: | |
| """Convert raw HTML into structured Markdown preserving headings, tables, and code blocks.""" | |
| parser = _HTMLToMarkdownParser() | |
| parser.feed(html_text) | |
| parser.close() | |
| return parser.get_markdown() | |
| def csv_to_markdown_table(csv_text: str, delimiter: str = ",") -> str: | |
| """Convert CSV/TSV text into a normalized Markdown table.""" | |
| reader = csv.reader(io.StringIO(csv_text.strip()), delimiter=delimiter) | |
| rows = [[cell.strip().replace("|", "\\|") for cell in row] for row in reader if any(c.strip() for c in row)] | |
| if not rows: | |
| return "" | |
| cols = max(len(r) for r in rows) | |
| padded = [r + [""] * (cols - len(r)) for r in rows] | |
| header = "| " + " | ".join(padded[0]) + " |" | |
| sep = "| " + " | ".join(["---"] * cols) + " |" | |
| body = ["| " + " | ".join(r) + " |" for r in padded[1:]] | |
| return "\n".join([header, sep, *body]) | |
| _HEADING_RE = re.compile(r"^(#{1,6})\s+(.+?)\s*$") | |
| _FENCE_RE = re.compile(r"^```") | |
| _TABLE_ROW_RE = re.compile(r"^\|.*\|$") | |
| _TABLE_SEP_RE = re.compile(r"^\|\s*:?-{2,}:?\s*(\|\s*:?-{2,}:?\s*)*\|?$") | |
| class DocumentMorpher: | |
| """Structure-aware document chunker that preserves code blocks and repeats table headers across splits.""" | |
| SUPPORTED_EXTENSIONS = frozenset({".md", ".markdown", ".txt", ".html", ".htm", ".csv", ".tsv", ".json"}) | |
| def __init__(self, max_chars: int = 1000) -> None: | |
| if max_chars < 80: | |
| raise ValueError("max_chars must be >= 80") | |
| self.max_chars = max_chars | |
| def morph_text( | |
| self, | |
| doc_id: str, | |
| text: str, | |
| *, | |
| fmt: str = "markdown", | |
| namespace: str = "default", | |
| metadata: dict[str, Any] | None = None, | |
| ) -> list[DocChunk]: | |
| """Normalize text from markdown/html/csv/json/txt and split into structure-preserving chunks.""" | |
| fmt_lower = fmt.lower().lstrip(".") | |
| if fmt_lower in {"html", "htm"}: | |
| normalized = html_to_markdown(text) | |
| elif fmt_lower == "csv": | |
| normalized = csv_to_markdown_table(text, delimiter=",") | |
| elif fmt_lower == "tsv": | |
| normalized = csv_to_markdown_table(text, delimiter="\t") | |
| elif fmt_lower == "json": | |
| try: | |
| parsed = json.loads(text) | |
| normalized = f"```json\n{json.dumps(parsed, ensure_ascii=False, indent=2)}\n```" | |
| except json.JSONDecodeError: | |
| normalized = text | |
| else: | |
| normalized = text | |
| return self._chunk_markdown(doc_id, normalized, namespace=namespace, base_metadata=metadata) | |
| def morph_file( | |
| self, | |
| path: str | Path, | |
| *, | |
| root_dir: str | Path | None = None, | |
| namespace: str = "default", | |
| ) -> list[DocChunk]: | |
| """Parse and chunk a single file on disk.""" | |
| file_path = Path(path).resolve() | |
| rel_id = str(file_path.relative_to(Path(root_dir).resolve())) if root_dir else file_path.name | |
| ext = file_path.suffix.lower() | |
| raw_text = file_path.read_text(encoding="utf-8", errors="replace") | |
| return self.morph_text( | |
| doc_id=rel_id.replace("\\", "/"), | |
| text=raw_text, | |
| fmt=ext, | |
| namespace=namespace, | |
| metadata={"source_path": str(file_path)}, | |
| ) | |
| def morph_directory( | |
| self, | |
| directory: str | Path, | |
| *, | |
| namespace: str = "default", | |
| ) -> list[DocChunk]: | |
| """Recursively parse and chunk all supported documents in a directory.""" | |
| root = Path(directory).resolve() | |
| if root.is_file(): | |
| return self.morph_file(root, namespace=namespace) | |
| all_chunks: list[DocChunk] = [] | |
| for path in sorted(root.rglob("*")): | |
| if any(part.startswith(".") for part in path.relative_to(root).parts): | |
| continue | |
| if path.is_file() and path.suffix.lower() in self.SUPPORTED_EXTENSIONS: | |
| all_chunks.extend(self.morph_file(path, root_dir=root, namespace=namespace)) | |
| return all_chunks | |
| def _chunk_markdown( | |
| self, | |
| doc_id: str, | |
| text: str, | |
| *, | |
| namespace: str, | |
| base_metadata: dict[str, Any] | None, | |
| ) -> list[DocChunk]: | |
| meta = dict(base_metadata or {}) | |
| heading_stack: list[tuple[int, str]] = [] | |
| chunks: list[DocChunk] = [] | |
| prose_buf: list[str] = [] | |
| lines = text.splitlines() | |
| idx = 0 | |
| def current_section() -> tuple[str, ...]: | |
| return tuple(title for _, title in heading_stack) | |
| def emit(content: str, kind: str) -> None: | |
| cleaned = content.strip() | |
| if not cleaned: | |
| return | |
| sec = current_section() | |
| chunk_idx = len(chunks) | |
| digest = hashlib.sha256( | |
| f"{namespace}|{doc_id}|{chunk_idx}|{'/'.join(sec)}|{kind}|{cleaned}".encode("utf-8") | |
| ).hexdigest()[:12] | |
| chunks.append( | |
| DocChunk( | |
| chunk_id=f"{doc_id}#{chunk_idx}:{digest}", | |
| doc_id=doc_id, | |
| chunk_index=chunk_idx, | |
| section_path=sec, | |
| content=cleaned, | |
| kind=kind, | |
| namespace=namespace, | |
| metadata=dict(meta), | |
| ) | |
| ) | |
| def flush_prose() -> None: | |
| if not prose_buf: | |
| return | |
| joined = "\n".join(prose_buf).strip() | |
| prose_buf.clear() | |
| if not joined: | |
| return | |
| while len(joined) > self.max_chars: | |
| split_at = joined.rfind("\n\n", 0, self.max_chars) | |
| if split_at <= 0: | |
| split_at = joined.rfind("\n", 0, self.max_chars) | |
| if split_at <= 0: | |
| split_at = self.max_chars | |
| emit(joined[:split_at], "prose") | |
| joined = joined[split_at:].lstrip() | |
| if joined: | |
| emit(joined, "prose") | |
| def emit_table(table_lines: list[str]) -> None: | |
| full_table = "\n".join(table_lines).strip() | |
| if len(full_table) <= self.max_chars or len(table_lines) <= 2: | |
| emit(full_table, "table") | |
| return | |
| has_header = len(table_lines) >= 2 and bool(_TABLE_SEP_RE.match(table_lines[1].strip())) | |
| header_prefix = table_lines[:2] if has_header else [] | |
| data_rows = table_lines[2:] if has_header else table_lines | |
| current_batch = list(header_prefix) | |
| current_len = sum(len(r) + 1 for r in current_batch) | |
| for row in data_rows: | |
| row_len = len(row) + 1 | |
| if len(current_batch) > len(header_prefix) and (current_len + row_len > self.max_chars): | |
| emit("\n".join(current_batch), "table") | |
| current_batch = list(header_prefix) | |
| current_len = sum(len(r) + 1 for r in current_batch) | |
| current_batch.append(row) | |
| current_len += row_len | |
| if len(current_batch) > len(header_prefix): | |
| emit("\n".join(current_batch), "table") | |
| while idx < len(lines): | |
| line = lines[idx] | |
| heading_match = _HEADING_RE.match(line) | |
| if heading_match: | |
| flush_prose() | |
| level = len(heading_match.group(1)) | |
| title = heading_match.group(2).strip() | |
| while heading_stack and heading_stack[-1][0] >= level: | |
| heading_stack.pop() | |
| heading_stack.append((level, title)) | |
| idx += 1 | |
| continue | |
| if _FENCE_RE.match(line.strip()): | |
| flush_prose() | |
| code_lines = [line] | |
| idx += 1 | |
| while idx < len(lines): | |
| code_lines.append(lines[idx]) | |
| if _FENCE_RE.match(lines[idx].strip()): | |
| idx += 1 | |
| break | |
| idx += 1 | |
| emit("\n".join(code_lines), "code") | |
| continue | |
| if _TABLE_ROW_RE.match(line.strip()): | |
| flush_prose() | |
| table_lines = [line] | |
| idx += 1 | |
| while idx < len(lines) and _TABLE_ROW_RE.match(lines[idx].strip()): | |
| table_lines.append(lines[idx]) | |
| idx += 1 | |
| emit_table(table_lines) | |
| continue | |
| prose_buf.append(line) | |
| idx += 1 | |
| flush_prose() | |
| return chunks | |