Spaces:
Running on Zero
Running on Zero
File size: 3,977 Bytes
53d9817 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 | from __future__ import annotations
from collections.abc import Iterator
from pathlib import Path
from typing import ClassVar
from selectolax.parser import HTMLParser
from universal_parser.core.router import register
from universal_parser.core.schema import Element, TableData
from universal_parser.core.sniffer import FileType
from universal_parser.extractors.base import BaseExtractor
@register
class HTMLExtractor(BaseExtractor):
"""
Ultra-fast HTML/XHTML extractor using selectolax.
Handles:
- Headings (h1 - h6) preserving document tree hierarchy
- Text paragraphs (<p>, <li>)
- HTML <table> tags -> TableData + Markdown tables
- Strips script, style, and navigation junk tags
"""
supported_types: ClassVar[list[FileType]] = [FileType.HTML]
def stream(self, path: str | Path) -> Iterator[Element]:
path_obj = Path(path)
try:
with open(path_obj, encoding="utf-8", errors="replace") as f:
html_content = f.read()
if not html_content.strip():
return
parser = HTMLParser(html_content)
# Strip noise (script, style, svg)
for tag in parser.css("script, style, noscript, svg"):
tag.decompose()
body = parser.body or parser.root
if body is None:
return
for node in body.iter():
tag_name = node.tag.lower() if node.tag else ""
# Headings
if tag_name in ("h1", "h2", "h3", "h4", "h5", "h6"):
level = int(tag_name[1])
text = node.text(strip=True)
if text:
yield Element(
type="heading",
level=level,
text=text,
markdown_repr=f"{'#' * level} {text}",
confidence=1.0,
)
# Paragraphs and Lists
elif tag_name in ("p", "li"):
text = node.text(strip=True)
if text:
prefix = "- " if tag_name == "li" else ""
elem_type = "list_item" if tag_name == "li" else "paragraph"
yield Element(
type=elem_type,
text=text,
markdown_repr=f"{prefix}{text}",
confidence=1.0,
)
# Tables
elif tag_name == "table":
headers = []
rows = []
for th in node.css("th"):
h_text = th.text(strip=True)
if h_text:
headers.append(h_text)
for tr in node.css("tr"):
tds = [td.text(strip=True) for td in tr.css("td")]
if any(tds):
rows.append(tds)
if not headers and rows:
headers = [f"Column_{i + 1}" for i in range(len(rows[0]))]
if headers or rows:
md_header = "| " + " | ".join(headers) + " |"
md_separator = "| " + " | ".join(["---"] * len(headers)) + " |"
md_rows = ["| " + " | ".join(r) + " |" for r in rows]
markdown_repr = "\n".join([md_header, md_separator] + md_rows)
yield Element(
type="table",
text=path_obj.name,
data=TableData(headers=headers, rows=rows),
markdown_repr=markdown_repr,
confidence=1.0,
)
except Exception: # noqa: BLE001
return
|