clairic-4bit / clairic_runtime /source_rows.py
lucataco's picture
Clairic, MLX 4-bit, with runtime, server and conversion script
1a48d6b verified
Raw History Blame Contribute Delete
5.08 kB
from __future__ import annotations
import csv
import io
import re
from collections.abc import Callable
from html.parser import HTMLParser
from typing import Any
_ONE_PER_ROW = re.compile(
"\\bone\\s+(?:[\\w-]+\\s+){0,5}per\\s+(?:[\\w-]+\\s+){0,5}row\\b", re.IGNORECASE
)
class _Table(HTMLParser):
def __init__(self) -> None:
super().__init__(convert_charrefs=False)
self.tables = self.depth = 0
self.rows: list[set[str]] = []
self.closed = False
self.row_header = False
self.in_cell = False
self.ambiguous = False
def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
if self.in_cell and tag in ("br", "div", "p", "ul", "ol", "li", "table"):
self.ambiguous = True
if tag == "table":
self.tables += 1
self.depth += 1
if self.depth == 1:
if tag == "tr":
self.rows.append(set())
elif tag in ("td", "th") and self.rows:
self.in_cell = True
if any(
(
key in ("rowspan", "colspan") and value != "1"
for (key, value) in attrs
)
):
self.ambiguous = True
self.rows[-1].add(tag)
if len(self.rows) == 1 and (
dict(attrs).get("scope") or ""
).casefold() in ("row", "rowgroup"):
self.row_header = True
def handle_endtag(self, tag: str) -> None:
if tag in ("td", "th"):
self.in_cell = False
if tag == "table":
self.depth -= 1
self.closed = self.depth == 0
def handle_data(self, data: str) -> None:
if (
self.in_cell
and len([line for line in data.splitlines() if line.strip()]) > 1
):
self.ambiguous = True
def table_rows(source: str) -> int | None:
if re.search("<table(?:\\s|>)", source, re.IGNORECASE):
parser = _Table()
try:
parser.feed(source)
parser.close()
except (ValueError, AssertionError):
return None
if (
parser.tables != 1
or not parser.closed
or parser.depth != 0
or (not parser.rows)
or (parser.rows[0] != {"th"})
or parser.row_header
or parser.ambiguous
):
return None
return len(parser.rows) - 1
lines = source.splitlines()
if len(lines) < 2:
return None
for delimiter in (",", "\t", ";", "|"):
if delimiter not in lines[0]:
continue
try:
rows = list(
csv.reader(io.StringIO(source), delimiter=delimiter, strict=True)
)
except csv.Error:
continue
if (
not rows
or len(rows[0]) < 2
or any((not row or len(row) != len(rows[0]) for row in rows))
or any((not cell.strip() or re.search("\\d", cell) for cell in rows[0]))
or (not any((re.search("\\d", cell) for row in rows[1:] for cell in row)))
):
continue
return len(rows) - 1
return None
def list_bounds(
source: str, fields: dict[str, dict[str, Any]], *, has_images: bool = False
) -> dict[str, int]:
if has_images:
return {}
parents = {}
for name, spec in fields.items():
instructions = spec.get("instructions", "")
if (
spec.get("type") == "list"
and spec.get("items") in ("text", "number")
and (not spec.get("knowledge"))
and _ONE_PER_ROW.search(instructions)
and (
not re.search(
"\\b(?:not|never|multiple|two|three)\\b|don't",
instructions,
re.IGNORECASE,
)
)
):
parents[name] = spec
count = table_rows(source) if parents else None
return {name: count for name in parents} if count is not None else {}
def bounded_predictions(
predictions: dict[str, dict[str, Any]],
fields: dict[str, dict[str, Any]],
bounds: dict[str, int],
item_name: Callable[[str, int], str],
) -> dict[str, dict[str, Any]]:
if not bounds:
return predictions
result = dict(predictions)
for parent, count in bounds.items():
name = item_name(parent, count + 1)
if name not in fields or name not in result:
continue
previous = result[name]
if previous.get("value") is None and previous.get("valid", True):
continue
result[name] = {
**previous,
"value": None,
"valid": True,
"complete": True,
"span": None,
"confidence": 1.0,
"log_probability": 0.0,
"confidence_method": "source_structure",
"grounding_method": "source_row_bound",
}
return result