from __future__ import annotations import csv import io import re from collections.abc import Callable from html.parser import HTMLParser from typing import Any _ONE_PER_ROW = re.compile( "\\bone\\s+(?:[\\w-]+\\s+){0,5}per\\s+(?:[\\w-]+\\s+){0,5}row\\b", re.IGNORECASE ) class _Table(HTMLParser): def __init__(self) -> None: super().__init__(convert_charrefs=False) self.tables = self.depth = 0 self.rows: list[set[str]] = [] self.closed = False self.row_header = False self.in_cell = False self.ambiguous = False def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None: if self.in_cell and tag in ("br", "div", "p", "ul", "ol", "li", "table"): self.ambiguous = True if tag == "table": self.tables += 1 self.depth += 1 if self.depth == 1: if tag == "tr": self.rows.append(set()) elif tag in ("td", "th") and self.rows: self.in_cell = True if any( ( key in ("rowspan", "colspan") and value != "1" for (key, value) in attrs ) ): self.ambiguous = True self.rows[-1].add(tag) if len(self.rows) == 1 and ( dict(attrs).get("scope") or "" ).casefold() in ("row", "rowgroup"): self.row_header = True def handle_endtag(self, tag: str) -> None: if tag in ("td", "th"): self.in_cell = False if tag == "table": self.depth -= 1 self.closed = self.depth == 0 def handle_data(self, data: str) -> None: if ( self.in_cell and len([line for line in data.splitlines() if line.strip()]) > 1 ): self.ambiguous = True def table_rows(source: str) -> int | None: if re.search(")", source, re.IGNORECASE): parser = _Table() try: parser.feed(source) parser.close() except (ValueError, AssertionError): return None if ( parser.tables != 1 or not parser.closed or parser.depth != 0 or (not parser.rows) or (parser.rows[0] != {"th"}) or parser.row_header or parser.ambiguous ): return None return len(parser.rows) - 1 lines = source.splitlines() if len(lines) < 2: return None for delimiter in (",", "\t", ";", "|"): if delimiter not in lines[0]: continue try: rows = list( csv.reader(io.StringIO(source), delimiter=delimiter, strict=True) ) except csv.Error: continue if ( not rows or len(rows[0]) < 2 or any((not row or len(row) != len(rows[0]) for row in rows)) or any((not cell.strip() or re.search("\\d", cell) for cell in rows[0])) or (not any((re.search("\\d", cell) for row in rows[1:] for cell in row))) ): continue return len(rows) - 1 return None def list_bounds( source: str, fields: dict[str, dict[str, Any]], *, has_images: bool = False ) -> dict[str, int]: if has_images: return {} parents = {} for name, spec in fields.items(): instructions = spec.get("instructions", "") if ( spec.get("type") == "list" and spec.get("items") in ("text", "number") and (not spec.get("knowledge")) and _ONE_PER_ROW.search(instructions) and ( not re.search( "\\b(?:not|never|multiple|two|three)\\b|don't", instructions, re.IGNORECASE, ) ) ): parents[name] = spec count = table_rows(source) if parents else None return {name: count for name in parents} if count is not None else {} def bounded_predictions( predictions: dict[str, dict[str, Any]], fields: dict[str, dict[str, Any]], bounds: dict[str, int], item_name: Callable[[str, int], str], ) -> dict[str, dict[str, Any]]: if not bounds: return predictions result = dict(predictions) for parent, count in bounds.items(): name = item_name(parent, count + 1) if name not in fields or name not in result: continue previous = result[name] if previous.get("value") is None and previous.get("valid", True): continue result[name] = { **previous, "value": None, "valid": True, "complete": True, "span": None, "confidence": 1.0, "log_probability": 0.0, "confidence_method": "source_structure", "grounding_method": "source_row_bound", } return result