File size: 5,080 Bytes
1a48d6b
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
from __future__ import annotations
import csv
import io
import re
from collections.abc import Callable
from html.parser import HTMLParser
from typing import Any

_ONE_PER_ROW = re.compile(
    "\\bone\\s+(?:[\\w-]+\\s+){0,5}per\\s+(?:[\\w-]+\\s+){0,5}row\\b", re.IGNORECASE
)


class _Table(HTMLParser):
    def __init__(self) -> None:
        super().__init__(convert_charrefs=False)
        self.tables = self.depth = 0
        self.rows: list[set[str]] = []
        self.closed = False
        self.row_header = False
        self.in_cell = False
        self.ambiguous = False

    def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
        if self.in_cell and tag in ("br", "div", "p", "ul", "ol", "li", "table"):
            self.ambiguous = True
        if tag == "table":
            self.tables += 1
            self.depth += 1
        if self.depth == 1:
            if tag == "tr":
                self.rows.append(set())
            elif tag in ("td", "th") and self.rows:
                self.in_cell = True
                if any(
                    (
                        key in ("rowspan", "colspan") and value != "1"
                        for (key, value) in attrs
                    )
                ):
                    self.ambiguous = True
                self.rows[-1].add(tag)
                if len(self.rows) == 1 and (
                    dict(attrs).get("scope") or ""
                ).casefold() in ("row", "rowgroup"):
                    self.row_header = True

    def handle_endtag(self, tag: str) -> None:
        if tag in ("td", "th"):
            self.in_cell = False
        if tag == "table":
            self.depth -= 1
            self.closed = self.depth == 0

    def handle_data(self, data: str) -> None:
        if (
            self.in_cell
            and len([line for line in data.splitlines() if line.strip()]) > 1
        ):
            self.ambiguous = True


def table_rows(source: str) -> int | None:
    if re.search("<table(?:\\s|>)", source, re.IGNORECASE):
        parser = _Table()
        try:
            parser.feed(source)
            parser.close()
        except (ValueError, AssertionError):
            return None
        if (
            parser.tables != 1
            or not parser.closed
            or parser.depth != 0
            or (not parser.rows)
            or (parser.rows[0] != {"th"})
            or parser.row_header
            or parser.ambiguous
        ):
            return None
        return len(parser.rows) - 1
    lines = source.splitlines()
    if len(lines) < 2:
        return None
    for delimiter in (",", "\t", ";", "|"):
        if delimiter not in lines[0]:
            continue
        try:
            rows = list(
                csv.reader(io.StringIO(source), delimiter=delimiter, strict=True)
            )
        except csv.Error:
            continue
        if (
            not rows
            or len(rows[0]) < 2
            or any((not row or len(row) != len(rows[0]) for row in rows))
            or any((not cell.strip() or re.search("\\d", cell) for cell in rows[0]))
            or (not any((re.search("\\d", cell) for row in rows[1:] for cell in row)))
        ):
            continue
        return len(rows) - 1
    return None


def list_bounds(
    source: str, fields: dict[str, dict[str, Any]], *, has_images: bool = False
) -> dict[str, int]:
    if has_images:
        return {}
    parents = {}
    for name, spec in fields.items():
        instructions = spec.get("instructions", "")
        if (
            spec.get("type") == "list"
            and spec.get("items") in ("text", "number")
            and (not spec.get("knowledge"))
            and _ONE_PER_ROW.search(instructions)
            and (
                not re.search(
                    "\\b(?:not|never|multiple|two|three)\\b|don't",
                    instructions,
                    re.IGNORECASE,
                )
            )
        ):
            parents[name] = spec
    count = table_rows(source) if parents else None
    return {name: count for name in parents} if count is not None else {}


def bounded_predictions(
    predictions: dict[str, dict[str, Any]],
    fields: dict[str, dict[str, Any]],
    bounds: dict[str, int],
    item_name: Callable[[str, int], str],
) -> dict[str, dict[str, Any]]:
    if not bounds:
        return predictions
    result = dict(predictions)
    for parent, count in bounds.items():
        name = item_name(parent, count + 1)
        if name not in fields or name not in result:
            continue
        previous = result[name]
        if previous.get("value") is None and previous.get("valid", True):
            continue
        result[name] = {
            **previous,
            "value": None,
            "valid": True,
            "complete": True,
            "span": None,
            "confidence": 1.0,
            "log_probability": 0.0,
            "confidence_method": "source_structure",
            "grounding_method": "source_row_bound",
        }
    return result