File size: 13,730 Bytes
dc7d8aa
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
from __future__ import annotations
from collections.abc import Callable
from typing import TYPE_CHECKING, Any

if TYPE_CHECKING:
    from .api import PredictRequest
    from .backend import EncodedRequest
import hashlib
import math
import re
from copy import deepcopy
from dataclasses import dataclass
from itertools import pairwise
import numpy as np
from PIL import Image
from . import image_table_rows, table_rows
from .schema import MAX_LIST_ITEMS

VIEW_ID = "ragged-table-v1"
TARGET_GATE = "target_is_billed_item"
TARGET_ROW = "target_row"
_BILLED_SCOPE = re.compile(
    "\\b(?:line[ _-]?items?|billed\\s+(?:products?|services?|items?|goods))\\b",
    re.IGNORECASE,
)
_FINANCIAL_SCOPE = re.compile(
    "\\b(?:tax(?:es)?|fees?|surcharges?|payments?)\\b", re.IGNORECASE
)
TARGET_INSTRUCTION = "Image 1 is the complete source document. Image 2 isolates one physical row from image 1. Classify and read only the target row in image 2. Use image 1 to identify the table column labels. A column header is not a billed item. Do not select a neighboring row or count row positions. "


@dataclass(frozen=True)
class RaggedRows:
    rows: tuple[tuple[int, int], ...]

    def views(self, image: Image.Image) -> tuple[Image.Image, ...]:
        return tuple(
            (
                image.crop(
                    (0, max(0, top - 3), image.width, min(image.height, bottom + 3))
                )
                for (top, bottom) in self.rows
            )
        )


def detect(image: Image.Image) -> RaggedRows | None:
    if image_table_rows.detect(image) is not None:
        return None
    if image.mode != "RGB" or image.width < 100 or image.height < 60:
        return None
    grey = np.asarray(image.convert("L"))
    if np.mean(grey < 245) > 0.22:
        return None
    ink = grey < 200
    candidates = []
    for index, (top, bottom) in enumerate(image_table_rows._runs(ink.any(axis=1))):
        height = bottom - top
        if height < 10:
            continue
        band = ink[top:bottom]
        columns = band.any(axis=0)
        separators = []
        for left, right in image_table_rows._runs(band.sum(axis=0) >= 0.9 * height):
            if (
                right - left > max(3, height * 0.18)
                or left < 3
                or right > image.width - 3
                or columns[left - 3 : left].any()
                or columns[right : right + 3].any()
            ):
                continue
            separators.append((left, right))
        if len(separators) >= 2:
            candidates.append((index, top, bottom, separators, columns))
    if len(candidates) < 2:
        return None
    if any((right[0] != left[0] + 1 for (left, right) in pairwise(candidates))):
        return None
    if len({len(row[3]) for row in candidates}) != 1:
        return None
    if any((row[4][row[3][-1][1] + 3 :].any() for row in candidates)):
        return None
    heights = np.array([bottom - top for (_, top, bottom, _, _) in candidates])
    spacing = np.diff([row[1] for row in candidates])
    if np.max(heights) > 1.2 * np.min(heights) or np.max(spacing) > 1.3 * np.min(
        spacing
    ):
        return None
    return RaggedRows(tuple(((row[1], row[2]) for row in candidates)))


def _eligible(spec: dict[str, Any]) -> bool:
    item = spec.get("items")
    properties = item.get("properties", {}) if isinstance(item, dict) else {}
    (scope, *exclusions) = re.split(
        "\\bExclude\\b", spec.get("instructions", ""), maxsplit=1, flags=re.IGNORECASE
    )
    if exclusions:
        if re.search(
            "\\b(?:not|never|\\w+n['’]t)(?:\\s+\\w+){0,2}\\s*$", scope, re.IGNORECASE
        ):
            return False
        (excluded, *later) = re.split("[.;\\n]", exclusions[0], maxsplit=1)
        if (
            later
            and _FINANCIAL_SCOPE.search(later[0])
            or (
                _FINANCIAL_SCOPE.search(excluded)
                and re.search(
                    "\\b(?:include|including|keep|retain|except|unless|not|no|none)\\b",
                    excluded,
                    re.IGNORECASE,
                )
            )
        ):
            return False
    return (
        spec.get("type") == "list"
        and isinstance(item, dict)
        and (item.get("type") == "object")
        and (not table_rows.join_wrapped_descriptions(spec.get("instructions", "")))
        and (not spec.get("knowledge"))
        and (len(properties) >= 3)
        and bool(_BILLED_SCOPE.search(scope))
        and (not _FINANCIAL_SCOPE.search(scope))
        and all(
            (
                child.get("type") in ("text", "number") and (not child.get("knowledge"))
                for child in properties.values()
            )
        )
    )


def row_fields(spec: dict[str, Any]) -> dict[str, dict[str, Any]]:
    wrapped = table_rows.join_wrapped_descriptions(spec.get("instructions", ""))
    membership = (
        "A logical item can span several printed bands. Include it exactly once, at its first description band. Return false for a later continuation band belonging to that preceding item, even when its numeric cells are printed on the continuation band. Distinct separate source rows remain separate even if they repeat. "
        if wrapped
        else ""
    )
    readout = (
        "For the target item's first band, use the full document to read its complete wrapped description and cells from its continuation bands. Never borrow from a neighboring logical item. "
        if wrapped
        else ""
    )
    return {
        TARGET_GATE: {
            "type": "boolean",
            "nullable": False,
            "instructions": TARGET_INSTRUCTION
            + membership
            + "Is the target row a billed product or service row? Return false for the column header, totals, tax, fees, or payment rows. "
            + spec.get("instructions", ""),
        },
        TARGET_ROW: {
            "type": "object",
            "nullable": False,
            "properties": deepcopy(spec["items"]["properties"]),
            "instructions": TARGET_INSTRUCTION
            + readout
            + "Read the complete printed target row as an object. Whether it belongs in the requested list is decided separately. Always return an object with every requested cell. Missing cells and absent columns are null. Do not relabel an Amount column as Unit price. Never compute missing values.",
        },
    }


def prepare(
    request: PredictRequest,
    encoded: EncodedRequest,
    encode: Callable[..., EncodedRequest],
    *,
    scale: float = 2,
) -> EncodedRequest:
    (record, forecasts, tasks, typed, eos) = encoded
    annotations = record["_meta"]["extraction_contract"]["fields"]
    parents = [
        name
        for (name, spec) in request.fields.items()
        if _eligible(spec) and annotations.get(name, {}).get("image_row_fields")
    ]
    if not parents or len(request.images) != 1 or request.documents:
        return encoded
    originals = record.get("_original_images")
    image = originals[0] if originals else request.images[0].to_pil()
    plan = detect(image)
    if plan is None:
        return encoded
    views = []
    for view in plan.views(image):
        factor = min(scale, (1600000 / (view.width * view.height)) ** 0.5)
        if factor > 1:
            view = view.resize(
                (int(view.width * factor), int(view.height * factor)),
                Image.Resampling.BICUBIC,
            )
        views.append(view)
    salts = dict(record.get("_ar_task_salts", {}))
    gates = list(record.get("_source_row_gates", []))
    changed = False
    for name in parents:
        (spec, annotation) = (request.fields[name], annotations[name])
        previous = annotation["image_row_fields"]
        maximum = spec.get("max_items", MAX_LIST_ITEMS)
        if len(views) > maximum + 1 or len(views) > len(previous):
            continue
        (replacements, pairs) = ([], [])
        for index, (view, band) in enumerate(zip(views, plan.rows, strict=True)):
            child = previous[index]
            digest = hashlib.sha256(child.encode()).hexdigest()[:16]
            gate = "ragged_gate_" + digest
            while gate in record["fields"]:
                gate += "_"
            row_request = request.model_copy(update={"fields": row_fields(spec)})
            (row_record, _, row_tasks, _, _) = encode(
                row_request,
                physical_row_view=True,
                image_views=lambda view=view: (image, view),
                image_view_key=f"{VIEW_ID}:{band}:scale{scale}",
            )
            assert len(row_tasks) == 2
            names = {TARGET_GATE: gate, TARGET_ROW: child}
            for source, ids, payload in row_tasks:
                target = names[source]
                record["fields"][target] = row_record["fields"][source]
                record["_ar_constraints"][target] = row_record["_ar_constraints"][
                    source
                ]
                salts[target] = row_record["_cache_salt"]
                annotations[target] = {"parent": name}
                replacements.append((target, ids, payload))
            gates.append(gate)
            pairs.append((gate, child))
        tasks = [
            task for task in tasks if task[0] not in {name, *previous}
        ] + replacements
        retained = {copy for (_, copy) in pairs}
        for child in set(previous) - retained:
            record["fields"].pop(child, None)
            record["_ar_constraints"].pop(child, None)
            annotations.pop(child, None)
            salts.pop(child, None)
        annotation.pop("image_row_fields")
        annotation["ragged_row_fields"] = pairs
        annotation["ragged_row_max_items"] = maximum
        record["fields"][name].pop("_image_row_inventory", None)
        changed = True
    if not changed:
        return encoded
    record.update(
        _ar_tasks=tasks,
        _ar_task_salts=salts,
        _source_row_gates=gates,
        _ar_prefix=[],
        _ar_group_prefixes=[],
    )
    record.pop("_ar_media_groups", None)
    return (record, forecasts, tasks, typed, eos)


def _score(value: object, *, probability: bool = False) -> bool:
    return (
        isinstance(value, (int, float))
        and (not isinstance(value, bool))
        and math.isfinite(value)
        and (0 <= value <= 1 if probability else value <= 0)
    )


def assemble(
    record: dict[str, Any], predictions: dict[str, dict[str, Any]]
) -> dict[str, dict[str, Any]]:
    result = {}
    for name, annotation in record["_meta"]["extraction_contract"]["fields"].items():
        pairs = annotation.get("ragged_row_fields")
        if not pairs:
            continue
        rows = [(predictions[gate], predictions[copy]) for (gate, copy) in pairs]
        selected = [(gate, copy) for (gate, copy) in rows if gate.get("value") is True]
        complete = all((gate.get("complete", False) for (gate, _) in rows)) and all(
            (copy.get("complete", False) for (_, copy) in selected)
        )
        exclusion_scored = all(
            (_score(gate.get("p_excluded"), probability=True) for (gate, _) in rows)
        )
        relevant = [gate for (gate, _) in rows] + [copy for (_, copy) in selected]
        scored = exclusion_scored and all(
            (_score(answer.get("log_probability")) for answer in relevant)
        )
        probability = (
            sum((answer["log_probability"] for answer in relevant)) if scored else None
        )
        if probability is not None and (not math.isfinite(probability)):
            (scored, probability) = (False, None)
        valid = (
            complete
            and scored
            and all(
                (
                    gate.get("valid", False) and type(gate.get("value")) is bool
                    for (gate, _) in rows
                )
            )
            and all(
                (
                    copy.get("valid", False) and isinstance(copy.get("value"), dict)
                    for (_, copy) in selected
                )
            )
        )
        within_limit = len(selected) <= annotation["ragged_row_max_items"]
        valid &= within_limit
        answer = {
            "value": [copy["value"] for (_, copy) in selected],
            "valid": valid,
            "complete": complete,
            "confidence": math.exp(probability)
            if valid and probability is not None
            else 0.0,
            "confidence_method": "ar_physical_image_row_eligibility_and_copy_probability",
            "confidences": [
                math.exp(gate["log_probability"] + copy["log_probability"])
                if scored
                else 0.0
                for (gate, copy) in selected
            ],
            "p_null": math.prod((gate["p_excluded"] for (gate, _) in rows))
            if exclusion_scored
            else 0.0,
            "p_null_method": "product_physical_image_row_exclusion_probabilities"
            if exclusion_scored
            else "unavailable_physical_image_row_exclusion_probability",
            "p_nulls": [
                gate["p_excluded"] if exclusion_scored else 0.0
                for (gate, _) in selected
            ],
            "spans": [None] * len(selected),
        }
        if probability is not None:
            answer["log_probability"] = probability
        if not valid:
            answer["failure_reason"] = (
                "ragged_row_output_exceeds_max_items"
                if not within_limit
                else "ragged_row_unscored"
                if not scored
                else "ragged_row_read_incomplete"
            )
        result[name] = answer
    return result