File size: 10,136 Bytes
ae97c5d
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
from __future__ import annotations
import math
import re
from collections.abc import Collection
from decimal import Decimal, DecimalException
from typing import Any
from jsonschema import Draft202012Validator, ValidationError
from . import ar

COPY_INSTRUCTION = " Copy the complete requested number exactly as printed, including every digit, currency symbol, grouping separator, sign or parentheses. Do not calculate, normalize or round. Return null only if this requested value is absent."
PAYMENT_PRESENCE_INSTRUCTION = " Every separately printed payment actually received is a present row, including an amount of zero, a negative amount, or a few cents. Copy all matching printed received-payment rows in their printed order, preserving duplicate rows. Do not return null or end the list while a matching received-payment row remains."
_NUMBER = re.compile(
    "\\s*(?P<open>\\()?\\s*(?P<before>[+-]?)\\s*(?:[$€£¥]|USD|EUR|GBP|CAD|AUD|JPY)?\\s*(?P<after>[+-]?)\\s*(?P<number>(?:\\d{1,3}(?:,\\d{3})+|\\d+)(?:\\.\\d+)?(?:[eE][+-]?\\d+)?)\\s*(?P<close>\\))?\\s*",
    re.IGNORECASE,
)
_TRANSFORM = re.compile(
    "\\b(?:calculat\\w*|comput\\w*|deriv\\w*|convert\\w*|summar\\w*|aggregat\\w*|group\\w*|combin\\w*|deduplicat\\w*|normaliz\\w*)\\b",
    re.IGNORECASE,
)


def _literal(spec: dict[str, Any]) -> bool:
    instructions = spec.get("instructions", "")
    if not isinstance(instructions, str) or spec.get("knowledge"):
        return False
    instructions = re.sub(
        "\\b(?:do\\s+not|never|don't)\\s+(?:calculate\\w*|compute\\w*|derive\\w*|convert\\w*|summarize\\w*|aggregate\\w*|group\\w*|combine\\w*|deduplicate\\w*|normalize\\w*)\\b",
        "",
        instructions,
        flags=re.IGNORECASE,
    )
    return not _TRANSFORM.search(instructions)


def _payment_property(spec: dict[str, Any]) -> str | None:
    properties = spec.get("properties", {})
    if (
        spec.get("type") != "object"
        or len(properties) != 2
        or (not _literal(spec))
        or (
            not all(
                (
                    isinstance(child, dict) and _literal(child)
                    for child in properties.values()
                )
            )
        )
        or (sum((child.get("type") == "text" for child in properties.values())) != 1)
        or (sum((child.get("type") == "number" for child in properties.values())) != 1)
    ):
        return None
    return next(
        (name for (name, child) in properties.items() if child.get("type") == "number")
    )


def payment_lists(fields: dict[str, dict[str, Any]]) -> list[str]:
    selected = []
    for name, spec in fields.items():
        item = spec.get("items")
        if (
            spec.get("type") != "list"
            or not isinstance(item, dict)
            or (not _literal(spec))
            or (_payment_property(item) is None)
        ):
            continue
        positive = re.split(
            "[.\\n]|\\bExclude\\b",
            spec.get("instructions", ""),
            maxsplit=1,
            flags=re.IGNORECASE,
        )[0]
        if (
            re.search("\\bpayments?\\b[^.\\n]*\\breceived\\b", positive, re.IGNORECASE)
            or re.search(
                "\\breceived\\b[^.\\n]*\\bpayments?\\b", positive, re.IGNORECASE
            )
        ) and (
            not re.search(
                "\\b(?:not|never|exclude|excluding|ignore|without)\\b",
                positive,
                re.IGNORECASE,
            )
        ):
            selected.append(name)
    return selected


def normalize(value: object, spec: dict[str, Any]) -> int | float | None:
    if value is None:
        return None
    if not isinstance(value, str):
        raise TypeError("A printed number must be a string")
    raw = value.strip()
    unit = spec.get("unit")
    if unit and raw.endswith(unit):
        raw = raw[: -len(unit)].rstrip()
    match = _NUMBER.fullmatch(raw)
    if (
        match is None
        or bool(match["open"]) != bool(match["close"])
        or (match["before"] and match["after"])
        or (match["open"] and (match["before"] or match["after"]))
    ):
        raise ValueError("Unsupported printed number")
    try:
        number = Decimal(match["number"].replace(",", ""))
        if not math.isfinite(float(number)):
            raise ValueError("Nonfinite numeric answer")
    except DecimalException as error:
        raise ValueError("Unsupported printed number") from error
    if match["open"] or "-" in (match["before"], match["after"]):
        number = number.copy_negate()
    result = ar.normalize_number(number, spec)
    if isinstance(result, float) and (not math.isfinite(result)):
        raise ValueError("Nonfinite numeric answer")
    Draft202012Validator(ar.schema(spec)).validate(result)
    return result


def _text_spec(spec: dict[str, Any]) -> dict[str, Any]:
    return {
        "type": "text",
        "nullable": spec.get("nullable", True),
        "instructions": spec.get("instructions", "") + COPY_INSTRUCTION,
    }


def prepare(
    fields: dict[str, dict[str, Any]], *, payment_rows: Collection[str] = ()
) -> tuple[dict[str, dict[str, Any]], dict[str, dict[str, Any]]]:
    fields = {name: dict(spec) for (name, spec) in fields.items()}
    contracts = {}
    payment_rows = set(payment_rows)
    for parent in payment_lists(fields):
        prefix = parent + ".item"
        children = [
            name
            for name in payment_rows
            if name.startswith(prefix) and name[len(prefix) :].isdigit()
        ]
        if not children:
            continue
        instructions = fields[parent].get("instructions", "")
        if PAYMENT_PRESENCE_INSTRUCTION in instructions:
            continue
        fields[parent] = {
            **fields[parent],
            "instructions": instructions + PAYMENT_PRESENCE_INSTRUCTION,
        }
        for child in children:
            if child not in fields or _payment_property(fields[child]) is None:
                continue
            requested = fields[child].get("instructions", "")
            fields[child] = {
                **fields[child],
                "instructions": requested.replace(
                    instructions, instructions + PAYMENT_PRESENCE_INSTRUCTION, 1
                )
                if instructions and instructions in requested
                else requested + PAYMENT_PRESENCE_INSTRUCTION,
            }
    for name, spec in fields.items():
        if spec["type"] == "number" and spec.get("source_only"):
            contracts[name] = {"scalar": spec}
            fields[name] = _text_spec(spec)
        elif (
            spec["type"] == "list"
            and spec.get("source_only")
            and (spec["items"] == "number")
        ):
            fields[name] = {
                **spec,
                "items": "text",
                "instructions": spec.get("instructions", "") + COPY_INSTRUCTION,
            }
        elif (
            name in payment_rows
            and (number_name := _payment_property(spec)) is not None
        ):
            number_spec = spec["properties"][number_name]
            contracts[name] = {"row": spec, "properties": {number_name: number_spec}}
            fields[name] = {
                **spec,
                "properties": {
                    **spec["properties"],
                    number_name: _text_spec(number_spec),
                },
            }
    return (fields, contracts)


def _normalize_row(
    raw: object, contract: dict[str, Any], record: dict[str, Any], name: str
) -> dict[str, Any] | None:
    if raw is None:
        result = None
    elif isinstance(raw, dict):
        result = dict(raw)
        current = record.get("fields", {}).get(name, {}).get("properties", {})
        for property_name, spec in contract["properties"].items():
            value = raw[property_name]
            if current.get(property_name, {}).get("type") == "number":
                if value is not None and (
                    isinstance(value, bool)
                    or not isinstance(value, (int, float, Decimal))
                ):
                    raise TypeError("A numeric row value must be a number")
                result[property_name] = ar.normalize_number(value, spec)
            else:
                if value is not None and (not record.get("_media_payload")):
                    if not isinstance(value, str) or value not in record["input"]:
                        raise ValueError("not_in_source")
                result[property_name] = normalize(value, spec)
    else:
        raise TypeError("A printed payment row must be an object")
    Draft202012Validator(ar.schema(contract["row"])).validate(result)
    return result


def normalize_predictions(
    record: dict[str, Any], predictions: dict[str, dict[str, Any]]
) -> dict[str, dict[str, Any]]:
    result = dict(predictions)
    for name, contract in record.get("_meta", {}).get("printed_numbers", {}).items():
        answer = dict(result[name])
        result[name] = answer
        if not answer.get("valid"):
            continue
        raw = answer["value"]
        try:
            if "properties" in contract:
                answer["value"] = _normalize_row(raw, contract, record, name)
            else:
                if raw is not None and (not record.get("_media_payload")):
                    if not isinstance(raw, str) or raw not in record["input"]:
                        raise ValueError("not_in_source")
                answer["value"] = normalize(raw, contract["scalar"])
            answer["printed_value"] = raw
            answer["normalization_method"] = "printed_number_v1"
        except (
            ValueError,
            TypeError,
            KeyError,
            OverflowError,
            ValidationError,
        ) as error:
            answer.update(
                value=None,
                valid=False,
                confidence=0.0,
                span=None,
                failure_reason="not_in_source"
                if str(error) == "not_in_source"
                else "invalid_printed_number",
            )
    return result