File size: 6,448 Bytes
f65336d
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
from __future__ import annotations
import re
from datetime import datetime
from typing import Any

TYPES = ("date", "time", "datetime")
COPY_INSTRUCTION = " Copy only a date or time explicitly printed for this exact requested role. Return null if this role has no printed date or time. Do not use invoice or due dates for service or billing period boundaries. Do not infer a period from its duration. Copy the requested date or time exactly as printed, including AM or PM when present. Preserve every digit. Do not normalize or convert the value. Return null only if the requested value is absent."
_LITERAL = re.compile("\\b(?:print\\w*|explicit\\w*|verbatim|shown)\\b", re.IGNORECASE)
_DERIVED = re.compile(
    "\\b(?:calculat\\w*|comput\\w*|deriv\\w*|infer\\w*|after|before|ago|next|previous|following|prior|add|subtract|plus|minus|earlier|later|offset|duration|terms)\\b|\\bnet\\s*\\d+\\b|\\b(?:first|last)\\s+day\\s+of\\b",
    re.IGNORECASE,
)
_DOCUMENT_DATE = re.compile(
    "\\b(?:invoice|billing|service|subscription|document|receipt)\\b", re.IGNORECASE
)
_DATE_ROLES = {
    "invoice_date",
    "due_date",
    "period_start",
    "period_end",
    "start_date",
    "end_date",
}


def explicit_fields(fields: dict[str, dict[str, Any]]) -> dict[str, dict[str, Any]]:
    result = dict(fields)
    for name, spec in fields.items():
        date_list = spec.get("type") == "list" and spec.get("items") == "date"
        if (
            spec.get("type") not in TYPES
            and (not date_list)
            or "source_only" in spec
            or "null_threshold" in spec
            or spec.get("knowledge")
        ):
            continue
        instructions = spec.get("instructions", "")
        if not isinstance(instructions, str):
            continue
        role = name.casefold() in _DATE_ROLES and _DOCUMENT_DATE.search(instructions)
        if (_LITERAL.search(instructions) or role) and (
            not _DERIVED.search(instructions)
        ):
            result[name] = {**spec, "source_only": True}
    return result


def clock(value: str) -> str:
    match = re.fullmatch(
        "\\s*(\\d{1,2}):(\\d{2})(?::(\\d{2}))?\\s*([AP]\\.?M\\.?)?\\s*",
        value,
        re.IGNORECASE,
    )
    if match is None:
        raise ValueError("Unsupported printed time")
    (hour, minute, second) = map(int, (match[1], match[2], match[3] or "0"))
    meridiem = match[4]
    if meridiem:
        if not 1 <= hour <= 12:
            raise ValueError("Invalid 12-hour clock")
        hour = hour % 12 + (12 if meridiem.replace(".", "").upper() == "PM" else 0)
    if not 0 <= hour < 24 or not 0 <= minute < 60 or (not 0 <= second < 60):
        raise ValueError("Invalid printed time")
    return f"{hour:02}:{minute:02}:{second:02}"


def day(value: str) -> str:
    value = value.strip()
    for fmt in (
        "%Y-%m-%d",
        "%Y/%m/%d",
        "%Y.%m.%d",
        "%B %d, %Y",
        "%b %d, %Y",
        "%B %d %Y",
        "%b %d %Y",
        "%d %B %Y",
        "%d %b %Y",
    ):
        try:
            return datetime.strptime(value, fmt).date().isoformat()
        except ValueError:
            pass
    numeric = re.fullmatch("(\\d{1,2})([-/.])(\\d{1,2})\\2(\\d{4})", value)
    if numeric:
        (first, second, year) = (int(numeric[1]), int(numeric[3]), int(numeric[4]))
        readings = set()
        for month, date in ((first, second), (second, first)):
            try:
                readings.add(datetime(year, month, date).date().isoformat())
            except ValueError:
                pass
        if len(readings) == 1:
            return readings.pop()
    raise ValueError("Unsupported or ambiguous printed date")


def normalize(value: object, kind: str) -> str | None:
    if value is None:
        return None
    if not isinstance(value, str):
        raise TypeError("A printed temporal answer must be a string")
    if kind == "time":
        return clock(value)
    if kind == "date":
        return day(value)
    if kind == "datetime":
        match = re.fullmatch(
            "(.+?)(?:T|\\s+(?:at\\s+)?)(\\d{1,2}:\\d{2}(?::\\d{2})?\\s*(?:[AP]\\.?M\\.?)?)",
            value.strip(),
            re.IGNORECASE,
        )
        if match is None:
            raise ValueError("Unsupported printed datetime")
        return day(match[1]) + "T" + clock(match[2])
    raise ValueError("Unsupported temporal type")


def prepare(
    fields: dict[str, dict[str, Any]],
) -> tuple[dict[str, dict[str, Any]], dict[str, str]]:
    fields = {name: dict(spec) for (name, spec) in fields.items()}
    kinds = {}
    for name, spec in fields.items():
        if not spec.get("source_only"):
            continue
        if spec["type"] == "list" and spec["items"] == "date":
            spec["items"] = "text"
            spec["instructions"] = spec.get("instructions", "") + COPY_INSTRUCTION
            continue
        kind = spec["type"]
        if kind not in TYPES:
            continue
        kinds[name] = kind
        fields[name] = {
            "type": "text",
            "nullable": spec.get("nullable", True),
            "instructions": spec.get("instructions", "") + COPY_INSTRUCTION,
        }
    return (fields, kinds)


def normalize_predictions(
    record: dict[str, Any], predictions: dict[str, dict[str, Any]]
) -> dict[str, dict[str, Any]]:
    result = dict(predictions)
    for name, kind in record.get("_meta", {}).get("printed_temporal", {}).items():
        answer = dict(result[name])
        result[name] = answer
        if not answer.get("valid"):
            continue
        printed = answer["value"]
        answer["printed_value"] = printed
        if (
            printed is not None
            and (not record.get("_media_payload"))
            and (not isinstance(printed, str) or printed not in record["input"])
        ):
            answer.update(
                value=None,
                valid=False,
                confidence=0.0,
                span=None,
                failure_reason="not_in_source",
            )
            continue
        try:
            answer["value"] = normalize(printed, kind)
            answer["normalization_method"] = "printed_temporal_v1"
        except (TypeError, ValueError):
            answer.update(
                value=None,
                valid=False,
                confidence=0.0,
                failure_reason="invalid_printed_temporal",
            )
    return result