clairic-4bit / clairic_runtime /printed_numbers.py
lucataco's picture
Clairic, MLX 4-bit, with runtime, server and conversion script
1a48d6b verified
Raw History Blame Contribute Delete
10.1 kB
from __future__ import annotations
import math
import re
from collections.abc import Collection
from decimal import Decimal, DecimalException
from typing import Any
from jsonschema import Draft202012Validator, ValidationError
from . import ar
COPY_INSTRUCTION = " Copy the complete requested number exactly as printed, including every digit, currency symbol, grouping separator, sign or parentheses. Do not calculate, normalize or round. Return null only if this requested value is absent."
PAYMENT_PRESENCE_INSTRUCTION = " Every separately printed payment actually received is a present row, including an amount of zero, a negative amount, or a few cents. Copy all matching printed received-payment rows in their printed order, preserving duplicate rows. Do not return null or end the list while a matching received-payment row remains."
_NUMBER = re.compile(
"\\s*(?P<open>\\()?\\s*(?P<before>[+-]?)\\s*(?:[$€£¥]|USD|EUR|GBP|CAD|AUD|JPY)?\\s*(?P<after>[+-]?)\\s*(?P<number>(?:\\d{1,3}(?:,\\d{3})+|\\d+)(?:\\.\\d+)?(?:[eE][+-]?\\d+)?)\\s*(?P<close>\\))?\\s*",
re.IGNORECASE,
)
_TRANSFORM = re.compile(
"\\b(?:calculat\\w*|comput\\w*|deriv\\w*|convert\\w*|summar\\w*|aggregat\\w*|group\\w*|combin\\w*|deduplicat\\w*|normaliz\\w*)\\b",
re.IGNORECASE,
)
def _literal(spec: dict[str, Any]) -> bool:
instructions = spec.get("instructions", "")
if not isinstance(instructions, str) or spec.get("knowledge"):
return False
instructions = re.sub(
"\\b(?:do\\s+not|never|don't)\\s+(?:calculate\\w*|compute\\w*|derive\\w*|convert\\w*|summarize\\w*|aggregate\\w*|group\\w*|combine\\w*|deduplicate\\w*|normalize\\w*)\\b",
"",
instructions,
flags=re.IGNORECASE,
)
return not _TRANSFORM.search(instructions)
def _payment_property(spec: dict[str, Any]) -> str | None:
properties = spec.get("properties", {})
if (
spec.get("type") != "object"
or len(properties) != 2
or (not _literal(spec))
or (
not all(
(
isinstance(child, dict) and _literal(child)
for child in properties.values()
)
)
)
or (sum((child.get("type") == "text" for child in properties.values())) != 1)
or (sum((child.get("type") == "number" for child in properties.values())) != 1)
):
return None
return next(
(name for (name, child) in properties.items() if child.get("type") == "number")
)
def payment_lists(fields: dict[str, dict[str, Any]]) -> list[str]:
selected = []
for name, spec in fields.items():
item = spec.get("items")
if (
spec.get("type") != "list"
or not isinstance(item, dict)
or (not _literal(spec))
or (_payment_property(item) is None)
):
continue
positive = re.split(
"[.\\n]|\\bExclude\\b",
spec.get("instructions", ""),
maxsplit=1,
flags=re.IGNORECASE,
)[0]
if (
re.search("\\bpayments?\\b[^.\\n]*\\breceived\\b", positive, re.IGNORECASE)
or re.search(
"\\breceived\\b[^.\\n]*\\bpayments?\\b", positive, re.IGNORECASE
)
) and (
not re.search(
"\\b(?:not|never|exclude|excluding|ignore|without)\\b",
positive,
re.IGNORECASE,
)
):
selected.append(name)
return selected
def normalize(value: object, spec: dict[str, Any]) -> int | float | None:
if value is None:
return None
if not isinstance(value, str):
raise TypeError("A printed number must be a string")
raw = value.strip()
unit = spec.get("unit")
if unit and raw.endswith(unit):
raw = raw[: -len(unit)].rstrip()
match = _NUMBER.fullmatch(raw)
if (
match is None
or bool(match["open"]) != bool(match["close"])
or (match["before"] and match["after"])
or (match["open"] and (match["before"] or match["after"]))
):
raise ValueError("Unsupported printed number")
try:
number = Decimal(match["number"].replace(",", ""))
if not math.isfinite(float(number)):
raise ValueError("Nonfinite numeric answer")
except DecimalException as error:
raise ValueError("Unsupported printed number") from error
if match["open"] or "-" in (match["before"], match["after"]):
number = number.copy_negate()
result = ar.normalize_number(number, spec)
if isinstance(result, float) and (not math.isfinite(result)):
raise ValueError("Nonfinite numeric answer")
Draft202012Validator(ar.schema(spec)).validate(result)
return result
def _text_spec(spec: dict[str, Any]) -> dict[str, Any]:
return {
"type": "text",
"nullable": spec.get("nullable", True),
"instructions": spec.get("instructions", "") + COPY_INSTRUCTION,
}
def prepare(
fields: dict[str, dict[str, Any]], *, payment_rows: Collection[str] = ()
) -> tuple[dict[str, dict[str, Any]], dict[str, dict[str, Any]]]:
fields = {name: dict(spec) for (name, spec) in fields.items()}
contracts = {}
payment_rows = set(payment_rows)
for parent in payment_lists(fields):
prefix = parent + ".item"
children = [
name
for name in payment_rows
if name.startswith(prefix) and name[len(prefix) :].isdigit()
]
if not children:
continue
instructions = fields[parent].get("instructions", "")
if PAYMENT_PRESENCE_INSTRUCTION in instructions:
continue
fields[parent] = {
**fields[parent],
"instructions": instructions + PAYMENT_PRESENCE_INSTRUCTION,
}
for child in children:
if child not in fields or _payment_property(fields[child]) is None:
continue
requested = fields[child].get("instructions", "")
fields[child] = {
**fields[child],
"instructions": requested.replace(
instructions, instructions + PAYMENT_PRESENCE_INSTRUCTION, 1
)
if instructions and instructions in requested
else requested + PAYMENT_PRESENCE_INSTRUCTION,
}
for name, spec in fields.items():
if spec["type"] == "number" and spec.get("source_only"):
contracts[name] = {"scalar": spec}
fields[name] = _text_spec(spec)
elif (
spec["type"] == "list"
and spec.get("source_only")
and (spec["items"] == "number")
):
fields[name] = {
**spec,
"items": "text",
"instructions": spec.get("instructions", "") + COPY_INSTRUCTION,
}
elif (
name in payment_rows
and (number_name := _payment_property(spec)) is not None
):
number_spec = spec["properties"][number_name]
contracts[name] = {"row": spec, "properties": {number_name: number_spec}}
fields[name] = {
**spec,
"properties": {
**spec["properties"],
number_name: _text_spec(number_spec),
},
}
return (fields, contracts)
def _normalize_row(
raw: object, contract: dict[str, Any], record: dict[str, Any], name: str
) -> dict[str, Any] | None:
if raw is None:
result = None
elif isinstance(raw, dict):
result = dict(raw)
current = record.get("fields", {}).get(name, {}).get("properties", {})
for property_name, spec in contract["properties"].items():
value = raw[property_name]
if current.get(property_name, {}).get("type") == "number":
if value is not None and (
isinstance(value, bool)
or not isinstance(value, (int, float, Decimal))
):
raise TypeError("A numeric row value must be a number")
result[property_name] = ar.normalize_number(value, spec)
else:
if value is not None and (not record.get("_media_payload")):
if not isinstance(value, str) or value not in record["input"]:
raise ValueError("not_in_source")
result[property_name] = normalize(value, spec)
else:
raise TypeError("A printed payment row must be an object")
Draft202012Validator(ar.schema(contract["row"])).validate(result)
return result
def normalize_predictions(
record: dict[str, Any], predictions: dict[str, dict[str, Any]]
) -> dict[str, dict[str, Any]]:
result = dict(predictions)
for name, contract in record.get("_meta", {}).get("printed_numbers", {}).items():
answer = dict(result[name])
result[name] = answer
if not answer.get("valid"):
continue
raw = answer["value"]
try:
if "properties" in contract:
answer["value"] = _normalize_row(raw, contract, record, name)
else:
if raw is not None and (not record.get("_media_payload")):
if not isinstance(raw, str) or raw not in record["input"]:
raise ValueError("not_in_source")
answer["value"] = normalize(raw, contract["scalar"])
answer["printed_value"] = raw
answer["normalization_method"] = "printed_number_v1"
except (
ValueError,
TypeError,
KeyError,
OverflowError,
ValidationError,
) as error:
answer.update(
value=None,
valid=False,
confidence=0.0,
span=None,
failure_reason="not_in_source"
if str(error) == "not_in_source"
else "invalid_printed_number",
)
return result