clairic-flash / clairic_runtime /image_financial_lines.py
areneau's picture
Add complete Clairic Transformers package
dc7d8aa verified
Raw History Blame Contribute Delete
13.2 kB
from __future__ import annotations
from collections.abc import Callable
from typing import TYPE_CHECKING, Any
if TYPE_CHECKING:
from .api import PredictRequest
from .backend import EncodedRequest
import math
import re
from copy import deepcopy
import numpy as np
from jsonschema import Draft202012Validator, ValidationError
from PIL import Image
from . import ar, image_ragged_rows, image_table_rows, printed_numbers
from .schema import MAX_LIST_ITEMS
LINE = {
"type": "text",
"nullable": False,
"instructions": "Copy the complete single printed line verbatim, including all words, prefixes, punctuation and numbers. Do not extract a name or normalize any part. Return only the complete line.",
}
_TAX = re.compile("\\btax(?:es)?(?:\\s+(?:refund|credit|adjustment))?$", re.IGNORECASE)
_FEE = re.compile(
"\\b(?:fees?|surcharges?)(?:\\s+(?:refund|credit|adjustment))?$", re.IGNORECASE
)
_SUMMARY = re.compile(
"(?:sub[ -]?total|grand total|total|amount due|balance(?: due| forward)?|tax total|fee total)",
re.IGNORECASE,
)
_PAYMENT = re.compile(
"^(?:payments?\\s+(?:received|instructions?|due|method|status)|(?:received|posted|applied)\\s+payments?)\\b|\\bpayments?$",
re.IGNORECASE,
)
_TRANSFORM = re.compile(
"\\b(?:deduplicat\\w*|unique|aggregat\\w*|summari[sz]\\w*|combin\\w*|group\\w*|averag\\w*|calculat\\w*|normaliz\\w*|convert\\w*|except|unless|instead|knowledge)\\b",
re.IGNORECASE,
)
_CELL = "\\(?[+-]?\\$?[\\d,]+\\.\\d{2}\\)?"
_SCOPE_WORDS = {
"only",
"explicit",
"printed",
"separately",
"tax",
"taxes",
"row",
"rows",
"fee",
"fees",
"billed",
"goods",
"product",
"products",
"services",
"service",
"charge",
"charges",
"exclude",
"excluding",
"all",
"subtotal",
"subtotals",
"total",
"totals",
"grand",
"payment",
"payments",
"amount",
"due",
"and",
"or",
"null",
"if",
"no",
"when",
"the",
"document",
"has",
"a",
"must",
"be",
"surcharge",
"surcharges",
"return",
"instructions",
}
def role(label: str) -> str:
label = " ".join(label.split()).rstrip(": ")
if _SUMMARY.fullmatch(label):
return "summary"
if _PAYMENT.search(label):
return "payment"
if _TAX.search(label):
return "tax"
return "fee" if _FEE.search(label) else "charge"
def scope(spec: dict[str, Any]) -> str | None:
item = spec.get("items")
if (
spec.get("type") != "list"
or spec.get("knowledge")
or (not isinstance(item, dict))
):
return None
properties = item.get("properties", {})
if item.get("type") != "object" or len(properties) != 2:
return None
if any(
(
not isinstance(child, dict) or child.get("knowledge")
for child in properties.values()
)
):
return None
if sorted((child.get("type", "") for child in properties.values())) != [
"number",
"text",
]:
return None
amount = next((child for child in properties.values() if child["type"] == "number"))
if amount.get("decimals") != 2 or amount.get("unit") or amount.get("integer"):
return None
instructions = spec.get("instructions", "")
if not isinstance(instructions, str) or _TRANSFORM.search(instructions):
return None
if set(re.findall("[a-z]+", instructions.lower())) - _SCOPE_WORDS:
return None
if any(
(
set(child) - {"type", "instructions", "nullable", "decimals"}
for child in properties.values()
)
):
return None
if any(
(
_TRANSFORM.search(child.get("instructions", ""))
for child in properties.values()
)
):
return None
positive = re.split(
"[.\\n]|\\bExclude\\b", instructions, maxsplit=1, flags=re.IGNORECASE
)[0].strip()
if re.fullmatch(
"Only\\s+(?:(?:explicit|printed|separately\\s+printed)\\s+)?tax(?:es)?\\s+rows",
positive,
re.IGNORECASE,
):
return "tax"
if re.fullmatch(
"Only\\s+(?:(?:explicit|printed|separately\\s+printed)\\s+)?fees?\\s+rows",
positive,
re.IGNORECASE,
):
return "fee"
if (
re.fullmatch(
"Only\\s+billed\\s+(?:(?:goods?|products?|services?)\\s+(?:(?:and|or)\\s+)?)+(?:charge\\s+)?rows",
positive,
re.IGNORECASE,
)
and all(
(
re.search(
"\\bExclude\\b[^.\\n]*\\b" + token + "\\b",
instructions,
re.IGNORECASE,
)
for token in ("tax(?:es)?", "fees?", "payments?")
)
)
and (
not re.search("\\b(?:not|never)\\s+exclude\\b", instructions, re.IGNORECASE)
)
):
return "charge"
return None
def parse(line: object) -> tuple[str, int | float | None] | None:
if not isinstance(line, str):
raise TypeError("Missing literal image line")
match = re.fullmatch("(.*?)\\s*(?:\\||:)\\s*(" + _CELL + ")\\s*\\|?\\s*", line)
if not match:
match = re.fullmatch("(.*?)\\s+(\\(?[+-]?\\$[\\d,]+\\.\\d{2}\\)?)\\s*", line)
if not match:
return None
label = match[1].rstrip(" |:")
if not label or "|" in label:
return None
return (
label,
printed_numbers.normalize(match[2], {"type": "number", "decimals": 2}),
)
def bands(image: Image.Image) -> list[tuple[int, int]] | None:
if (
image.mode != "RGB"
or image.width < 100
or image.getpixel((0, 0)) != (255, 255, 255)
):
return None
ink = np.asarray(image.convert("L")) < 200
if np.mean(ink) > 0.22:
return None
rows = image_table_rows._runs(ink.any(axis=1))
if not 2 <= len(rows) <= MAX_LIST_ITEMS or any(
(not 8 <= bottom - top <= 32 for (top, bottom) in rows)
):
return None
return rows
def prepare(
request: PredictRequest,
encoded: EncodedRequest,
encode: Callable[..., EncodedRequest],
) -> EncodedRequest:
(record, forecasts, tasks, typed, eos) = encoded
annotations = record["_meta"]["extraction_contract"]["fields"]
selected = {
name: kind
for (name, spec) in request.fields.items()
if (kind := scope(spec)) and annotations.get(name, {}).get("query_fields")
}
if not selected or len(request.images) != 1 or request.documents:
return encoded
originals = record.get("_original_images")
image = originals[0] if originals else request.images[0].to_pil()
rows = bands(image)
if rows is None:
return encoded
if (
image_table_rows.detect(image) is not None
or image_ragged_rows.detect(image) is not None
):
selected = {name: kind for (name, kind) in selected.items() if kind != "charge"}
if not selected:
return encoded
salts = dict(record.get("_ar_task_salts", {}))
def add(child_request: PredictRequest, **options: Any) -> list[str]:
(child, _, prepared, _, _) = encode(
child_request, physical_row_view=True, **options
)
names = []
for source, ids, payload in prepared:
target = "financial_line_" + str(len(record["fields"]))
while target in record["fields"]:
target += "_"
record["fields"][target] = child["fields"][source]
record["_ar_constraints"][target] = child["_ar_constraints"][source]
salts[target] = child["_cache_salt"]
tasks.append((target, ids, payload))
names.append(target)
return names
tasks = list(tasks)
queries = []
for top, bottom in rows:
view = image.crop(
(0, max(0, top - 3), image.width, min(image.height, bottom + 3))
)
view = view.resize((view.width * 2, view.height * 2), Image.Resampling.BICUBIC)
child_request = request.model_copy(update={"fields": {"printed_line": LINE}})
queries.extend(
add(
child_request,
image_views=lambda view=view: (view,),
image_view_key=f"financial-line-v1:{top}:{bottom}",
)
)
plans = {}
for name, kind in selected.items():
spec = request.fields[name]
presence = {
"type": "enum",
"enum": [True, None],
"nullable": True,
"instructions": "Decide whether at least one printed row satisfies this requested list scope. "
+ spec.get("instructions", "")
+ " Return true if any matching printed row exists; return null only if there are no matching rows. A matching row with amount zero or a negative amount is present. Check the row's own label; a neighboring tax or fee does not remove an eligible service row. This asks only whether a matching row exists, not its value or row count.",
}
gate = add(request.model_copy(update={"fields": {name: presence}}))[0]
previous = annotations[name].pop("query_fields")
tasks = [task for task in tasks if task[0] not in {name, *previous}]
for child in previous:
record["fields"].pop(child)
record["_ar_constraints"].pop(child, None)
annotations.pop(child, None)
plans[name] = {"kind": kind, "presence": gate, "spec": deepcopy(spec)}
record.update(
_financial_line_queries=queries,
_financial_line_plans=plans,
_ar_tasks=tasks,
_ar_task_salts=salts,
_ar_prefix=[],
_ar_group_prefixes=[],
)
return (record, forecasts, tasks, typed, eos)
def assemble(
record: dict[str, Any], predictions: dict[str, dict[str, Any]]
) -> dict[str, dict[str, Any]]:
queries = record.get("_financial_line_queries", [])
if not queries:
return {}
copies = [predictions[query] for query in queries]
(parsed, parse_valid) = ([], True)
for answer in copies:
try:
parsed.append(
parse(answer["value"]) if isinstance(answer.get("value"), str) else None
)
except (ValueError, TypeError, OverflowError):
parsed.append(None)
parse_valid = False
result = {}
for name, plan in record["_financial_line_plans"].items():
spec = plan["spec"]
properties = spec["items"]["properties"]
text = next(
(key for (key, value) in properties.items() if value["type"] == "text")
)
number = next(
(key for (key, value) in properties.items() if value["type"] == "number")
)
selected = [
(answer, row)
for (answer, row) in zip(copies, parsed, strict=True)
if row is not None and role(row[0]) == plan["kind"]
]
values = [{text: row[0], number: row[1]} for (_, row) in selected]
presence = predictions[plan["presence"]]
relevant = [*copies, presence]
complete = all((answer.get("complete", False) for answer in relevant))
valid = (
parse_valid
and complete
and all((answer.get("valid", False) for answer in relevant))
)
valid &= len(values) <= spec.get("max_items", MAX_LIST_ITEMS)
try:
for value in values:
Draft202012Validator(ar.schema(spec["items"])).validate(value)
except ValidationError:
valid = False
p_null = presence["p_null"]
if (
type(p_null) not in (int, float)
or not math.isfinite(p_null)
or (not 0 <= p_null <= 1)
or any(
(
type(answer.get("log_probability")) not in (int, float)
or not math.isfinite(answer["log_probability"])
or answer["log_probability"] > 0
for answer in copies
)
)
):
raise ValueError("Invalid financial-line model scores")
event = 1 - p_null if values else p_null
probability = (
sum((answer["log_probability"] for answer in copies)) + math.log(event)
if event > 0
else None
)
answer = {
"value": values,
"valid": bool(valid),
"complete": complete,
"confidence": 0.0,
"confidence_method": "ar_literal_image_lines_and_scope_presence_probability",
"confidences": [copy["confidence"] for (copy, _) in selected],
"p_null": p_null,
"p_null_method": "ar_parent_scope_presence_null_probability",
"p_nulls": [copy["p_null"] for (copy, _) in selected],
"spans": [None] * len(selected),
}
if probability is not None:
answer.update(
log_probability=probability,
confidence=math.exp(probability) if valid else 0.0,
)
result[name] = answer
return result