from __future__ import annotations from collections.abc import Callable from typing import TYPE_CHECKING, Any if TYPE_CHECKING: from .api import PredictRequest from .backend import EncodedRequest import math import re from copy import deepcopy import numpy as np from jsonschema import Draft202012Validator, ValidationError from PIL import Image from . import ar, image_ragged_rows, image_table_rows, printed_numbers from .schema import MAX_LIST_ITEMS LINE = { "type": "text", "nullable": False, "instructions": "Copy the complete single printed line verbatim, including all words, prefixes, punctuation and numbers. Do not extract a name or normalize any part. Return only the complete line.", } _TAX = re.compile("\\btax(?:es)?(?:\\s+(?:refund|credit|adjustment))?$", re.IGNORECASE) _FEE = re.compile( "\\b(?:fees?|surcharges?)(?:\\s+(?:refund|credit|adjustment))?$", re.IGNORECASE ) _SUMMARY = re.compile( "(?:sub[ -]?total|grand total|total|amount due|balance(?: due| forward)?|tax total|fee total)", re.IGNORECASE, ) _PAYMENT = re.compile( "^(?:payments?\\s+(?:received|instructions?|due|method|status)|(?:received|posted|applied)\\s+payments?)\\b|\\bpayments?$", re.IGNORECASE, ) _TRANSFORM = re.compile( "\\b(?:deduplicat\\w*|unique|aggregat\\w*|summari[sz]\\w*|combin\\w*|group\\w*|averag\\w*|calculat\\w*|normaliz\\w*|convert\\w*|except|unless|instead|knowledge)\\b", re.IGNORECASE, ) _CELL = "\\(?[+-]?\\$?[\\d,]+\\.\\d{2}\\)?" _SCOPE_WORDS = { "only", "explicit", "printed", "separately", "tax", "taxes", "row", "rows", "fee", "fees", "billed", "goods", "product", "products", "services", "service", "charge", "charges", "exclude", "excluding", "all", "subtotal", "subtotals", "total", "totals", "grand", "payment", "payments", "amount", "due", "and", "or", "null", "if", "no", "when", "the", "document", "has", "a", "must", "be", "surcharge", "surcharges", "return", "instructions", } def role(label: str) -> str: label = " ".join(label.split()).rstrip(": ") if _SUMMARY.fullmatch(label): return "summary" if _PAYMENT.search(label): return "payment" if _TAX.search(label): return "tax" return "fee" if _FEE.search(label) else "charge" def scope(spec: dict[str, Any]) -> str | None: item = spec.get("items") if ( spec.get("type") != "list" or spec.get("knowledge") or (not isinstance(item, dict)) ): return None properties = item.get("properties", {}) if item.get("type") != "object" or len(properties) != 2: return None if any( ( not isinstance(child, dict) or child.get("knowledge") for child in properties.values() ) ): return None if sorted((child.get("type", "") for child in properties.values())) != [ "number", "text", ]: return None amount = next((child for child in properties.values() if child["type"] == "number")) if amount.get("decimals") != 2 or amount.get("unit") or amount.get("integer"): return None instructions = spec.get("instructions", "") if not isinstance(instructions, str) or _TRANSFORM.search(instructions): return None if set(re.findall("[a-z]+", instructions.lower())) - _SCOPE_WORDS: return None if any( ( set(child) - {"type", "instructions", "nullable", "decimals"} for child in properties.values() ) ): return None if any( ( _TRANSFORM.search(child.get("instructions", "")) for child in properties.values() ) ): return None positive = re.split( "[.\\n]|\\bExclude\\b", instructions, maxsplit=1, flags=re.IGNORECASE )[0].strip() if re.fullmatch( "Only\\s+(?:(?:explicit|printed|separately\\s+printed)\\s+)?tax(?:es)?\\s+rows", positive, re.IGNORECASE, ): return "tax" if re.fullmatch( "Only\\s+(?:(?:explicit|printed|separately\\s+printed)\\s+)?fees?\\s+rows", positive, re.IGNORECASE, ): return "fee" if ( re.fullmatch( "Only\\s+billed\\s+(?:(?:goods?|products?|services?)\\s+(?:(?:and|or)\\s+)?)+(?:charge\\s+)?rows", positive, re.IGNORECASE, ) and all( ( re.search( "\\bExclude\\b[^.\\n]*\\b" + token + "\\b", instructions, re.IGNORECASE, ) for token in ("tax(?:es)?", "fees?", "payments?") ) ) and ( not re.search("\\b(?:not|never)\\s+exclude\\b", instructions, re.IGNORECASE) ) ): return "charge" return None def parse(line: object) -> tuple[str, int | float | None] | None: if not isinstance(line, str): raise TypeError("Missing literal image line") match = re.fullmatch("(.*?)\\s*(?:\\||:)\\s*(" + _CELL + ")\\s*\\|?\\s*", line) if not match: match = re.fullmatch("(.*?)\\s+(\\(?[+-]?\\$[\\d,]+\\.\\d{2}\\)?)\\s*", line) if not match: return None label = match[1].rstrip(" |:") if not label or "|" in label: return None return ( label, printed_numbers.normalize(match[2], {"type": "number", "decimals": 2}), ) def bands(image: Image.Image) -> list[tuple[int, int]] | None: if ( image.mode != "RGB" or image.width < 100 or image.getpixel((0, 0)) != (255, 255, 255) ): return None ink = np.asarray(image.convert("L")) < 200 if np.mean(ink) > 0.22: return None rows = image_table_rows._runs(ink.any(axis=1)) if not 2 <= len(rows) <= MAX_LIST_ITEMS or any( (not 8 <= bottom - top <= 32 for (top, bottom) in rows) ): return None return rows def prepare( request: PredictRequest, encoded: EncodedRequest, encode: Callable[..., EncodedRequest], ) -> EncodedRequest: (record, forecasts, tasks, typed, eos) = encoded annotations = record["_meta"]["extraction_contract"]["fields"] selected = { name: kind for (name, spec) in request.fields.items() if (kind := scope(spec)) and annotations.get(name, {}).get("query_fields") } if not selected or len(request.images) != 1 or request.documents: return encoded originals = record.get("_original_images") image = originals[0] if originals else request.images[0].to_pil() rows = bands(image) if rows is None: return encoded if ( image_table_rows.detect(image) is not None or image_ragged_rows.detect(image) is not None ): selected = {name: kind for (name, kind) in selected.items() if kind != "charge"} if not selected: return encoded salts = dict(record.get("_ar_task_salts", {})) def add(child_request: PredictRequest, **options: Any) -> list[str]: (child, _, prepared, _, _) = encode( child_request, physical_row_view=True, **options ) names = [] for source, ids, payload in prepared: target = "financial_line_" + str(len(record["fields"])) while target in record["fields"]: target += "_" record["fields"][target] = child["fields"][source] record["_ar_constraints"][target] = child["_ar_constraints"][source] salts[target] = child["_cache_salt"] tasks.append((target, ids, payload)) names.append(target) return names tasks = list(tasks) queries = [] for top, bottom in rows: view = image.crop( (0, max(0, top - 3), image.width, min(image.height, bottom + 3)) ) view = view.resize((view.width * 2, view.height * 2), Image.Resampling.BICUBIC) child_request = request.model_copy(update={"fields": {"printed_line": LINE}}) queries.extend( add( child_request, image_views=lambda view=view: (view,), image_view_key=f"financial-line-v1:{top}:{bottom}", ) ) plans = {} for name, kind in selected.items(): spec = request.fields[name] presence = { "type": "enum", "enum": [True, None], "nullable": True, "instructions": "Decide whether at least one printed row satisfies this requested list scope. " + spec.get("instructions", "") + " Return true if any matching printed row exists; return null only if there are no matching rows. A matching row with amount zero or a negative amount is present. Check the row's own label; a neighboring tax or fee does not remove an eligible service row. This asks only whether a matching row exists, not its value or row count.", } gate = add(request.model_copy(update={"fields": {name: presence}}))[0] previous = annotations[name].pop("query_fields") tasks = [task for task in tasks if task[0] not in {name, *previous}] for child in previous: record["fields"].pop(child) record["_ar_constraints"].pop(child, None) annotations.pop(child, None) plans[name] = {"kind": kind, "presence": gate, "spec": deepcopy(spec)} record.update( _financial_line_queries=queries, _financial_line_plans=plans, _ar_tasks=tasks, _ar_task_salts=salts, _ar_prefix=[], _ar_group_prefixes=[], ) return (record, forecasts, tasks, typed, eos) def assemble( record: dict[str, Any], predictions: dict[str, dict[str, Any]] ) -> dict[str, dict[str, Any]]: queries = record.get("_financial_line_queries", []) if not queries: return {} copies = [predictions[query] for query in queries] (parsed, parse_valid) = ([], True) for answer in copies: try: parsed.append( parse(answer["value"]) if isinstance(answer.get("value"), str) else None ) except (ValueError, TypeError, OverflowError): parsed.append(None) parse_valid = False result = {} for name, plan in record["_financial_line_plans"].items(): spec = plan["spec"] properties = spec["items"]["properties"] text = next( (key for (key, value) in properties.items() if value["type"] == "text") ) number = next( (key for (key, value) in properties.items() if value["type"] == "number") ) selected = [ (answer, row) for (answer, row) in zip(copies, parsed, strict=True) if row is not None and role(row[0]) == plan["kind"] ] values = [{text: row[0], number: row[1]} for (_, row) in selected] presence = predictions[plan["presence"]] relevant = [*copies, presence] complete = all((answer.get("complete", False) for answer in relevant)) valid = ( parse_valid and complete and all((answer.get("valid", False) for answer in relevant)) ) valid &= len(values) <= spec.get("max_items", MAX_LIST_ITEMS) try: for value in values: Draft202012Validator(ar.schema(spec["items"])).validate(value) except ValidationError: valid = False p_null = presence["p_null"] if ( type(p_null) not in (int, float) or not math.isfinite(p_null) or (not 0 <= p_null <= 1) or any( ( type(answer.get("log_probability")) not in (int, float) or not math.isfinite(answer["log_probability"]) or answer["log_probability"] > 0 for answer in copies ) ) ): raise ValueError("Invalid financial-line model scores") event = 1 - p_null if values else p_null probability = ( sum((answer["log_probability"] for answer in copies)) + math.log(event) if event > 0 else None ) answer = { "value": values, "valid": bool(valid), "complete": complete, "confidence": 0.0, "confidence_method": "ar_literal_image_lines_and_scope_presence_probability", "confidences": [copy["confidence"] for (copy, _) in selected], "p_null": p_null, "p_null_method": "ar_parent_scope_presence_null_probability", "p_nulls": [copy["p_null"] for (copy, _) in selected], "spans": [None] * len(selected), } if probability is not None: answer.update( log_probability=probability, confidence=math.exp(probability) if valid else 0.0, ) result[name] = answer return result