from __future__ import annotations import hashlib import json import math import re import unicodedata from collections.abc import Iterable, Iterator from decimal import Decimal, localcontext from itertools import pairwise from typing import Any from . import ar, source_dates from . import source_sums as sums MAX_BLOCKS = 128 MAX_LABEL_AMOUNT_GAP = 24 MAX_SUPPORT_BLOCKS = 24 MAX_SUPPORT_PRECISION = 256 _RECEIPT = re.compile("\\breceipts?\\b", re.IGNORECASE) _REQUESTED_TOTAL = re.compile("\\btotals?\\b", re.IGNORECASE) _COMPOUND = re.compile( "\\b(?:averag\\w*|mean|median|subtract\\w*|deduct\\w*|minus|less|plus|add(?:s|ed|ing)?|multipl\\w*|times|divid\\w*|quotient|ratio|percent(?:age)?|discount\\w*|surcharge\\w*|convert\\w*|conversion\\w*|exchange|fx|clamp\\w*|cap(?:ped)?|net[ \\t]+of)\\b|[*/×÷%=]|(?<=\\s)[+-][ \\t]*\\d|\\b(?:in|to)[ \\t]+cents?\\b", re.IGNORECASE, ) _ROUND = "(?:round\\w*|ceil\\w*|floor\\w*|truncat\\w*)" _ROUND_EACH = re.compile( "\\b" + _ROUND + "\\b[^.;\\n]*(?:\\b(?:each|every|per|individually|before)\\b|\\breceipts?\\b)|\\b(?:each|every|per|individual(?:ly)?)\\b[^.;\\n]*\\b" + _ROUND + "\\b", re.IGNORECASE, ) _TOTAL = re.compile( "\\b(?:amount[ \\t]+charged|(?:grand[ \\t]+)?total(?:[ \\t]+(?:charges|claimed|paid|due|amount))?)\\b", re.IGNORECASE, ) _SEPARATOR = re.compile("\\r?\\n[ \\t]*\\r?\\n") _RECORD = re.compile("\\b(?:receipt|payment)\\b", re.IGNORECASE) _REFERENCE = re.compile( "\\b(?:(?:room|receipt)[ \\t]+\\d+|(?:odometer[ \\t]+(?:start|end)|(?:trip[ \\t]+)?miles)[ \\t]+\\d+|[a-z][a-z0-9_]*[-/#][a-z0-9_/-]*\\d[a-z0-9_/-]*|[a-z]+\\d+[a-z0-9]*)\\b", re.IGNORECASE, ) _MEASUREMENT = re.compile( "\\b\\d+(?:\\.\\d+)?[ \\t]+(?:miles?|nights?)\\b", re.IGNORECASE ) _DISTANCE_RATE = re.compile( "(? bytes: return json.dumps( value, sort_keys=True, ensure_ascii=False, separators=(",", ":"), allow_nan=False, ).encode() def _paragraphs(source: str) -> Iterator[tuple[int, int]]: start = 0 for boundary in list(_SEPARATOR.finditer(source)) + [None]: end = len(source) if boundary is None else boundary.start() (left, right) = (start, end) while left < right and source[left].isspace(): left += 1 while right > left and source[right - 1].isspace(): right -= 1 if left < right: yield (left, right) if boundary is not None: start = boundary.end() def _blocks( task: dict[str, Any], *, paragraphs: Iterable[tuple[int, int]] | None = None ) -> list[dict[str, Any]] | None: source = task["input_text"] gap: str | list[str] metadata = [(item.start, item.end) for item in source_dates.candidates(source)] metadata.extend( ( match.span() for match in _REFERENCE.finditer(source) if re.match("[a-z]+", match[0], re.IGNORECASE)[0].upper() not in sums._CURRENCY_CODES ) ) metadata.extend( ( match.span() for pattern in ( _MEASUREMENT, _DISTANCE_RATE, _CARD_REFERENCE, _CONFIRMATION, _CLOCK, _SHORT_DATE, _DATE_CELL, ) for match in pattern.finditer(source) ) ) def non_amount(item: dict[str, Any]) -> bool: (left, right) = item["span"] return any( ( right <= end and (start <= left or not source[left:start].strip(" \t+-")) for (start, end) in metadata ) ) blocks = [] for start, end in _paragraphs(source) if paragraphs is None else paragraphs: labels = list(_TOTAL.finditer(source, start, end)) numbers = [ item for item in task["candidates"] if start <= item["span"][0] < end ] if not labels: text = source[start:end] if ( any((not non_amount(item) for item in numbers)) or _RECORD.search(text) or _CURRENCY_CODE.search(text) or any((unicodedata.category(char) == "Sc" for char in text)) ): return None continue if len(labels) != 1 or len(list(_RECORD.finditer(source, start, end))) > 1: return None label = labels[0] linked = [] for item in task["candidates"]: (left, right) = item["span"] if ( not label.end() <= left <= label.end() + MAX_LABEL_AMOUNT_GAP or right > end ): continue gap = "".join( ( "$" if unicodedata.category(char) == "Sc" else char for char in source[label.end() : left] ) ) if _BRIDGE.fullmatch(gap): linked.append(item) if not linked: for item in task["candidates"]: (left, right) = item["span"] if ( left < start or not label.start() - MAX_LABEL_AMOUNT_GAP <= right <= label.start() ): continue gap = "".join( ( "$" if unicodedata.category(char) == "Sc" else char for char in source[right : label.start()] ) ) if _BRIDGE.fullmatch(gap): linked.append(item) if len(linked) != 1: return None item = linked[0] for other in numbers: if other is item or non_amount(other): continue (left, _) = other["span"] components = _COMPONENT.finditer(source, max(start, left - 48), left) if not any( ( _BRIDGE.fullmatch( "".join( ( "$" if unicodedata.category(char) == "Sc" else char for char in source[component.end() : left] ) ) ) for component in components ) ): return None monetary = [other for other in numbers if not non_amount(other)] for previous, following in pairwise(monetary): (left, right) = (previous["span"][1], following["span"][0]) gap = list(source[left:right]) for meta_left, meta_right in metadata: for offset in range(max(left, meta_left), min(right, meta_right)): gap[offset - left] = " " gap = "".join( (" " if unicodedata.category(char) == "Sc" else char for char in gap) ) for pattern in (_TOTAL, _COMPONENT, _COMPONENT_CONNECTOR, _CURRENCY_CODE): gap = pattern.sub(" ", gap) if not _AMOUNT_GAP.fullmatch(gap): return None blocks.append( { "block_span": [start, end], "source_block": source[start:end], "total_label": label[0], "total_label_span": [label.start(), label.end()], "total_occurrence": { key: item[key] for key in ("id", "span", "printed") }, } ) if len(blocks) > MAX_BLOCKS: return None return blocks or None def _compatible_currencies( task: dict[str, Any], blocks: Iterable[dict[str, Any]] ) -> bool: allowed = set(sums._CURRENCY_CODES) unit = task["field_spec"].get("unit") if unit not in (None, ""): if not isinstance(unit, str): return False normalized = unit.strip().upper() if normalized in sums._CURRENCY_CODES: allowed.intersection_update({normalized}) elif unit.strip() in _CURRENCY_SYMBOLS: allowed.intersection_update(_CURRENCY_SYMBOLS[unit.strip()]) else: return False for text in [ task["field_spec"]["instructions"], *(block["source_block"] for block in blocks), ]: for code in _CURRENCY_CODE.finditer(text): allowed.intersection_update({code[0].upper()}) for char in text: if unicodedata.category(char) == "Sc": allowed.intersection_update(_CURRENCY_SYMBOLS.get(char, ())) if not allowed: return False return True def _from_sum_task(task: dict[str, Any]) -> dict[str, Any] | None: blocks = _blocks(task) if blocks is None or not _compatible_currencies(task, blocks): return None prefix = ( "Classify each source block using the entire supplied document.\nOriginal document:\n" + task["input_text"] + "\n\nRequested numeric field: " + task["field"] + "\nOriginal field schema and instructions: " + json.dumps(task["field_spec"], ensure_ascii=False, separators=(",", ":")) + "\nDecide whether the candidate block is one receipt record in the requested sum. Use its category, entity, date and the original requested scope. Return true if and only if this record's explicit total is one requested operand. A total label alone does not establish inclusion. Hotel, airline and claim-summary blocks remain candidates; include them only when the requested receipt scope includes them. A claim summary is not an additional underlying receipt. Do not separately select food, subtotal, tax, tip or other components already included in a total. A zero or refund remains a present amount. Distinct source records remain distinct even with equal totals. Do not calculate the sum. Return false for an excluded record; return null if the evidence does not determine its inclusion. No expected values are supplied. Return exactly one JSON Boolean or null, then end immediately.\n" ) tasks = [] for block in blocks: suffix = "\nCandidate source block: " + json.dumps( block, ensure_ascii=False, separators=(",", ":") ) tasks.append( { "occurrence_id": block["total_occurrence"]["id"], "suffix": suffix, "prompt": prefix + suffix, "json_schema": {"type": ["boolean", "null"]}, "grammar": {"ebnf": 'root ::= "true" | "false" | "null"'}, } ) return { "version": 1, "kind": "source-receipt-block-boolean-sum-diagnostic-v1", "sum_task": task, "shared_prefix": prefix, "shared_prefix_sha256": hashlib.sha256(prefix.encode()).hexdigest(), "tasks": tasks, "blocks": blocks, "all_candidates_require_complete_decisions": True, "native_typed_boolean_head": False, "runtime_changes": False, "semantic_completeness_proven_by_grounding": False, } def eligible( record: dict[str, Any], field_name: str, *, has_media: bool = False ) -> bool: if not sums.eligible(record, field_name, has_media=has_media): return False instructions = record["fields"][field_name]["instructions"] return bool( _RECEIPT.search(instructions) and _REQUESTED_TOTAL.search(instructions) and (not _COMPOUND.search(instructions)) and (not _ROUND_EACH.search(instructions)) ) def prepare_task( record: dict[str, Any], field_name: str, *, has_media: bool = False ) -> dict[str, Any] | None: if not eligible(record, field_name, has_media=has_media): return None from . import receipt_hybrid_inventory task = sums.prepare_task(record, field_name, has_media=has_media) if task is None: return None legacy = _from_sum_task(task) if legacy is not None: return legacy packet = receipt_hybrid_inventory._from_sum_task(task) if ( packet is None or not packet["currency_compatible"] or packet["unresolved_fragments_without_numeric_candidates"] or (len(packet["tasks"]) > MAX_BLOCKS) ): return None return packet def supports_sum(packet: dict[str, Any], primary_value: object) -> bool | None: if isinstance(primary_value, bool) or not isinstance( primary_value, (int, float, Decimal) ): return None try: target = Decimal(str(primary_value)) if not target.is_finite(): return None task = packet["sum_task"] hybrid = packet.get("kind") == "receipt-hybrid-paragraph-runtime-v1" blocks = packet["descriptors" if hybrid else "blocks"] if not 0 < len(blocks) <= MAX_SUPPORT_BLOCKS: return None recomputed = prepare_task( { "input": task["input_text"], "fields": {task["field"]: task["field_spec"]}, }, task["field"], ) if recomputed is None or canonical(recomputed) != canonical(packet): return None indexed = {item["id"]: item for item in task["candidates"]} amounts = [] for block in blocks: item = indexed[block["occurrence" if hybrid else "total_occurrence"]["id"]] if item["decimal"] is None: return None amount = Decimal(item["decimal"]) if not amount.is_finite(): return None amounts.append(amount) values = [target, *amounts] highest = max((value.adjusted() for value in values)) lowest: int = min((value.as_tuple().exponent for value in values)) required_digits = highest - lowest + len(str(len(amounts))) + 3 precision = max(28, required_digits) if precision > MAX_SUPPORT_PRECISION: return None def subsets(items: list[Decimal]) -> dict[Decimal, bool]: totals = {Decimal(0): False} for amount in items: previous = list(totals) for value in previous: totals[value + amount] = True return totals with localcontext() as context: context.prec = precision spec = task["field_spec"] def normalized(value: Decimal) -> Decimal: return Decimal(str(ar.normalize_number(value, spec))) if normalized(target) != target: return None middle = len(amounts) // 2 (left, right) = (subsets(amounts[:middle]), subsets(amounts[middle:])) for value, nonempty in left.items(): rest = target - value if ( rest in right and (nonempty or right[rest]) and (normalized(value + rest) == target) ): return True if any((normalized(amount) != amount for amount in amounts)): return None if required_digits > 15 and any( (amount != amount.to_integral_value() for amount in amounts) ): return None return False except (ArithmeticError, AttributeError, KeyError, TypeError, ValueError): return None def _selection_scores( decisions: dict[str, dict[str, Any]], expected: Iterable[str] ) -> dict[str, Any] | None: scores = [] for identifier in expected: row = decisions[identifier] score = row.get("log_probability") if ( row.get("valid") is not True or row.get("complete") is not True or type(row.get("value")) is not bool or (row["value"] is not json.loads(row["raw_json"])) or (row.get("decoder") != ar.VERSION) or (row.get("confidence_method") != "ar_sequence_probability") or isinstance(score, bool) or (not isinstance(score, (int, float))) ): return None try: if not math.isfinite(score) or score > 0: return None scores.append(float(score)) except (OverflowError, ValueError): return None try: total = math.fsum(scores) except OverflowError: return None if not math.isfinite(total): return None return { "log_probability": total, "confidence": math.exp(total), "confidence_method": "receipt_block_boolean_joint_sequence_probability", "score_scope": "all_receipt_block_inclusion_decisions", "decision_count": len(scores), } def validate_result( packet: dict[str, Any], decisions: dict[str, dict[str, Any]] ) -> dict[str, Any]: if packet.get("kind") == "receipt-hybrid-paragraph-runtime-v1": from . import receipt_hybrid_inventory return receipt_hybrid_inventory.validate_result(packet, decisions) result: dict[str, Any] = { "valid": False, "value": None, "operands": [], "decimal_sum": None, "operand_count": 0, "error": None, "selection_method": "per_receipt_block_boolean_diagnostic", "decision_count": 0, "required_decision_count": 0, "semantic_completeness_proven_by_grounding": False, } try: task = packet["sum_task"] result["required_decision_count"] = len(packet["tasks"]) recomputed = prepare_task( { "input": task["input_text"], "fields": {task["field"]: task["field_spec"]}, }, task["field"], ) if recomputed is None or canonical(recomputed) != canonical(packet): raise ValueError("receipt_block_prompt_or_source_mapping_changed") expected = [row["occurrence_id"] for row in packet["tasks"]] if not isinstance(decisions, dict) or set(decisions) != set(expected): raise ValueError("receipt_block_coverage_incomplete_or_extra") selected = [] for identifier in expected: row = decisions[identifier] if not isinstance(row, dict) or row.get("complete") is not True: raise ValueError("incomplete_boolean_generation:" + identifier) value = json.loads( row["raw_json"], parse_constant=lambda _: (_ for _ in ()).throw( ValueError("nonfinite_json") ), ) if value is None: raise ValueError("unresolved_boolean_inclusion:" + identifier) if type(value) is not bool: raise ValueError("invalid_boolean_scalar:" + identifier) result["decision_count"] += 1 if value: selected.append(identifier) result.update( sums.validate_result( task, json.dumps(selected) if selected else "null", complete=True ) ) result["selected_ids"] = selected if ( result["valid"] and (scores := _selection_scores(decisions, expected)) is not None ): result["selection_scores"] = scores except Exception as error: result.update( valid=False, value=None, operands=[], decimal_sum=None, operand_count=0, error=str(error) or type(error).__name__, ) return result