from __future__ import annotations from collections.abc import Callable from typing import TYPE_CHECKING, Any, cast if TYPE_CHECKING: from .api import PredictRequest from numpy.typing import ArrayLike from .backend import EncodedRequest import math from dataclasses import dataclass import numpy as np from PIL import Image from .schema import MAX_LIST_ITEMS @dataclass(frozen=True) class RowTable: header: tuple[int, int] rows: tuple[tuple[int, int], ...] def views(self, image: Image.Image) -> list[Image.Image]: result = [] for top, bottom in self.rows: header = image.crop( ( 0, max(0, self.header[0] - 3), image.width, min(image.height, self.header[1] + 3), ) ) row = image.crop( (0, max(0, top - 3), image.width, min(image.height, bottom + 3)) ) view = Image.new( "RGB", (image.width, header.height + row.height + 12), "white" ) view.paste(header, (0, 0)) view.paste(row, (0, header.height + 12)) result.append(view) return result def _runs(values: ArrayLike) -> list[tuple[int, int]]: padded = np.pad(np.asarray(values, dtype=np.int8), (1, 1)) edges = np.flatnonzero(np.diff(padded)) return cast( list[tuple[int, int]], [tuple(pair) for pair in edges.reshape(-1, 2).tolist()] ) def detect(image: Image.Image) -> RowTable | None: if image.mode != "RGB" or image.width < 100 or image.height < 60: return None grey = np.asarray(image.convert("L")) if np.mean(grey < 245) > 0.22: return None ink = grey < 200 bands = _runs(ink.any(axis=1)) height: float candidates = [] for index, (top, bottom) in enumerate(bands): height = bottom - top if height < 10: continue band = ink[top:bottom] columns = band.any(axis=0) separators = [] for left, right in _runs(band.sum(axis=0) >= 0.9 * height): if ( right - left > max(3, height * 0.18) or left < 3 or right > image.width - 3 or columns[left - 3 : left].any() or columns[right : right + 3].any() ): continue separators.append((left + right) / 2) if len(separators) >= 2: candidates.append((index, top, bottom, separators)) if len(candidates) < 4: return None groups: list[list[tuple[int, int, int, list[float]]]] = [] for candidate in candidates: if not groups or candidate[0] != groups[-1][-1][0] + 1: groups.append([]) groups[-1].append(candidate) if len(groups) != 1: return None eligible = [] for group in groups: if len(group) < 4 or len({len(row[3]) for row in group}) != 1: continue (header, body) = (group[0], group[1:]) heights = np.array([bottom - top for (_, top, bottom, _) in group]) height = float(np.median(heights)) positions = np.array([row[3] for row in body]) middle = np.median(positions, axis=0) spacing = np.diff([row[1] for row in group]) if ( np.max(heights) > 1.2 * np.min(heights) or np.max(spacing) > 1.3 * np.min(spacing) or np.max(np.abs(positions - middle)) > height * 1.2 or (abs(header[3][0] - middle[0]) < height * 2) ): continue eligible.append( RowTable((header[1], header[2]), tuple(((row[1], row[2]) for row in body))) ) return eligible[0] if len(eligible) == 1 else None def prepare( request: PredictRequest, encoded: EncodedRequest, encode: Callable[..., EncodedRequest], *, scale: float = 2, ) -> EncodedRequest: (record, forecasts, tasks, typed, eos) = encoded annotations = record["_meta"]["extraction_contract"]["fields"] parents = [ name for (name, annotation) in annotations.items() if annotation.get("image_row_fields") ] if not parents or len(request.images) != 1 or request.documents: return encoded originals = record.get("_original_images") image = originals[0] if originals else request.images[0].to_pil() table = detect(image) if table is None: return encoded views = table.views(image) salts = dict(record.get("_ar_task_salts", {})) for name in parents: spec = request.fields[name] if len(views) > spec.get("max_items", MAX_LIST_ITEMS): continue annotation = annotations[name] previous = annotation["image_row_fields"] replacements = [] for index, view in enumerate(views): factor = min(scale, (1600000 / (view.width * view.height)) ** 0.5) if factor > 1: view = view.resize( (int(view.width * factor), int(view.height * factor)), Image.Resampling.BICUBIC, ) child = previous[index] view_field = "source_row" row_spec = { "type": "object", "nullable": True, "properties": spec["items"]["properties"], "instructions": "Check whether the single data row below the column header matches the requested scope. Return null for an excluded row. Otherwise return the complete printed row. " + spec.get("instructions", "") + " Missing cells are null. Do not count or select a different row.", } row_request = request.model_copy(update={"fields": {view_field: row_spec}}) (row_record, _, row_tasks, _, _) = encode( row_request, physical_row_view=True, image_views=lambda view=view: (view,), image_view_key=f"table-row-v1:{table.header}:{table.rows[index]}:{scale}", ) assert len(row_tasks) == 1 record["fields"][child] = row_record["fields"][view_field] record["_ar_constraints"][child] = row_record["_ar_constraints"][view_field] salts[child] = row_record["_cache_salt"] (_, ids, payload) = row_tasks[0] replacements.append((child, ids, payload)) replaced = {name, *previous} tasks = [task for task in tasks if task[0] not in replaced] + replacements retained = {task[0] for task in replacements} for child in set(previous) - retained: record["fields"].pop(child) record["_ar_constraints"].pop(child, None) annotations.pop(child, None) annotation["geometric_row_fields"] = [task[0] for task in replacements] annotation.pop("image_row_fields") record["fields"][name].pop("_image_row_inventory", None) if not any( (annotation.get("geometric_row_fields") for annotation in annotations.values()) ): return encoded record.update( _ar_tasks=tasks, _ar_task_salts=salts, _ar_prefix=[], _ar_group_prefixes=[] ) return (record, forecasts, tasks, typed, eos) def assemble( record: dict[str, Any], predictions: dict[str, dict[str, Any]] ) -> dict[str, dict[str, Any]]: result = {} for name, annotation in record["_meta"]["extraction_contract"]["fields"].items(): queries = annotation.get("geometric_row_fields") if not queries: continue rows = [predictions[query] for query in queries] selected = [row for row in rows if row.get("value") is not None] complete = all((row.get("complete", False) for row in rows)) valid = complete and all((row.get("valid", False) for row in rows)) answer = { "value": [row["value"] for row in selected], "complete": complete, "valid": valid, "confidence": 0.0, "confidences": [row.get("confidence", 0.0) for row in selected], "confidence_method": "ar_physical_image_row_sequence_probability", "p_null": math.prod((row["p_null"] for row in rows)), "p_nulls": [row["p_null"] for row in selected], "p_null_method": "product_physical_image_row_exclusion_probabilities", "spans": [None] * len(selected), } if valid and all(("log_probability" in row for row in rows)): probability = sum((row["log_probability"] for row in rows)) if math.isfinite(probability): answer.update( log_probability=probability, confidence=math.exp(min(0.0, probability)), ) result[name] = answer return result