clairic-4bit / clairic_runtime /image_table_rows.py
lucataco's picture
Clairic, MLX 4-bit, with runtime, server and conversion script
1a48d6b verified
Raw History Blame Contribute Delete
8.87 kB
from __future__ import annotations
from collections.abc import Callable
from typing import TYPE_CHECKING, Any, cast
if TYPE_CHECKING:
from .api import PredictRequest
from numpy.typing import ArrayLike
from .backend import EncodedRequest
import math
from dataclasses import dataclass
import numpy as np
from PIL import Image
from .schema import MAX_LIST_ITEMS
@dataclass(frozen=True)
class RowTable:
header: tuple[int, int]
rows: tuple[tuple[int, int], ...]
def views(self, image: Image.Image) -> list[Image.Image]:
result = []
for top, bottom in self.rows:
header = image.crop(
(
0,
max(0, self.header[0] - 3),
image.width,
min(image.height, self.header[1] + 3),
)
)
row = image.crop(
(0, max(0, top - 3), image.width, min(image.height, bottom + 3))
)
view = Image.new(
"RGB", (image.width, header.height + row.height + 12), "white"
)
view.paste(header, (0, 0))
view.paste(row, (0, header.height + 12))
result.append(view)
return result
def _runs(values: ArrayLike) -> list[tuple[int, int]]:
padded = np.pad(np.asarray(values, dtype=np.int8), (1, 1))
edges = np.flatnonzero(np.diff(padded))
return cast(
list[tuple[int, int]], [tuple(pair) for pair in edges.reshape(-1, 2).tolist()]
)
def detect(image: Image.Image) -> RowTable | None:
if image.mode != "RGB" or image.width < 100 or image.height < 60:
return None
grey = np.asarray(image.convert("L"))
if np.mean(grey < 245) > 0.22:
return None
ink = grey < 200
bands = _runs(ink.any(axis=1))
height: float
candidates = []
for index, (top, bottom) in enumerate(bands):
height = bottom - top
if height < 10:
continue
band = ink[top:bottom]
columns = band.any(axis=0)
separators = []
for left, right in _runs(band.sum(axis=0) >= 0.9 * height):
if (
right - left > max(3, height * 0.18)
or left < 3
or right > image.width - 3
or columns[left - 3 : left].any()
or columns[right : right + 3].any()
):
continue
separators.append((left + right) / 2)
if len(separators) >= 2:
candidates.append((index, top, bottom, separators))
if len(candidates) < 4:
return None
groups: list[list[tuple[int, int, int, list[float]]]] = []
for candidate in candidates:
if not groups or candidate[0] != groups[-1][-1][0] + 1:
groups.append([])
groups[-1].append(candidate)
if len(groups) != 1:
return None
eligible = []
for group in groups:
if len(group) < 4 or len({len(row[3]) for row in group}) != 1:
continue
(header, body) = (group[0], group[1:])
heights = np.array([bottom - top for (_, top, bottom, _) in group])
height = float(np.median(heights))
positions = np.array([row[3] for row in body])
middle = np.median(positions, axis=0)
spacing = np.diff([row[1] for row in group])
if (
np.max(heights) > 1.2 * np.min(heights)
or np.max(spacing) > 1.3 * np.min(spacing)
or np.max(np.abs(positions - middle)) > height * 1.2
or (abs(header[3][0] - middle[0]) < height * 2)
):
continue
eligible.append(
RowTable((header[1], header[2]), tuple(((row[1], row[2]) for row in body)))
)
return eligible[0] if len(eligible) == 1 else None
def prepare(
request: PredictRequest,
encoded: EncodedRequest,
encode: Callable[..., EncodedRequest],
*,
scale: float = 2,
) -> EncodedRequest:
(record, forecasts, tasks, typed, eos) = encoded
annotations = record["_meta"]["extraction_contract"]["fields"]
parents = [
name
for (name, annotation) in annotations.items()
if annotation.get("image_row_fields")
]
if not parents or len(request.images) != 1 or request.documents:
return encoded
originals = record.get("_original_images")
image = originals[0] if originals else request.images[0].to_pil()
table = detect(image)
if table is None:
return encoded
views = table.views(image)
salts = dict(record.get("_ar_task_salts", {}))
for name in parents:
spec = request.fields[name]
if len(views) > spec.get("max_items", MAX_LIST_ITEMS):
continue
annotation = annotations[name]
previous = annotation["image_row_fields"]
replacements = []
for index, view in enumerate(views):
factor = min(scale, (1600000 / (view.width * view.height)) ** 0.5)
if factor > 1:
view = view.resize(
(int(view.width * factor), int(view.height * factor)),
Image.Resampling.BICUBIC,
)
child = previous[index]
view_field = "source_row"
row_spec = {
"type": "object",
"nullable": True,
"properties": spec["items"]["properties"],
"instructions": "Check whether the single data row below the column header matches the requested scope. Return null for an excluded row. Otherwise return the complete printed row. "
+ spec.get("instructions", "")
+ " Missing cells are null. Do not count or select a different row.",
}
row_request = request.model_copy(update={"fields": {view_field: row_spec}})
(row_record, _, row_tasks, _, _) = encode(
row_request,
physical_row_view=True,
image_views=lambda view=view: (view,),
image_view_key=f"table-row-v1:{table.header}:{table.rows[index]}:{scale}",
)
assert len(row_tasks) == 1
record["fields"][child] = row_record["fields"][view_field]
record["_ar_constraints"][child] = row_record["_ar_constraints"][view_field]
salts[child] = row_record["_cache_salt"]
(_, ids, payload) = row_tasks[0]
replacements.append((child, ids, payload))
replaced = {name, *previous}
tasks = [task for task in tasks if task[0] not in replaced] + replacements
retained = {task[0] for task in replacements}
for child in set(previous) - retained:
record["fields"].pop(child)
record["_ar_constraints"].pop(child, None)
annotations.pop(child, None)
annotation["geometric_row_fields"] = [task[0] for task in replacements]
annotation.pop("image_row_fields")
record["fields"][name].pop("_image_row_inventory", None)
if not any(
(annotation.get("geometric_row_fields") for annotation in annotations.values())
):
return encoded
record.update(
_ar_tasks=tasks, _ar_task_salts=salts, _ar_prefix=[], _ar_group_prefixes=[]
)
return (record, forecasts, tasks, typed, eos)
def assemble(
record: dict[str, Any], predictions: dict[str, dict[str, Any]]
) -> dict[str, dict[str, Any]]:
result = {}
for name, annotation in record["_meta"]["extraction_contract"]["fields"].items():
queries = annotation.get("geometric_row_fields")
if not queries:
continue
rows = [predictions[query] for query in queries]
selected = [row for row in rows if row.get("value") is not None]
complete = all((row.get("complete", False) for row in rows))
valid = complete and all((row.get("valid", False) for row in rows))
answer = {
"value": [row["value"] for row in selected],
"complete": complete,
"valid": valid,
"confidence": 0.0,
"confidences": [row.get("confidence", 0.0) for row in selected],
"confidence_method": "ar_physical_image_row_sequence_probability",
"p_null": math.prod((row["p_null"] for row in rows)),
"p_nulls": [row["p_null"] for row in selected],
"p_null_method": "product_physical_image_row_exclusion_probabilities",
"spans": [None] * len(selected),
}
if valid and all(("log_probability" in row for row in rows)):
probability = sum((row["log_probability"] for row in rows))
if math.isfinite(probability):
answer.update(
log_probability=probability,
confidence=math.exp(min(0.0, probability)),
)
result[name] = answer
return result