clairic-flash / clairic_runtime /image_identifier.py
areneau's picture
Add complete Clairic Transformers package
dc7d8aa verified
Raw History Blame Contribute Delete
5.72 kB
from __future__ import annotations
from collections.abc import Callable
from typing import TYPE_CHECKING, Any
if TYPE_CHECKING:
from .api import PredictRequest
from .backend import EncodedRequest
import math
import re
from PIL import Image, ImageChops, ImageStat
from .image_focus import focus_document
VIEW_ID = "identifier-content-band-v2"
_ROLE = re.compile(
"\\b(?:invoice|receipt|purchase order|order|tracking|reference|transaction|account|application)\\s+(?:number|identifier|id|code)\\b",
re.IGNORECASE,
)
_EXCLUDED_ROLE = re.compile(
"\\b(?:not|exclude|excluding|ignore|rather than)\\b[^.;:\\n]*$", re.IGNORECASE
)
def _requested_identifier(value: str) -> bool:
return any(
(
not _EXCLUDED_ROLE.search(value[: match.start()])
for match in _ROLE.finditer(value)
)
)
def fields(fields: dict[str, dict[str, Any]]) -> list[str]:
selected = []
for name, spec in fields.items():
if spec.get("type") != "text" or spec.get("knowledge"):
continue
label = re.sub("[_-]+", " ", name)
if _requested_identifier(label) or _requested_identifier(
spec.get("instructions", "")
):
selected.append(name)
return selected
def detail_view(image: Image.Image, *, max_pixels: int = 1600000) -> Image.Image:
return focus_document(image, max_pixels=max_pixels, max_scale=3.0)
def header_view(image: Image.Image, *, max_pixels: int = 1600000) -> Image.Image | None:
if image.mode != "RGB":
return None
if (
ImageChops.difference(
image, Image.new("RGB", image.size, image.getpixel((0, 0)))
).getbbox()
is None
):
return None
header = image.crop((0, 0, image.width, max(1, math.ceil(image.height * 0.4))))
scale = min(3.0, math.sqrt(max_pixels / (header.width * header.height)))
size = (max(1, int(header.width * scale)), max(1, int(header.height * scale)))
return header.resize(size, Image.Resampling.BICUBIC)
def views(
image: Image.Image,
*,
max_pixels: int = 1600000,
unfocused_image: Image.Image | None = None,
) -> tuple[Image.Image, ...]:
focused = (
image
if image is unfocused_image and max_pixels == 1600000
else detail_view(image, max_pixels=max_pixels)
)
if focused is not image:
return (focused,)
header = header_view(image, max_pixels=max_pixels)
return (image, header) if header is not None else (image,)
def content_view(
image: Image.Image, *, max_pixels: int = 1600000
) -> Image.Image | None:
if image.mode != "RGB":
return None
background = tuple(ImageStat.Stat(image.resize((64, 64))).median)
if min(background) < 160:
return None
difference = ImageChops.difference(image, Image.new("RGB", image.size, background))
channels = difference.split()
contrast = ImageChops.lighter(
ImageChops.lighter(channels[0], channels[1]), channels[2]
)
mask = contrast.point(lambda value: 255 if value >= 32 else 0)
bounds = mask.getbbox()
if bounds is None:
return None
(left, top, right, bottom) = bounds
bounds = (
max(0, left - 24),
max(0, top - 24),
min(image.width, right + 24),
min(image.height, bottom + 24),
)
(width, height) = (bounds[2] - bounds[0], bounds[3] - bounds[1])
if width * height > image.width * image.height / 2:
return None
scale = min(3.0, math.sqrt(max_pixels / (width * height)))
if scale <= 1:
return None
return image.crop(bounds).resize(
(max(1, int(width * scale)), max(1, int(height * scale))),
Image.Resampling.BICUBIC,
)
def content_views(
image: Image.Image,
*,
max_pixels: int = 1600000,
unfocused_image: Image.Image | None = None,
) -> tuple[Image.Image, ...]:
selected = (
views(image, max_pixels=max_pixels)
if unfocused_image is None
else views(image, max_pixels=max_pixels, unfocused_image=unfocused_image)
)
if selected[0] is not image:
return selected
content = content_view(image, max_pixels=max_pixels)
return (*selected, content) if content is not None else selected
def prepare(
request: PredictRequest,
encoded: EncodedRequest,
encode: Callable[..., EncodedRequest],
*,
unfocused_image: Image.Image | None = None,
) -> EncodedRequest:
selected = fields(request.fields)
if not selected or len(request.images) != 1 or request.documents:
return encoded
(record, forecasts, tasks, typed, eos) = encoded
child_fields = {name: request.fields[name] for name in selected}
child_request = request.model_copy(update={"fields": child_fields})
def image_views() -> tuple[Image.Image, ...]:
originals = record.get("_original_images")
image: Image.Image = originals[0] if originals else request.images[0].to_pil()
return (
content_views(image, unfocused_image=unfocused_image)
if image is unfocused_image
else content_views(image)
)
(child, _, replacements, _, _) = encode(
child_request, identifier_view=True, image_views=image_views
)
by_name = {task[0]: task for task in replacements}
tasks = [by_name.get(task[0], task) for task in tasks]
salts = dict(record.get("_ar_task_salts", {}))
for name in selected:
record["_ar_constraints"][name] = child["_ar_constraints"][name]
salts[name] = child["_cache_salt"]
record.update(
_ar_tasks=tasks, _ar_task_salts=salts, _ar_prefix=[], _ar_group_prefixes=[]
)
return (record, forecasts, tasks, typed, eos)