clairic / clairic_runtime /po_image_supplement.py
areneau's picture
Add complete Clairic Transformers package
f65336d verified
Raw History Blame Contribute Delete
8.2 kB
from __future__ import annotations
import base64
import io
import json
from typing import TYPE_CHECKING
from .api import ClairImage
from . import ar, grounding, identifier_spelling, image_focus
from PIL import Image
if TYPE_CHECKING:
from collections.abc import Callable, Sequence
from typing import Any
from .api import PredictRequest
from .backend import ARTask, EncodedRequest
from transformers import PreTrainedTokenizerBase
VERSION = "27b-po-all-content-horizontal2-supplement-v2"
MAX_PIXELS = 1600000
def supplement(images: Sequence[Image.Image]) -> ClairImage:
views: list[Image.Image] = []
for image in images:
focused = image_focus.focus_document(image)
views.append(
focused.resize(
(round(focused.width * 2.0), focused.height), Image.Resampling.BICUBIC
)
)
board = Image.new(
"RGB",
(
max((image.width for image in views)),
sum((image.height for image in views)) + 32 * (len(views) - 1),
),
"white",
)
y = 0
for image in views:
board.paste(image, (0, y))
y += image.height + 32
scale = min(1.0, (MAX_PIXELS / (board.width * board.height)) ** 0.5)
if scale < 1:
board = board.resize(
(max(1, int(board.width * scale)), max(1, int(board.height * scale))),
Image.Resampling.BICUBIC,
)
buffer = io.BytesIO()
board.save(buffer, format="PNG")
return ClairImage.model_validate(
{
"content_type": "image/png",
"base64": base64.b64encode(buffer.getvalue()).decode(),
}
)
def plain_task(
task: ARTask,
wire_spec: dict[str, Any],
public_spec: dict[str, Any],
tokenizer: PreTrainedTokenizerBase,
max_length: int,
) -> tuple[ARTask, dict[str, Any], dict[str, str]]:
(name, ids, media) = task
source_ids = list(ids)
text = tokenizer.decode(
ids, skip_special_tokens=False, clean_up_tokenization_spaces=False
)
if tokenizer.encode(text, add_special_tokens=False) != source_ids:
raise ValueError(
"PO scalar source prompt cannot make a lossless tokenizer roundtrip"
)
before = identifier_spelling.INSTRUCTION + "\n"
if text.count(before) != 1 or wire_spec.get("instructions", "").count(before) != 1:
raise ValueError(
"PO scalar source must contain exactly one spelling instruction"
)
scalar = {
**wire_spec,
"instructions": wire_spec["instructions"].replace(before, ""),
}
for key in ("minLength", "maxLength", "pattern"):
if key in public_spec:
scalar[key] = public_spec[key]
(old_schema, schema) = (ar.schema(wire_spec), ar.schema(scalar))
text = text.replace(before, "")
if schema != old_schema:
old = "\nSchema: " + json.dumps(
old_schema, ensure_ascii=False, separators=(",", ":")
)
new = "\nSchema: " + json.dumps(
schema, ensure_ascii=False, separators=(",", ":")
)
if text.count(old) != 1:
raise ValueError("PO scalar source must contain exactly one native schema")
text = text.replace(old, new)
plain_ids = tokenizer.encode(text, add_special_tokens=False)
if len(plain_ids) > max_length:
raise ValueError("PO scalar detail exceeds the complete model context")
return ((name, plain_ids, media), scalar, {"json_schema": json.dumps(schema)})
def selected_fields(
request: PredictRequest, encoded: EncodedRequest
) -> dict[str, dict[str, Any]]:
image_count = len(request.images)
if image_count not in (1, 2, 3) or request.documents:
return {}
(record, _, tasks, _, _) = encoded
contracts = record.get("_meta", {}).get("identifier_spelling", {})
active = {name for (name, _, media) in tasks if media is not None}
return {
name: request.fields[name]
for (name, contract) in contracts.items()
if name in active
and contract.get("wire") == identifier_spelling.VERSION
and (not request.fields[name].get("knowledge"))
and (not grounding.state_only(request.fields[name]))
}
def prepare(
request: PredictRequest,
encoded: EncodedRequest,
*,
encode_child: Callable[[PredictRequest], EncodedRequest],
tokenizer: PreTrainedTokenizerBase,
max_length: int,
media_groups: Callable[..., list[dict[str, Any]]],
) -> EncodedRequest:
selected = selected_fields(request, encoded)
if not selected:
return encoded
image_count = len(request.images)
(record, forecasts, tasks, typed, eos) = encoded
contracts = record["_meta"]["identifier_spelling"]
originals = record.get("_original_images", ())
if len(originals) != image_count:
raise ValueError("PO supplement must retain every original image in order")
fields = image_focus.preserve_scalar_fields(selected)
supplements = (
tuple((supplement([image]) for image in originals))
if image_count == 2
else (supplement(originals),)
)
child_request = request.model_copy(
update={"fields": fields, "images": (*request.images, *supplements)}
)
(child, _, replacements, child_typed, child_eos) = encode_child(child_request)
by_name = {task[0]: task for task in replacements}
if (
child_typed is not None
or child_eos != eos
or set(by_name) != set(selected)
or (len(replacements) != len(selected))
):
raise ValueError(
"PO supplement must retain the complete selected scalar task contract"
)
plain = image_count == 1
constraints: dict[str, dict[str, Any]]
remaining: dict[str, Any] | list[str]
(fields, constraints) = ({}, {})
for name, public_spec in selected.items():
if plain:
(by_name[name], fields[name], constraints[name]) = plain_task(
by_name[name], child["fields"][name], public_spec, tokenizer, max_length
)
else:
fields[name] = child["fields"][name]
constraints[name] = child["_ar_constraints"][name]
updated = [by_name.get(task[0], task) for task in tasks]
salts = dict(record.get("_ar_task_salts", {}))
for name in selected:
record["fields"][name] = fields[name]
record["_ar_constraints"][name] = constraints[name]
salts[name] = child.get("_ar_task_salts", {}).get(name, child["_cache_salt"])
if plain:
remaining = {
name: contract
for (name, contract) in contracts.items()
if name not in selected
}
if remaining:
record["_meta"]["identifier_spelling"] = remaining
else:
record["_meta"].pop("identifier_spelling", None)
for key in ("model_document_scalar_view", "identifier_spelling_image_view"):
previous = record["_meta"].get(key)
if previous is None:
continue
remaining = [name for name in previous["fields"] if name not in selected]
if not remaining:
record["_meta"].pop(key)
else:
retained = {**previous, "fields": remaining}
if "po_fields" in retained:
retained["po_fields"] = [
name for name in retained["po_fields"] if name not in selected
]
record["_meta"][key] = retained
record["_meta"]["model_po_supplement"] = {
"version": VERSION,
"fields": list(selected),
"original_image_count": image_count,
"native_image_count": len(child_request.images),
"horizontal_scale": 2.0,
"supplement_max_pixels": MAX_PIXELS,
"wire": "native_scalar" if plain else identifier_spelling.VERSION,
}
record.update(
_ar_tasks=updated, _ar_task_salts=salts, _ar_prefix=[], _ar_group_prefixes=[]
)
record["_ar_media_groups"] = media_groups(
updated,
default_salt=record["_cache_salt"],
task_salts=salts,
annotations=record["_meta"]["extraction_contract"]["fields"],
)
return (record, forecasts, updated, typed, eos)