File size: 8,203 Bytes
ae97c5d
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
from __future__ import annotations
import base64
import io
import json
from typing import TYPE_CHECKING
from .api import ClairImage
from . import ar, grounding, identifier_spelling, image_focus
from PIL import Image

if TYPE_CHECKING:
    from collections.abc import Callable, Sequence
    from typing import Any
    from .api import PredictRequest
    from .backend import ARTask, EncodedRequest
    from transformers import PreTrainedTokenizerBase
VERSION = "27b-po-all-content-horizontal2-supplement-v2"
MAX_PIXELS = 1600000


def supplement(images: Sequence[Image.Image]) -> ClairImage:
    views: list[Image.Image] = []
    for image in images:
        focused = image_focus.focus_document(image)
        views.append(
            focused.resize(
                (round(focused.width * 2.0), focused.height), Image.Resampling.BICUBIC
            )
        )
    board = Image.new(
        "RGB",
        (
            max((image.width for image in views)),
            sum((image.height for image in views)) + 32 * (len(views) - 1),
        ),
        "white",
    )
    y = 0
    for image in views:
        board.paste(image, (0, y))
        y += image.height + 32
    scale = min(1.0, (MAX_PIXELS / (board.width * board.height)) ** 0.5)
    if scale < 1:
        board = board.resize(
            (max(1, int(board.width * scale)), max(1, int(board.height * scale))),
            Image.Resampling.BICUBIC,
        )
    buffer = io.BytesIO()
    board.save(buffer, format="PNG")
    return ClairImage.model_validate(
        {
            "content_type": "image/png",
            "base64": base64.b64encode(buffer.getvalue()).decode(),
        }
    )


def plain_task(
    task: ARTask,
    wire_spec: dict[str, Any],
    public_spec: dict[str, Any],
    tokenizer: PreTrainedTokenizerBase,
    max_length: int,
) -> tuple[ARTask, dict[str, Any], dict[str, str]]:
    (name, ids, media) = task
    source_ids = list(ids)
    text = tokenizer.decode(
        ids, skip_special_tokens=False, clean_up_tokenization_spaces=False
    )
    if tokenizer.encode(text, add_special_tokens=False) != source_ids:
        raise ValueError(
            "PO scalar source prompt cannot make a lossless tokenizer roundtrip"
        )
    before = identifier_spelling.INSTRUCTION + "\n"
    if text.count(before) != 1 or wire_spec.get("instructions", "").count(before) != 1:
        raise ValueError(
            "PO scalar source must contain exactly one spelling instruction"
        )
    scalar = {
        **wire_spec,
        "instructions": wire_spec["instructions"].replace(before, ""),
    }
    for key in ("minLength", "maxLength", "pattern"):
        if key in public_spec:
            scalar[key] = public_spec[key]
    (old_schema, schema) = (ar.schema(wire_spec), ar.schema(scalar))
    text = text.replace(before, "")
    if schema != old_schema:
        old = "\nSchema: " + json.dumps(
            old_schema, ensure_ascii=False, separators=(",", ":")
        )
        new = "\nSchema: " + json.dumps(
            schema, ensure_ascii=False, separators=(",", ":")
        )
        if text.count(old) != 1:
            raise ValueError("PO scalar source must contain exactly one native schema")
        text = text.replace(old, new)
    plain_ids = tokenizer.encode(text, add_special_tokens=False)
    if len(plain_ids) > max_length:
        raise ValueError("PO scalar detail exceeds the complete model context")
    return ((name, plain_ids, media), scalar, {"json_schema": json.dumps(schema)})


def selected_fields(
    request: PredictRequest, encoded: EncodedRequest
) -> dict[str, dict[str, Any]]:
    image_count = len(request.images)
    if image_count not in (1, 2, 3) or request.documents:
        return {}
    (record, _, tasks, _, _) = encoded
    contracts = record.get("_meta", {}).get("identifier_spelling", {})
    active = {name for (name, _, media) in tasks if media is not None}
    return {
        name: request.fields[name]
        for (name, contract) in contracts.items()
        if name in active
        and contract.get("wire") == identifier_spelling.VERSION
        and (not request.fields[name].get("knowledge"))
        and (not grounding.state_only(request.fields[name]))
    }


def prepare(
    request: PredictRequest,
    encoded: EncodedRequest,
    *,
    encode_child: Callable[[PredictRequest], EncodedRequest],
    tokenizer: PreTrainedTokenizerBase,
    max_length: int,
    media_groups: Callable[..., list[dict[str, Any]]],
) -> EncodedRequest:
    selected = selected_fields(request, encoded)
    if not selected:
        return encoded
    image_count = len(request.images)
    (record, forecasts, tasks, typed, eos) = encoded
    contracts = record["_meta"]["identifier_spelling"]
    originals = record.get("_original_images", ())
    if len(originals) != image_count:
        raise ValueError("PO supplement must retain every original image in order")
    fields = image_focus.preserve_scalar_fields(selected)
    supplements = (
        tuple((supplement([image]) for image in originals))
        if image_count == 2
        else (supplement(originals),)
    )
    child_request = request.model_copy(
        update={"fields": fields, "images": (*request.images, *supplements)}
    )
    (child, _, replacements, child_typed, child_eos) = encode_child(child_request)
    by_name = {task[0]: task for task in replacements}
    if (
        child_typed is not None
        or child_eos != eos
        or set(by_name) != set(selected)
        or (len(replacements) != len(selected))
    ):
        raise ValueError(
            "PO supplement must retain the complete selected scalar task contract"
        )
    plain = image_count == 1
    constraints: dict[str, dict[str, Any]]
    remaining: dict[str, Any] | list[str]
    (fields, constraints) = ({}, {})
    for name, public_spec in selected.items():
        if plain:
            (by_name[name], fields[name], constraints[name]) = plain_task(
                by_name[name], child["fields"][name], public_spec, tokenizer, max_length
            )
        else:
            fields[name] = child["fields"][name]
            constraints[name] = child["_ar_constraints"][name]
    updated = [by_name.get(task[0], task) for task in tasks]
    salts = dict(record.get("_ar_task_salts", {}))
    for name in selected:
        record["fields"][name] = fields[name]
        record["_ar_constraints"][name] = constraints[name]
        salts[name] = child.get("_ar_task_salts", {}).get(name, child["_cache_salt"])
    if plain:
        remaining = {
            name: contract
            for (name, contract) in contracts.items()
            if name not in selected
        }
        if remaining:
            record["_meta"]["identifier_spelling"] = remaining
        else:
            record["_meta"].pop("identifier_spelling", None)
    for key in ("model_document_scalar_view", "identifier_spelling_image_view"):
        previous = record["_meta"].get(key)
        if previous is None:
            continue
        remaining = [name for name in previous["fields"] if name not in selected]
        if not remaining:
            record["_meta"].pop(key)
        else:
            retained = {**previous, "fields": remaining}
            if "po_fields" in retained:
                retained["po_fields"] = [
                    name for name in retained["po_fields"] if name not in selected
                ]
            record["_meta"][key] = retained
    record["_meta"]["model_po_supplement"] = {
        "version": VERSION,
        "fields": list(selected),
        "original_image_count": image_count,
        "native_image_count": len(child_request.images),
        "horizontal_scale": 2.0,
        "supplement_max_pixels": MAX_PIXELS,
        "wire": "native_scalar" if plain else identifier_spelling.VERSION,
    }
    record.update(
        _ar_tasks=updated, _ar_task_salts=salts, _ar_prefix=[], _ar_group_prefixes=[]
    )
    record["_ar_media_groups"] = media_groups(
        updated,
        default_salt=record["_cache_salt"],
        task_salts=salts,
        annotations=record["_meta"]["extraction_contract"]["fields"],
    )
    return (record, forecasts, updated, typed, eos)