clairic-4bit / clairic_runtime /media_state_prompt.py
lucataco's picture
Clairic, MLX 4-bit, with runtime, server and conversion script
1a48d6b verified
Raw History Blame Contribute Delete
4.47 kB
from __future__ import annotations
from typing import TYPE_CHECKING
if TYPE_CHECKING:
from collections.abc import Callable
from typing import Any
from .api import PredictRequest
from .backend import ARTask, EncodedRequest
from transformers import PreTrainedTokenizerBase
def eligible(request: PredictRequest) -> bool:
if not request.state or not (request.images or request.documents):
return False
if len(request.images) > 1:
return True
from . import grounding
return any(
(
spec.get("type") == "boolean"
and (not spec.get("knowledge"))
and (not grounding.state_only(spec))
for spec in request.fields.values()
)
)
def rewrite_media_evidence(
task: ARTask,
*,
record: dict[str, Any],
tokenizer: PreTrainedTokenizerBase,
max_length: int,
) -> ARTask:
from . import ar
(name, ids, media) = task
if media is None:
raise ValueError("Media evidence guidance requires actual task media")
source_ids = list(ids)
text = tokenizer.decode(
ids, skip_special_tokens=False, clean_up_tokenization_spaces=False
)
if tokenizer.encode(text, add_special_tokens=False) != source_ids:
raise ValueError(
"Media evidence prompt must make a lossless tokenizer roundtrip"
)
instruction = ar.instruction(record, name, compact_lists=False).rstrip()
boundary = instruction + "\n\nDocument:\n"
existing = (
ar.MEDIA_EVIDENCE_INSTRUCTION
+ "\n\n"
+ instruction
+ "\n\nAccompanying text:\n"
)
if text.count(existing) == 1:
return task
if text.count(boundary) != 1 or ar.MEDIA_EVIDENCE_INSTRUCTION in text:
raise ValueError(
"Expected exactly one original instruction and accompanying-state boundary"
)
revised = text.replace(boundary, existing)
revised_ids = tokenizer.encode(revised, add_special_tokens=False)
if (
tokenizer.decode(
revised_ids, skip_special_tokens=False, clean_up_tokenization_spaces=False
)
!= revised
):
raise ValueError(
"Media evidence revised prompt must make a lossless tokenizer roundtrip"
)
if len(revised_ids) > max_length:
raise ValueError("Media evidence prompt exceeds the complete model context")
return (name, revised_ids, media)
def prepare(
request: PredictRequest,
encoded: EncodedRequest,
*,
tokenizer: PreTrainedTokenizerBase,
max_length: int,
media_groups: Callable[..., list[dict[str, Any]]],
) -> EncodedRequest:
if not eligible(request):
return encoded
from . import grounding
(record, forecasts, tasks, typed, eos) = encoded
annotations = (
record.get("_meta", {}).get("extraction_contract", {}).get("fields", {})
)
meta = record.get("_meta", {})
protected = set(meta.get("identifier_spelling", {})) | set(
meta.get("model_po_supplement", {}).get("fields", ())
)
multi_image = len(request.images) > 1
updated: list[ARTask] = []
changed = False
for task in tasks:
(name, _, media) = task
parent = annotations.get(name, {}).get("parent", name)
spec = request.fields.get(parent)
selected = (
media is not None
and name not in protected
and (parent not in protected)
and (spec is not None)
and (not spec.get("knowledge"))
and (not grounding.state_only(spec))
and (multi_image or (name == parent and spec.get("type") == "boolean"))
)
revised = task
if selected:
try:
revised = rewrite_media_evidence(
task, record=record, tokenizer=tokenizer, max_length=max_length
)
except ValueError:
return encoded
updated.append(revised)
changed |= revised is not task
if not changed:
return encoded
revised_record = {
**record,
"_ar_tasks": updated,
"_ar_prefix": [],
"_ar_group_prefixes": [],
}
revised_record["_ar_media_groups"] = media_groups(
updated,
default_salt=record["_cache_salt"],
task_salts=record.get("_ar_task_salts", {}),
annotations=annotations,
)
return (revised_record, forecasts, updated, typed, eos)