clairic-flash-4bit / clairic_runtime /text_list_completion.py
lucataco's picture
Clairic Flash, MLX 4-bit, with runtime, server and conversion script
ae97c5d verified
Raw History Blame Contribute Delete
9.21 kB
from __future__ import annotations
from typing import TYPE_CHECKING
from . import batch_completion as shared
from . import ordinary_list_completion, schema
if TYPE_CHECKING:
from collections.abc import Callable
from typing import Any
from .backend import ARTask
from transformers import PreTrainedTokenizerBase
FRONTIER = 8
def _object_lists(
record: dict[str, Any], tasks: list[ARTask]
) -> dict[str, tuple[int, ...]]:
if record.get("_null_recovery"):
return {}
fields = record["fields"]
annotations = (
record.get("_meta", {}).get("extraction_contract", {}).get("fields", {})
)
names = [name for (name, _, _) in tasks]
if len(set(names)) != len(names):
raise ValueError("AR task names must be distinct")
indices = {name: index for (index, name) in enumerate(names)}
claimed: set[str]
groups: dict[str, tuple[int, ...]]
(claimed, groups) = (set(), {})
forbidden = {"_physical_row_scope", "_image_row_inventory", "_image_row_proposal"}
plans = (
record.get("_financial_line_plans", {}),
record.get("_list_source_bounds", {}),
)
for parent, annotation in annotations.items():
spec = fields.get(parent, {})
queries = annotation.get("query_fields")
item = spec.get("items")
if (
spec.get("type") != "list"
or not isinstance(item, dict)
or item.get("type") != "object"
or spec.get("knowledge")
or (set(annotation) != {"query_fields"})
or (not isinstance(queries, list))
or (len(queries) <= FRONTIER)
or any((spec.get(key) for key in forbidden))
or (parent in indices)
or any((parent in plan for plan in plans))
):
continue
valid = True
for ordinal, name in enumerate(queries, 1):
child = fields.get(name, {})
ownership = annotations.get(name, {})
if (
name != schema.list_item_name(parent, ordinal)
or name not in indices
or name in claimed
or (ownership != {"parent": parent, "index": ordinal})
or (child.get("type") != "object")
or (child.get("nullable", True) is not True)
or any((child.get(key) for key in forbidden))
or (name in record.get("_source_row_gates", ()))
):
valid = False
break
if "enum" in child and None not in child["enum"]:
valid = False
break
if valid:
groups[parent] = tuple((indices[name] for name in queries))
claimed.update(queries)
return groups
def _groups(record: dict[str, Any], tasks: list[ARTask]) -> dict[str, tuple[int, ...]]:
if (
record.get("_null_recovery")
or record.get("_meta", {}).get("construction") != schema.EVIDENCE_CONSTRUCTION
):
return {}
groups = _object_lists(record, tasks)
if (
not record.get("_media_payload")
and (not record.get("_original_images"))
and all((media is None for (_, _, media) in tasks))
):
groups.update(
{
parent: group
for (parent, group) in shared._ordinary_lists(
record, tasks, schema.list_item_name
).items()
if len(group) > FRONTIER
}
)
return groups
def eligible(record: dict[str, Any], tasks: list[ARTask]) -> bool:
return bool(_groups(record, tasks))
def completed_null(
result: dict[str, Any], eos: int, tokenizer: PreTrainedTokenizerBase
) -> bool:
try:
return shared._completed_null(result, eos, tokenizer)
except (KeyError, TypeError, ValueError, OverflowError):
return False
async def generate(
engine: Any,
record: dict[str, Any],
tasks: list[ARTask],
eos: int,
tokenizer: PreTrainedTokenizerBase,
*,
parse_result: Callable[..., dict[str, Any]] | None = None,
normalize: Callable[
[dict[str, Any], dict[str, dict[str, Any]]], dict[str, dict[str, Any]]
]
| None = None,
**kwargs: Any,
) -> list[dict[str, Any]]:
if eligible(record, tasks) and ordinary_list_completion.eligible(
record, tasks, kwargs
):
return await ordinary_list_completion.generate(
engine,
record,
tasks,
eos,
tokenizer,
initial_frontier=8,
next_frontier=32,
parse_result=parse_result,
normalize=normalize,
**kwargs,
)
media = kwargs.get("image_data")
if (
set(kwargs) - shared._BATCHED_KEYS - shared._SHARED_KEYS
or not eligible(record, tasks)
or (not isinstance(media, list))
or (len(media) != len(tasks))
or any(
(
actual is not task[2]
and (
not (task[2] is None and isinstance(actual, list) and (not actual))
)
for (actual, task) in zip(media, tasks, strict=True)
)
)
):
return await shared.generate(
engine,
record,
tasks,
eos,
tokenizer,
parse_result=parse_result,
normalize=normalize,
**kwargs,
)
groups = _groups(record, tasks)
size = len(tasks)
rids = kwargs.get("rid")
if (
not isinstance(rids, list)
or len(rids) != size
or len(set(rids)) != size
or (not all((isinstance(rid, str) and rid for rid in rids)))
or (kwargs.get("input_ids") != [ids for (_, ids, _) in tasks])
):
raise ValueError("List frontier requires the original distinct AR batch")
shared._select_kwargs(kwargs, list(range(size)), size)
deferred = {index for group in groups.values() for index in group[FRONTIER:]}
results: list[Any] = [None] * size
lengths = {name: len(ids) for (name, ids, _media) in tasks}
unexecuted: dict[str, int] = {}
async def submit(indices: list[int]) -> None:
if not indices:
return
selected = shared._select_kwargs(kwargs, indices, size)
subset = [tasks[index] for index in indices]
received = await shared.generate(
engine,
record,
subset,
eos,
tokenizer,
parse_result=parse_result,
normalize=normalize,
**selected,
)
accounted = record.get("_unexecuted_list_items", {})
if any(
(
name not in lengths or length != lengths[name]
for (name, length) in accounted.items()
)
):
raise RuntimeError("Shared scheduler changed original omission accounting")
if len(received) != len(indices):
raise RuntimeError(
"Native completion differs from its list-frontier submission"
)
for index, result, rid in zip(indices, received, selected["rid"], strict=True):
name = tasks[index][0]
if name in accounted:
if (
not result.get("_clair_unused_row_proposal")
or result.get("meta_info", {}).get("completion_tokens") != 0
):
raise RuntimeError(
"Shared scheduler omission lacks its unexecuted result slot"
)
elif result.get("meta_info", {}).get("id") != rid:
raise RuntimeError(
"Native completion differs from its list-frontier submission"
)
results[index] = result
for name, length in accounted.items():
if name in unexecuted and unexecuted[name] != length:
raise RuntimeError(
"Schedulers disagree on original omitted prompt length"
)
unexecuted[name] = length
await submit([index for index in range(size) if index not in deferred])
continued: list[int] = []
for group in groups.values():
if any(
(
completed_null(results[index], eos, tokenizer)
for index in group[:FRONTIER]
)
):
for index in group[FRONTIER:]:
(name, ids, _) = tasks[index]
if name in unexecuted:
raise RuntimeError("List frontier attempted to omit a slot twice")
unexecuted[name] = len(ids)
results[index] = {
"_clair_unused_row_proposal": True,
"_text_frontier_unexecuted": True,
"meta_info": {"completion_tokens": 0},
}
else:
continued.extend(group[FRONTIER:])
await submit(sorted(continued))
if any((result is None for result in results)):
raise RuntimeError("List frontier left an unaccounted result slot")
record["_unexecuted_list_items"] = unexecuted
return results