clairic-4bit / clairic_runtime /batch_completion.py
lucataco's picture
Clairic, MLX 4-bit, with runtime, server and conversion script
1a48d6b verified
Raw History Blame Contribute Delete
15.8 kB
from __future__ import annotations
from collections.abc import Callable
from typing import TYPE_CHECKING, Any, cast
if TYPE_CHECKING:
from types import ModuleType
from . import ar, image_rows, schema
def unused_prediction(result: dict[str, Any]) -> dict[str, Any]:
return {
"value": None,
"valid": False,
"complete": False,
"tokens": int(result.get("meta_info", {}).get("completion_tokens", 0)),
"_unused_row_proposal": True,
}
def _empty_inventory(result: dict[str, Any], eos: int, tokenizer: Any) -> bool:
finish = result.get("meta_info", {}).get("finish_reason", {}) or {}
if finish.get("type") != "stop" or finish.get("matched") != eos:
return False
ids = result.get("output_ids")
if ids is not None:
ids = ids[:-1] if ids and ids[-1] == eos else ids
text = tokenizer.decode(
ids, skip_special_tokens=False, clean_up_tokenization_spaces=False
)
else:
text = result.get("text", "").removesuffix(ar.EOS)
return text.strip() == "null"
async def _generate(
engine: Any,
record: dict[str, Any],
tasks: list[tuple[str, list[int], Any]],
eos: int,
tokenizer: Any,
**kwargs: Any,
) -> list[dict[str, Any]]:
return await engine.async_generate(**kwargs)
_BATCHED_KEYS = {"input_ids", "image_data", "rid", "cache_salt", "sampling_params"}
_SHARED_KEYS = {"return_logprob", "logprob_start_len", "token_ids_logprob"}
def _select_kwargs(
kwargs: dict[str, Any], indices: list[int], size: int
) -> dict[str, Any]:
if set(kwargs) - _BATCHED_KEYS - _SHARED_KEYS:
raise ValueError("Unknown native argument for first-item scheduling")
result = {}
for key, value in kwargs.items():
if key in _BATCHED_KEYS and isinstance(value, list):
if len(value) != size:
raise ValueError("Native batch argument cardinality differs: " + key)
result[key] = [value[index] for index in indices]
else:
result[key] = value
return result
def _ordinary_lists(
record: dict[str, Any],
tasks: list[tuple[str, list[int], Any]],
item_name: Callable[[str, int], str],
) -> dict[str, tuple[int, ...]]:
if record.get("_null_recovery"):
return {}
fields = record["fields"]
annotations = (
record.get("_meta", {}).get("extraction_contract", {}).get("fields", {})
)
names = [name for (name, _, _) in tasks]
if len(set(names)) != len(names):
raise ValueError("AR task names must be distinct")
indices = {name: index for (index, name) in enumerate(names)}
(claimed, groups) = (set(), {})
forbidden = {"_physical_row_scope", "_image_row_inventory", "_image_row_proposal"}
plans = (
record.get("_financial_line_plans", {}),
record.get("_list_source_bounds", {}),
)
for parent, annotation in annotations.items():
spec = fields.get(parent, {})
queries = annotation.get("query_fields")
item = spec.get("items")
kind = item.get("type") if isinstance(item, dict) else item
if (
spec.get("type") != "list"
or kind not in ("text", "number", "date")
or spec.get("knowledge")
or (set(annotation) != {"query_fields"})
or (not isinstance(queries, list))
or (len(queries) < 2)
or any((spec.get(key) for key in forbidden))
or (parent in indices)
or any((parent in plan for plan in plans))
):
continue
valid = True
for ordinal, name in enumerate(queries, 1):
child = fields.get(name, {})
ownership = annotations.get(name, {})
if (
name != item_name(parent, ordinal)
or name not in indices
or name in claimed
or (ownership != {"parent": parent, "index": ordinal})
or (child.get("type") not in ("text", "number", "date"))
or (child.get("nullable", True) is not True)
or any((child.get(key) for key in forbidden))
or (name in record.get("_source_row_gates", ()))
):
valid = False
break
if "enum" in child and None not in child["enum"]:
valid = False
break
if valid:
groups[parent] = tuple((indices[name] for name in queries))
claimed.update(queries)
return groups
def _completed_null(result: dict[str, Any], eos: int, tokenizer: Any) -> bool:
meta = result.get("meta_info", {})
finish = meta.get("finish_reason", {}) or {}
ids = result.get("output_ids")
if (
finish.get("type") != "stop"
or finish.get("matched") != eos
or (not isinstance(ids, list))
or (not ids)
or (ids[-1] != eos)
or (meta.get("completion_tokens") != len(ids))
):
return False
return (
tokenizer.decode(
ids[:-1], skip_special_tokens=False, clean_up_tokenization_spaces=False
).strip()
== "null"
)
def _row_cohorts(
record: dict[str, Any],
tasks: list[tuple[str, list[int], Any]],
item_name: Callable[[str, int], str],
) -> dict[str, dict[str, Any]]:
fields = record["fields"]
annotations = (
record.get("_meta", {}).get("extraction_contract", {}).get("fields", {})
)
indices = {name: index for (index, (name, _, _)) in enumerate(tasks)}
result = {}
for parent, annotation in annotations.items():
(queries, spec) = (annotation.get("image_row_fields"), fields.get(parent, {}))
if (
set(annotation) != {"image_row_fields"}
or not spec.get("_image_row_inventory")
or parent not in indices
or (not isinstance(queries, list))
or (len(queries) <= 8)
or (parent in record.get("_list_source_bounds", {}))
or (parent in record.get("_financial_line_plans", {}))
or (not isinstance(spec.get("items"), dict))
or (spec["items"].get("type") != "object")
):
continue
text_keys = [
key
for (key, child) in spec["items"].get("properties", {}).items()
if child.get("type") == "text"
]
if not text_keys:
continue
if any(
(
name != item_name(parent, ordinal)
or name not in indices
or annotations.get(name) != {"parent": parent, "index": ordinal}
or (fields.get(name, {}).get("type") != "object")
or (not fields[name].get("_image_row_proposal"))
for (ordinal, name) in enumerate(queries, 1)
)
):
continue
result[parent] = {
"inventory": indices[parent],
"proposals": tuple((indices[name] for name in queries)),
"text_keys": text_keys,
}
return result
def _cohort_proof(
record: dict[str, Any],
tasks: list[tuple[str, list[int], Any]],
results: list[dict[str, Any]],
parent: str,
cohort: dict[str, Any],
tokenizer: Any,
scalar_result: Callable[..., dict[str, Any]],
image_rows: ModuleType,
normalize: Callable[
[dict[str, Any], dict[str, dict[str, Any]]], dict[str, dict[str, Any]]
],
) -> dict[str, Any] | None:
spec = record["fields"][parent]
native = results[cohort["inventory"]]
inventory = scalar_result(native, spec, tokenizer, record.get("input", ""), None)
if not inventory.get("valid") or not inventory.get("complete"):
return None
value = inventory.get("value")
if value is not None and (
not isinstance(value, list)
or any(
(
not isinstance(row, dict)
or set(row) != {"label"}
or (not isinstance(row["label"], str))
for row in value
)
)
):
return None
labels = [row["label"] for row in value or []]
if len(labels) > 8 or len(labels) > spec.get("max_items", 32):
return None
retained = {parent: inventory}
for index in cohort["proposals"][: len(labels)]:
name = tasks[index][0]
result = results[index]
if result.get("_clair_unused_row_proposal"):
return None
parsed = scalar_result(
result, record["fields"][name], tokenizer, record.get("input", ""), None
)
retained[name] = parsed
normalized = normalize(record, retained)
normalized_inventory = normalized[parent]
if (
not normalized_inventory.get("valid")
or not normalized_inventory.get("complete")
or normalized_inventory.get("value") != value
):
return None
proposals = []
for index in cohort["proposals"][: len(labels)]:
parsed = normalized[tasks[index][0]]
if (
not parsed.get("valid")
or not parsed.get("complete")
or (not isinstance(parsed.get("value"), dict))
or (not any((value is not None for value in parsed["value"].values())))
):
return None
proposals.append(parsed)
(selected, conflicts) = image_rows._select(labels, proposals, cohort["text_keys"])
if conflicts or any(
(row is not proposals[index] for (index, row) in enumerate(selected))
):
return None
return {
"inventory_labels": labels,
"selected_ordinals": list(range(1, len(labels) + 1)),
"assignment_cost": 0,
"cohort": 8,
"original_proposal_capacity": len(cohort["proposals"]),
"inventory_valid_complete": True,
"same_ordinal_matches": True,
}
def _collector_record(
record: dict[str, Any], tasks: list[tuple[str, list[int], Any]]
) -> dict[str, Any]:
annotations = record["_meta"]["extraction_contract"]["fields"]
names = {name for (name, _, _) in tasks}
updated = dict(annotations)
for parent, annotation in annotations.items():
if parent in names and annotation.get("image_row_fields"):
updated[parent] = {
**annotation,
"image_row_fields": [
name for name in annotation["image_row_fields"] if name in names
],
}
return {
**record,
"_meta": {
**record["_meta"],
"extraction_contract": {
**record["_meta"]["extraction_contract"],
"fields": updated,
},
},
}
def _normalize_retained(
record: dict[str, Any],
predictions: dict[str, dict[str, Any]],
normalize: Callable[
[dict[str, Any], dict[str, dict[str, Any]]], dict[str, dict[str, Any]]
],
) -> dict[str, dict[str, Any]]:
meta = record.get("_meta", {})
view = {
**record,
"_meta": {
**meta,
**{
kind: {
name: spec
for (name, spec) in meta.get(kind, {}).items()
if name in predictions
}
for kind in ("printed_temporal", "printed_numbers")
},
},
}
return normalize(view, predictions)
async def generate(
engine: Any,
record: dict[str, Any],
tasks: list[tuple[str, list[int], Any]],
eos: int,
tokenizer: Any,
*,
parse_result: Callable[..., dict[str, Any]] | None = None,
normalize: Callable[
[dict[str, Any], dict[str, dict[str, Any]]], dict[str, dict[str, Any]]
]
| None = None,
**kwargs: Any,
) -> list[dict[str, Any]]:
if (
not record.get("_first_item_list_gating")
or record.get("_null_recovery")
or set(kwargs) - _BATCHED_KEYS - _SHARED_KEYS
):
return await _generate(engine, record, tasks, eos, tokenizer, **kwargs)
groups = _ordinary_lists(record, tasks, schema.list_item_name)
cohorts = (
_row_cohorts(record, tasks, schema.list_item_name)
if parse_result is not None and normalize is not None
else {}
)
if not groups and (not cohorts):
return await _generate(engine, record, tasks, eos, tokenizer, **kwargs)
size = len(tasks)
rids = kwargs.get("rid")
if (
not isinstance(rids, list)
or len(rids) != size
or len(set(rids)) != size
or (kwargs.get("input_ids") != [ids for (_, ids, _) in tasks])
):
raise ValueError("Cohort scheduling requires the original distinct AR batch")
_select_kwargs(kwargs, list(range(size)), size)
tails = {index for group in groups.values() for index in group[1:]}
tails.update(
(index for cohort in cohorts.values() for index in cohort["proposals"][8:])
)
results: list[Any] = [None] * size
async def submit(indices: list[int]) -> None:
if not indices:
return
selected = _select_kwargs(kwargs, indices, size)
subset = [tasks[index] for index in indices]
view = _collector_record(record, subset) if cohorts else record
received = await _generate(engine, view, subset, eos, tokenizer, **selected)
if "_completed_inventory_pruning" in view:
record["_completed_inventory_pruning"] = view[
"_completed_inventory_pruning"
]
if len(received) != len(indices) or any(
(
result.get("meta_info", {}).get("id") != rid
for (result, rid) in zip(received, selected["rid"], strict=True)
)
):
raise RuntimeError("Native completion differs from its cohort submission")
for index, result in zip(indices, received, strict=True):
results[index] = result
def omit(index: int, marker: str) -> None:
(name, ids, _) = tasks[index]
unexecuted[name] = len(ids)
results[index] = {
"_clair_unused_row_proposal": True,
marker: True,
"meta_info": {"completion_tokens": 0},
}
await submit([index for index in range(size) if index not in tails])
unexecuted: dict[str, int]
continued: list[int]
(unexecuted, continued) = ({}, [])
for group in groups.values():
if _completed_null(results[group[0]], eos, tokenizer):
for index in group[1:]:
omit(index, "_first_item_unexecuted")
else:
continued.extend(group[1:])
proofs = []
for parent, cohort in cohorts.items():
proof = _cohort_proof(
record,
tasks,
results,
parent,
cohort,
tokenizer,
cast(Callable[..., dict[str, Any]], parse_result),
image_rows,
lambda record, predictions: _normalize_retained(
record,
predictions,
cast(
Callable[
[dict[str, Any], dict[str, dict[str, Any]]],
dict[str, dict[str, Any]],
],
normalize,
),
),
)
proofs.append({"parent": parent, "proof": proof, "fallback": proof is None})
if proof is None:
continued.extend(cohort["proposals"][8:])
else:
for index in cohort["proposals"][8:]:
omit(index, "_row_cohort_unexecuted")
await submit(sorted(continued))
if any((result is None for result in results)):
raise RuntimeError("Cohort scheduling left an unaccounted result slot")
record["_unexecuted_list_items"] = unexecuted
record["_row_cohort_gating"] = proofs
return results