lucataco's picture
Clairic, MLX 4-bit, with runtime, server and conversion script
1a48d6b verified
Raw History Blame Contribute Delete
6.51 kB
from __future__ import annotations
import asyncio
import json
import uuid
from typing import TYPE_CHECKING
from . import (
ar,
batch_completion,
grounding,
image_rows,
printed_numbers,
printed_temporal,
)
from .backend import MAX_OUTPUT_TOKENS, ClairInferenceError, scalar_result
from .media import batch_image_data
from . import text_list_completion
if TYPE_CHECKING:
from typing import Any
from .large import Clairic27BModel
async def text_ar_all(
self: Clairic27BModel, record: dict[str, Any], eos: int
) -> dict[str, dict[str, Any]]:
tasks = record["_ar_tasks"]
salt = record["_cache_salt"]
row_gates = set(record.get("_source_row_gates", ()))
false_id = self.tokenizer.convert_tokens_to_ids("false") if row_gates else None
score_ids = [self.tokenizer.convert_tokens_to_ids("null")]
if false_id is not None:
score_ids.append(false_id)
results = await text_list_completion.generate(
self.engine,
record,
tasks,
eos,
self.tokenizer,
parse_result=scalar_result,
normalize=lambda record, predictions: printed_numbers.normalize_predictions(
record, printed_temporal.normalize_predictions(record, predictions)
),
input_ids=[ids for (_, ids, _) in tasks],
image_data=batch_image_data([media for (_, _, media) in tasks]),
rid=["ar-" + uuid.uuid4().hex for _ in tasks],
cache_salt=[
record.get("_ar_task_salts", {}).get(name, salt) for (name, _, _) in tasks
],
return_logprob=True,
logprob_start_len=-1,
token_ids_logprob=score_ids,
sampling_params=[
{
"temperature": 0,
"max_new_tokens": image_rows.output_limit(
record, name, len(ids), self.settings.max_length
)
if image_rows.is_inventory(record["fields"][name])
else record.get("_ar_max_output_tokens", MAX_OUTPUT_TOKENS),
"stop_token_ids": [eos],
**(
record["_ar_constraints"][name]
if name in record.get("_ar_constraints", {})
else {"json_schema": json.dumps(ar.schema(record["fields"][name]))}
),
"skip_special_tokens": False,
}
for (name, ids, _) in tasks
],
)
if usage := record.get("_completed_generation_usage"):
usage["tasks"].extend(tasks)
usage["readings"].extend(
(
{"tokens": int(result.get("meta_info", {}).get("completion_tokens", 0))}
for result in results
)
)
predictions = {
name: batch_completion.unused_prediction(result)
if result.get("_clair_unused_row_proposal")
else scalar_result(
result,
record["fields"][name],
self.tokenizer,
record["input"],
record.get("_ar_sources", {}).get(name, record.get("_ar_source"))
if grounding.applies(record["fields"][name], media)
else None,
excluded_token_id=false_id if name in row_gates else None,
provenance=record.get("_ar_sources", {}).get(name, record.get("_ar_source"))
if media is None and (not record["fields"][name].get("knowledge"))
else None,
)
for ((name, _, media), result) in zip(tasks, results, strict=True)
}
if any(
(
"confidence" not in prediction
for prediction in predictions.values()
if not prediction.get("_unused_row_proposal")
)
):
raise ClairInferenceError("Transformers omitted generation log probabilities")
if any(
(
"p_null" not in prediction
for prediction in predictions.values()
if not prediction.get("_unused_row_proposal")
)
):
raise ClairInferenceError("Transformers omitted null-token log probabilities")
if self.RECOVER_NULLS and (not record.get("_null_recovery")):
fields = {
name
for (name, prediction) in predictions.items()
if (
prediction["valid"]
and prediction["value"] is None
or prediction.get("failure_reason") == "not_in_source"
)
and (not record["fields"][name].get("knowledge"))
and (name not in record.get("_meta", {}).get("printed_temporal", {}))
and (name not in record.get("_meta", {}).get("printed_numbers", {}))
and (
record["fields"][name]["type"] == "number"
or (record["fields"][name]["type"] == "text" and record["input"])
)
and (
not record["_meta"]["extraction_contract"]["fields"]
.get(name, {})
.get("parent")
)
}
if fields:
repair = await asyncio.get_running_loop().run_in_executor(
self._encoder, self.null_recovery_record, record, fields
)
if repair["_ar_tasks"]:
alternatives = await self._ar_all(repair, eos)
retry_lengths = {
name: len(ids) for (name, ids, _) in repair["_ar_tasks"]
}
for name, alternative in alternatives.items():
original = predictions[name]
tokens = original["tokens"] + alternative["tokens"]
if (
alternative["valid"]
and alternative["value"] is not None
and (
alternative.get("grounding_method")
in ("literal_source", "whitespace_normalized_source")
or record["fields"][name]["type"] == "number"
or (
alternative.get("span") is not None
and alternative.get("confidence", 0.0)
> alternative.get("p_null", 1.0)
)
)
):
predictions[name] = {**alternative, "null_recovered": True}
predictions[name]["tokens"] = tokens
predictions[name]["retry_input_tokens"] = retry_lengths[name]
return predictions