Image-Text-to-Text
MLX
Safetensors
qwen3_5
clairic
cloudflare
systemone
structured-output
information-extraction
multimodal
custom-code
conversational
4-bit precision
Instructions to use mlx-community/clairic-4bit with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- MLX
How to use mlx-community/clairic-4bit with MLX:
# Make sure mlx-vlm is installed # pip install --upgrade mlx-vlm from mlx_vlm import load, generate from mlx_vlm.prompt_utils import apply_chat_template from mlx_vlm.utils import load_config # Load the model model, processor = load("mlx-community/clairic-4bit") config = load_config("mlx-community/clairic-4bit") # Prepare input image = ["http://images.cocodataset.org/val2017/000000039769.jpg"] prompt = "Describe this image." # Apply chat template formatted_prompt = apply_chat_template( processor, config, prompt, num_images=1 ) # Generate output output = generate(model, processor, formatted_prompt, image) print(output) - Notebooks
- Google Colab
- Kaggle
- Local Apps Settings
- LM Studio
- Pi
How to use mlx-community/clairic-4bit with Pi:
Start the MLX server
# Install MLX LM: uv tool install mlx-lm # Start a local OpenAI-compatible server: mlx_lm.server --model "mlx-community/clairic-4bit"
Configure the model in Pi
# Install Pi: npm install -g @earendil-works/pi-coding-agent # Add to ~/.pi/agent/models.json: { "providers": { "mlx-lm": { "baseUrl": "http://localhost:8080/v1", "api": "openai-completions", "apiKey": "none", "models": [ { "id": "mlx-community/clairic-4bit" } ] } } }Run Pi
# Start Pi in your project directory: pi
- Hermes Agent
How to use mlx-community/clairic-4bit with Hermes Agent:
Start the MLX server
# Install MLX LM: uv tool install mlx-lm # Start a local OpenAI-compatible server: mlx_lm.server --model "mlx-community/clairic-4bit"
Configure Hermes
# Install Hermes: curl -fsSL https://hermes-agent.nousresearch.com/install.sh | bash hermes setup # Point Hermes at the local server: hermes config set model.provider custom hermes config set model.base_url http://127.0.0.1:8080/v1 hermes config set model.default mlx-community/clairic-4bit
Run Hermes
hermes
- Atomic Chat
- OpenClaw
How to use mlx-community/clairic-4bit with OpenClaw:
Start the MLX server
# Install MLX LM: uv tool install mlx-lm # Start a local OpenAI-compatible server: mlx_lm.server --model "mlx-community/clairic-4bit"
Configure OpenClaw
# Install OpenClaw: npm install -g openclaw@latest # Register the local server and set it as the default model: openclaw onboard --non-interactive --mode local \ --auth-choice custom-api-key \ --custom-base-url http://127.0.0.1:8080/v1 \ --custom-model-id "mlx-community/clairic-4bit" \ --custom-provider-id mlx-lm \ --custom-compatibility openai \ --custom-text-input \ --accept-risk \ --skip-health
Run OpenClaw
openclaw agent --local --agent main --message "Hello from Hugging Face"
Download clairic_runtime/model.py from mlx-community/clairic-4bit: direct link, hf CLI and curl.
- Browser
- Download file 84 kB
-
https://huggingface.co/mlx-community/clairic-4bit/resolve/main/clairic_runtime/model.py
- Command line
-
hf download hf://mlx-community/clairic-4bit/clairic_runtime/model.py
-
curl -L -o model.py https://huggingface.co/mlx-community/clairic-4bit/resolve/main/clairic_runtime/model.py
84 kB
# Modified from the Cloudflare Clairic release (Apache-2.0).
# Changes: the typed head (ClairicHead) was moved to a separate MLX module, and
# the tensor code in this file was ported from PyTorch to NumPy.
from __future__ import annotations
import calendar
import datetime
import functools
import math
import os
import re
from bisect import bisect_left, bisect_right
from dataclasses import dataclass, field
from decimal import ROUND_HALF_UP, Decimal
from typing import Any, cast
import numpy as np
from . import schema as zs
SYSTEM_PREFIX = "<|im_start|>system\n" + zs.SYSTEM_PREAMBLE + "\nFields:\n"
EVIDENCE_SYSTEM_PREFIX = (
"<|im_start|>system\n" + zs.EVIDENCE_SYSTEM_PREAMBLE + "\nFields:\n"
)
USER_PREFIX = "<|im_end|>\n<|im_start|>user\n"
ASSISTANT_SUFFIX = "<|im_end|>\n<|im_start|>assistant\n<think>\n\n</think>\n"
ANSWER_CUE = "\nAnswer:"
@dataclass(frozen=True)
class PromptLayout:
schema_header: str
query_header: str
answer_cue: str
slot_header: str
PROMPT_LAYOUTS = (
PromptLayout("", "", ANSWER_CUE, ""),
PromptLayout(
"Field definition:\n", "\nField query:", "\nValue:", "\nAnswer field:"
),
PromptLayout(
"Requested field:\n", "\nRequested field:", "\nResult:", "\nResult for field:"
),
)
PROMPT_VARIANTS = (0, 1, 2)
def prompt_layout(
template_variant: int = 0, complete_field: bool = True
) -> PromptLayout:
if (
not isinstance(template_variant, int)
or isinstance(template_variant, bool)
or template_variant not in PROMPT_VARIANTS
):
raise ValueError("template_variant must be 0, 1, or 2")
if template_variant and (not complete_field):
raise ValueError("alternative layouts require the evidence construction")
return PROMPT_LAYOUTS[template_variant]
QUERY_TOKENS = 256
MAX_SPAN = 64
TRIM_CLASSES = 8
TRIM_INIT = 2.0
MIN_INPUT_TOKENS = 16
BINS = 256
POINTER_WIDTH = 256
NUMBER_DIGITS = 12
(MIN_EXPONENT, MAX_EXPONENT) = (-12, 15)
EXPONENT_CLASSES = MAX_EXPONENT - MIN_EXPONENT + 2
YEAR_DIGITS = 4
(YEAR_LOW, YEAR_HIGH) = (1800, 2199)
YEAR_OUTSIDE = YEAR_HIGH - YEAR_LOW + 1
YEAR_CLASSES = YEAR_OUTSIDE + 1
LIST_SLOTS = zs.MAX_LIST_ITEMS
ITEM_CUES = tuple((f"\n{index}." for index in range(1, LIST_SLOTS + 1)))
(INSIDE, BELOW, ABOVE) = (0, 1, 2)
OVERFLOW_INIT = -5.0
QUERY_ORDER_RATE = 0.8
LOOP_RANK = 256
LEXICAL_DIGITS = 4
ESTIMATE_GATE_INIT = 0.0
ESTIMATE_SPREAD_LIMIT = 2.0
(EDGE_NEGATIVE, EDGE_ZERO, EDGE_POSITIVE) = (0, 1, 2)
(TAG_OUTSIDE, TAG_BEGIN, TAG_INSIDE) = (0, 1, 2)
NUMBER_WIDTH = 2 + EXPONENT_CLASSES + NUMBER_DIGITS * 10
GROUP_DIGITS = 3
GROUP_CLASSES = 10**GROUP_DIGITS
(GROUP_LOW, GROUP_HIGH) = (MIN_EXPONENT // GROUP_DIGITS, MAX_EXPONENT // GROUP_DIGITS)
NUMBER_GROUPS = GROUP_HIGH - GROUP_LOW + 1
GROUP_RANK = 256
COPY_CATEGORIES: tuple[str | None, ...] | None = ("extraction", None)
COPY_GATE_INIT = -3.0
NUMERAL_RUN = re.compile(
"(?:(?<![\\d.])\\.(?=\\d))?\\d+(?:[,.'\\u00a0\\u202f\\u2009\\u2007 \\u066b\\u066c]\\d+)*"
)
NUMERAL_SEPARATORS = ",.'\xa0\u202f\u2009\u2007 ٫٬"
NUMERAL_READINGS = (
(re.compile("\\d{1,3}(?:,\\d{3})+(?:\\.\\d+)?|\\d+(?:\\.\\d+)?|\\.\\d+"), ",", "."),
(
re.compile(
"\\d{1,3}(?:[.\\u00a0\\u202f\\u2009\\u2007' ]\\d{3})+(?:,\\d+)?|\\d+(?:,\\d+)?"
),
".\xa0\u202f\u2009\u2007' ",
",",
),
(
re.compile("\\d{1,3}(?:[\\u00a0\\u202f\\u2009\\u2007' ]\\d{3})+(?:\\.\\d+)?"),
"\xa0\u202f\u2009\u2007' ",
".",
),
(re.compile("\\d{1,2}(?:,\\d{2})+,\\d{3}(?:\\.\\d+)?"), ",", "."),
(
re.compile("\\d{1,3}(?:\\u066c\\d{3})+(?:\\u066b\\d+)?|\\d+\\u066b\\d+"),
"٬",
"٫",
),
(re.compile("\\d{1,2}(?:\\u066c\\d{2})+\\u066c\\d{3}(?:\\u066b\\d+)?"), "٬", "٫"),
)
NUMERAL_SCALES = {
"thousand": 3,
"thousands": 3,
"k": 3,
"K": 3,
"million": 6,
"millions": 6,
"mn": 6,
"mln": 6,
"M": 6,
"MM": 6,
"billion": 9,
"billions": 9,
"bn": 9,
"B": 9,
"trillion": 12,
"tn": 12,
"lakh": 5,
"lakhs": 5,
"lac": 5,
"lacs": 5,
"crore": 7,
"crores": 7,
"cr": 7,
"Cr": 7,
"Mn": 6,
"Bn": 9,
"Lakh": 5,
"Lakhs": 5,
"Crore": 7,
"Crores": 7,
"Million": 6,
"Billion": 9,
"Thousand": 3,
"Trillion": 12,
"MILLION": 6,
"BILLION": 9,
"ألف": 3,
"مليون": 6,
"مليار": 9,
"हज़ार": 3,
"लाख": 5,
"करोड़": 7,
}
NUMERAL_SCALE = re.compile(
"(?:[ \\u00a0\\u202f\\u2009]?("
+ "|".join(
sorted(
(
word
for word in NUMERAL_SCALES
if len(word) > 2
or word in ("mn", "bn", "tn", "cr", "Cr", "Mn", "Bn", "MM")
),
key=len,
reverse=True,
)
)
+ ")|(k|K|M|B))(?![^\\W\\d_])"
)
NUMERAL_SPAN = 48
CONVENTION_LANGUAGE = re.compile(
"\\b(?:Rechnung|Gesamtbetrag|Betrag|Lieferschein|Factuur|Bedrag|Totaal|Pakbon|Fattura|Importo|Totale|Fatura|Nota fiscal|Factura|Importe|Albar\\u00e1n|Facture|Montant)\\b"
)
CONVENTION_LANGUAGE_US = re.compile("\\b(?:M\\u00e9xico|Mexico|MXN|CFDI|RFC)\\b")
CONVENTION_STATED = {
"eu": re.compile(
"european (?:number )?(?:notation|format)|(?:dot|period|point)s? (?:separates?|for|marks?) (?:the )?thousands|comma (?:separates?|for|marks?|as) (?:the )?decimal|comma decimals?|decimal comma|dot thousands|1\\.234,56|1 234,56",
re.IGNORECASE,
),
"us": re.compile(
"comma (?:separates?|for|marks?) (?:the )?thousands|(?:dot|period|point) (?:separates?|for|marks?|as) (?:the )?decimal|comma thousands|dot decimals?|1,234\\.56",
re.IGNORECASE,
),
}
COPY_CONFIDENT = 0.9
COPY_MIN_CONFIDENCE = 0.5
SLOT_TEXT = "_"
SLOT_COUNTS = {"text": 80, "text_list": 64, "number": 32}
SLOT_CUE_TOKENS = 32
DATE_WIDTH = YEAR_DIGITS * 10 + YEAR_CLASSES + 12 + 31
TIME_WIDTH = 24 + 60 + 60
MASKED = -1000000000.0
SCALAR_KINDS = ("text", "number", "date", "time", "datetime")
IMAGE_PLACEHOLDER = "<|vision_start|><|image_pad|><|vision_end|>"
MEDIA_BATCH_KEYS = ("pixel_values", "image_grid_thw")
MEDIA_TOKEN_KEYS = ("mm_token_type_ids",)
LIST_KINDS = ("number_list", "date_list")
def head_kind(spec: dict[str, Any]) -> str:
kind = spec["type"]
if kind == "duration":
return "number"
if kind == "list":
return f"{spec['items']}_list"
return kind
def number_components(value: float) -> tuple[int, int, list[int]] | None:
value = float(value)
if not math.isfinite(value):
return None
if value == 0:
return (0, 0, [0] * NUMBER_DIGITS)
(mantissa, exponent) = f"{abs(value):.{NUMBER_DIGITS - 1}e}".split("e")
power = int(exponent)
if not MIN_EXPONENT <= power <= MAX_EXPONENT:
return None
return (
int(value < 0),
power - MIN_EXPONENT + 1,
[int(digit) for digit in mantissa.replace(".", "")],
)
def edge_number(value: float) -> tuple[int, int, list[int], float]:
if value == 0 or not math.isfinite(value):
return (EDGE_ZERO, 1, [0] * LEXICAL_DIGITS, 0.0)
(mantissa, exponent) = f"{abs(value):.{LEXICAL_DIGITS + 8}e}".split("e")
(power, digits) = (int(exponent), mantissa.replace(".", ""))
sign = EDGE_NEGATIVE if value < 0 else EDGE_POSITIVE
if power < MIN_EXPONENT:
return (sign, 1, [0] * LEXICAL_DIGITS, 0.0)
if power > MAX_EXPONENT:
return (sign, EXPONENT_CLASSES - 1, [9] * LEXICAL_DIGITS, 1.0)
return (
sign,
power - MIN_EXPONENT + 1,
[int(digit) for digit in digits[:LEXICAL_DIGITS]],
float("0." + digits[LEXICAL_DIGITS:]),
)
@functools.lru_cache(maxsize=8192)
def _edge_parts(
low: float, high: float, scale: str, bins: int
) -> tuple[tuple[int, ...], tuple[int, ...], tuple[int, ...], tuple[float, ...]]:
parts = [
edge_number(zs.from_unit(index / bins, low, high, scale))
for index in range(bins + 1)
]
return (
tuple((part[0] for part in parts)),
tuple((part[1] for part in parts)),
tuple((digit for part in parts for digit in part[2])),
tuple((part[3] for part in parts)),
)
def lexical_estimate(spec: dict[str, Any]) -> bool:
return not zs.is_date_estimate(spec) or zs.date_granularity(spec) == "year"
def estimate_edge_parts(
spec: dict[str, Any], bins: int = BINS
) -> tuple[tuple[int, ...], tuple[int, ...], tuple[int, ...], tuple[float, ...]] | None:
if not lexical_estimate(spec):
return None
(low, high, scale) = zs.estimate_range(spec)
return _edge_parts(float(low), float(high), scale, bins)
@dataclass
class EncodedField:
name: str
kind: str
query: int
items: list[int] = field(default_factory=list)
prefix: tuple[int, int, tuple[int, ...]] | None = None
window: tuple[int, int] | None = None
answer_slots: list[int] = field(default_factory=list)
@dataclass(frozen=True)
class Commit:
text: str
done: bool = False
prefix: tuple[int, int, tuple[int, ...]] | None = None
probability: float = 1.0
@dataclass
class EncodedRecord:
input_ids: list[int]
input_start: int
offsets: list[tuple[int, int]]
fields: list[EncodedField]
text: str
record: dict[str, Any]
media: dict[str, Any] | None = None
image_expansion: int = 0
@dataclass
class EncodedMedia:
ids: list[int]
media: dict[str, Any] | None = None
expansion: int = 0
@property
def length(self) -> int:
return len(self.ids) + self.expansion
def configure_image_processor(processor: Any, max_pixels: int) -> dict[str, Any]:
image_processor = getattr(processor, "image_processor", None)
if image_processor is None or max_pixels <= 0:
raise ValueError("an image processor and positive pixel budget are required")
legacy = hasattr(image_processor, "max_pixels")
size = getattr(image_processor, "size", {})
sized = "longest_edge" in size
if not legacy and (not sized):
raise ValueError(
f"unsupported image pixel configuration: {type(image_processor).__name__}"
)
if legacy:
image_processor.max_pixels = max_pixels
if sized:
size["longest_edge"] = max_pixels
actual = size["longest_edge"] if sized else image_processor.max_pixels
if actual != max_pixels or (legacy and image_processor.max_pixels != max_pixels):
raise ValueError(
"the image processor did not retain its requested pixel budget"
)
return {"class": type(image_processor).__name__, "max_pixels": actual}
def encode_media(
tokenizer: Any,
images: list[Any],
processor: Any = None,
image_token_counts: list[int] | None = None,
) -> EncodedMedia:
if not images:
return EncodedMedia([])
text = IMAGE_PLACEHOLDER * len(images) + "\n"
if image_token_counts is not None:
if len(image_token_counts) != len(images) or any(
(count < 1 for count in image_token_counts)
):
raise ValueError("every image needs a positive token count")
return EncodedMedia(
_tokens(tokenizer, text),
None,
sum((count - 1 for count in image_token_counts)),
)
if processor is None:
raise ValueError("records with images need the model processor")
encoded = processor(text=[text], images=list(images), return_tensors="pt")
media = {key: encoded[key] for key in MEDIA_BATCH_KEYS if key in encoded}
for key in MEDIA_TOKEN_KEYS:
if key in encoded:
media[key] = encoded[key][0].tolist()
return EncodedMedia(encoded["input_ids"][0].tolist(), media)
def _tokens(tokenizer: Any, text: str) -> list[int]:
return tokenizer(text, add_special_tokens=False).input_ids
def _input_tokens(tokenizer: Any, text: str) -> tuple[list[int], list[tuple[int, int]]]:
try:
encoding = tokenizer(
text,
add_special_tokens=False,
return_offsets_mapping=True,
split_special_tokens=True,
)
except TypeError:
encoding = tokenizer(
text, add_special_tokens=False, return_offsets_mapping=True
)
return (list(encoding.input_ids), [tuple(pair) for pair in encoding.offset_mapping])
def schema_line(name: str, spec: dict[str, Any], complete_field: bool = False) -> str:
return "- " + zs.field_line(name, spec, preserve_instructions=complete_field) + "\n"
def reference_line(record: dict[str, Any]) -> str:
reference = record.get("reference_date")
return f"Reference date: {reference}\n" if reference else ""
def query_block(
tokenizer: Any,
name: str,
spec: dict[str, Any],
commit: Commit | None = None,
complete_field: bool = False,
template_variant: int = 0,
) -> tuple[list[int], int, list[int]]:
layout = prompt_layout(template_variant, complete_field)
line = _tokens(
tokenizer,
"\n" + zs.field_line(name, spec, preserve_instructions=complete_field),
)
ids = (
_tokens(tokenizer, layout.query_header)
+ (line if complete_field else line[:QUERY_TOKENS])
+ _tokens(tokenizer, layout.answer_cue)
)
if commit is not None:
ids.extend(_tokens(tokenizer, " " + commit.text))
query = len(ids) - 1
items: list[int] = []
if head_kind(spec) in LIST_KINDS and commit is None:
for cue in ITEM_CUES:
ids.extend(_tokens(tokenizer, cue))
items.append(len(ids) - 1)
return (ids, query, items)
def clean_cut(
text: str, offsets: list[tuple[int, int]], budget: int, lookback: int = 32
) -> int:
lowest = max(0, budget - lookback)
for keep in range(budget, lowest, -1):
end = offsets[keep - 1][1]
if end >= len(text) or (
end > 0 and text[end].isspace() and (not text[end - 1].isspace())
):
return keep
for keep in range(budget, lowest, -1):
end = offsets[keep - 1][1]
if end > 0 and (text[end].isspace() or text[end - 1].isspace()):
return keep
return budget
def char_span_to_tokens(
starts: list[int], ends: list[int], char_start: int, char_end: int
) -> tuple[int, int] | None:
first = bisect_right(ends, char_start)
last = bisect_left(starts, char_end) - 1
if first >= len(starts) or last < first:
return None
return (first, last)
def numeral_values(printed: str, convention: str | None = None) -> list[Decimal]:
values: list[Decimal] = []
both = convention is not None and all(
(form.fullmatch(printed) for (form, _, _) in NUMERAL_READINGS[:2])
)
for index, (form, separators, mark) in enumerate(NUMERAL_READINGS):
if both and index == (1 if convention == "us" else 0):
continue
if form.fullmatch(printed):
plain = printed.translate(
{ord(character): None for character in separators}
).replace(mark, ".")
try:
value = Decimal(plain)
except ArithmeticError:
continue
if value not in values:
values.append(value)
return values
@functools.lru_cache(maxsize=256)
def numeral_convention(text: str) -> str | None:
stated = {
name
for (name, pattern) in CONVENTION_STATED.items()
if pattern.search(text or "")
}
if len(stated) == 1:
return stated.pop()
shown: list[str] = []
for run in NUMERAL_RUN.finditer(text or ""):
printed = run.group()
if not any((mark in printed for mark in ",.")):
continue
(us, eu) = (
bool(form.fullmatch(printed)) for (form, _, _) in NUMERAL_READINGS[:2]
)
if us != eu:
shown.append("us" if us else "eu")
if len(shown) >= 2 and len(set(shown)) == 1:
return shown[0]
if (
not shown
and CONVENTION_LANGUAGE.search(text or "")
and (not CONVENTION_LANGUAGE_US.search(text or ""))
):
return "eu"
return None
@functools.lru_cache(maxsize=256)
def _numeral_candidates(text: str) -> tuple[tuple[list[Decimal], int, int, bool], ...]:
found: list[tuple[list[Decimal], int, int, bool]] = []
for run in NUMERAL_RUN.finditer(text or ""):
(begin, finish) = (run.start(), run.end())
cuts = [
index
for index in range(begin, finish)
if text[index] in NUMERAL_SEPARATORS and index > begin
]
starts = [begin] + [index + 1 for index in cuts]
ends = cuts + [finish]
convention = numeral_convention(text)
scale = NUMERAL_SCALE.match(text, finish)
exponent = (
None if scale is None else NUMERAL_SCALES[scale.group(1) or scale.group(2)]
)
run_found = []
for start in starts:
for end in ends[bisect_left(ends, start + 1) :]:
if end - start > NUMERAL_SPAN:
break
values = numeral_values(text[start:end], convention)
if not values:
continue
if end == finish and exponent is not None:
scaled = [value.scaleb(exponent) for value in values]
run_found.append(
(
scaled + [value for value in values if value not in scaled],
start,
cast(re.Match[str], scale).end(),
)
)
else:
run_found.append((values, start, end))
by_start: dict[int, tuple[list[Decimal], int, int]] = {}
for item in run_found:
if item[1] not in by_start or item[2] > by_start[item[1]][2]:
by_start[item[1]] = item
(position, whole) = (begin, set())
while True:
longest = by_start.get(position)
following = bisect_left(
starts, position + 1 if longest is None else longest[2]
)
if longest is not None:
whole.add((longest[1], longest[2]))
if following >= len(starts):
break
position = starts[following]
found.extend(
(
(values, start, end, (start, end) in whole)
for (values, start, end) in run_found
)
)
return tuple(found)
def read_numerals(
text: str, every: bool = False
) -> list[tuple[list[Decimal], int, int]]:
found = [
(values, start, end)
for (values, start, end, whole) in _numeral_candidates(text)
if every or whole
]
return sorted(found, key=lambda item: (item[1], -item[2]))
def numeral_spans(text: str, value: Any, every: bool = True) -> list[tuple[int, int]]:
try:
target = abs(Decimal(str(value)))
except ArithmeticError:
return []
candidates = _numeral_candidates(text)
whole = [
(start, end)
for (values, start, end, longest) in candidates
if longest and target in values
]
if whole or not every:
return sorted(whole)
return sorted(
{(start, end) for (values, start, end, _) in candidates if target in values},
key=lambda item: (item[0], -item[1]),
)
def slot_token_id(tokenizer: Any) -> int:
ids = _tokens(tokenizer, SLOT_TEXT)
if len(ids) != 1:
raise ValueError(f"answer slots need {SLOT_TEXT!r} to be one token")
return ids[0]
def slot_block(
tokenizer: Any,
name: str,
spec: dict[str, Any],
complete_field: bool = False,
template_variant: int = 0,
) -> list[int]:
count = SLOT_COUNTS.get(head_kind(spec))
layout = prompt_layout(template_variant, complete_field)
if count is None:
return []
cue = _tokens(tokenizer, "\n" + name + ":")
return (
_tokens(tokenizer, layout.slot_header)
+ (cue if complete_field else cue[:SLOT_CUE_TOKENS])
+ [slot_token_id(tokenizer)] * count
)
def system_prefix(record: dict[str, Any], text_answer_slots: bool = False) -> str:
return (
EVIDENCE_SYSTEM_PREFIX
if evidence_construction(record, text_answer_slots)
else SYSTEM_PREFIX
)
def evidence_construction(
record: dict[str, Any], text_answer_slots: bool = False
) -> bool:
return (
text_answer_slots
or (record.get("_meta") or {}).get("construction") == zs.EVIDENCE_CONSTRUCTION
)
def encode_record(
tokenizer: Any,
record: dict[str, Any],
max_length: int = 8192,
media: EncodedMedia | None = None,
commits: dict[str, Commit] | None = None,
answer_slots: bool = False,
text_answer_slots: bool = False,
template_variant: int = 0,
) -> EncodedRecord | None:
if text_answer_slots and (not answer_slots):
raise ValueError("text_answer_slots needs answer_slots")
commits = commits or {}
if record.get("images") and media is None:
raise ValueError("records with images need their encoded media")
media = media or EncodedMedia([])
items = list(record["fields"].items())
complete_field = evidence_construction(record, text_answer_slots)
layout = prompt_layout(template_variant, complete_field)
ids = _tokens(tokenizer, system_prefix(record, text_answer_slots))
for name, spec in items:
ids.extend(_tokens(tokenizer, layout.schema_header))
ids.extend(_tokens(tokenizer, schema_line(name, spec, complete_field)))
ids.extend(_tokens(tokenizer, reference_line(record)))
ids.extend(_tokens(tokenizer, USER_PREFIX))
media_start = len(ids)
ids.extend(media.ids)
input_start = len(ids)
suffix = _tokens(tokenizer, ASSISTANT_SUFFIX)
blocks = [
query_block(
tokenizer, name, spec, commits.get(name), complete_field, template_variant
)
for (name, spec) in items
]
slot_blocks: dict[int, list[int]] = {}
if answer_slots and (media.ids or text_answer_slots):
for index, (name, spec) in enumerate(items):
if not (name in commits and commits[name].done):
block = slot_block(
tokenizer, name, spec, complete_field, template_variant
)
if block:
slot_blocks[index] = block
budget = (
max_length
- len(ids)
- media.expansion
- len(suffix)
- sum((len(block[0]) for block in blocks))
- sum((len(block) for block in slot_blocks.values()))
)
if budget < MIN_INPUT_TOKENS:
return None
text = record["input"]
if media.ids and (not text.strip()):
text = ""
(input_ids, offsets) = (_tokens(tokenizer, "\n"), [(0, 0)])
else:
(input_ids, offsets) = _input_tokens(tokenizer, text)
if len(input_ids) > budget:
keep = clean_cut(text, offsets, budget)
text = text[: offsets[keep - 1][1]]
(input_ids, offsets) = _input_tokens(tokenizer, text)
if len(input_ids) > budget:
(input_ids, offsets) = (input_ids[:budget], offsets[:budget])
text = text[: offsets[-1][1]]
if not input_ids:
return None
ids.extend(input_ids)
ids.extend(suffix)
order = list(range(len(items)))
done = {
index
for (index, (name, _)) in enumerate(items)
if name in commits and commits[name].done
}
if done:
order = [index for index in order if index in done] + [
index for index in order if index not in done
]
queries: dict[int, int] = {}
slots: dict[int, list[int]] = {}
for index in order:
(block, query, item_offsets) = blocks[index]
start = len(ids)
ids.extend(block)
queries[index] = start + query
slots[index] = [start + offset for offset in item_offsets]
answer_positions: dict[int, list[int]] = {}
for index in order:
if index in slot_blocks:
ids.extend(slot_blocks[index])
count = SLOT_COUNTS[head_kind(items[index][1])]
answer_positions[index] = list(range(len(ids) - count, len(ids)))
fields: list[EncodedField] = []
for index, (name, spec) in enumerate(items):
if index in done:
continue
encoded = EncodedField(name, head_kind(spec), queries[index], slots[index])
if index in answer_positions:
encoded.answer_slots = answer_positions[index]
commit = commits.get(name)
if (
commit is not None
and commit.prefix is not None
and (encoded.kind in ("number", "estimate"))
):
encoded.prefix = commit.prefix
if encoded.kind == "estimate":
encoded.window = prefix_window(spec, commit.prefix)
fields.append(encoded)
if not fields:
return None
record_media = None
if media.media is not None:
record_media = {**media.media, "token_offset": media_start}
return EncodedRecord(
ids, input_start, offsets, fields, text, record, record_media, media.expansion
)
def span_base(text: str, starts: list[int], ends: list[int], first: int) -> int:
base = starts[first]
while base < ends[first] - 1 and text[base].isspace():
base += 1
return base
def trimmed_span(
text: str,
offsets: list[tuple[int, int]],
first: int,
last: int,
trims: list[list[int]] | None,
) -> tuple[int, int]:
(char_start, char_end) = (offsets[first][0], offsets[last][1])
while char_start < offsets[first][1] - 1 and text[char_start].isspace():
char_start += 1
if trims is not None:
start = max(
char_start, min(char_start + int(trims[first][0]), offsets[first][1] - 1)
)
end = min(char_end, max(char_end - int(trims[last][1]), offsets[last][0] + 1))
if start < end:
(char_start, char_end) = (start, end)
while char_start < char_end - 1 and text[char_start].isspace():
char_start += 1
return (char_start, char_end)
def plain_number(value: Any) -> str:
number = Decimal(str(value))
if not number.is_finite():
raise ValueError("A plain number must be finite.")
text = format(number, "f")
return text.rstrip("0").rstrip(".") if "." in text else text
def prefix_window(
spec: dict[str, Any], prefix: tuple[int, int, tuple[int, ...]], bins: int = BINS
) -> tuple[int, int] | None:
(sign, exponent_class, digits) = prefix
power = exponent_class - 1 + MIN_EXPONENT
unit = Decimal(1).scaleb(power - len(digits) + 1)
start = Decimal(int("".join((str(digit) for digit in digits)))) * unit
(first, last) = (float(start), float(start + unit))
if sign:
(first, last) = (-last, -first)
(low, high, scale) = zs.estimate_range(spec)
(lower, upper) = sorted(
(zs.raw_unit(first, low, high, scale), zs.raw_unit(last, low, high, scale))
)
if upper <= 0.0 or lower >= 1.0:
return None
first_bin = max(0, min(bins - 1, math.floor(lower * bins)))
last_bin = max(first_bin, min(bins - 1, math.ceil(upper * bins) - 1))
return (first_bin, last_bin)
def _masked_fill(values: np.ndarray, mask: Any, fill: float) -> np.ndarray:
filled = np.array(values, copy=True)
filled[np.broadcast_to(np.asarray(mask, dtype=bool), filled.shape)] = fill
return filled
def _log_softmax(values: Any, axis: int = -1) -> np.ndarray:
values = np.asarray(values)
shifted = values - values.max(axis=axis, keepdims=True)
return shifted - np.log(np.exp(shifted).sum(axis=axis, keepdims=True))
def _softmax(values: Any, axis: int = -1) -> np.ndarray:
values = np.asarray(values)
exps = np.exp(values - values.max(axis=axis, keepdims=True))
return exps * (1 / exps.sum(axis=axis, keepdims=True))
def _logsumexp(values: Any, axis: int = -1) -> np.ndarray:
values = np.asarray(values)
maxes = values.max(axis=axis, keepdims=True)
maxes = np.where(np.abs(maxes) == np.inf, np.zeros_like(maxes), maxes)
summed = np.exp(values - maxes).sum(axis=axis)
return np.log(summed) + np.squeeze(maxes, axis=axis)
def _sigmoid(values: Any) -> np.ndarray:
values = np.asarray(values)
with np.errstate(over="ignore"):
return 1 / (1 + np.exp(-values))
def _log_sigmoid(values: Any) -> np.ndarray:
values = np.asarray(values)
return np.minimum(values, 0) - np.log1p(np.exp(-np.abs(values)))
def _cumsum(values: Any, axis: int = -1) -> np.ndarray:
# torch's CPU cumsum accumulates float32 in double precision.
values = np.asarray(values)
if values.dtype == np.float32:
return np.cumsum(values.astype(np.float64), axis=axis).astype(np.float32)
return np.cumsum(values, axis=axis)
def _f32(values: Any) -> np.ndarray:
return np.asarray(values, dtype=np.float32)
def _flatten1(values: np.ndarray) -> np.ndarray:
return values.reshape(values.shape[0], math.prod(values.shape[1:]))
def span_scores(
start: np.ndarray, end: np.ndarray, length_bias: np.ndarray
) -> np.ndarray:
(start, end, length_bias) = (
np.asarray(start),
np.asarray(end),
np.asarray(length_bias),
)
width = length_bias.shape[0]
pad = [(0, 0)] * (end.ndim - 1) + [(0, width - 1)]
padded = np.pad(end, pad, constant_values=float("-inf"))
windows = np.lib.stride_tricks.sliding_window_view(padded, width, axis=-1)
return start[..., None] + windows + length_bias
DIGIT_SLOTS = NUMBER_DIGITS + YEAR_DIGITS + 2 + 2 + 2 + 2
(YEAR_SLOT, DAY_SLOT, HOUR_SLOT, MINUTE_SLOT, SECOND_SLOT) = (
NUMBER_DIGITS,
NUMBER_DIGITS + YEAR_DIGITS,
NUMBER_DIGITS + YEAR_DIGITS + 2,
NUMBER_DIGITS + YEAR_DIGITS + 4,
NUMBER_DIGITS + YEAR_DIGITS + 6,
)
MONTH_WORDS = tuple((" " + name for name in calendar.month_name[1:]))
SIGN_WORDS = (" ", " -")
ROUTE_KINDS = (
"text",
"tags",
"number",
"date",
"time",
"estimate",
"number_list",
"date_list",
)
def two_digit_index(values: range) -> np.ndarray:
return np.array(
[[value // 10 for value in values], [value % 10 for value in values]],
dtype=np.int64,
)
def compose_two_digits(slots: np.ndarray, index: np.ndarray) -> np.ndarray:
slots = np.asarray(slots)
return np.take(slots[..., 0, :], index[0], axis=-1) + np.take(
slots[..., 1, :], index[1], axis=-1
)
def compose_years(slots: np.ndarray, index: np.ndarray) -> np.ndarray:
slots = np.asarray(slots)
inside = cast(
np.ndarray,
sum(
(
np.take(slots[..., position, :], index[position], axis=-1)
for position in range(YEAR_DIGITS)
)
),
)
total = _logsumexp(slots, axis=-1).sum(axis=-1)
gap = np.minimum(_logsumexp(inside, axis=-1) - total, -1e-06)
outside = total + np.log(-np.expm1(gap))
return np.concatenate([inside, outside[..., None]], axis=-1)
SPELL_END = "\n"
SPELL_SEPARATOR = " | "
SPELL_MIN_CONFIDENCE = 0.3
def spelled_answer(
ids: list[int], log_probs: list[float], tokenizer: Any, end_id: int
) -> tuple[str, float, list[tuple[str, float]]]:
stop = ids.index(end_id) if end_id in ids else len(ids)
pieces = [tokenizer.decode([token]) for token in ids[:stop]]
items: list[tuple[list[int], list[float]]] = []
current: list[int] = []
scores: list[float] = []
for token, piece, score in zip(ids[:stop], pieces, log_probs[:stop]):
if piece.strip() == SPELL_SEPARATOR.strip():
items.append((current, scores))
(current, scores) = ([], [])
else:
current.append(token)
scores.append(score)
items.append((current, scores))
parts = []
for item_ids, item_scores in items:
words = tokenizer.decode(item_ids).split()
if words:
parts.append(
(" ".join(words), math.exp(sum(item_scores) / len(item_scores)))
)
text = SPELL_SEPARATOR.join((part for (part, _) in parts))
return (text, math.exp(sum(log_probs[: min(stop + 1, len(ids))])), parts)
def evidence_text_answer(
prediction: dict[str, Any],
ids: list[int],
log_probs: list[float],
tokenizer: Any,
end_id: int,
text: str,
spec: dict[str, Any],
has_images: bool = False,
) -> dict[str, Any]:
if end_id not in ids:
return prediction
stop = ids.index(end_id)
value = tokenizer.decode(ids[:stop]).strip()
confidence = math.exp(sum(log_probs[: stop + 1]))
exact = (
[
span
for span in zs.find_spans(text, value)
if text[span[0] : span[1]] == value
]
if value
else []
)
present = prediction["p_null"] < zs.null_threshold(spec)
return {
**prediction,
"guess": value or None,
"value": value if value and present else None,
"span": list(exact[0]) if exact else None,
"confidence": confidence,
"source": "input" if exact else "image" if has_images else "readout",
}
def _simple(value: Any) -> str:
return "".join(
(character for character in str(value).lower() if character.isalnum())
)
def _similarity(left: Any, right: Any) -> float:
import difflib
(left, right) = (_simple(left), _simple(right))
return difflib.SequenceMatcher(None, left, right).ratio() if left and right else 0.0
def _state_match(
text: str, value: str, threshold: float = 0.8, max_words: int = 4000
) -> tuple[int, int] | None:
if not text or not value:
return None
spans = zs.find_spans(text, value)
if spans:
return spans[0]
import re
words = [match.span() for match in re.finditer("\\S+", text)][:max_words]
size = len(value.split())
(best, best_span) = (threshold, None)
for width in range(max(1, size - 1), size + 2):
for first in range(max(0, len(words) - width + 1)):
(start, end) = (words[first][0], words[first + width - 1][1])
score = _similarity(text[start:end].strip(".,;:()[]\"'"), value)
if score >= best:
(best, best_span) = (score, (start, end))
return best_span
def _stutters(value: Any) -> bool:
import re
simple = _simple(value)
return bool(re.search("(.)\\1\\1", simple) or re.search("([a-z]{4,})\\1", simple))
def _digit_code(value: Any) -> bool:
simple = _simple(value)
return bool(simple) and 2 * sum(
(character.isdigit() for character in simple)
) >= len(simple)
PLACEHOLDER_WORDS = frozenset(
(
"a",
"above",
"an",
"app",
"attach",
"attached",
"attachment",
"below",
"doc",
"document",
"documents",
"file",
"heic",
"here",
"image",
"images",
"is",
"jpeg",
"jpg",
"mobile",
"page",
"pdf",
"photo",
"photos",
"pic",
"picture",
"please",
"png",
"scan",
"scanned",
"screenshot",
"see",
"the",
"this",
"upload",
"uploaded",
"via",
)
)
def _placeholder_text(value: Any) -> bool:
import re
words = re.findall("[a-z0-9]+", str(value or "").lower())
return bool(words) and all((word in PLACEHOLDER_WORDS for word in words))
def _defers(pointer_value: Any, spelled: str) -> bool:
if pointer_value is None:
return False
return _stutters(spelled) or (
_digit_code(spelled)
and any((character.isdigit() for character in str(pointer_value)))
)
def route_spelled(
prediction: dict[str, Any],
kind: str,
spelled: str,
items: list[tuple[str, float]],
text: str,
defer_lists: bool = False,
) -> dict[str, Any]:
if kind == "text":
if _placeholder_text(prediction.get("guess")):
prediction = {
**prediction,
"value": None,
"guess": None,
"span": None,
"confidence": 0.0,
}
items = [
(item, score) for (item, score) in items if not _placeholder_text(item)
]
if not items:
return prediction
(value, confidence) = (
items[0][0] if len(items) == 1 else spelled,
min((score for (_, score) in items)),
)
routed = {**prediction, "spell_confidence": confidence}
if _similarity(prediction.get("guess"), value) >= 0.6:
if (
routed.get("value") is None
and prediction.get("guess")
and (confidence >= SPELL_MIN_CONFIDENCE)
):
return {**routed, "value": prediction["guess"]}
return routed
span = _state_match(text, value)
if span is not None:
return {
**routed,
"value": text[span[0] : span[1]],
"guess": text[span[0] : span[1]],
"span": list(span),
}
if confidence >= SPELL_MIN_CONFIDENCE and (
not _defers(prediction.get("value"), value)
):
return {
**routed,
"value": value,
"guess": value,
"span": None,
"confidence": confidence,
"source": "image",
}
return routed
(pointer_values, pointer_spans) = (
list(prediction.get("value") or []),
list(prediction.get("spans") or []),
)
if any((_placeholder_text(value) for value in pointer_values)):
kept = [
(value, pointer_spans[index] if index < len(pointer_spans) else None)
for (index, value) in enumerate(pointer_values)
if not _placeholder_text(value)
]
(pointer_values, pointer_spans) = (
[value for (value, _) in kept],
[span for (_, span) in kept],
)
items = [(item, score) for (item, score) in items if not _placeholder_text(item)]
(values, spans, used) = ([], [], set())
for item, confidence in items:
match = max(
range(len(pointer_values)),
key=lambda index: _similarity(pointer_values[index], item),
default=None,
)
if (
match is not None
and match not in used
and (_similarity(pointer_values[match], item) >= 0.6)
):
used.add(match)
values.append(pointer_values[match])
spans.append(pointer_spans[match] if match < len(pointer_spans) else None)
continue
span = _state_match(text, item)
if span is not None:
value = text[span[0] : span[1]]
if not any((_simple(value) == _simple(other) for other in values)):
values.append(value)
spans.append(list(span))
elif (
confidence >= SPELL_MIN_CONFIDENCE
and (
not (
pointer_values and (defer_lists or _defers(pointer_values[0], item))
)
)
and (not any((_simple(item) == _simple(other) for other in values)))
):
values.append(item)
spans.append(None)
for index, (value, span) in enumerate(zip(pointer_values, pointer_spans)):
if index not in used and (
not any((_simple(value) == _simple(other) for other in values))
):
values.append(value)
spans.append(span)
routed = {**prediction, "value": values, "spans": spans}
if any((span is None for span in spans)):
routed["source"] = "image"
return routed
def number_parts(
logits: np.ndarray,
) -> tuple[np.ndarray, np.ndarray, np.ndarray]:
logits = np.asarray(logits)
sign = logits[..., :2]
exponent = logits[..., 2 : 2 + EXPONENT_CLASSES]
digits = logits[..., 2 + EXPONENT_CLASSES :]
if logits.shape[-1] != 2 + EXPONENT_CLASSES + NUMBER_DIGITS * 10:
raise RuntimeError("number logits do not match the number layout")
return (sign, exponent, digits.reshape(*digits.shape[:-1], NUMBER_DIGITS, 10))
def group_class_digits(device: Any = None) -> np.ndarray:
classes = np.arange(GROUP_CLASSES, dtype=np.int64)
return np.stack(
[
classes // 10 ** (GROUP_DIGITS - 1 - place) % 10
for place in range(GROUP_DIGITS)
],
axis=-1,
)
def group_slots(exponent_class: np.ndarray) -> tuple[np.ndarray, np.ndarray]:
exponent_class = np.asarray(exponent_class).astype(np.int64)
power = (exponent_class - 1 + MIN_EXPONENT).reshape(-1, 1, 1)
groups = np.arange(GROUP_LOW, GROUP_HIGH + 1, dtype=np.int64).reshape(1, -1, 1)
places = np.arange(GROUP_DIGITS - 1, -1, -1, dtype=np.int64).reshape(1, 1, -1)
slots = power - (groups * GROUP_DIGITS + places)
inside = (
(slots >= 0)
& (slots < NUMBER_DIGITS)
& (exponent_class > 0).reshape(-1, 1, 1)
)
return (
np.where(inside, slots, np.full_like(slots, NUMBER_DIGITS)),
inside.any(axis=-1),
)
def number_group_logits(
digit_logits: np.ndarray,
exponent_class: np.ndarray,
learned: np.ndarray | None = None,
) -> tuple[np.ndarray, np.ndarray]:
digit_logits = np.asarray(digit_logits)
count = digit_logits.shape[0]
leading_zero = np.zeros((NUMBER_DIGITS, 10), dtype=bool)
leading_zero[0, 0] = True
log = _masked_fill(_log_softmax(_f32(digit_logits), axis=-1), leading_zero, MASKED)
held = np.full((count, 1, 10), MASKED, dtype=log.dtype)
held[..., 0] = 0.0
padded = np.concatenate([log, held], axis=1)
(slots, groups) = group_slots(exponent_class)
picked = padded[np.arange(count)[:, None], _flatten1(slots)]
picked = picked.reshape(count, NUMBER_GROUPS, GROUP_DIGITS, 10)
table = group_class_digits()
scores = cast(
np.ndarray,
sum(
(
np.take(picked[:, :, place, :], table[:, place], axis=-1)
for place in range(GROUP_DIGITS)
)
),
)
if learned is not None:
scores = scores + _f32(learned)
return (scores, groups)
def temper_number(logits: np.ndarray, spread: np.ndarray) -> np.ndarray:
(sign, exponent, digits) = number_parts(logits)
temperature = np.exp(
ESTIMATE_SPREAD_LIMIT * np.tanh(_f32(spread) / ESTIMATE_SPREAD_LIMIT)
)
exponent = exponent / temperature[:, :1]
digits = np.concatenate(
[
digits[:, :LEXICAL_DIGITS] / temperature[:, 1:][..., None],
digits[:, LEXICAL_DIGITS:],
],
axis=1,
)
return np.concatenate(
[sign, exponent, _flatten1(digits)], axis=-1
)
def apply_prefix(
logits: np.ndarray, views: dict[str, np.ndarray], group: str
) -> np.ndarray:
length = views.get(f"{group}_prefix_length")
logits = np.asarray(logits)
if length is None or not logits.shape[0]:
return logits
(sign_logits, exponent_logits, digit_logits) = number_parts(logits)
slots = np.arange(NUMBER_DIGITS, dtype=np.int64)
length = np.asarray(length).astype(np.int64)
relative = np.maximum(slots[None, :] - length[:, None], 0)
shifted = digit_logits[np.arange(digit_logits.shape[0])[:, None], relative]
forced_digits = np.where(
np.arange(10, dtype=np.int64)
== np.asarray(views[f"{group}_prefix_digits"]).astype(np.int64)[..., None],
0.0,
MASKED,
).astype(logits.dtype)
digit_logits = np.where(
(slots[None, :] < length[:, None])[..., None], forced_digits, shifted
)
written = (length > 0)[:, None]
forced_sign = np.where(
np.arange(2, dtype=np.int64)
== np.asarray(views[f"{group}_prefix_sign"]).astype(np.int64)[:, None],
0.0,
MASKED,
).astype(logits.dtype)
forced_exponent = np.where(
np.arange(EXPONENT_CLASSES, dtype=np.int64)
== np.asarray(views[f"{group}_prefix_exponent"]).astype(np.int64)[:, None],
0.0,
MASKED,
).astype(logits.dtype)
return np.concatenate(
[
np.where(written, forced_sign, sign_logits),
np.where(written, forced_exponent, exponent_logits),
_flatten1(digit_logits),
],
axis=-1,
)
def window_log_probs(
log_probs: np.ndarray, window: np.ndarray
) -> tuple[np.ndarray, np.ndarray]:
log_probs = np.asarray(log_probs)
bins = np.arange(log_probs.shape[-1], dtype=np.int64)
window = np.asarray(window).astype(np.int64)
inside = (bins >= window[:, :1]) & (bins <= window[:, 1:])
return (_log_softmax(_masked_fill(log_probs, ~inside, MASKED), axis=-1), inside)
def date_parts(
logits: np.ndarray,
) -> tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray]:
logits = np.asarray(logits)
if logits.shape[-1] != YEAR_DIGITS * 10 + YEAR_CLASSES + 12 + 31:
raise RuntimeError("date logits do not match the date layout")
(digits, year, month, day) = np.split(
logits,
np.cumsum([YEAR_DIGITS * 10, YEAR_CLASSES, 12]).tolist(),
axis=-1,
)
return (digits.reshape(*digits.shape[:-1], YEAR_DIGITS, 10), year, month, day)
def time_parts(logits: np.ndarray) -> tuple[np.ndarray, np.ndarray, np.ndarray]:
logits = np.asarray(logits)
if logits.shape[-1] != 24 + 60 + 60:
raise RuntimeError("time logits do not match the time layout")
(hour, minute, second) = np.split(logits, [24, 84], axis=-1)
return (hour, minute, second)
def lexical_bin_log_probs(
number_logits: np.ndarray,
edge_sign: np.ndarray,
edge_class: np.ndarray,
edge_digits: np.ndarray,
edge_frac: np.ndarray,
) -> np.ndarray:
(sign, exponent, digits) = number_parts(
np.asarray(number_logits).astype(np.float64)
)
(edge_sign, edge_class, edge_digits) = (
np.asarray(edge_sign),
np.asarray(edge_class).astype(np.int64),
np.asarray(edge_digits).astype(np.int64),
)
sign_probs = _softmax(sign, axis=-1)
exponent_probs = _softmax(exponent, axis=-1)
zero = exponent_probs[:, :1]
nonzero = exponent_probs[:, 1:] / np.maximum(1.0 - zero, 1e-300)
below_class = _cumsum(nonzero, axis=-1) - nonzero
leading = _masked_fill(digits[:, :1], np.arange(10) == 0, float("-inf"))
digit_probs = _softmax(
np.concatenate([leading, digits[:, 1:LEXICAL_DIGITS]], axis=1), axis=-1
)
below_digit = _cumsum(digit_probs, axis=-1) - digit_probs
classes = np.clip(edge_class - 1, 0, nonzero.shape[1] - 1)
within = np.asarray(edge_frac).astype(np.float64)
for position in reversed(range(LEXICAL_DIGITS)):
index = edge_digits[..., position]
within = (
np.take_along_axis(below_digit[:, position], index, axis=1)
+ np.take_along_axis(digit_probs[:, position], index, axis=1) * within
)
magnitude = (
np.take_along_axis(below_class, classes, axis=1)
+ np.take_along_axis(nonzero, classes, axis=1) * within
)
negative = sign_probs[:, 1:] * (1.0 - zero)
positive = sign_probs[:, :1] * (1.0 - zero)
cdf = np.where(
edge_sign == EDGE_NEGATIVE,
negative * (1.0 - magnitude),
np.where(
edge_sign == EDGE_ZERO,
np.broadcast_to(negative, magnitude.shape),
negative + zero + positive * magnitude,
),
)
mass = np.maximum(cdf[:, 1:] - cdf[:, :-1], 0.0) + 1e-30
return (np.log(mass) - np.log(mass.sum(axis=-1, keepdims=True))).astype(
np.float32
)
def estimate_log_probs(
range_logits: np.ndarray,
number_logits: np.ndarray | None = None,
gate: np.ndarray | None = None,
lexical: np.ndarray | None = None,
edge_sign: np.ndarray | None = None,
edge_class: np.ndarray | None = None,
edge_digits: np.ndarray | None = None,
edge_frac: np.ndarray | None = None,
) -> np.ndarray:
range_part = _log_softmax(_f32(range_logits), axis=-1)
if number_logits is None or not np.asarray(number_logits).shape[0]:
return range_part
lexical_part = lexical_bin_log_probs(
number_logits,
cast(np.ndarray, edge_sign),
cast(np.ndarray, edge_class),
cast(np.ndarray, edge_digits),
cast(np.ndarray, edge_frac),
)
use = np.asarray(cast(np.ndarray, lexical)).astype(bool)[:, None]
gate = _f32(cast(np.ndarray, gate))[:, None]
mixed = np.logaddexp(
_log_sigmoid(gate) + lexical_part,
_log_sigmoid(-gate) + range_part,
)
return np.where(use, mixed, range_part)
def decode_number(
logits: np.ndarray, spec: dict[str, Any], groups: np.ndarray | None = None
) -> tuple[float | int, float]:
(sign_logits, exponent_logits, digit_logits) = number_parts(_f32(logits))
decimals = zs.allowed_decimals(spec)
exponent_log = _log_softmax(exponent_logits, axis=-1)
exponent_class = int(exponent_log.argmax())
log_probability = float(exponent_log[exponent_class])
if exponent_class == 0:
return (0, math.exp(log_probability))
power = exponent_class - 1 + MIN_EXPONENT
kept = NUMBER_DIGITS if decimals is None else power + decimals + 1
if kept < 0:
return (0, math.exp(log_probability))
leading_zero = np.zeros((NUMBER_DIGITS, 10), dtype=bool)
leading_zero[0, 0] = True
digit_log = _masked_fill(_log_softmax(digit_logits, axis=-1), leading_zero, MASKED)
digits = digit_log.argmax(axis=-1).tolist()
sign_log = _log_softmax(sign_logits, axis=-1)
sign = int(sign_log.argmax())
kept = min(kept, NUMBER_DIGITS)
log_probability += float(sign_log[sign])
if groups is None:
log_probability += sum(
(float(digit_log[position, digits[position]]) for position in range(kept))
)
if kept < NUMBER_DIGITS:
rounding = digit_log[kept, 5:] if digits[kept] >= 5 else digit_log[kept, :5]
log_probability += float(_logsumexp(rounding, axis=0))
else:
exponent_tensor = np.array([exponent_class], dtype=np.int64)
(scores, present) = number_group_logits(
digit_logits[None],
exponent_tensor,
_f32(groups)[None],
)
group_log = _log_softmax(scores[0], axis=-1)
slots = group_slots(exponent_tensor)[0][0].tolist()
table = group_class_digits()
counted = set()
for group in range(NUMBER_GROUPS):
if not bool(present[0, group]):
continue
best = int(group_log[group].argmax())
for place, slot in enumerate(slots[group]):
if slot < NUMBER_DIGITS:
digits[slot] = int(table[best, place])
if any((slot < kept for slot in slots[group])):
counted.add(group)
log_probability += float(group_log[group, best])
if kept < NUMBER_DIGITS:
for group in range(NUMBER_GROUPS):
if kept in slots[group]:
if group not in counted:
place = slots[group].index(kept)
same = (
table[:, place] >= 5
if digits[kept] >= 5
else table[:, place] < 5
)
log_probability += float(
_logsumexp(group_log[group][same], axis=0)
)
break
else:
rounding = (
digit_log[kept, 5:] if digits[kept] >= 5 else digit_log[kept, :5]
)
log_probability += float(_logsumexp(rounding, axis=0))
magnitude = Decimal(int("".join((str(digit) for digit in digits)))).scaleb(
power - NUMBER_DIGITS + 1
)
if decimals is not None:
magnitude = magnitude.quantize(
Decimal(1).scaleb(-decimals), rounding=ROUND_HALF_UP
)
if magnitude == 0:
return (0, math.exp(log_probability))
value = -magnitude if sign else magnitude
if decimals == 0 or (value == value.to_integral_value() and abs(value) < 2**53):
return (int(value), math.exp(log_probability))
return (float(value), math.exp(log_probability))
def number_from_numeral(
magnitude: Decimal,
logits: np.ndarray,
spec: dict[str, Any],
check_exponent: bool = True,
) -> float | int | None:
(sign_logits, exponent_logits, _) = number_parts(_f32(logits))
if check_exponent:
target = number_components(float(magnitude))
if target is None or int(exponent_logits.argmax()) != target[1]:
return None
decimals = zs.allowed_decimals(spec)
if decimals is not None:
magnitude = magnitude.quantize(
Decimal(1).scaleb(-decimals), rounding=ROUND_HALF_UP
)
if magnitude == 0:
return 0
value = -magnitude if int(sign_logits.argmax()) else magnitude
if decimals == 0 or (value == value.to_integral_value() and abs(value) < 2**53):
return int(value)
return float(value)
def number_from_numerals(
values: list[Decimal],
logits: np.ndarray,
spec: dict[str, Any],
readout: float | None,
confident: bool = False,
) -> float | int | None:
agreeing = [
value
for value in (
number_from_numeral(magnitude, logits, spec) for magnitude in values
)
if value is not None
]
if not agreeing and confident and values:
def distance(magnitude: Decimal) -> float:
if readout is None or not readout or (not magnitude):
return 0.0
return abs(math.log10(float(magnitude)) - math.log10(abs(float(readout))))
agreeing = [
value
for value in (
number_from_numeral(min(values, key=distance), logits, spec, False),
)
if value is not None
]
if not agreeing:
return None
if readout is None:
return agreeing[0]
return min(agreeing, key=lambda value: abs(float(value) - float(readout)))
def copied_number(
output: dict[str, Any],
row: int,
record: EncodedRecord,
spec: dict[str, Any],
readout: float | None = None,
) -> tuple[float | int, float, list[int]] | None:
if (
"number_copy_gate" not in output
or record.record.get("images")
or (not record.text)
):
return None
if os.environ.get("SZ_DISABLE_NUMBER_COPY") == "1":
return None
gate = float(_sigmoid(_f32(output["number_copy_gate"][row])))
if gate < 0.5:
return None
if "number_copy_best" in output:
(best, probability) = (
output["number_copy_best"][row],
output["number_copy_probability"][row],
)
else:
scores = _f32(output["number_copy"][row]).flatten()
(best, probability) = (
int(scores.argmax()),
float(_softmax(scores, axis=0).max()),
)
(start, length) = divmod(int(best), MAX_SPAN)
last = min(start + length, len(record.offsets) - 1)
if start >= len(record.offsets):
return None
(char_start, char_end) = (record.offsets[start][0], record.offsets[last][1])
while char_start < char_end and (not record.text[char_start].isalnum()):
char_start += 1
around = [
found
for found in read_numerals(record.text, every=True)
if found[1] < char_end and found[2] > char_start
]
if not around:
return None
nearest = min(
around,
key=lambda found: (
abs(found[1] - char_start) + abs(found[2] - char_end),
-(found[2] - found[1]),
),
)
whole = [
found
for found in read_numerals(record.text)
if found[1] <= char_start
and found[2] >= char_end
and (found[2] - found[1] > nearest[2] - nearest[1])
]
choices = whole[:1] + [nearest]
if gate * probability < COPY_MIN_CONFIDENCE:
return None
sure = gate >= COPY_CONFIDENT
(value, numeral_start, numeral_end) = (None, nearest[1], nearest[2])
for values, start, end in choices:
run = NUMERAL_RUN.match(record.text, start)
printed = record.text[start : min(end, run.end())] if run else ""
settled = (
numeral_convention(record.text) is not None
and len(numeral_values(printed)) > 1
)
enclosing = bool(whole) and (start, end) == (whole[0][1], whole[0][2])
value = number_from_numerals(
values,
output["number"][row],
spec,
readout,
confident=settled or (sure and enclosing and (len(values) == 1)),
)
if value is not None:
(numeral_start, numeral_end) = (start, end)
break
if value is None and sure:
(values, numeral_start, numeral_end) = choices[0]
value = number_from_numerals(
values, output["number"][row], spec, readout, confident=True
)
if value is None or (readout is not None and rescaled(readout, value)):
return None
return (value, gate * probability, [numeral_start, numeral_end])
def rescaled(readout: float, copied: float) -> bool:
try:
(first, second) = (
abs(Decimal(str(readout))).normalize(),
abs(Decimal(str(copied))).normalize(),
)
except ArithmeticError:
return False
if not first or not second or first == second:
return False
return first.as_tuple().digits == second.as_tuple().digits
def decode_date(logits: np.ndarray) -> tuple[str, float]:
(digit_logits, year_logits, month_logits, day_logits) = date_parts(_f32(logits))
class_log = _log_softmax(year_logits, axis=-1)
year_class = int(class_log.argmax())
log_probability = float(class_log[year_class])
if year_class < YEAR_OUTSIDE:
year = YEAR_LOW + year_class
else:
digit_log = _log_softmax(digit_logits, axis=-1)
digits = digit_log.argmax(axis=-1).tolist()
if digits == [0, 0, 0, 0]:
masked = digit_log[3].copy()
masked[0] = MASKED
digits[3] = int(masked.argmax())
year = int("".join((str(digit) for digit in digits)))
log_probability += float(
sum((digit_log[position, digit] for (position, digit) in enumerate(digits)))
)
(month_log, day_log) = (
_log_softmax(month_logits, axis=-1),
_log_softmax(day_logits, axis=-1),
)
joint = month_log[:, None] + day_log[None, :]
lengths = np.array(
[calendar.monthrange(year, month)[1] for month in range(1, 13)],
dtype=np.int64,
)
joint = _masked_fill(joint, np.arange(31)[None, :] >= lengths[:, None], MASKED)
best = int(joint.flatten().argmax())
(month, day) = divmod(best, 31)
log_probability += float(joint.flatten()[best])
return (
datetime.date(year, month + 1, day + 1).isoformat(),
math.exp(log_probability),
)
def decode_time(logits: np.ndarray) -> tuple[str, float]:
parts = [_log_softmax(part, axis=-1) for part in time_parts(_f32(logits))]
values = [int(part.argmax()) for part in parts]
return (
zs.format_time(*values),
math.exp(sum((float(part[value]) for (part, value) in zip(parts, values)))),
)
def decode_spans(
labels: list[int],
offsets: list[tuple[int, int]],
text: str,
trims: list[list[int]] | None = None,
) -> tuple[list[str], list[list[int]]]:
spans: list[list[int]] = []
current: list[int] | None = None
for position, label in enumerate(labels):
if label == TAG_BEGIN:
if current is not None:
spans.append(current)
current = [position, position]
elif label == TAG_INSIDE and current is not None:
current[1] = position
elif current is not None:
spans.append(current)
current = None
if current is not None:
spans.append(current)
values: list[str] = []
char_spans: list[list[int]] = []
for first, last in spans:
(char_start, char_end) = trimmed_span(text, offsets, first, last, trims)
if not text[char_start:char_end].strip():
continue
char_spans.append([char_start, char_end])
if text[char_start:char_end] not in values:
values.append(text[char_start:char_end])
return (values, char_spans)
def quantile_from_cdf(
cdf: list[float], probabilities: list[float], fraction: float, bins: int
) -> float:
index = min(bisect_left(cdf, fraction), bins - 1)
previous = cdf[index - 1] if index > 0 else 0.0
inside = (
(fraction - previous) / probabilities[index]
if probabilities[index] > 0
else 0.5
)
return (index + min(1.0, max(0.0, inside))) / bins
def decode_estimate(
logits: np.ndarray,
offsets: np.ndarray,
overflow: np.ndarray,
spec: dict[str, Any],
) -> dict[str, Any]:
probabilities = _softmax(_f32(logits), axis=-1)
cdf = _cumsum(probabilities, axis=-1).tolist()
row = probabilities.tolist()
bins = len(row)
shift = _sigmoid(_f32(offsets)).tolist()
(low, high, scale) = zs.estimate_range(spec)
centers = [(index + shift[index]) / bins for index in range(bins)]
mean = sum(
(
probability * zs.from_unit(center, low, high, scale)
for (probability, center) in zip(row, centers)
)
)
quantiles: list[Any] = [
zs.from_unit(
quantile_from_cdf(cdf, row, fraction / 100.0, bins), low, high, scale
)
for fraction in range(1, 100)
]
value = (
zs.from_unit(
sum((probability * center for (probability, center) in zip(row, centers))),
low,
high,
scale,
)
if scale == "log"
else mean
)
step = zs.estimate_step(spec)
if step > 0:
quantiles = [
zs.snap_quantile(quantile, step, low, high) for quantile in quantiles
]
value = zs.snap(mean, step, low, high)
if zs.is_date_estimate(spec):
(value, mean) = (zs.estimate_value(spec, value), zs.estimate_value(spec, mean))
quantiles = [zs.estimate_value(spec, quantile) for quantile in quantiles]
prediction = {
"value": value,
"mean": mean,
"median": quantiles[49],
"p10": quantiles[9],
"p90": quantiles[89],
"quantiles": quantiles,
}
if not zs.is_date_estimate(spec):
overflow = np.asarray(overflow)
sides = _softmax(
np.concatenate([np.zeros(1, dtype=overflow.dtype), _f32(overflow)]), axis=0
).tolist()
(prediction["p_below"], prediction["p_above"]) = (sides[BELOW], sides[ABOVE])
return prediction
def estimate_row_log_probs(
output: dict[str, Any],
row: int,
spec: dict[str, Any],
window: tuple[int, int] | None = None,
) -> np.ndarray:
log_probs = _estimate_row_log_probs(output, row, spec)
if window is None:
return log_probs
return window_log_probs(log_probs[None], np.array([window], dtype=np.int64))[0][0]
def _estimate_row_log_probs(
output: dict[str, Any], row: int, spec: dict[str, Any]
) -> np.ndarray:
logits = _f32(output["estimate"][row])
parts = (
estimate_edge_parts(spec, logits.shape[-1])
if "estimate_number" in output
else None
)
if parts is None:
return _log_softmax(logits, axis=-1)
(sign, classes, digits, fractions) = parts
count = len(sign)
return estimate_log_probs(
logits[None],
_f32(output["estimate_number"][row])[None],
_f32(output["estimate_gate"][row]).reshape(1),
np.ones(1, dtype=np.float32),
np.array([sign], dtype=np.int64),
np.array([classes], dtype=np.int64),
np.array(digits, dtype=np.int64).reshape(1, count, LEXICAL_DIGITS),
np.array([fractions], dtype=np.float64),
)[0]
def decode_record(
output: dict[str, Any],
record: EncodedRecord,
tokenizer: Any = None,
end_id: int | None = None,
) -> dict[str, dict[str, Any]]:
fields = record.fields
specs = {item.name: record.record["fields"][item.name] for item in fields}
p_null = _sigmoid(_f32(output["null"])).tolist()
predictions: dict[str, dict[str, Any]] = {}
value: Any
rows_of = {
group: [
position for (position, item) in enumerate(fields) if item.kind in kinds
]
for (group, kinds) in (
("text", ("text",)),
("tags", ("text_list",)),
("number", ("number",)),
("date", ("date", "datetime")),
("time", ("time", "datetime")),
("estimate", ("estimate",)),
("number_list", ("number_list",)),
("date_list", ("date_list",)),
)
}
index_of = {
group: {position: row for (row, position) in enumerate(rows)}
for (group, rows) in rows_of.items()
}
if rows_of["text"] and "text_best" in output:
(best, probabilities) = (output["text_best"], output["text_probability"])
elif rows_of["text"]:
scores = _f32(output["text"])
scores = _flatten1(scores)
best = scores.argmax(axis=1).tolist()
probabilities = _softmax(scores, axis=1).max(axis=1).tolist()
if rows_of["tags"]:
labels = (
output["tags_labels"]
if "tags_labels" in output
else np.asarray(output["tags"]).argmax(axis=-1).tolist()
)
trims = output.get("trim_labels")
if trims is None and "trim" in output:
trims = np.asarray(output["trim"]).argmax(axis=-1).tolist()
for position, item in enumerate(fields):
spec = specs[item.name]
kind = item.kind
if kind == "estimate":
row = index_of["estimate"][position]
predictions[item.name] = decode_estimate(
estimate_row_log_probs(output, row, spec, item.window),
output["estimate_offset"][row],
output["estimate_overflow"][row],
spec,
)
continue
if kind == "text_list":
row = index_of["tags"][position]
(values, spans) = decode_spans(
labels[row], record.offsets, record.text, trims
)
predictions[item.name] = {"value": values, "spans": spans}
if "tags_log_probability" in output:
predictions[item.name]["confidence"] = math.exp(
output["tags_log_probability"][row]
)
elif "tags" in output:
label_log = _log_softmax(_f32(output["tags"][row]), axis=-1)
predictions[item.name]["confidence"] = math.exp(
float(label_log.max(axis=-1).sum())
)
continue
if kind in LIST_KINDS:
row = index_of[kind][position]
ends = _sigmoid(_f32(output[f"{kind}_end"][row])).tolist()
(values, confidences) = ([], [])
log_probability = 0.0
for slot in range(LIST_SLOTS):
if ends[slot] >= 0.5:
log_probability += math.log(max(ends[slot], 1e-12))
break
log_probability += math.log(max(1.0 - ends[slot], 1e-12))
if kind == "number_list":
(value, confidence) = decode_number(
output[kind][row, slot],
spec,
output["number_list_groups"][row, slot]
if "number_list_groups" in output
else None,
)
else:
(value, confidence) = decode_date(output[kind][row, slot])
values.append(value)
confidences.append(confidence)
log_probability += math.log(max(confidence, 1e-12))
predictions[item.name] = {
"value": values,
"confidences": confidences,
"confidence": math.exp(log_probability),
}
continue
prediction: dict[str, Any] = {"p_null": p_null[position]}
if kind == "text":
row = index_of["text"][position]
(start, length) = divmod(best[row], MAX_SPAN)
last = min(start + length, len(record.offsets) - 1)
(char_start, char_end) = trimmed_span(
record.text, record.offsets, start, last, trims
)
(value, confidence) = (record.text[char_start:char_end], probabilities[row])
prediction["span"] = [char_start, char_end]
if not record.text:
(value, prediction["span"]) = (None, None)
elif kind == "number":
number_row = index_of["number"][position]
(value, confidence) = decode_number(
output["number"][number_row],
spec,
output["number_groups"][number_row]
if "number_groups" in output
else None,
)
copied = copied_number(output, number_row, record, spec, value)
if (
copied is not None
and value is not None
and (os.environ.get("SZ_NUMBER_COPY_VETO", "1") == "1")
and numeral_spans(record.text, value, every=False)
):
copied = None
if copied is not None:
(value, confidence, prediction["span"]) = copied
prediction["source"] = "input"
if spec["type"] == "duration":
prediction["unit"] = zs.duration_unit(spec)
elif kind == "date":
(value, confidence) = decode_date(
output["date"][index_of["date"][position]]
)
elif kind == "time":
(value, confidence) = decode_time(
output["time"][index_of["time"][position]]
)
else:
(day, day_confidence) = decode_date(
output["date"][index_of["date"][position]]
)
(clock, clock_confidence) = decode_time(
output["time"][index_of["time"][position]]
)
(value, confidence) = (f"{day}T{clock}", day_confidence * clock_confidence)
prediction["guess"] = value
prediction["confidence"] = confidence
prediction["value"] = (
None if p_null[position] >= zs.null_threshold(spec) else value
)
predictions[item.name] = prediction
if tokenizer is not None and end_id is not None and ("spell_ids" in output):
(ids, log_probs) = (
np.asarray(output["spell_ids"]).tolist(),
_f32(output["spell_log_probs"]).tolist(),
)
for position, item in enumerate(fields):
if item.kind == "text":
row = index_of["text"][position]
elif item.kind == "text_list":
row = len(rows_of["text"]) + index_of["tags"][position]
else:
continue
(spelled, _, parts) = spelled_answer(
ids[row], log_probs[row], tokenizer, end_id
)
predictions[item.name] = route_spelled(
predictions[item.name], item.kind, spelled, parts, record.text
)
if tokenizer is not None and "slot_ids" in output:
slot_end = _tokens(tokenizer, SPELL_END)[0]
(slot_ids, slot_log_probs) = (
np.asarray(output["slot_ids"]).tolist(),
_f32(output["slot_log_probs"]).tolist(),
)
slot_rows = [
position for (position, item) in enumerate(fields) if item.answer_slots
]
for row, position in enumerate(slot_rows):
item = fields[position]
if item.kind not in ("text", "text_list"):
continue
count = len(item.answer_slots)
(spelled, _probability, parts) = spelled_answer(
slot_ids[row][:count], slot_log_probs[row][:count], tokenizer, slot_end
)
prediction = predictions[item.name]
if (record.record.get("_meta") or {}).get(
"construction"
) == zs.EVIDENCE_CONSTRUCTION and item.kind == "text":
predictions[item.name] = evidence_text_answer(
prediction,
slot_ids[row][:count],
slot_log_probs[row][:count],
tokenizer,
slot_end,
record.text,
specs[item.name],
bool(record.record.get("images")),
)
continue
if item.kind in ("text", "text_list"):
predictions[item.name] = route_spelled(
prediction, item.kind, spelled, parts, record.text, defer_lists=True
)
continue
return predictions
FIELD_KINDS = SCALAR_KINDS + ("estimate", "text_list") + LIST_KINDS
VIEW_GROUPS = (
("text", ("text",)),
("tags", ("text_list",)),
("number", ("number",)),
("date", ("date", "datetime")),
("time", ("time", "datetime")),
("estimate", ("estimate",)),
)
COMPACT_KEYS = (
"number",
"date",
"time",
"estimate",
"estimate_offset",
"estimate_overflow",
"number_list_end",
"number_list",
"date_list_end",
"date_list",
"estimate_number",
"estimate_gate",
"number_groups",
"number_list_groups",
"number_copy_gate",
)
COMPACT_GROUPS = {
"number": "number",
"date": "date",
"time": "time",
"estimate": "estimate",
"estimate_offset": "estimate",
"estimate_overflow": "estimate",
"number_list_end": "number_list",
"number_list": "number_list",
"date_list_end": "date_list",
"date_list": "date_list",
"estimate_number": "estimate",
"estimate_gate": "estimate",
"number_groups": "number",
"number_list_groups": "number_list",
"number_copy_gate": "number",
}
def prediction_layout(record: EncodedRecord) -> list[int]:
layout = [
len(record.input_ids),
record.input_start,
len(record.offsets),
len(record.fields),
]
for item in record.fields:
layout += [
FIELD_KINDS.index(item.kind),
item.query,
len(item.items),
*item.items,
]
slotted = [
(index, item.answer_slots)
for (index, item) in enumerate(record.fields)
if item.answer_slots
]
if slotted:
layout.append(len(slotted))
for index, slots in slotted:
layout += [index, len(slots), *slots]
return layout
def parse_prediction_layout(
layout: list[int], num_tokens: int, with_slots: bool = False
) -> tuple[Any, ...]:
if len(layout) < 4 or any((type(value) is not int for value in layout)):
raise ValueError("prediction_layout must be a list of ints")
(prompt_length, input_start, input_length, count) = layout[:4]
shift = num_tokens - prompt_length
if (
shift < 0
or count < 1
or input_length < 1
or (not 0 <= input_start < input_start + input_length <= prompt_length)
):
raise ValueError("prediction_layout does not match the prompt")
kinds: list[str] = []
queries: list[int] = []
items: list[list[int]] = []
cursor = 4
for _ in range(count):
if cursor + 3 > len(layout):
raise ValueError("prediction_layout is truncated")
(kind, query, size) = layout[cursor : cursor + 3]
cursor += 3
if (
not 0 <= kind < len(FIELD_KINDS)
or not input_start + input_length <= query < prompt_length
):
raise ValueError("prediction_layout has an invalid field")
expected = LIST_SLOTS if FIELD_KINDS[kind] in LIST_KINDS else 0
slots = layout[cursor : cursor + size]
cursor += size
if (
size != expected
or len(slots) != size
or any((not query < slot < prompt_length for slot in slots))
):
raise ValueError("prediction_layout has invalid item positions")
kinds.append(FIELD_KINDS[kind])
queries.append(query + shift)
items.append([slot + shift for slot in slots])
slot_fields: list[int] = []
answer_slots: list[list[int]] = []
if cursor < len(layout):
slotted = layout[cursor]
cursor += 1
if not 0 < slotted <= count:
raise ValueError("prediction_layout has invalid answer slots")
for _ in range(slotted):
if cursor + 2 > len(layout):
raise ValueError("prediction_layout is truncated")
(index, size) = layout[cursor : cursor + 2]
cursor += 2
positions = layout[cursor : cursor + size]
cursor += size
if (
not 0 <= index < count
or (slot_fields and index <= slot_fields[-1])
or size < 1
or (len(positions) != size)
or any(
(
not queries[index] - shift < slot < prompt_length
for slot in positions
)
)
):
raise ValueError("prediction_layout has invalid answer slots")
slot_fields.append(index)
answer_slots.append([slot + shift for slot in positions])
if cursor != len(layout):
raise ValueError("prediction_layout has trailing values")
parsed = (input_start + shift, input_length, kinds, queries, items)
return (*parsed, slot_fields, answer_slots) if with_slots else parsed
def prediction_views(
kinds: list[str], queries: list[int], items: list[list[int]], device: Any = None
) -> dict[str, np.ndarray]:
views = {"query_positions": np.array(queries, dtype=np.int64)}
for group, members in VIEW_GROUPS:
views[f"{group}_rows"] = np.array(
[row for (row, kind) in enumerate(kinds) if kind in members],
dtype=np.int64,
)
for kind in LIST_KINDS:
rows = [row for (row, field_kind) in enumerate(kinds) if field_kind == kind]
views[f"{kind}_rows"] = np.array(rows, dtype=np.int64)
views[f"{kind}_items"] = np.array(
[slot for row in rows for slot in items[row]],
dtype=np.int64,
).reshape(len(rows), LIST_SLOTS)
return views
def compact_output(output: dict[str, Any]) -> np.ndarray:
parts = [_f32(output["null"]).flatten()]
if "text" in output:
scores = _f32(output["text"])
scores = _flatten1(scores)
parts += [
scores.argmax(axis=1).astype(np.float32),
_softmax(scores, axis=1).max(axis=1),
]
if "tags" in output:
tag_log = _log_softmax(_f32(output["tags"]), axis=-1)
parts.append(tag_log.argmax(axis=-1).astype(np.float32).flatten())
parts.append(tag_log.max(axis=-1).sum(axis=-1))
if "trim" in output:
parts.append(np.asarray(output["trim"]).argmax(axis=-1).astype(np.float32).flatten())
if "number_copy" in output:
scores = _f32(output["number_copy"])
scores = _flatten1(scores)
parts += [
scores.argmax(axis=1).astype(np.float32),
_softmax(scores, axis=1).max(axis=1),
]
parts += [_f32(output[key]).flatten() for key in COMPACT_KEYS if key in output]
if "slot_ids" in output:
parts += [
_f32(output["slot_ids"]).flatten(),
_f32(output["slot_log_probs"]).flatten(),
]
return np.concatenate(parts).astype(np.float32, copy=False)
def expand_output(
flat: np.ndarray,
record: EncodedRecord,
bins: int = BINS,
estimate_lexical: bool = False,
span_trim: bool = False,
number_groups: bool = False,
number_pointer: bool = False,
) -> dict[str, Any]:
flat = _f32(flat).reshape(-1)
kinds = [item.kind for item in record.fields]
rows = {
group: sum((kind in members for kind in kinds))
for (group, members) in VIEW_GROUPS
}
rows.update({kind: kinds.count(kind) for kind in LIST_KINDS})
cursor = 0
def take(*shape: int) -> np.ndarray:
nonlocal cursor
size = math.prod(shape)
if cursor + size > flat.size:
raise ValueError("compact output is shorter than the layout needs")
value = flat[cursor : cursor + size].reshape(*shape)
cursor += size
return value
output: dict[str, Any] = {"null": take(len(kinds))}
if rows["text"]:
output["text_best"] = take(rows["text"]).round().astype(np.int64).tolist()
output["text_probability"] = take(rows["text"]).tolist()
if rows["tags"]:
output["tags_labels"] = (
take(rows["tags"], len(record.offsets)).round().astype(np.int64).tolist()
)
output["tags_log_probability"] = take(rows["tags"]).tolist()
if span_trim and (rows["text"] or rows["tags"]):
output["trim_labels"] = take(len(record.offsets), 2).round().astype(np.int64).tolist()
if number_pointer and rows["number"]:
output["number_copy_best"] = take(rows["number"]).round().astype(np.int64).tolist()
output["number_copy_probability"] = take(rows["number"]).tolist()
widths = {
"number": (NUMBER_WIDTH,),
"date": (DATE_WIDTH,),
"time": (TIME_WIDTH,),
"estimate": (bins,),
"estimate_offset": (bins,),
"estimate_overflow": (2,),
"number_list_end": (LIST_SLOTS,),
"number_list": (LIST_SLOTS, NUMBER_WIDTH),
"date_list_end": (LIST_SLOTS,),
"date_list": (LIST_SLOTS, DATE_WIDTH),
"estimate_number": (NUMBER_WIDTH,),
"estimate_gate": (),
"number_groups": (NUMBER_GROUPS, GROUP_CLASSES),
"number_list_groups": (LIST_SLOTS, NUMBER_GROUPS, GROUP_CLASSES),
"number_copy_gate": (),
}
for key in COMPACT_KEYS:
if key in ("estimate_number", "estimate_gate") and (not estimate_lexical):
continue
if key in ("number_groups", "number_list_groups") and (not number_groups):
continue
if key == "number_copy_gate" and (not number_pointer):
continue
count = rows[COMPACT_GROUPS[key]]
if count:
output[key] = take(count, *widths[key])
slotted = [len(item.answer_slots) for item in record.fields if item.answer_slots]
if slotted:
output["slot_ids"] = take(len(slotted), max(slotted)).round().astype(np.int64)
output["slot_log_probs"] = take(len(slotted), max(slotted))
if cursor != flat.size:
raise ValueError("compact output is longer than the layout needs")
return output
def merge_forecasts(
predictions: dict[str, dict[str, Any]], groups: dict[str, list[str]]
) -> dict[str, dict[str, Any]]:
merged = {
name: prediction
for (name, prediction) in predictions.items()
if not any((name in names for names in groups.values()))
}
for name, names in groups.items():
steps = [predictions[step] for step in names]
merged[name] = {
key: [step[key] for step in steps]
for key in ("value", "median", "p10", "p90")
}
merged[name]["steps"] = steps
return merged
# Modified from the Cloudflare Clairic release (Apache-2.0).
# Changes: the typed head (ClairicHead) was moved to a separate MLX module, and
# the tensor code in this file was ported from PyTorch to NumPy.
from __future__ import annotations
import calendar
import datetime
import functools
import math
import os
import re
from bisect import bisect_left, bisect_right
from dataclasses import dataclass, field
from decimal import ROUND_HALF_UP, Decimal
from typing import Any, cast
import numpy as np
from . import schema as zs
SYSTEM_PREFIX = "<|im_start|>system\n" + zs.SYSTEM_PREAMBLE + "\nFields:\n"
EVIDENCE_SYSTEM_PREFIX = (
"<|im_start|>system\n" + zs.EVIDENCE_SYSTEM_PREAMBLE + "\nFields:\n"
)
USER_PREFIX = "<|im_end|>\n<|im_start|>user\n"
ASSISTANT_SUFFIX = "<|im_end|>\n<|im_start|>assistant\n<think>\n\n</think>\n"
ANSWER_CUE = "\nAnswer:"
class PromptLayout:
schema_header: str
query_header: str
answer_cue: str
slot_header: str
PROMPT_LAYOUTS = (
PromptLayout("", "", ANSWER_CUE, ""),
PromptLayout(
"Field definition:\n", "\nField query:", "\nValue:", "\nAnswer field:"
),
PromptLayout(
"Requested field:\n", "\nRequested field:", "\nResult:", "\nResult for field:"
),
)
PROMPT_VARIANTS = (0, 1, 2)
def prompt_layout(
template_variant: int = 0, complete_field: bool = True
) -> PromptLayout:
if (
not isinstance(template_variant, int)
or isinstance(template_variant, bool)
or template_variant not in PROMPT_VARIANTS
):
raise ValueError("template_variant must be 0, 1, or 2")
if template_variant and (not complete_field):
raise ValueError("alternative layouts require the evidence construction")
return PROMPT_LAYOUTS[template_variant]
QUERY_TOKENS = 256
MAX_SPAN = 64
TRIM_CLASSES = 8
TRIM_INIT = 2.0
MIN_INPUT_TOKENS = 16
BINS = 256
POINTER_WIDTH = 256
NUMBER_DIGITS = 12
(MIN_EXPONENT, MAX_EXPONENT) = (-12, 15)
EXPONENT_CLASSES = MAX_EXPONENT - MIN_EXPONENT + 2
YEAR_DIGITS = 4
(YEAR_LOW, YEAR_HIGH) = (1800, 2199)
YEAR_OUTSIDE = YEAR_HIGH - YEAR_LOW + 1
YEAR_CLASSES = YEAR_OUTSIDE + 1
LIST_SLOTS = zs.MAX_LIST_ITEMS
ITEM_CUES = tuple((f"\n{index}." for index in range(1, LIST_SLOTS + 1)))
(INSIDE, BELOW, ABOVE) = (0, 1, 2)
OVERFLOW_INIT = -5.0
QUERY_ORDER_RATE = 0.8
LOOP_RANK = 256
LEXICAL_DIGITS = 4
ESTIMATE_GATE_INIT = 0.0
ESTIMATE_SPREAD_LIMIT = 2.0
(EDGE_NEGATIVE, EDGE_ZERO, EDGE_POSITIVE) = (0, 1, 2)
(TAG_OUTSIDE, TAG_BEGIN, TAG_INSIDE) = (0, 1, 2)
NUMBER_WIDTH = 2 + EXPONENT_CLASSES + NUMBER_DIGITS * 10
GROUP_DIGITS = 3
GROUP_CLASSES = 10**GROUP_DIGITS
(GROUP_LOW, GROUP_HIGH) = (MIN_EXPONENT // GROUP_DIGITS, MAX_EXPONENT // GROUP_DIGITS)
NUMBER_GROUPS = GROUP_HIGH - GROUP_LOW + 1
GROUP_RANK = 256
COPY_CATEGORIES: tuple[str | None, ...] | None = ("extraction", None)
COPY_GATE_INIT = -3.0
NUMERAL_RUN = re.compile(
"(?:(?<![\\d.])\\.(?=\\d))?\\d+(?:[,.'\\u00a0\\u202f\\u2009\\u2007 \\u066b\\u066c]\\d+)*"
)
NUMERAL_SEPARATORS = ",.'\xa0\u202f\u2009\u2007 ٫٬"
NUMERAL_READINGS = (
(re.compile("\\d{1,3}(?:,\\d{3})+(?:\\.\\d+)?|\\d+(?:\\.\\d+)?|\\.\\d+"), ",", "."),
(