# Modified from the Cloudflare Clairic release (Apache-2.0). # Changes: the typed head (ClairicHead) was moved to a separate MLX module, and # the tensor code in this file was ported from PyTorch to NumPy. from __future__ import annotations import calendar import datetime import functools import math import os import re from bisect import bisect_left, bisect_right from dataclasses import dataclass, field from decimal import ROUND_HALF_UP, Decimal from typing import Any, cast import numpy as np from . import schema as zs SYSTEM_PREFIX = "<|im_start|>system\n" + zs.SYSTEM_PREAMBLE + "\nFields:\n" EVIDENCE_SYSTEM_PREFIX = ( "<|im_start|>system\n" + zs.EVIDENCE_SYSTEM_PREAMBLE + "\nFields:\n" ) USER_PREFIX = "<|im_end|>\n<|im_start|>user\n" ASSISTANT_SUFFIX = "<|im_end|>\n<|im_start|>assistant\n\n\n\n" ANSWER_CUE = "\nAnswer:" @dataclass(frozen=True) class PromptLayout: schema_header: str query_header: str answer_cue: str slot_header: str PROMPT_LAYOUTS = ( PromptLayout("", "", ANSWER_CUE, ""), PromptLayout( "Field definition:\n", "\nField query:", "\nValue:", "\nAnswer field:" ), PromptLayout( "Requested field:\n", "\nRequested field:", "\nResult:", "\nResult for field:" ), ) PROMPT_VARIANTS = (0, 1, 2) def prompt_layout( template_variant: int = 0, complete_field: bool = True ) -> PromptLayout: if ( not isinstance(template_variant, int) or isinstance(template_variant, bool) or template_variant not in PROMPT_VARIANTS ): raise ValueError("template_variant must be 0, 1, or 2") if template_variant and (not complete_field): raise ValueError("alternative layouts require the evidence construction") return PROMPT_LAYOUTS[template_variant] QUERY_TOKENS = 256 MAX_SPAN = 64 TRIM_CLASSES = 8 TRIM_INIT = 2.0 MIN_INPUT_TOKENS = 16 BINS = 256 POINTER_WIDTH = 256 NUMBER_DIGITS = 12 (MIN_EXPONENT, MAX_EXPONENT) = (-12, 15) EXPONENT_CLASSES = MAX_EXPONENT - MIN_EXPONENT + 2 YEAR_DIGITS = 4 (YEAR_LOW, YEAR_HIGH) = (1800, 2199) YEAR_OUTSIDE = YEAR_HIGH - YEAR_LOW + 1 YEAR_CLASSES = YEAR_OUTSIDE + 1 LIST_SLOTS = zs.MAX_LIST_ITEMS ITEM_CUES = tuple((f"\n{index}." for index in range(1, LIST_SLOTS + 1))) (INSIDE, BELOW, ABOVE) = (0, 1, 2) OVERFLOW_INIT = -5.0 QUERY_ORDER_RATE = 0.8 LOOP_RANK = 256 LEXICAL_DIGITS = 4 ESTIMATE_GATE_INIT = 0.0 ESTIMATE_SPREAD_LIMIT = 2.0 (EDGE_NEGATIVE, EDGE_ZERO, EDGE_POSITIVE) = (0, 1, 2) (TAG_OUTSIDE, TAG_BEGIN, TAG_INSIDE) = (0, 1, 2) NUMBER_WIDTH = 2 + EXPONENT_CLASSES + NUMBER_DIGITS * 10 GROUP_DIGITS = 3 GROUP_CLASSES = 10**GROUP_DIGITS (GROUP_LOW, GROUP_HIGH) = (MIN_EXPONENT // GROUP_DIGITS, MAX_EXPONENT // GROUP_DIGITS) NUMBER_GROUPS = GROUP_HIGH - GROUP_LOW + 1 GROUP_RANK = 256 COPY_CATEGORIES: tuple[str | None, ...] | None = ("extraction", None) COPY_GATE_INIT = -3.0 NUMERAL_RUN = re.compile( "(?:(? 2 or word in ("mn", "bn", "tn", "cr", "Cr", "Mn", "Bn", "MM") ), key=len, reverse=True, ) ) + ")|(k|K|M|B))(?![^\\W\\d_])" ) NUMERAL_SPAN = 48 CONVENTION_LANGUAGE = re.compile( "\\b(?:Rechnung|Gesamtbetrag|Betrag|Lieferschein|Factuur|Bedrag|Totaal|Pakbon|Fattura|Importo|Totale|Fatura|Nota fiscal|Factura|Importe|Albar\\u00e1n|Facture|Montant)\\b" ) CONVENTION_LANGUAGE_US = re.compile("\\b(?:M\\u00e9xico|Mexico|MXN|CFDI|RFC)\\b") CONVENTION_STATED = { "eu": re.compile( "european (?:number )?(?:notation|format)|(?:dot|period|point)s? (?:separates?|for|marks?) (?:the )?thousands|comma (?:separates?|for|marks?|as) (?:the )?decimal|comma decimals?|decimal comma|dot thousands|1\\.234,56|1 234,56", re.IGNORECASE, ), "us": re.compile( "comma (?:separates?|for|marks?) (?:the )?thousands|(?:dot|period|point) (?:separates?|for|marks?|as) (?:the )?decimal|comma thousands|dot decimals?|1,234\\.56", re.IGNORECASE, ), } COPY_CONFIDENT = 0.9 COPY_MIN_CONFIDENCE = 0.5 SLOT_TEXT = "_" SLOT_COUNTS = {"text": 80, "text_list": 64, "number": 32} SLOT_CUE_TOKENS = 32 DATE_WIDTH = YEAR_DIGITS * 10 + YEAR_CLASSES + 12 + 31 TIME_WIDTH = 24 + 60 + 60 MASKED = -1000000000.0 SCALAR_KINDS = ("text", "number", "date", "time", "datetime") IMAGE_PLACEHOLDER = "<|vision_start|><|image_pad|><|vision_end|>" MEDIA_BATCH_KEYS = ("pixel_values", "image_grid_thw") MEDIA_TOKEN_KEYS = ("mm_token_type_ids",) LIST_KINDS = ("number_list", "date_list") def head_kind(spec: dict[str, Any]) -> str: kind = spec["type"] if kind == "duration": return "number" if kind == "list": return f"{spec['items']}_list" return kind def number_components(value: float) -> tuple[int, int, list[int]] | None: value = float(value) if not math.isfinite(value): return None if value == 0: return (0, 0, [0] * NUMBER_DIGITS) (mantissa, exponent) = f"{abs(value):.{NUMBER_DIGITS - 1}e}".split("e") power = int(exponent) if not MIN_EXPONENT <= power <= MAX_EXPONENT: return None return ( int(value < 0), power - MIN_EXPONENT + 1, [int(digit) for digit in mantissa.replace(".", "")], ) def edge_number(value: float) -> tuple[int, int, list[int], float]: if value == 0 or not math.isfinite(value): return (EDGE_ZERO, 1, [0] * LEXICAL_DIGITS, 0.0) (mantissa, exponent) = f"{abs(value):.{LEXICAL_DIGITS + 8}e}".split("e") (power, digits) = (int(exponent), mantissa.replace(".", "")) sign = EDGE_NEGATIVE if value < 0 else EDGE_POSITIVE if power < MIN_EXPONENT: return (sign, 1, [0] * LEXICAL_DIGITS, 0.0) if power > MAX_EXPONENT: return (sign, EXPONENT_CLASSES - 1, [9] * LEXICAL_DIGITS, 1.0) return ( sign, power - MIN_EXPONENT + 1, [int(digit) for digit in digits[:LEXICAL_DIGITS]], float("0." + digits[LEXICAL_DIGITS:]), ) @functools.lru_cache(maxsize=8192) def _edge_parts( low: float, high: float, scale: str, bins: int ) -> tuple[tuple[int, ...], tuple[int, ...], tuple[int, ...], tuple[float, ...]]: parts = [ edge_number(zs.from_unit(index / bins, low, high, scale)) for index in range(bins + 1) ] return ( tuple((part[0] for part in parts)), tuple((part[1] for part in parts)), tuple((digit for part in parts for digit in part[2])), tuple((part[3] for part in parts)), ) def lexical_estimate(spec: dict[str, Any]) -> bool: return not zs.is_date_estimate(spec) or zs.date_granularity(spec) == "year" def estimate_edge_parts( spec: dict[str, Any], bins: int = BINS ) -> tuple[tuple[int, ...], tuple[int, ...], tuple[int, ...], tuple[float, ...]] | None: if not lexical_estimate(spec): return None (low, high, scale) = zs.estimate_range(spec) return _edge_parts(float(low), float(high), scale, bins) @dataclass class EncodedField: name: str kind: str query: int items: list[int] = field(default_factory=list) prefix: tuple[int, int, tuple[int, ...]] | None = None window: tuple[int, int] | None = None answer_slots: list[int] = field(default_factory=list) @dataclass(frozen=True) class Commit: text: str done: bool = False prefix: tuple[int, int, tuple[int, ...]] | None = None probability: float = 1.0 @dataclass class EncodedRecord: input_ids: list[int] input_start: int offsets: list[tuple[int, int]] fields: list[EncodedField] text: str record: dict[str, Any] media: dict[str, Any] | None = None image_expansion: int = 0 @dataclass class EncodedMedia: ids: list[int] media: dict[str, Any] | None = None expansion: int = 0 @property def length(self) -> int: return len(self.ids) + self.expansion def configure_image_processor(processor: Any, max_pixels: int) -> dict[str, Any]: image_processor = getattr(processor, "image_processor", None) if image_processor is None or max_pixels <= 0: raise ValueError("an image processor and positive pixel budget are required") legacy = hasattr(image_processor, "max_pixels") size = getattr(image_processor, "size", {}) sized = "longest_edge" in size if not legacy and (not sized): raise ValueError( f"unsupported image pixel configuration: {type(image_processor).__name__}" ) if legacy: image_processor.max_pixels = max_pixels if sized: size["longest_edge"] = max_pixels actual = size["longest_edge"] if sized else image_processor.max_pixels if actual != max_pixels or (legacy and image_processor.max_pixels != max_pixels): raise ValueError( "the image processor did not retain its requested pixel budget" ) return {"class": type(image_processor).__name__, "max_pixels": actual} def encode_media( tokenizer: Any, images: list[Any], processor: Any = None, image_token_counts: list[int] | None = None, ) -> EncodedMedia: if not images: return EncodedMedia([]) text = IMAGE_PLACEHOLDER * len(images) + "\n" if image_token_counts is not None: if len(image_token_counts) != len(images) or any( (count < 1 for count in image_token_counts) ): raise ValueError("every image needs a positive token count") return EncodedMedia( _tokens(tokenizer, text), None, sum((count - 1 for count in image_token_counts)), ) if processor is None: raise ValueError("records with images need the model processor") encoded = processor(text=[text], images=list(images), return_tensors="pt") media = {key: encoded[key] for key in MEDIA_BATCH_KEYS if key in encoded} for key in MEDIA_TOKEN_KEYS: if key in encoded: media[key] = encoded[key][0].tolist() return EncodedMedia(encoded["input_ids"][0].tolist(), media) def _tokens(tokenizer: Any, text: str) -> list[int]: return tokenizer(text, add_special_tokens=False).input_ids def _input_tokens(tokenizer: Any, text: str) -> tuple[list[int], list[tuple[int, int]]]: try: encoding = tokenizer( text, add_special_tokens=False, return_offsets_mapping=True, split_special_tokens=True, ) except TypeError: encoding = tokenizer( text, add_special_tokens=False, return_offsets_mapping=True ) return (list(encoding.input_ids), [tuple(pair) for pair in encoding.offset_mapping]) def schema_line(name: str, spec: dict[str, Any], complete_field: bool = False) -> str: return "- " + zs.field_line(name, spec, preserve_instructions=complete_field) + "\n" def reference_line(record: dict[str, Any]) -> str: reference = record.get("reference_date") return f"Reference date: {reference}\n" if reference else "" def query_block( tokenizer: Any, name: str, spec: dict[str, Any], commit: Commit | None = None, complete_field: bool = False, template_variant: int = 0, ) -> tuple[list[int], int, list[int]]: layout = prompt_layout(template_variant, complete_field) line = _tokens( tokenizer, "\n" + zs.field_line(name, spec, preserve_instructions=complete_field), ) ids = ( _tokens(tokenizer, layout.query_header) + (line if complete_field else line[:QUERY_TOKENS]) + _tokens(tokenizer, layout.answer_cue) ) if commit is not None: ids.extend(_tokens(tokenizer, " " + commit.text)) query = len(ids) - 1 items: list[int] = [] if head_kind(spec) in LIST_KINDS and commit is None: for cue in ITEM_CUES: ids.extend(_tokens(tokenizer, cue)) items.append(len(ids) - 1) return (ids, query, items) def clean_cut( text: str, offsets: list[tuple[int, int]], budget: int, lookback: int = 32 ) -> int: lowest = max(0, budget - lookback) for keep in range(budget, lowest, -1): end = offsets[keep - 1][1] if end >= len(text) or ( end > 0 and text[end].isspace() and (not text[end - 1].isspace()) ): return keep for keep in range(budget, lowest, -1): end = offsets[keep - 1][1] if end > 0 and (text[end].isspace() or text[end - 1].isspace()): return keep return budget def char_span_to_tokens( starts: list[int], ends: list[int], char_start: int, char_end: int ) -> tuple[int, int] | None: first = bisect_right(ends, char_start) last = bisect_left(starts, char_end) - 1 if first >= len(starts) or last < first: return None return (first, last) def numeral_values(printed: str, convention: str | None = None) -> list[Decimal]: values: list[Decimal] = [] both = convention is not None and all( (form.fullmatch(printed) for (form, _, _) in NUMERAL_READINGS[:2]) ) for index, (form, separators, mark) in enumerate(NUMERAL_READINGS): if both and index == (1 if convention == "us" else 0): continue if form.fullmatch(printed): plain = printed.translate( {ord(character): None for character in separators} ).replace(mark, ".") try: value = Decimal(plain) except ArithmeticError: continue if value not in values: values.append(value) return values @functools.lru_cache(maxsize=256) def numeral_convention(text: str) -> str | None: stated = { name for (name, pattern) in CONVENTION_STATED.items() if pattern.search(text or "") } if len(stated) == 1: return stated.pop() shown: list[str] = [] for run in NUMERAL_RUN.finditer(text or ""): printed = run.group() if not any((mark in printed for mark in ",.")): continue (us, eu) = ( bool(form.fullmatch(printed)) for (form, _, _) in NUMERAL_READINGS[:2] ) if us != eu: shown.append("us" if us else "eu") if len(shown) >= 2 and len(set(shown)) == 1: return shown[0] if ( not shown and CONVENTION_LANGUAGE.search(text or "") and (not CONVENTION_LANGUAGE_US.search(text or "")) ): return "eu" return None @functools.lru_cache(maxsize=256) def _numeral_candidates(text: str) -> tuple[tuple[list[Decimal], int, int, bool], ...]: found: list[tuple[list[Decimal], int, int, bool]] = [] for run in NUMERAL_RUN.finditer(text or ""): (begin, finish) = (run.start(), run.end()) cuts = [ index for index in range(begin, finish) if text[index] in NUMERAL_SEPARATORS and index > begin ] starts = [begin] + [index + 1 for index in cuts] ends = cuts + [finish] convention = numeral_convention(text) scale = NUMERAL_SCALE.match(text, finish) exponent = ( None if scale is None else NUMERAL_SCALES[scale.group(1) or scale.group(2)] ) run_found = [] for start in starts: for end in ends[bisect_left(ends, start + 1) :]: if end - start > NUMERAL_SPAN: break values = numeral_values(text[start:end], convention) if not values: continue if end == finish and exponent is not None: scaled = [value.scaleb(exponent) for value in values] run_found.append( ( scaled + [value for value in values if value not in scaled], start, cast(re.Match[str], scale).end(), ) ) else: run_found.append((values, start, end)) by_start: dict[int, tuple[list[Decimal], int, int]] = {} for item in run_found: if item[1] not in by_start or item[2] > by_start[item[1]][2]: by_start[item[1]] = item (position, whole) = (begin, set()) while True: longest = by_start.get(position) following = bisect_left( starts, position + 1 if longest is None else longest[2] ) if longest is not None: whole.add((longest[1], longest[2])) if following >= len(starts): break position = starts[following] found.extend( ( (values, start, end, (start, end) in whole) for (values, start, end) in run_found ) ) return tuple(found) def read_numerals( text: str, every: bool = False ) -> list[tuple[list[Decimal], int, int]]: found = [ (values, start, end) for (values, start, end, whole) in _numeral_candidates(text) if every or whole ] return sorted(found, key=lambda item: (item[1], -item[2])) def numeral_spans(text: str, value: Any, every: bool = True) -> list[tuple[int, int]]: try: target = abs(Decimal(str(value))) except ArithmeticError: return [] candidates = _numeral_candidates(text) whole = [ (start, end) for (values, start, end, longest) in candidates if longest and target in values ] if whole or not every: return sorted(whole) return sorted( {(start, end) for (values, start, end, _) in candidates if target in values}, key=lambda item: (item[0], -item[1]), ) def slot_token_id(tokenizer: Any) -> int: ids = _tokens(tokenizer, SLOT_TEXT) if len(ids) != 1: raise ValueError(f"answer slots need {SLOT_TEXT!r} to be one token") return ids[0] def slot_block( tokenizer: Any, name: str, spec: dict[str, Any], complete_field: bool = False, template_variant: int = 0, ) -> list[int]: count = SLOT_COUNTS.get(head_kind(spec)) layout = prompt_layout(template_variant, complete_field) if count is None: return [] cue = _tokens(tokenizer, "\n" + name + ":") return ( _tokens(tokenizer, layout.slot_header) + (cue if complete_field else cue[:SLOT_CUE_TOKENS]) + [slot_token_id(tokenizer)] * count ) def system_prefix(record: dict[str, Any], text_answer_slots: bool = False) -> str: return ( EVIDENCE_SYSTEM_PREFIX if evidence_construction(record, text_answer_slots) else SYSTEM_PREFIX ) def evidence_construction( record: dict[str, Any], text_answer_slots: bool = False ) -> bool: return ( text_answer_slots or (record.get("_meta") or {}).get("construction") == zs.EVIDENCE_CONSTRUCTION ) def encode_record( tokenizer: Any, record: dict[str, Any], max_length: int = 8192, media: EncodedMedia | None = None, commits: dict[str, Commit] | None = None, answer_slots: bool = False, text_answer_slots: bool = False, template_variant: int = 0, ) -> EncodedRecord | None: if text_answer_slots and (not answer_slots): raise ValueError("text_answer_slots needs answer_slots") commits = commits or {} if record.get("images") and media is None: raise ValueError("records with images need their encoded media") media = media or EncodedMedia([]) items = list(record["fields"].items()) complete_field = evidence_construction(record, text_answer_slots) layout = prompt_layout(template_variant, complete_field) ids = _tokens(tokenizer, system_prefix(record, text_answer_slots)) for name, spec in items: ids.extend(_tokens(tokenizer, layout.schema_header)) ids.extend(_tokens(tokenizer, schema_line(name, spec, complete_field))) ids.extend(_tokens(tokenizer, reference_line(record))) ids.extend(_tokens(tokenizer, USER_PREFIX)) media_start = len(ids) ids.extend(media.ids) input_start = len(ids) suffix = _tokens(tokenizer, ASSISTANT_SUFFIX) blocks = [ query_block( tokenizer, name, spec, commits.get(name), complete_field, template_variant ) for (name, spec) in items ] slot_blocks: dict[int, list[int]] = {} if answer_slots and (media.ids or text_answer_slots): for index, (name, spec) in enumerate(items): if not (name in commits and commits[name].done): block = slot_block( tokenizer, name, spec, complete_field, template_variant ) if block: slot_blocks[index] = block budget = ( max_length - len(ids) - media.expansion - len(suffix) - sum((len(block[0]) for block in blocks)) - sum((len(block) for block in slot_blocks.values())) ) if budget < MIN_INPUT_TOKENS: return None text = record["input"] if media.ids and (not text.strip()): text = "" (input_ids, offsets) = (_tokens(tokenizer, "\n"), [(0, 0)]) else: (input_ids, offsets) = _input_tokens(tokenizer, text) if len(input_ids) > budget: keep = clean_cut(text, offsets, budget) text = text[: offsets[keep - 1][1]] (input_ids, offsets) = _input_tokens(tokenizer, text) if len(input_ids) > budget: (input_ids, offsets) = (input_ids[:budget], offsets[:budget]) text = text[: offsets[-1][1]] if not input_ids: return None ids.extend(input_ids) ids.extend(suffix) order = list(range(len(items))) done = { index for (index, (name, _)) in enumerate(items) if name in commits and commits[name].done } if done: order = [index for index in order if index in done] + [ index for index in order if index not in done ] queries: dict[int, int] = {} slots: dict[int, list[int]] = {} for index in order: (block, query, item_offsets) = blocks[index] start = len(ids) ids.extend(block) queries[index] = start + query slots[index] = [start + offset for offset in item_offsets] answer_positions: dict[int, list[int]] = {} for index in order: if index in slot_blocks: ids.extend(slot_blocks[index]) count = SLOT_COUNTS[head_kind(items[index][1])] answer_positions[index] = list(range(len(ids) - count, len(ids))) fields: list[EncodedField] = [] for index, (name, spec) in enumerate(items): if index in done: continue encoded = EncodedField(name, head_kind(spec), queries[index], slots[index]) if index in answer_positions: encoded.answer_slots = answer_positions[index] commit = commits.get(name) if ( commit is not None and commit.prefix is not None and (encoded.kind in ("number", "estimate")) ): encoded.prefix = commit.prefix if encoded.kind == "estimate": encoded.window = prefix_window(spec, commit.prefix) fields.append(encoded) if not fields: return None record_media = None if media.media is not None: record_media = {**media.media, "token_offset": media_start} return EncodedRecord( ids, input_start, offsets, fields, text, record, record_media, media.expansion ) def span_base(text: str, starts: list[int], ends: list[int], first: int) -> int: base = starts[first] while base < ends[first] - 1 and text[base].isspace(): base += 1 return base def trimmed_span( text: str, offsets: list[tuple[int, int]], first: int, last: int, trims: list[list[int]] | None, ) -> tuple[int, int]: (char_start, char_end) = (offsets[first][0], offsets[last][1]) while char_start < offsets[first][1] - 1 and text[char_start].isspace(): char_start += 1 if trims is not None: start = max( char_start, min(char_start + int(trims[first][0]), offsets[first][1] - 1) ) end = min(char_end, max(char_end - int(trims[last][1]), offsets[last][0] + 1)) if start < end: (char_start, char_end) = (start, end) while char_start < char_end - 1 and text[char_start].isspace(): char_start += 1 return (char_start, char_end) def plain_number(value: Any) -> str: number = Decimal(str(value)) if not number.is_finite(): raise ValueError("A plain number must be finite.") text = format(number, "f") return text.rstrip("0").rstrip(".") if "." in text else text def prefix_window( spec: dict[str, Any], prefix: tuple[int, int, tuple[int, ...]], bins: int = BINS ) -> tuple[int, int] | None: (sign, exponent_class, digits) = prefix power = exponent_class - 1 + MIN_EXPONENT unit = Decimal(1).scaleb(power - len(digits) + 1) start = Decimal(int("".join((str(digit) for digit in digits)))) * unit (first, last) = (float(start), float(start + unit)) if sign: (first, last) = (-last, -first) (low, high, scale) = zs.estimate_range(spec) (lower, upper) = sorted( (zs.raw_unit(first, low, high, scale), zs.raw_unit(last, low, high, scale)) ) if upper <= 0.0 or lower >= 1.0: return None first_bin = max(0, min(bins - 1, math.floor(lower * bins))) last_bin = max(first_bin, min(bins - 1, math.ceil(upper * bins) - 1)) return (first_bin, last_bin) def _masked_fill(values: np.ndarray, mask: Any, fill: float) -> np.ndarray: filled = np.array(values, copy=True) filled[np.broadcast_to(np.asarray(mask, dtype=bool), filled.shape)] = fill return filled def _log_softmax(values: Any, axis: int = -1) -> np.ndarray: values = np.asarray(values) shifted = values - values.max(axis=axis, keepdims=True) return shifted - np.log(np.exp(shifted).sum(axis=axis, keepdims=True)) def _softmax(values: Any, axis: int = -1) -> np.ndarray: values = np.asarray(values) exps = np.exp(values - values.max(axis=axis, keepdims=True)) return exps * (1 / exps.sum(axis=axis, keepdims=True)) def _logsumexp(values: Any, axis: int = -1) -> np.ndarray: values = np.asarray(values) maxes = values.max(axis=axis, keepdims=True) maxes = np.where(np.abs(maxes) == np.inf, np.zeros_like(maxes), maxes) summed = np.exp(values - maxes).sum(axis=axis) return np.log(summed) + np.squeeze(maxes, axis=axis) def _sigmoid(values: Any) -> np.ndarray: values = np.asarray(values) with np.errstate(over="ignore"): return 1 / (1 + np.exp(-values)) def _log_sigmoid(values: Any) -> np.ndarray: values = np.asarray(values) return np.minimum(values, 0) - np.log1p(np.exp(-np.abs(values))) def _cumsum(values: Any, axis: int = -1) -> np.ndarray: # torch's CPU cumsum accumulates float32 in double precision. values = np.asarray(values) if values.dtype == np.float32: return np.cumsum(values.astype(np.float64), axis=axis).astype(np.float32) return np.cumsum(values, axis=axis) def _f32(values: Any) -> np.ndarray: return np.asarray(values, dtype=np.float32) def _flatten1(values: np.ndarray) -> np.ndarray: return values.reshape(values.shape[0], math.prod(values.shape[1:])) def span_scores( start: np.ndarray, end: np.ndarray, length_bias: np.ndarray ) -> np.ndarray: (start, end, length_bias) = ( np.asarray(start), np.asarray(end), np.asarray(length_bias), ) width = length_bias.shape[0] pad = [(0, 0)] * (end.ndim - 1) + [(0, width - 1)] padded = np.pad(end, pad, constant_values=float("-inf")) windows = np.lib.stride_tricks.sliding_window_view(padded, width, axis=-1) return start[..., None] + windows + length_bias DIGIT_SLOTS = NUMBER_DIGITS + YEAR_DIGITS + 2 + 2 + 2 + 2 (YEAR_SLOT, DAY_SLOT, HOUR_SLOT, MINUTE_SLOT, SECOND_SLOT) = ( NUMBER_DIGITS, NUMBER_DIGITS + YEAR_DIGITS, NUMBER_DIGITS + YEAR_DIGITS + 2, NUMBER_DIGITS + YEAR_DIGITS + 4, NUMBER_DIGITS + YEAR_DIGITS + 6, ) MONTH_WORDS = tuple((" " + name for name in calendar.month_name[1:])) SIGN_WORDS = (" ", " -") ROUTE_KINDS = ( "text", "tags", "number", "date", "time", "estimate", "number_list", "date_list", ) def two_digit_index(values: range) -> np.ndarray: return np.array( [[value // 10 for value in values], [value % 10 for value in values]], dtype=np.int64, ) def compose_two_digits(slots: np.ndarray, index: np.ndarray) -> np.ndarray: slots = np.asarray(slots) return np.take(slots[..., 0, :], index[0], axis=-1) + np.take( slots[..., 1, :], index[1], axis=-1 ) def compose_years(slots: np.ndarray, index: np.ndarray) -> np.ndarray: slots = np.asarray(slots) inside = cast( np.ndarray, sum( ( np.take(slots[..., position, :], index[position], axis=-1) for position in range(YEAR_DIGITS) ) ), ) total = _logsumexp(slots, axis=-1).sum(axis=-1) gap = np.minimum(_logsumexp(inside, axis=-1) - total, -1e-06) outside = total + np.log(-np.expm1(gap)) return np.concatenate([inside, outside[..., None]], axis=-1) SPELL_END = "\n" SPELL_SEPARATOR = " | " SPELL_MIN_CONFIDENCE = 0.3 def spelled_answer( ids: list[int], log_probs: list[float], tokenizer: Any, end_id: int ) -> tuple[str, float, list[tuple[str, float]]]: stop = ids.index(end_id) if end_id in ids else len(ids) pieces = [tokenizer.decode([token]) for token in ids[:stop]] items: list[tuple[list[int], list[float]]] = [] current: list[int] = [] scores: list[float] = [] for token, piece, score in zip(ids[:stop], pieces, log_probs[:stop]): if piece.strip() == SPELL_SEPARATOR.strip(): items.append((current, scores)) (current, scores) = ([], []) else: current.append(token) scores.append(score) items.append((current, scores)) parts = [] for item_ids, item_scores in items: words = tokenizer.decode(item_ids).split() if words: parts.append( (" ".join(words), math.exp(sum(item_scores) / len(item_scores))) ) text = SPELL_SEPARATOR.join((part for (part, _) in parts)) return (text, math.exp(sum(log_probs[: min(stop + 1, len(ids))])), parts) def evidence_text_answer( prediction: dict[str, Any], ids: list[int], log_probs: list[float], tokenizer: Any, end_id: int, text: str, spec: dict[str, Any], has_images: bool = False, ) -> dict[str, Any]: if end_id not in ids: return prediction stop = ids.index(end_id) value = tokenizer.decode(ids[:stop]).strip() confidence = math.exp(sum(log_probs[: stop + 1])) exact = ( [ span for span in zs.find_spans(text, value) if text[span[0] : span[1]] == value ] if value else [] ) present = prediction["p_null"] < zs.null_threshold(spec) return { **prediction, "guess": value or None, "value": value if value and present else None, "span": list(exact[0]) if exact else None, "confidence": confidence, "source": "input" if exact else "image" if has_images else "readout", } def _simple(value: Any) -> str: return "".join( (character for character in str(value).lower() if character.isalnum()) ) def _similarity(left: Any, right: Any) -> float: import difflib (left, right) = (_simple(left), _simple(right)) return difflib.SequenceMatcher(None, left, right).ratio() if left and right else 0.0 def _state_match( text: str, value: str, threshold: float = 0.8, max_words: int = 4000 ) -> tuple[int, int] | None: if not text or not value: return None spans = zs.find_spans(text, value) if spans: return spans[0] import re words = [match.span() for match in re.finditer("\\S+", text)][:max_words] size = len(value.split()) (best, best_span) = (threshold, None) for width in range(max(1, size - 1), size + 2): for first in range(max(0, len(words) - width + 1)): (start, end) = (words[first][0], words[first + width - 1][1]) score = _similarity(text[start:end].strip(".,;:()[]\"'"), value) if score >= best: (best, best_span) = (score, (start, end)) return best_span def _stutters(value: Any) -> bool: import re simple = _simple(value) return bool(re.search("(.)\\1\\1", simple) or re.search("([a-z]{4,})\\1", simple)) def _digit_code(value: Any) -> bool: simple = _simple(value) return bool(simple) and 2 * sum( (character.isdigit() for character in simple) ) >= len(simple) PLACEHOLDER_WORDS = frozenset( ( "a", "above", "an", "app", "attach", "attached", "attachment", "below", "doc", "document", "documents", "file", "heic", "here", "image", "images", "is", "jpeg", "jpg", "mobile", "page", "pdf", "photo", "photos", "pic", "picture", "please", "png", "scan", "scanned", "screenshot", "see", "the", "this", "upload", "uploaded", "via", ) ) def _placeholder_text(value: Any) -> bool: import re words = re.findall("[a-z0-9]+", str(value or "").lower()) return bool(words) and all((word in PLACEHOLDER_WORDS for word in words)) def _defers(pointer_value: Any, spelled: str) -> bool: if pointer_value is None: return False return _stutters(spelled) or ( _digit_code(spelled) and any((character.isdigit() for character in str(pointer_value))) ) def route_spelled( prediction: dict[str, Any], kind: str, spelled: str, items: list[tuple[str, float]], text: str, defer_lists: bool = False, ) -> dict[str, Any]: if kind == "text": if _placeholder_text(prediction.get("guess")): prediction = { **prediction, "value": None, "guess": None, "span": None, "confidence": 0.0, } items = [ (item, score) for (item, score) in items if not _placeholder_text(item) ] if not items: return prediction (value, confidence) = ( items[0][0] if len(items) == 1 else spelled, min((score for (_, score) in items)), ) routed = {**prediction, "spell_confidence": confidence} if _similarity(prediction.get("guess"), value) >= 0.6: if ( routed.get("value") is None and prediction.get("guess") and (confidence >= SPELL_MIN_CONFIDENCE) ): return {**routed, "value": prediction["guess"]} return routed span = _state_match(text, value) if span is not None: return { **routed, "value": text[span[0] : span[1]], "guess": text[span[0] : span[1]], "span": list(span), } if confidence >= SPELL_MIN_CONFIDENCE and ( not _defers(prediction.get("value"), value) ): return { **routed, "value": value, "guess": value, "span": None, "confidence": confidence, "source": "image", } return routed (pointer_values, pointer_spans) = ( list(prediction.get("value") or []), list(prediction.get("spans") or []), ) if any((_placeholder_text(value) for value in pointer_values)): kept = [ (value, pointer_spans[index] if index < len(pointer_spans) else None) for (index, value) in enumerate(pointer_values) if not _placeholder_text(value) ] (pointer_values, pointer_spans) = ( [value for (value, _) in kept], [span for (_, span) in kept], ) items = [(item, score) for (item, score) in items if not _placeholder_text(item)] (values, spans, used) = ([], [], set()) for item, confidence in items: match = max( range(len(pointer_values)), key=lambda index: _similarity(pointer_values[index], item), default=None, ) if ( match is not None and match not in used and (_similarity(pointer_values[match], item) >= 0.6) ): used.add(match) values.append(pointer_values[match]) spans.append(pointer_spans[match] if match < len(pointer_spans) else None) continue span = _state_match(text, item) if span is not None: value = text[span[0] : span[1]] if not any((_simple(value) == _simple(other) for other in values)): values.append(value) spans.append(list(span)) elif ( confidence >= SPELL_MIN_CONFIDENCE and ( not ( pointer_values and (defer_lists or _defers(pointer_values[0], item)) ) ) and (not any((_simple(item) == _simple(other) for other in values))) ): values.append(item) spans.append(None) for index, (value, span) in enumerate(zip(pointer_values, pointer_spans)): if index not in used and ( not any((_simple(value) == _simple(other) for other in values)) ): values.append(value) spans.append(span) routed = {**prediction, "value": values, "spans": spans} if any((span is None for span in spans)): routed["source"] = "image" return routed def number_parts( logits: np.ndarray, ) -> tuple[np.ndarray, np.ndarray, np.ndarray]: logits = np.asarray(logits) sign = logits[..., :2] exponent = logits[..., 2 : 2 + EXPONENT_CLASSES] digits = logits[..., 2 + EXPONENT_CLASSES :] if logits.shape[-1] != 2 + EXPONENT_CLASSES + NUMBER_DIGITS * 10: raise RuntimeError("number logits do not match the number layout") return (sign, exponent, digits.reshape(*digits.shape[:-1], NUMBER_DIGITS, 10)) def group_class_digits(device: Any = None) -> np.ndarray: classes = np.arange(GROUP_CLASSES, dtype=np.int64) return np.stack( [ classes // 10 ** (GROUP_DIGITS - 1 - place) % 10 for place in range(GROUP_DIGITS) ], axis=-1, ) def group_slots(exponent_class: np.ndarray) -> tuple[np.ndarray, np.ndarray]: exponent_class = np.asarray(exponent_class).astype(np.int64) power = (exponent_class - 1 + MIN_EXPONENT).reshape(-1, 1, 1) groups = np.arange(GROUP_LOW, GROUP_HIGH + 1, dtype=np.int64).reshape(1, -1, 1) places = np.arange(GROUP_DIGITS - 1, -1, -1, dtype=np.int64).reshape(1, 1, -1) slots = power - (groups * GROUP_DIGITS + places) inside = ( (slots >= 0) & (slots < NUMBER_DIGITS) & (exponent_class > 0).reshape(-1, 1, 1) ) return ( np.where(inside, slots, np.full_like(slots, NUMBER_DIGITS)), inside.any(axis=-1), ) def number_group_logits( digit_logits: np.ndarray, exponent_class: np.ndarray, learned: np.ndarray | None = None, ) -> tuple[np.ndarray, np.ndarray]: digit_logits = np.asarray(digit_logits) count = digit_logits.shape[0] leading_zero = np.zeros((NUMBER_DIGITS, 10), dtype=bool) leading_zero[0, 0] = True log = _masked_fill(_log_softmax(_f32(digit_logits), axis=-1), leading_zero, MASKED) held = np.full((count, 1, 10), MASKED, dtype=log.dtype) held[..., 0] = 0.0 padded = np.concatenate([log, held], axis=1) (slots, groups) = group_slots(exponent_class) picked = padded[np.arange(count)[:, None], _flatten1(slots)] picked = picked.reshape(count, NUMBER_GROUPS, GROUP_DIGITS, 10) table = group_class_digits() scores = cast( np.ndarray, sum( ( np.take(picked[:, :, place, :], table[:, place], axis=-1) for place in range(GROUP_DIGITS) ) ), ) if learned is not None: scores = scores + _f32(learned) return (scores, groups) def temper_number(logits: np.ndarray, spread: np.ndarray) -> np.ndarray: (sign, exponent, digits) = number_parts(logits) temperature = np.exp( ESTIMATE_SPREAD_LIMIT * np.tanh(_f32(spread) / ESTIMATE_SPREAD_LIMIT) ) exponent = exponent / temperature[:, :1] digits = np.concatenate( [ digits[:, :LEXICAL_DIGITS] / temperature[:, 1:][..., None], digits[:, LEXICAL_DIGITS:], ], axis=1, ) return np.concatenate( [sign, exponent, _flatten1(digits)], axis=-1 ) def apply_prefix( logits: np.ndarray, views: dict[str, np.ndarray], group: str ) -> np.ndarray: length = views.get(f"{group}_prefix_length") logits = np.asarray(logits) if length is None or not logits.shape[0]: return logits (sign_logits, exponent_logits, digit_logits) = number_parts(logits) slots = np.arange(NUMBER_DIGITS, dtype=np.int64) length = np.asarray(length).astype(np.int64) relative = np.maximum(slots[None, :] - length[:, None], 0) shifted = digit_logits[np.arange(digit_logits.shape[0])[:, None], relative] forced_digits = np.where( np.arange(10, dtype=np.int64) == np.asarray(views[f"{group}_prefix_digits"]).astype(np.int64)[..., None], 0.0, MASKED, ).astype(logits.dtype) digit_logits = np.where( (slots[None, :] < length[:, None])[..., None], forced_digits, shifted ) written = (length > 0)[:, None] forced_sign = np.where( np.arange(2, dtype=np.int64) == np.asarray(views[f"{group}_prefix_sign"]).astype(np.int64)[:, None], 0.0, MASKED, ).astype(logits.dtype) forced_exponent = np.where( np.arange(EXPONENT_CLASSES, dtype=np.int64) == np.asarray(views[f"{group}_prefix_exponent"]).astype(np.int64)[:, None], 0.0, MASKED, ).astype(logits.dtype) return np.concatenate( [ np.where(written, forced_sign, sign_logits), np.where(written, forced_exponent, exponent_logits), _flatten1(digit_logits), ], axis=-1, ) def window_log_probs( log_probs: np.ndarray, window: np.ndarray ) -> tuple[np.ndarray, np.ndarray]: log_probs = np.asarray(log_probs) bins = np.arange(log_probs.shape[-1], dtype=np.int64) window = np.asarray(window).astype(np.int64) inside = (bins >= window[:, :1]) & (bins <= window[:, 1:]) return (_log_softmax(_masked_fill(log_probs, ~inside, MASKED), axis=-1), inside) def date_parts( logits: np.ndarray, ) -> tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray]: logits = np.asarray(logits) if logits.shape[-1] != YEAR_DIGITS * 10 + YEAR_CLASSES + 12 + 31: raise RuntimeError("date logits do not match the date layout") (digits, year, month, day) = np.split( logits, np.cumsum([YEAR_DIGITS * 10, YEAR_CLASSES, 12]).tolist(), axis=-1, ) return (digits.reshape(*digits.shape[:-1], YEAR_DIGITS, 10), year, month, day) def time_parts(logits: np.ndarray) -> tuple[np.ndarray, np.ndarray, np.ndarray]: logits = np.asarray(logits) if logits.shape[-1] != 24 + 60 + 60: raise RuntimeError("time logits do not match the time layout") (hour, minute, second) = np.split(logits, [24, 84], axis=-1) return (hour, minute, second) def lexical_bin_log_probs( number_logits: np.ndarray, edge_sign: np.ndarray, edge_class: np.ndarray, edge_digits: np.ndarray, edge_frac: np.ndarray, ) -> np.ndarray: (sign, exponent, digits) = number_parts( np.asarray(number_logits).astype(np.float64) ) (edge_sign, edge_class, edge_digits) = ( np.asarray(edge_sign), np.asarray(edge_class).astype(np.int64), np.asarray(edge_digits).astype(np.int64), ) sign_probs = _softmax(sign, axis=-1) exponent_probs = _softmax(exponent, axis=-1) zero = exponent_probs[:, :1] nonzero = exponent_probs[:, 1:] / np.maximum(1.0 - zero, 1e-300) below_class = _cumsum(nonzero, axis=-1) - nonzero leading = _masked_fill(digits[:, :1], np.arange(10) == 0, float("-inf")) digit_probs = _softmax( np.concatenate([leading, digits[:, 1:LEXICAL_DIGITS]], axis=1), axis=-1 ) below_digit = _cumsum(digit_probs, axis=-1) - digit_probs classes = np.clip(edge_class - 1, 0, nonzero.shape[1] - 1) within = np.asarray(edge_frac).astype(np.float64) for position in reversed(range(LEXICAL_DIGITS)): index = edge_digits[..., position] within = ( np.take_along_axis(below_digit[:, position], index, axis=1) + np.take_along_axis(digit_probs[:, position], index, axis=1) * within ) magnitude = ( np.take_along_axis(below_class, classes, axis=1) + np.take_along_axis(nonzero, classes, axis=1) * within ) negative = sign_probs[:, 1:] * (1.0 - zero) positive = sign_probs[:, :1] * (1.0 - zero) cdf = np.where( edge_sign == EDGE_NEGATIVE, negative * (1.0 - magnitude), np.where( edge_sign == EDGE_ZERO, np.broadcast_to(negative, magnitude.shape), negative + zero + positive * magnitude, ), ) mass = np.maximum(cdf[:, 1:] - cdf[:, :-1], 0.0) + 1e-30 return (np.log(mass) - np.log(mass.sum(axis=-1, keepdims=True))).astype( np.float32 ) def estimate_log_probs( range_logits: np.ndarray, number_logits: np.ndarray | None = None, gate: np.ndarray | None = None, lexical: np.ndarray | None = None, edge_sign: np.ndarray | None = None, edge_class: np.ndarray | None = None, edge_digits: np.ndarray | None = None, edge_frac: np.ndarray | None = None, ) -> np.ndarray: range_part = _log_softmax(_f32(range_logits), axis=-1) if number_logits is None or not np.asarray(number_logits).shape[0]: return range_part lexical_part = lexical_bin_log_probs( number_logits, cast(np.ndarray, edge_sign), cast(np.ndarray, edge_class), cast(np.ndarray, edge_digits), cast(np.ndarray, edge_frac), ) use = np.asarray(cast(np.ndarray, lexical)).astype(bool)[:, None] gate = _f32(cast(np.ndarray, gate))[:, None] mixed = np.logaddexp( _log_sigmoid(gate) + lexical_part, _log_sigmoid(-gate) + range_part, ) return np.where(use, mixed, range_part) def decode_number( logits: np.ndarray, spec: dict[str, Any], groups: np.ndarray | None = None ) -> tuple[float | int, float]: (sign_logits, exponent_logits, digit_logits) = number_parts(_f32(logits)) decimals = zs.allowed_decimals(spec) exponent_log = _log_softmax(exponent_logits, axis=-1) exponent_class = int(exponent_log.argmax()) log_probability = float(exponent_log[exponent_class]) if exponent_class == 0: return (0, math.exp(log_probability)) power = exponent_class - 1 + MIN_EXPONENT kept = NUMBER_DIGITS if decimals is None else power + decimals + 1 if kept < 0: return (0, math.exp(log_probability)) leading_zero = np.zeros((NUMBER_DIGITS, 10), dtype=bool) leading_zero[0, 0] = True digit_log = _masked_fill(_log_softmax(digit_logits, axis=-1), leading_zero, MASKED) digits = digit_log.argmax(axis=-1).tolist() sign_log = _log_softmax(sign_logits, axis=-1) sign = int(sign_log.argmax()) kept = min(kept, NUMBER_DIGITS) log_probability += float(sign_log[sign]) if groups is None: log_probability += sum( (float(digit_log[position, digits[position]]) for position in range(kept)) ) if kept < NUMBER_DIGITS: rounding = digit_log[kept, 5:] if digits[kept] >= 5 else digit_log[kept, :5] log_probability += float(_logsumexp(rounding, axis=0)) else: exponent_tensor = np.array([exponent_class], dtype=np.int64) (scores, present) = number_group_logits( digit_logits[None], exponent_tensor, _f32(groups)[None], ) group_log = _log_softmax(scores[0], axis=-1) slots = group_slots(exponent_tensor)[0][0].tolist() table = group_class_digits() counted = set() for group in range(NUMBER_GROUPS): if not bool(present[0, group]): continue best = int(group_log[group].argmax()) for place, slot in enumerate(slots[group]): if slot < NUMBER_DIGITS: digits[slot] = int(table[best, place]) if any((slot < kept for slot in slots[group])): counted.add(group) log_probability += float(group_log[group, best]) if kept < NUMBER_DIGITS: for group in range(NUMBER_GROUPS): if kept in slots[group]: if group not in counted: place = slots[group].index(kept) same = ( table[:, place] >= 5 if digits[kept] >= 5 else table[:, place] < 5 ) log_probability += float( _logsumexp(group_log[group][same], axis=0) ) break else: rounding = ( digit_log[kept, 5:] if digits[kept] >= 5 else digit_log[kept, :5] ) log_probability += float(_logsumexp(rounding, axis=0)) magnitude = Decimal(int("".join((str(digit) for digit in digits)))).scaleb( power - NUMBER_DIGITS + 1 ) if decimals is not None: magnitude = magnitude.quantize( Decimal(1).scaleb(-decimals), rounding=ROUND_HALF_UP ) if magnitude == 0: return (0, math.exp(log_probability)) value = -magnitude if sign else magnitude if decimals == 0 or (value == value.to_integral_value() and abs(value) < 2**53): return (int(value), math.exp(log_probability)) return (float(value), math.exp(log_probability)) def number_from_numeral( magnitude: Decimal, logits: np.ndarray, spec: dict[str, Any], check_exponent: bool = True, ) -> float | int | None: (sign_logits, exponent_logits, _) = number_parts(_f32(logits)) if check_exponent: target = number_components(float(magnitude)) if target is None or int(exponent_logits.argmax()) != target[1]: return None decimals = zs.allowed_decimals(spec) if decimals is not None: magnitude = magnitude.quantize( Decimal(1).scaleb(-decimals), rounding=ROUND_HALF_UP ) if magnitude == 0: return 0 value = -magnitude if int(sign_logits.argmax()) else magnitude if decimals == 0 or (value == value.to_integral_value() and abs(value) < 2**53): return int(value) return float(value) def number_from_numerals( values: list[Decimal], logits: np.ndarray, spec: dict[str, Any], readout: float | None, confident: bool = False, ) -> float | int | None: agreeing = [ value for value in ( number_from_numeral(magnitude, logits, spec) for magnitude in values ) if value is not None ] if not agreeing and confident and values: def distance(magnitude: Decimal) -> float: if readout is None or not readout or (not magnitude): return 0.0 return abs(math.log10(float(magnitude)) - math.log10(abs(float(readout)))) agreeing = [ value for value in ( number_from_numeral(min(values, key=distance), logits, spec, False), ) if value is not None ] if not agreeing: return None if readout is None: return agreeing[0] return min(agreeing, key=lambda value: abs(float(value) - float(readout))) def copied_number( output: dict[str, Any], row: int, record: EncodedRecord, spec: dict[str, Any], readout: float | None = None, ) -> tuple[float | int, float, list[int]] | None: if ( "number_copy_gate" not in output or record.record.get("images") or (not record.text) ): return None if os.environ.get("SZ_DISABLE_NUMBER_COPY") == "1": return None gate = float(_sigmoid(_f32(output["number_copy_gate"][row]))) if gate < 0.5: return None if "number_copy_best" in output: (best, probability) = ( output["number_copy_best"][row], output["number_copy_probability"][row], ) else: scores = _f32(output["number_copy"][row]).flatten() (best, probability) = ( int(scores.argmax()), float(_softmax(scores, axis=0).max()), ) (start, length) = divmod(int(best), MAX_SPAN) last = min(start + length, len(record.offsets) - 1) if start >= len(record.offsets): return None (char_start, char_end) = (record.offsets[start][0], record.offsets[last][1]) while char_start < char_end and (not record.text[char_start].isalnum()): char_start += 1 around = [ found for found in read_numerals(record.text, every=True) if found[1] < char_end and found[2] > char_start ] if not around: return None nearest = min( around, key=lambda found: ( abs(found[1] - char_start) + abs(found[2] - char_end), -(found[2] - found[1]), ), ) whole = [ found for found in read_numerals(record.text) if found[1] <= char_start and found[2] >= char_end and (found[2] - found[1] > nearest[2] - nearest[1]) ] choices = whole[:1] + [nearest] if gate * probability < COPY_MIN_CONFIDENCE: return None sure = gate >= COPY_CONFIDENT (value, numeral_start, numeral_end) = (None, nearest[1], nearest[2]) for values, start, end in choices: run = NUMERAL_RUN.match(record.text, start) printed = record.text[start : min(end, run.end())] if run else "" settled = ( numeral_convention(record.text) is not None and len(numeral_values(printed)) > 1 ) enclosing = bool(whole) and (start, end) == (whole[0][1], whole[0][2]) value = number_from_numerals( values, output["number"][row], spec, readout, confident=settled or (sure and enclosing and (len(values) == 1)), ) if value is not None: (numeral_start, numeral_end) = (start, end) break if value is None and sure: (values, numeral_start, numeral_end) = choices[0] value = number_from_numerals( values, output["number"][row], spec, readout, confident=True ) if value is None or (readout is not None and rescaled(readout, value)): return None return (value, gate * probability, [numeral_start, numeral_end]) def rescaled(readout: float, copied: float) -> bool: try: (first, second) = ( abs(Decimal(str(readout))).normalize(), abs(Decimal(str(copied))).normalize(), ) except ArithmeticError: return False if not first or not second or first == second: return False return first.as_tuple().digits == second.as_tuple().digits def decode_date(logits: np.ndarray) -> tuple[str, float]: (digit_logits, year_logits, month_logits, day_logits) = date_parts(_f32(logits)) class_log = _log_softmax(year_logits, axis=-1) year_class = int(class_log.argmax()) log_probability = float(class_log[year_class]) if year_class < YEAR_OUTSIDE: year = YEAR_LOW + year_class else: digit_log = _log_softmax(digit_logits, axis=-1) digits = digit_log.argmax(axis=-1).tolist() if digits == [0, 0, 0, 0]: masked = digit_log[3].copy() masked[0] = MASKED digits[3] = int(masked.argmax()) year = int("".join((str(digit) for digit in digits))) log_probability += float( sum((digit_log[position, digit] for (position, digit) in enumerate(digits))) ) (month_log, day_log) = ( _log_softmax(month_logits, axis=-1), _log_softmax(day_logits, axis=-1), ) joint = month_log[:, None] + day_log[None, :] lengths = np.array( [calendar.monthrange(year, month)[1] for month in range(1, 13)], dtype=np.int64, ) joint = _masked_fill(joint, np.arange(31)[None, :] >= lengths[:, None], MASKED) best = int(joint.flatten().argmax()) (month, day) = divmod(best, 31) log_probability += float(joint.flatten()[best]) return ( datetime.date(year, month + 1, day + 1).isoformat(), math.exp(log_probability), ) def decode_time(logits: np.ndarray) -> tuple[str, float]: parts = [_log_softmax(part, axis=-1) for part in time_parts(_f32(logits))] values = [int(part.argmax()) for part in parts] return ( zs.format_time(*values), math.exp(sum((float(part[value]) for (part, value) in zip(parts, values)))), ) def decode_spans( labels: list[int], offsets: list[tuple[int, int]], text: str, trims: list[list[int]] | None = None, ) -> tuple[list[str], list[list[int]]]: spans: list[list[int]] = [] current: list[int] | None = None for position, label in enumerate(labels): if label == TAG_BEGIN: if current is not None: spans.append(current) current = [position, position] elif label == TAG_INSIDE and current is not None: current[1] = position elif current is not None: spans.append(current) current = None if current is not None: spans.append(current) values: list[str] = [] char_spans: list[list[int]] = [] for first, last in spans: (char_start, char_end) = trimmed_span(text, offsets, first, last, trims) if not text[char_start:char_end].strip(): continue char_spans.append([char_start, char_end]) if text[char_start:char_end] not in values: values.append(text[char_start:char_end]) return (values, char_spans) def quantile_from_cdf( cdf: list[float], probabilities: list[float], fraction: float, bins: int ) -> float: index = min(bisect_left(cdf, fraction), bins - 1) previous = cdf[index - 1] if index > 0 else 0.0 inside = ( (fraction - previous) / probabilities[index] if probabilities[index] > 0 else 0.5 ) return (index + min(1.0, max(0.0, inside))) / bins def decode_estimate( logits: np.ndarray, offsets: np.ndarray, overflow: np.ndarray, spec: dict[str, Any], ) -> dict[str, Any]: probabilities = _softmax(_f32(logits), axis=-1) cdf = _cumsum(probabilities, axis=-1).tolist() row = probabilities.tolist() bins = len(row) shift = _sigmoid(_f32(offsets)).tolist() (low, high, scale) = zs.estimate_range(spec) centers = [(index + shift[index]) / bins for index in range(bins)] mean = sum( ( probability * zs.from_unit(center, low, high, scale) for (probability, center) in zip(row, centers) ) ) quantiles: list[Any] = [ zs.from_unit( quantile_from_cdf(cdf, row, fraction / 100.0, bins), low, high, scale ) for fraction in range(1, 100) ] value = ( zs.from_unit( sum((probability * center for (probability, center) in zip(row, centers))), low, high, scale, ) if scale == "log" else mean ) step = zs.estimate_step(spec) if step > 0: quantiles = [ zs.snap_quantile(quantile, step, low, high) for quantile in quantiles ] value = zs.snap(mean, step, low, high) if zs.is_date_estimate(spec): (value, mean) = (zs.estimate_value(spec, value), zs.estimate_value(spec, mean)) quantiles = [zs.estimate_value(spec, quantile) for quantile in quantiles] prediction = { "value": value, "mean": mean, "median": quantiles[49], "p10": quantiles[9], "p90": quantiles[89], "quantiles": quantiles, } if not zs.is_date_estimate(spec): overflow = np.asarray(overflow) sides = _softmax( np.concatenate([np.zeros(1, dtype=overflow.dtype), _f32(overflow)]), axis=0 ).tolist() (prediction["p_below"], prediction["p_above"]) = (sides[BELOW], sides[ABOVE]) return prediction def estimate_row_log_probs( output: dict[str, Any], row: int, spec: dict[str, Any], window: tuple[int, int] | None = None, ) -> np.ndarray: log_probs = _estimate_row_log_probs(output, row, spec) if window is None: return log_probs return window_log_probs(log_probs[None], np.array([window], dtype=np.int64))[0][0] def _estimate_row_log_probs( output: dict[str, Any], row: int, spec: dict[str, Any] ) -> np.ndarray: logits = _f32(output["estimate"][row]) parts = ( estimate_edge_parts(spec, logits.shape[-1]) if "estimate_number" in output else None ) if parts is None: return _log_softmax(logits, axis=-1) (sign, classes, digits, fractions) = parts count = len(sign) return estimate_log_probs( logits[None], _f32(output["estimate_number"][row])[None], _f32(output["estimate_gate"][row]).reshape(1), np.ones(1, dtype=np.float32), np.array([sign], dtype=np.int64), np.array([classes], dtype=np.int64), np.array(digits, dtype=np.int64).reshape(1, count, LEXICAL_DIGITS), np.array([fractions], dtype=np.float64), )[0] def decode_record( output: dict[str, Any], record: EncodedRecord, tokenizer: Any = None, end_id: int | None = None, ) -> dict[str, dict[str, Any]]: fields = record.fields specs = {item.name: record.record["fields"][item.name] for item in fields} p_null = _sigmoid(_f32(output["null"])).tolist() predictions: dict[str, dict[str, Any]] = {} value: Any rows_of = { group: [ position for (position, item) in enumerate(fields) if item.kind in kinds ] for (group, kinds) in ( ("text", ("text",)), ("tags", ("text_list",)), ("number", ("number",)), ("date", ("date", "datetime")), ("time", ("time", "datetime")), ("estimate", ("estimate",)), ("number_list", ("number_list",)), ("date_list", ("date_list",)), ) } index_of = { group: {position: row for (row, position) in enumerate(rows)} for (group, rows) in rows_of.items() } if rows_of["text"] and "text_best" in output: (best, probabilities) = (output["text_best"], output["text_probability"]) elif rows_of["text"]: scores = _f32(output["text"]) scores = _flatten1(scores) best = scores.argmax(axis=1).tolist() probabilities = _softmax(scores, axis=1).max(axis=1).tolist() if rows_of["tags"]: labels = ( output["tags_labels"] if "tags_labels" in output else np.asarray(output["tags"]).argmax(axis=-1).tolist() ) trims = output.get("trim_labels") if trims is None and "trim" in output: trims = np.asarray(output["trim"]).argmax(axis=-1).tolist() for position, item in enumerate(fields): spec = specs[item.name] kind = item.kind if kind == "estimate": row = index_of["estimate"][position] predictions[item.name] = decode_estimate( estimate_row_log_probs(output, row, spec, item.window), output["estimate_offset"][row], output["estimate_overflow"][row], spec, ) continue if kind == "text_list": row = index_of["tags"][position] (values, spans) = decode_spans( labels[row], record.offsets, record.text, trims ) predictions[item.name] = {"value": values, "spans": spans} if "tags_log_probability" in output: predictions[item.name]["confidence"] = math.exp( output["tags_log_probability"][row] ) elif "tags" in output: label_log = _log_softmax(_f32(output["tags"][row]), axis=-1) predictions[item.name]["confidence"] = math.exp( float(label_log.max(axis=-1).sum()) ) continue if kind in LIST_KINDS: row = index_of[kind][position] ends = _sigmoid(_f32(output[f"{kind}_end"][row])).tolist() (values, confidences) = ([], []) log_probability = 0.0 for slot in range(LIST_SLOTS): if ends[slot] >= 0.5: log_probability += math.log(max(ends[slot], 1e-12)) break log_probability += math.log(max(1.0 - ends[slot], 1e-12)) if kind == "number_list": (value, confidence) = decode_number( output[kind][row, slot], spec, output["number_list_groups"][row, slot] if "number_list_groups" in output else None, ) else: (value, confidence) = decode_date(output[kind][row, slot]) values.append(value) confidences.append(confidence) log_probability += math.log(max(confidence, 1e-12)) predictions[item.name] = { "value": values, "confidences": confidences, "confidence": math.exp(log_probability), } continue prediction: dict[str, Any] = {"p_null": p_null[position]} if kind == "text": row = index_of["text"][position] (start, length) = divmod(best[row], MAX_SPAN) last = min(start + length, len(record.offsets) - 1) (char_start, char_end) = trimmed_span( record.text, record.offsets, start, last, trims ) (value, confidence) = (record.text[char_start:char_end], probabilities[row]) prediction["span"] = [char_start, char_end] if not record.text: (value, prediction["span"]) = (None, None) elif kind == "number": number_row = index_of["number"][position] (value, confidence) = decode_number( output["number"][number_row], spec, output["number_groups"][number_row] if "number_groups" in output else None, ) copied = copied_number(output, number_row, record, spec, value) if ( copied is not None and value is not None and (os.environ.get("SZ_NUMBER_COPY_VETO", "1") == "1") and numeral_spans(record.text, value, every=False) ): copied = None if copied is not None: (value, confidence, prediction["span"]) = copied prediction["source"] = "input" if spec["type"] == "duration": prediction["unit"] = zs.duration_unit(spec) elif kind == "date": (value, confidence) = decode_date( output["date"][index_of["date"][position]] ) elif kind == "time": (value, confidence) = decode_time( output["time"][index_of["time"][position]] ) else: (day, day_confidence) = decode_date( output["date"][index_of["date"][position]] ) (clock, clock_confidence) = decode_time( output["time"][index_of["time"][position]] ) (value, confidence) = (f"{day}T{clock}", day_confidence * clock_confidence) prediction["guess"] = value prediction["confidence"] = confidence prediction["value"] = ( None if p_null[position] >= zs.null_threshold(spec) else value ) predictions[item.name] = prediction if tokenizer is not None and end_id is not None and ("spell_ids" in output): (ids, log_probs) = ( np.asarray(output["spell_ids"]).tolist(), _f32(output["spell_log_probs"]).tolist(), ) for position, item in enumerate(fields): if item.kind == "text": row = index_of["text"][position] elif item.kind == "text_list": row = len(rows_of["text"]) + index_of["tags"][position] else: continue (spelled, _, parts) = spelled_answer( ids[row], log_probs[row], tokenizer, end_id ) predictions[item.name] = route_spelled( predictions[item.name], item.kind, spelled, parts, record.text ) if tokenizer is not None and "slot_ids" in output: slot_end = _tokens(tokenizer, SPELL_END)[0] (slot_ids, slot_log_probs) = ( np.asarray(output["slot_ids"]).tolist(), _f32(output["slot_log_probs"]).tolist(), ) slot_rows = [ position for (position, item) in enumerate(fields) if item.answer_slots ] for row, position in enumerate(slot_rows): item = fields[position] if item.kind not in ("text", "text_list"): continue count = len(item.answer_slots) (spelled, _probability, parts) = spelled_answer( slot_ids[row][:count], slot_log_probs[row][:count], tokenizer, slot_end ) prediction = predictions[item.name] if (record.record.get("_meta") or {}).get( "construction" ) == zs.EVIDENCE_CONSTRUCTION and item.kind == "text": predictions[item.name] = evidence_text_answer( prediction, slot_ids[row][:count], slot_log_probs[row][:count], tokenizer, slot_end, record.text, specs[item.name], bool(record.record.get("images")), ) continue if item.kind in ("text", "text_list"): predictions[item.name] = route_spelled( prediction, item.kind, spelled, parts, record.text, defer_lists=True ) continue return predictions FIELD_KINDS = SCALAR_KINDS + ("estimate", "text_list") + LIST_KINDS VIEW_GROUPS = ( ("text", ("text",)), ("tags", ("text_list",)), ("number", ("number",)), ("date", ("date", "datetime")), ("time", ("time", "datetime")), ("estimate", ("estimate",)), ) COMPACT_KEYS = ( "number", "date", "time", "estimate", "estimate_offset", "estimate_overflow", "number_list_end", "number_list", "date_list_end", "date_list", "estimate_number", "estimate_gate", "number_groups", "number_list_groups", "number_copy_gate", ) COMPACT_GROUPS = { "number": "number", "date": "date", "time": "time", "estimate": "estimate", "estimate_offset": "estimate", "estimate_overflow": "estimate", "number_list_end": "number_list", "number_list": "number_list", "date_list_end": "date_list", "date_list": "date_list", "estimate_number": "estimate", "estimate_gate": "estimate", "number_groups": "number", "number_list_groups": "number_list", "number_copy_gate": "number", } def prediction_layout(record: EncodedRecord) -> list[int]: layout = [ len(record.input_ids), record.input_start, len(record.offsets), len(record.fields), ] for item in record.fields: layout += [ FIELD_KINDS.index(item.kind), item.query, len(item.items), *item.items, ] slotted = [ (index, item.answer_slots) for (index, item) in enumerate(record.fields) if item.answer_slots ] if slotted: layout.append(len(slotted)) for index, slots in slotted: layout += [index, len(slots), *slots] return layout def parse_prediction_layout( layout: list[int], num_tokens: int, with_slots: bool = False ) -> tuple[Any, ...]: if len(layout) < 4 or any((type(value) is not int for value in layout)): raise ValueError("prediction_layout must be a list of ints") (prompt_length, input_start, input_length, count) = layout[:4] shift = num_tokens - prompt_length if ( shift < 0 or count < 1 or input_length < 1 or (not 0 <= input_start < input_start + input_length <= prompt_length) ): raise ValueError("prediction_layout does not match the prompt") kinds: list[str] = [] queries: list[int] = [] items: list[list[int]] = [] cursor = 4 for _ in range(count): if cursor + 3 > len(layout): raise ValueError("prediction_layout is truncated") (kind, query, size) = layout[cursor : cursor + 3] cursor += 3 if ( not 0 <= kind < len(FIELD_KINDS) or not input_start + input_length <= query < prompt_length ): raise ValueError("prediction_layout has an invalid field") expected = LIST_SLOTS if FIELD_KINDS[kind] in LIST_KINDS else 0 slots = layout[cursor : cursor + size] cursor += size if ( size != expected or len(slots) != size or any((not query < slot < prompt_length for slot in slots)) ): raise ValueError("prediction_layout has invalid item positions") kinds.append(FIELD_KINDS[kind]) queries.append(query + shift) items.append([slot + shift for slot in slots]) slot_fields: list[int] = [] answer_slots: list[list[int]] = [] if cursor < len(layout): slotted = layout[cursor] cursor += 1 if not 0 < slotted <= count: raise ValueError("prediction_layout has invalid answer slots") for _ in range(slotted): if cursor + 2 > len(layout): raise ValueError("prediction_layout is truncated") (index, size) = layout[cursor : cursor + 2] cursor += 2 positions = layout[cursor : cursor + size] cursor += size if ( not 0 <= index < count or (slot_fields and index <= slot_fields[-1]) or size < 1 or (len(positions) != size) or any( ( not queries[index] - shift < slot < prompt_length for slot in positions ) ) ): raise ValueError("prediction_layout has invalid answer slots") slot_fields.append(index) answer_slots.append([slot + shift for slot in positions]) if cursor != len(layout): raise ValueError("prediction_layout has trailing values") parsed = (input_start + shift, input_length, kinds, queries, items) return (*parsed, slot_fields, answer_slots) if with_slots else parsed def prediction_views( kinds: list[str], queries: list[int], items: list[list[int]], device: Any = None ) -> dict[str, np.ndarray]: views = {"query_positions": np.array(queries, dtype=np.int64)} for group, members in VIEW_GROUPS: views[f"{group}_rows"] = np.array( [row for (row, kind) in enumerate(kinds) if kind in members], dtype=np.int64, ) for kind in LIST_KINDS: rows = [row for (row, field_kind) in enumerate(kinds) if field_kind == kind] views[f"{kind}_rows"] = np.array(rows, dtype=np.int64) views[f"{kind}_items"] = np.array( [slot for row in rows for slot in items[row]], dtype=np.int64, ).reshape(len(rows), LIST_SLOTS) return views def compact_output(output: dict[str, Any]) -> np.ndarray: parts = [_f32(output["null"]).flatten()] if "text" in output: scores = _f32(output["text"]) scores = _flatten1(scores) parts += [ scores.argmax(axis=1).astype(np.float32), _softmax(scores, axis=1).max(axis=1), ] if "tags" in output: tag_log = _log_softmax(_f32(output["tags"]), axis=-1) parts.append(tag_log.argmax(axis=-1).astype(np.float32).flatten()) parts.append(tag_log.max(axis=-1).sum(axis=-1)) if "trim" in output: parts.append(np.asarray(output["trim"]).argmax(axis=-1).astype(np.float32).flatten()) if "number_copy" in output: scores = _f32(output["number_copy"]) scores = _flatten1(scores) parts += [ scores.argmax(axis=1).astype(np.float32), _softmax(scores, axis=1).max(axis=1), ] parts += [_f32(output[key]).flatten() for key in COMPACT_KEYS if key in output] if "slot_ids" in output: parts += [ _f32(output["slot_ids"]).flatten(), _f32(output["slot_log_probs"]).flatten(), ] return np.concatenate(parts).astype(np.float32, copy=False) def expand_output( flat: np.ndarray, record: EncodedRecord, bins: int = BINS, estimate_lexical: bool = False, span_trim: bool = False, number_groups: bool = False, number_pointer: bool = False, ) -> dict[str, Any]: flat = _f32(flat).reshape(-1) kinds = [item.kind for item in record.fields] rows = { group: sum((kind in members for kind in kinds)) for (group, members) in VIEW_GROUPS } rows.update({kind: kinds.count(kind) for kind in LIST_KINDS}) cursor = 0 def take(*shape: int) -> np.ndarray: nonlocal cursor size = math.prod(shape) if cursor + size > flat.size: raise ValueError("compact output is shorter than the layout needs") value = flat[cursor : cursor + size].reshape(*shape) cursor += size return value output: dict[str, Any] = {"null": take(len(kinds))} if rows["text"]: output["text_best"] = take(rows["text"]).round().astype(np.int64).tolist() output["text_probability"] = take(rows["text"]).tolist() if rows["tags"]: output["tags_labels"] = ( take(rows["tags"], len(record.offsets)).round().astype(np.int64).tolist() ) output["tags_log_probability"] = take(rows["tags"]).tolist() if span_trim and (rows["text"] or rows["tags"]): output["trim_labels"] = take(len(record.offsets), 2).round().astype(np.int64).tolist() if number_pointer and rows["number"]: output["number_copy_best"] = take(rows["number"]).round().astype(np.int64).tolist() output["number_copy_probability"] = take(rows["number"]).tolist() widths = { "number": (NUMBER_WIDTH,), "date": (DATE_WIDTH,), "time": (TIME_WIDTH,), "estimate": (bins,), "estimate_offset": (bins,), "estimate_overflow": (2,), "number_list_end": (LIST_SLOTS,), "number_list": (LIST_SLOTS, NUMBER_WIDTH), "date_list_end": (LIST_SLOTS,), "date_list": (LIST_SLOTS, DATE_WIDTH), "estimate_number": (NUMBER_WIDTH,), "estimate_gate": (), "number_groups": (NUMBER_GROUPS, GROUP_CLASSES), "number_list_groups": (LIST_SLOTS, NUMBER_GROUPS, GROUP_CLASSES), "number_copy_gate": (), } for key in COMPACT_KEYS: if key in ("estimate_number", "estimate_gate") and (not estimate_lexical): continue if key in ("number_groups", "number_list_groups") and (not number_groups): continue if key == "number_copy_gate" and (not number_pointer): continue count = rows[COMPACT_GROUPS[key]] if count: output[key] = take(count, *widths[key]) slotted = [len(item.answer_slots) for item in record.fields if item.answer_slots] if slotted: output["slot_ids"] = take(len(slotted), max(slotted)).round().astype(np.int64) output["slot_log_probs"] = take(len(slotted), max(slotted)) if cursor != flat.size: raise ValueError("compact output is longer than the layout needs") return output def merge_forecasts( predictions: dict[str, dict[str, Any]], groups: dict[str, list[str]] ) -> dict[str, dict[str, Any]]: merged = { name: prediction for (name, prediction) in predictions.items() if not any((name in names for names in groups.values())) } for name, names in groups.items(): steps = [predictions[step] for step in names] merged[name] = { key: [step[key] for step in steps] for key in ("value", "median", "p10", "p90") } merged[name]["steps"] = steps return merged