diff --git "a/clairic_runtime/model.py" "b/clairic_runtime/model.py" new file mode 100644--- /dev/null +++ "b/clairic_runtime/model.py" @@ -0,0 +1,2999 @@ +from __future__ import annotations +import calendar +import datetime +import functools +import math +import os +import re +from bisect import bisect_left, bisect_right +from dataclasses import dataclass, field +from decimal import ROUND_HALF_UP, Decimal +from typing import Any, cast +import torch +from torch import nn +from torch.nn import functional +from . import schema as zs + +SYSTEM_PREFIX = "<|im_start|>system\n" + zs.SYSTEM_PREAMBLE + "\nFields:\n" +EVIDENCE_SYSTEM_PREFIX = ( + "<|im_start|>system\n" + zs.EVIDENCE_SYSTEM_PREAMBLE + "\nFields:\n" +) +USER_PREFIX = "<|im_end|>\n<|im_start|>user\n" +ASSISTANT_SUFFIX = "<|im_end|>\n<|im_start|>assistant\n\n\n\n" +ANSWER_CUE = "\nAnswer:" + + +@dataclass(frozen=True) +class PromptLayout: + schema_header: str + query_header: str + answer_cue: str + slot_header: str + + +PROMPT_LAYOUTS = ( + PromptLayout("", "", ANSWER_CUE, ""), + PromptLayout( + "Field definition:\n", "\nField query:", "\nValue:", "\nAnswer field:" + ), + PromptLayout( + "Requested field:\n", "\nRequested field:", "\nResult:", "\nResult for field:" + ), +) +PROMPT_VARIANTS = (0, 1, 2) + + +def prompt_layout( + template_variant: int = 0, complete_field: bool = True +) -> PromptLayout: + if ( + not isinstance(template_variant, int) + or isinstance(template_variant, bool) + or template_variant not in PROMPT_VARIANTS + ): + raise ValueError("template_variant must be 0, 1, or 2") + if template_variant and (not complete_field): + raise ValueError("alternative layouts require the evidence construction") + return PROMPT_LAYOUTS[template_variant] + + +QUERY_TOKENS = 256 +MAX_SPAN = 64 +TRIM_CLASSES = 8 +TRIM_INIT = 2.0 +MIN_INPUT_TOKENS = 16 +BINS = 256 +POINTER_WIDTH = 256 +NUMBER_DIGITS = 12 +(MIN_EXPONENT, MAX_EXPONENT) = (-12, 15) +EXPONENT_CLASSES = MAX_EXPONENT - MIN_EXPONENT + 2 +YEAR_DIGITS = 4 +(YEAR_LOW, YEAR_HIGH) = (1800, 2199) +YEAR_OUTSIDE = YEAR_HIGH - YEAR_LOW + 1 +YEAR_CLASSES = YEAR_OUTSIDE + 1 +LIST_SLOTS = zs.MAX_LIST_ITEMS +ITEM_CUES = tuple((f"\n{index}." for index in range(1, LIST_SLOTS + 1))) +(INSIDE, BELOW, ABOVE) = (0, 1, 2) +OVERFLOW_INIT = -5.0 +QUERY_ORDER_RATE = 0.8 +LOOP_RANK = 256 +LEXICAL_DIGITS = 4 +ESTIMATE_GATE_INIT = 0.0 +ESTIMATE_SPREAD_LIMIT = 2.0 +(EDGE_NEGATIVE, EDGE_ZERO, EDGE_POSITIVE) = (0, 1, 2) +(TAG_OUTSIDE, TAG_BEGIN, TAG_INSIDE) = (0, 1, 2) +NUMBER_WIDTH = 2 + EXPONENT_CLASSES + NUMBER_DIGITS * 10 +GROUP_DIGITS = 3 +GROUP_CLASSES = 10**GROUP_DIGITS +(GROUP_LOW, GROUP_HIGH) = (MIN_EXPONENT // GROUP_DIGITS, MAX_EXPONENT // GROUP_DIGITS) +NUMBER_GROUPS = GROUP_HIGH - GROUP_LOW + 1 +GROUP_RANK = 256 +COPY_CATEGORIES: tuple[str | None, ...] | None = ("extraction", None) +COPY_GATE_INIT = -3.0 +NUMERAL_RUN = re.compile( + "(?:(? 2 + or word in ("mn", "bn", "tn", "cr", "Cr", "Mn", "Bn", "MM") + ), + key=len, + reverse=True, + ) + ) + + ")|(k|K|M|B))(?![^\\W\\d_])" +) +NUMERAL_SPAN = 48 +CONVENTION_LANGUAGE = re.compile( + "\\b(?:Rechnung|Gesamtbetrag|Betrag|Lieferschein|Factuur|Bedrag|Totaal|Pakbon|Fattura|Importo|Totale|Fatura|Nota fiscal|Factura|Importe|Albar\\u00e1n|Facture|Montant)\\b" +) +CONVENTION_LANGUAGE_US = re.compile("\\b(?:M\\u00e9xico|Mexico|MXN|CFDI|RFC)\\b") +CONVENTION_STATED = { + "eu": re.compile( + "european (?:number )?(?:notation|format)|(?:dot|period|point)s? (?:separates?|for|marks?) (?:the )?thousands|comma (?:separates?|for|marks?|as) (?:the )?decimal|comma decimals?|decimal comma|dot thousands|1\\.234,56|1 234,56", + re.IGNORECASE, + ), + "us": re.compile( + "comma (?:separates?|for|marks?) (?:the )?thousands|(?:dot|period|point) (?:separates?|for|marks?|as) (?:the )?decimal|comma thousands|dot decimals?|1,234\\.56", + re.IGNORECASE, + ), +} +COPY_CONFIDENT = 0.9 +COPY_MIN_CONFIDENCE = 0.5 +SLOT_TEXT = "_" +SLOT_COUNTS = {"text": 80, "text_list": 64, "number": 32} +SLOT_CUE_TOKENS = 32 +DATE_WIDTH = YEAR_DIGITS * 10 + YEAR_CLASSES + 12 + 31 +TIME_WIDTH = 24 + 60 + 60 +MASKED = -1000000000.0 +SCALAR_KINDS = ("text", "number", "date", "time", "datetime") +IMAGE_PLACEHOLDER = "<|vision_start|><|image_pad|><|vision_end|>" +MEDIA_BATCH_KEYS = ("pixel_values", "image_grid_thw") +MEDIA_TOKEN_KEYS = ("mm_token_type_ids",) +LIST_KINDS = ("number_list", "date_list") + + +def head_kind(spec: dict[str, Any]) -> str: + kind = spec["type"] + if kind == "duration": + return "number" + if kind == "list": + return f"{spec['items']}_list" + return kind + + +def number_components(value: float) -> tuple[int, int, list[int]] | None: + value = float(value) + if not math.isfinite(value): + return None + if value == 0: + return (0, 0, [0] * NUMBER_DIGITS) + (mantissa, exponent) = f"{abs(value):.{NUMBER_DIGITS - 1}e}".split("e") + power = int(exponent) + if not MIN_EXPONENT <= power <= MAX_EXPONENT: + return None + return ( + int(value < 0), + power - MIN_EXPONENT + 1, + [int(digit) for digit in mantissa.replace(".", "")], + ) + + +def edge_number(value: float) -> tuple[int, int, list[int], float]: + if value == 0 or not math.isfinite(value): + return (EDGE_ZERO, 1, [0] * LEXICAL_DIGITS, 0.0) + (mantissa, exponent) = f"{abs(value):.{LEXICAL_DIGITS + 8}e}".split("e") + (power, digits) = (int(exponent), mantissa.replace(".", "")) + sign = EDGE_NEGATIVE if value < 0 else EDGE_POSITIVE + if power < MIN_EXPONENT: + return (sign, 1, [0] * LEXICAL_DIGITS, 0.0) + if power > MAX_EXPONENT: + return (sign, EXPONENT_CLASSES - 1, [9] * LEXICAL_DIGITS, 1.0) + return ( + sign, + power - MIN_EXPONENT + 1, + [int(digit) for digit in digits[:LEXICAL_DIGITS]], + float("0." + digits[LEXICAL_DIGITS:]), + ) + + +@functools.lru_cache(maxsize=8192) +def _edge_parts( + low: float, high: float, scale: str, bins: int +) -> tuple[tuple[int, ...], tuple[int, ...], tuple[int, ...], tuple[float, ...]]: + parts = [ + edge_number(zs.from_unit(index / bins, low, high, scale)) + for index in range(bins + 1) + ] + return ( + tuple((part[0] for part in parts)), + tuple((part[1] for part in parts)), + tuple((digit for part in parts for digit in part[2])), + tuple((part[3] for part in parts)), + ) + + +def lexical_estimate(spec: dict[str, Any]) -> bool: + return not zs.is_date_estimate(spec) or zs.date_granularity(spec) == "year" + + +def estimate_edge_parts( + spec: dict[str, Any], bins: int = BINS +) -> tuple[tuple[int, ...], tuple[int, ...], tuple[int, ...], tuple[float, ...]] | None: + if not lexical_estimate(spec): + return None + (low, high, scale) = zs.estimate_range(spec) + return _edge_parts(float(low), float(high), scale, bins) + + +@dataclass +class EncodedField: + name: str + kind: str + query: int + items: list[int] = field(default_factory=list) + prefix: tuple[int, int, tuple[int, ...]] | None = None + window: tuple[int, int] | None = None + answer_slots: list[int] = field(default_factory=list) + + +@dataclass(frozen=True) +class Commit: + text: str + done: bool = False + prefix: tuple[int, int, tuple[int, ...]] | None = None + probability: float = 1.0 + + +@dataclass +class EncodedRecord: + input_ids: list[int] + input_start: int + offsets: list[tuple[int, int]] + fields: list[EncodedField] + text: str + record: dict[str, Any] + media: dict[str, Any] | None = None + image_expansion: int = 0 + + +@dataclass +class EncodedMedia: + ids: list[int] + media: dict[str, Any] | None = None + expansion: int = 0 + + @property + def length(self) -> int: + return len(self.ids) + self.expansion + + +def configure_image_processor(processor: Any, max_pixels: int) -> dict[str, Any]: + image_processor = getattr(processor, "image_processor", None) + if image_processor is None or max_pixels <= 0: + raise ValueError("an image processor and positive pixel budget are required") + legacy = hasattr(image_processor, "max_pixels") + size = getattr(image_processor, "size", {}) + sized = "longest_edge" in size + if not legacy and (not sized): + raise ValueError( + f"unsupported image pixel configuration: {type(image_processor).__name__}" + ) + if legacy: + image_processor.max_pixels = max_pixels + if sized: + size["longest_edge"] = max_pixels + actual = size["longest_edge"] if sized else image_processor.max_pixels + if actual != max_pixels or (legacy and image_processor.max_pixels != max_pixels): + raise ValueError( + "the image processor did not retain its requested pixel budget" + ) + return {"class": type(image_processor).__name__, "max_pixels": actual} + + +def encode_media( + tokenizer: Any, + images: list[Any], + processor: Any = None, + image_token_counts: list[int] | None = None, +) -> EncodedMedia: + if not images: + return EncodedMedia([]) + text = IMAGE_PLACEHOLDER * len(images) + "\n" + if image_token_counts is not None: + if len(image_token_counts) != len(images) or any( + (count < 1 for count in image_token_counts) + ): + raise ValueError("every image needs a positive token count") + return EncodedMedia( + _tokens(tokenizer, text), + None, + sum((count - 1 for count in image_token_counts)), + ) + if processor is None: + raise ValueError("records with images need the model processor") + encoded = processor(text=[text], images=list(images), return_tensors="pt") + media = {key: encoded[key] for key in MEDIA_BATCH_KEYS if key in encoded} + for key in MEDIA_TOKEN_KEYS: + if key in encoded: + media[key] = encoded[key][0].tolist() + return EncodedMedia(encoded["input_ids"][0].tolist(), media) + + +def _tokens(tokenizer: Any, text: str) -> list[int]: + return tokenizer(text, add_special_tokens=False).input_ids + + +def _input_tokens(tokenizer: Any, text: str) -> tuple[list[int], list[tuple[int, int]]]: + try: + encoding = tokenizer( + text, + add_special_tokens=False, + return_offsets_mapping=True, + split_special_tokens=True, + ) + except TypeError: + encoding = tokenizer( + text, add_special_tokens=False, return_offsets_mapping=True + ) + return (list(encoding.input_ids), [tuple(pair) for pair in encoding.offset_mapping]) + + +def schema_line(name: str, spec: dict[str, Any], complete_field: bool = False) -> str: + return "- " + zs.field_line(name, spec, preserve_instructions=complete_field) + "\n" + + +def reference_line(record: dict[str, Any]) -> str: + reference = record.get("reference_date") + return f"Reference date: {reference}\n" if reference else "" + + +def query_block( + tokenizer: Any, + name: str, + spec: dict[str, Any], + commit: Commit | None = None, + complete_field: bool = False, + template_variant: int = 0, +) -> tuple[list[int], int, list[int]]: + layout = prompt_layout(template_variant, complete_field) + line = _tokens( + tokenizer, + "\n" + zs.field_line(name, spec, preserve_instructions=complete_field), + ) + ids = ( + _tokens(tokenizer, layout.query_header) + + (line if complete_field else line[:QUERY_TOKENS]) + + _tokens(tokenizer, layout.answer_cue) + ) + if commit is not None: + ids.extend(_tokens(tokenizer, " " + commit.text)) + query = len(ids) - 1 + items: list[int] = [] + if head_kind(spec) in LIST_KINDS and commit is None: + for cue in ITEM_CUES: + ids.extend(_tokens(tokenizer, cue)) + items.append(len(ids) - 1) + return (ids, query, items) + + +def clean_cut( + text: str, offsets: list[tuple[int, int]], budget: int, lookback: int = 32 +) -> int: + lowest = max(0, budget - lookback) + for keep in range(budget, lowest, -1): + end = offsets[keep - 1][1] + if end >= len(text) or ( + end > 0 and text[end].isspace() and (not text[end - 1].isspace()) + ): + return keep + for keep in range(budget, lowest, -1): + end = offsets[keep - 1][1] + if end > 0 and (text[end].isspace() or text[end - 1].isspace()): + return keep + return budget + + +def char_span_to_tokens( + starts: list[int], ends: list[int], char_start: int, char_end: int +) -> tuple[int, int] | None: + first = bisect_right(ends, char_start) + last = bisect_left(starts, char_end) - 1 + if first >= len(starts) or last < first: + return None + return (first, last) + + +def numeral_values(printed: str, convention: str | None = None) -> list[Decimal]: + values: list[Decimal] = [] + both = convention is not None and all( + (form.fullmatch(printed) for (form, _, _) in NUMERAL_READINGS[:2]) + ) + for index, (form, separators, mark) in enumerate(NUMERAL_READINGS): + if both and index == (1 if convention == "us" else 0): + continue + if form.fullmatch(printed): + plain = printed.translate( + {ord(character): None for character in separators} + ).replace(mark, ".") + try: + value = Decimal(plain) + except ArithmeticError: + continue + if value not in values: + values.append(value) + return values + + +@functools.lru_cache(maxsize=256) +def numeral_convention(text: str) -> str | None: + stated = { + name + for (name, pattern) in CONVENTION_STATED.items() + if pattern.search(text or "") + } + if len(stated) == 1: + return stated.pop() + shown: list[str] = [] + for run in NUMERAL_RUN.finditer(text or ""): + printed = run.group() + if not any((mark in printed for mark in ",.")): + continue + (us, eu) = ( + bool(form.fullmatch(printed)) for (form, _, _) in NUMERAL_READINGS[:2] + ) + if us != eu: + shown.append("us" if us else "eu") + if len(shown) >= 2 and len(set(shown)) == 1: + return shown[0] + if ( + not shown + and CONVENTION_LANGUAGE.search(text or "") + and (not CONVENTION_LANGUAGE_US.search(text or "")) + ): + return "eu" + return None + + +@functools.lru_cache(maxsize=256) +def _numeral_candidates(text: str) -> tuple[tuple[list[Decimal], int, int, bool], ...]: + found: list[tuple[list[Decimal], int, int, bool]] = [] + for run in NUMERAL_RUN.finditer(text or ""): + (begin, finish) = (run.start(), run.end()) + cuts = [ + index + for index in range(begin, finish) + if text[index] in NUMERAL_SEPARATORS and index > begin + ] + starts = [begin] + [index + 1 for index in cuts] + ends = cuts + [finish] + convention = numeral_convention(text) + scale = NUMERAL_SCALE.match(text, finish) + exponent = ( + None if scale is None else NUMERAL_SCALES[scale.group(1) or scale.group(2)] + ) + run_found = [] + for start in starts: + for end in ends[bisect_left(ends, start + 1) :]: + if end - start > NUMERAL_SPAN: + break + values = numeral_values(text[start:end], convention) + if not values: + continue + if end == finish and exponent is not None: + scaled = [value.scaleb(exponent) for value in values] + run_found.append( + ( + scaled + [value for value in values if value not in scaled], + start, + cast(re.Match[str], scale).end(), + ) + ) + else: + run_found.append((values, start, end)) + by_start: dict[int, tuple[list[Decimal], int, int]] = {} + for item in run_found: + if item[1] not in by_start or item[2] > by_start[item[1]][2]: + by_start[item[1]] = item + (position, whole) = (begin, set()) + while True: + longest = by_start.get(position) + following = bisect_left( + starts, position + 1 if longest is None else longest[2] + ) + if longest is not None: + whole.add((longest[1], longest[2])) + if following >= len(starts): + break + position = starts[following] + found.extend( + ( + (values, start, end, (start, end) in whole) + for (values, start, end) in run_found + ) + ) + return tuple(found) + + +def read_numerals( + text: str, every: bool = False +) -> list[tuple[list[Decimal], int, int]]: + found = [ + (values, start, end) + for (values, start, end, whole) in _numeral_candidates(text) + if every or whole + ] + return sorted(found, key=lambda item: (item[1], -item[2])) + + +def numeral_spans(text: str, value: Any, every: bool = True) -> list[tuple[int, int]]: + try: + target = abs(Decimal(str(value))) + except ArithmeticError: + return [] + candidates = _numeral_candidates(text) + whole = [ + (start, end) + for (values, start, end, longest) in candidates + if longest and target in values + ] + if whole or not every: + return sorted(whole) + return sorted( + {(start, end) for (values, start, end, _) in candidates if target in values}, + key=lambda item: (item[0], -item[1]), + ) + + +def slot_token_id(tokenizer: Any) -> int: + ids = _tokens(tokenizer, SLOT_TEXT) + if len(ids) != 1: + raise ValueError(f"answer slots need {SLOT_TEXT!r} to be one token") + return ids[0] + + +def slot_block( + tokenizer: Any, + name: str, + spec: dict[str, Any], + complete_field: bool = False, + template_variant: int = 0, +) -> list[int]: + count = SLOT_COUNTS.get(head_kind(spec)) + layout = prompt_layout(template_variant, complete_field) + if count is None: + return [] + cue = _tokens(tokenizer, "\n" + name + ":") + return ( + _tokens(tokenizer, layout.slot_header) + + (cue if complete_field else cue[:SLOT_CUE_TOKENS]) + + [slot_token_id(tokenizer)] * count + ) + + +def system_prefix(record: dict[str, Any], text_answer_slots: bool = False) -> str: + return ( + EVIDENCE_SYSTEM_PREFIX + if evidence_construction(record, text_answer_slots) + else SYSTEM_PREFIX + ) + + +def evidence_construction( + record: dict[str, Any], text_answer_slots: bool = False +) -> bool: + return ( + text_answer_slots + or (record.get("_meta") or {}).get("construction") == zs.EVIDENCE_CONSTRUCTION + ) + + +def encode_record( + tokenizer: Any, + record: dict[str, Any], + max_length: int = 8192, + media: EncodedMedia | None = None, + commits: dict[str, Commit] | None = None, + answer_slots: bool = False, + text_answer_slots: bool = False, + template_variant: int = 0, +) -> EncodedRecord | None: + if text_answer_slots and (not answer_slots): + raise ValueError("text_answer_slots needs answer_slots") + commits = commits or {} + if record.get("images") and media is None: + raise ValueError("records with images need their encoded media") + media = media or EncodedMedia([]) + items = list(record["fields"].items()) + complete_field = evidence_construction(record, text_answer_slots) + layout = prompt_layout(template_variant, complete_field) + ids = _tokens(tokenizer, system_prefix(record, text_answer_slots)) + for name, spec in items: + ids.extend(_tokens(tokenizer, layout.schema_header)) + ids.extend(_tokens(tokenizer, schema_line(name, spec, complete_field))) + ids.extend(_tokens(tokenizer, reference_line(record))) + ids.extend(_tokens(tokenizer, USER_PREFIX)) + media_start = len(ids) + ids.extend(media.ids) + input_start = len(ids) + suffix = _tokens(tokenizer, ASSISTANT_SUFFIX) + blocks = [ + query_block( + tokenizer, name, spec, commits.get(name), complete_field, template_variant + ) + for (name, spec) in items + ] + slot_blocks: dict[int, list[int]] = {} + if answer_slots and (media.ids or text_answer_slots): + for index, (name, spec) in enumerate(items): + if not (name in commits and commits[name].done): + block = slot_block( + tokenizer, name, spec, complete_field, template_variant + ) + if block: + slot_blocks[index] = block + budget = ( + max_length + - len(ids) + - media.expansion + - len(suffix) + - sum((len(block[0]) for block in blocks)) + - sum((len(block) for block in slot_blocks.values())) + ) + if budget < MIN_INPUT_TOKENS: + return None + text = record["input"] + if media.ids and (not text.strip()): + text = "" + (input_ids, offsets) = (_tokens(tokenizer, "\n"), [(0, 0)]) + else: + (input_ids, offsets) = _input_tokens(tokenizer, text) + if len(input_ids) > budget: + keep = clean_cut(text, offsets, budget) + text = text[: offsets[keep - 1][1]] + (input_ids, offsets) = _input_tokens(tokenizer, text) + if len(input_ids) > budget: + (input_ids, offsets) = (input_ids[:budget], offsets[:budget]) + text = text[: offsets[-1][1]] + if not input_ids: + return None + ids.extend(input_ids) + ids.extend(suffix) + order = list(range(len(items))) + done = { + index + for (index, (name, _)) in enumerate(items) + if name in commits and commits[name].done + } + if done: + order = [index for index in order if index in done] + [ + index for index in order if index not in done + ] + queries: dict[int, int] = {} + slots: dict[int, list[int]] = {} + for index in order: + (block, query, item_offsets) = blocks[index] + start = len(ids) + ids.extend(block) + queries[index] = start + query + slots[index] = [start + offset for offset in item_offsets] + answer_positions: dict[int, list[int]] = {} + for index in order: + if index in slot_blocks: + ids.extend(slot_blocks[index]) + count = SLOT_COUNTS[head_kind(items[index][1])] + answer_positions[index] = list(range(len(ids) - count, len(ids))) + fields: list[EncodedField] = [] + for index, (name, spec) in enumerate(items): + if index in done: + continue + encoded = EncodedField(name, head_kind(spec), queries[index], slots[index]) + if index in answer_positions: + encoded.answer_slots = answer_positions[index] + commit = commits.get(name) + if ( + commit is not None + and commit.prefix is not None + and (encoded.kind in ("number", "estimate")) + ): + encoded.prefix = commit.prefix + if encoded.kind == "estimate": + encoded.window = prefix_window(spec, commit.prefix) + fields.append(encoded) + if not fields: + return None + record_media = None + if media.media is not None: + record_media = {**media.media, "token_offset": media_start} + return EncodedRecord( + ids, input_start, offsets, fields, text, record, record_media, media.expansion + ) + + +def span_base(text: str, starts: list[int], ends: list[int], first: int) -> int: + base = starts[first] + while base < ends[first] - 1 and text[base].isspace(): + base += 1 + return base + + +def trimmed_span( + text: str, + offsets: list[tuple[int, int]], + first: int, + last: int, + trims: list[list[int]] | None, +) -> tuple[int, int]: + (char_start, char_end) = (offsets[first][0], offsets[last][1]) + while char_start < offsets[first][1] - 1 and text[char_start].isspace(): + char_start += 1 + if trims is not None: + start = max( + char_start, min(char_start + int(trims[first][0]), offsets[first][1] - 1) + ) + end = min(char_end, max(char_end - int(trims[last][1]), offsets[last][0] + 1)) + if start < end: + (char_start, char_end) = (start, end) + while char_start < char_end - 1 and text[char_start].isspace(): + char_start += 1 + return (char_start, char_end) + + +def plain_number(value: Any) -> str: + number = Decimal(str(value)) + if not number.is_finite(): + raise ValueError("A plain number must be finite.") + text = format(number, "f") + return text.rstrip("0").rstrip(".") if "." in text else text + + +def prefix_window( + spec: dict[str, Any], prefix: tuple[int, int, tuple[int, ...]], bins: int = BINS +) -> tuple[int, int] | None: + (sign, exponent_class, digits) = prefix + power = exponent_class - 1 + MIN_EXPONENT + unit = Decimal(1).scaleb(power - len(digits) + 1) + start = Decimal(int("".join((str(digit) for digit in digits)))) * unit + (first, last) = (float(start), float(start + unit)) + if sign: + (first, last) = (-last, -first) + (low, high, scale) = zs.estimate_range(spec) + (lower, upper) = sorted( + (zs.raw_unit(first, low, high, scale), zs.raw_unit(last, low, high, scale)) + ) + if upper <= 0.0 or lower >= 1.0: + return None + first_bin = max(0, min(bins - 1, math.floor(lower * bins))) + last_bin = max(first_bin, min(bins - 1, math.ceil(upper * bins) - 1)) + return (first_bin, last_bin) + + +def span_scores( + start: torch.Tensor, end: torch.Tensor, length_bias: torch.Tensor +) -> torch.Tensor: + width = length_bias.shape[0] + padded = functional.pad(end, (0, width - 1), value=float("-inf")) + return start.unsqueeze(-1) + padded.unfold(-1, width, 1) + length_bias + + +DIGIT_SLOTS = NUMBER_DIGITS + YEAR_DIGITS + 2 + 2 + 2 + 2 +(YEAR_SLOT, DAY_SLOT, HOUR_SLOT, MINUTE_SLOT, SECOND_SLOT) = ( + NUMBER_DIGITS, + NUMBER_DIGITS + YEAR_DIGITS, + NUMBER_DIGITS + YEAR_DIGITS + 2, + NUMBER_DIGITS + YEAR_DIGITS + 4, + NUMBER_DIGITS + YEAR_DIGITS + 6, +) +MONTH_WORDS = tuple((" " + name for name in calendar.month_name[1:])) +SIGN_WORDS = (" ", " -") +ROUTE_KINDS = ( + "text", + "tags", + "number", + "date", + "time", + "estimate", + "number_list", + "date_list", +) + + +def two_digit_index(values: range) -> torch.Tensor: + return torch.tensor( + [[value // 10 for value in values], [value % 10 for value in values]], + dtype=torch.long, + ) + + +def compose_two_digits(slots: torch.Tensor, index: torch.Tensor) -> torch.Tensor: + return slots[..., 0, :].index_select(-1, index[0]) + slots[..., 1, :].index_select( + -1, index[1] + ) + + +def compose_years(slots: torch.Tensor, index: torch.Tensor) -> torch.Tensor: + inside = cast( + torch.Tensor, + sum( + ( + slots[..., position, :].index_select(-1, index[position]) + for position in range(YEAR_DIGITS) + ) + ), + ) + total = torch.logsumexp(slots, dim=-1).sum(dim=-1) + gap = (torch.logsumexp(inside, dim=-1) - total).clamp(max=-1e-06) + outside = total + torch.log(-torch.expm1(gap)) + return torch.cat([inside, outside.unsqueeze(-1)], dim=-1) + + +class ClairicHead(nn.Module): + digit_rows: torch.Tensor + month_rows: torch.Tensor + sign_rows: torch.Tensor + year_index: torch.Tensor + day_index: torch.Tensor + hour_index: torch.Tensor + sixty_index: torch.Tensor + + def __init__( + self, + hidden_size: int, + bins: int = BINS, + pointer_width: int = POINTER_WIDTH, + readout: str = "linear", + route_width: int = 1024, + route_layers: int = 2, + route_heads: int = 8, + slot_rank: int = 32, + estimate_lexical: bool = False, + span_trim: bool = False, + loop_feedback: bool = False, + loop_scale: float = 1.0, + number_groups: bool = False, + number_pointer: bool = False, + answer_slots: bool = False, + answer_slot_routing: bool = False, + ) -> None: + super().__init__() + if readout not in ("linear", "lexical"): + raise ValueError(f"unknown readout {readout!r}") + if (number_groups or number_pointer) and readout != "lexical": + raise ValueError( + "number_groups and number_pointer need the lexical readout" + ) + if estimate_lexical and readout != "lexical": + raise ValueError("estimate_lexical needs the lexical readout") + if answer_slot_routing and (readout != "lexical" or not answer_slots): + raise ValueError( + "answer_slot_routing needs lexical readout and answer_slots" + ) + self.config: dict[str, Any] = { + "hidden_size": hidden_size, + "bins": bins, + "pointer_width": pointer_width, + } + if readout == "lexical": + self.config.update( + readout=readout, + route_width=route_width, + route_layers=route_layers, + route_heads=route_heads, + slot_rank=slot_rank, + ) + if estimate_lexical: + self.config.update(estimate_lexical=True) + if span_trim: + self.config.update(span_trim=True) + if loop_feedback: + self.config.update(loop_feedback=True, loop_scale=float(loop_scale)) + self.number_groups = False + self.number_pointer = False + self.answer_slots = bool(answer_slots) + if answer_slots: + self.config["answer_slots"] = True + self.answer_slot_routing = bool(answer_slot_routing) + if answer_slot_routing: + self.config["answer_slot_routing"] = True + self.slot_route_scale = nn.Parameter(torch.ones(len(ROUTE_KINDS))) + self.readout = readout + self.loop_feedback = loop_feedback + self.loop_scale = float(loop_scale) + self.span_trim = span_trim + (self.estimate_lexical,) = (estimate_lexical,) + (self.pointer_width,) = (pointer_width,) + self.norm = nn.LayerNorm(hidden_size) + self.token_norm = nn.LayerNorm(hidden_size) + self.null = nn.Linear(hidden_size, 1) + self.span_query = nn.Linear(hidden_size, 2 * pointer_width) + self.span_key = nn.Linear(hidden_size, 2 * pointer_width) + self.length_bias = nn.Parameter( + -0.05 * torch.arange(MAX_SPAN, dtype=torch.float32) + ) + self.tag_query = nn.Linear(hidden_size, 2 * pointer_width) + self.tag_key = nn.Linear(hidden_size, pointer_width) + self.tag_bias = nn.Parameter(torch.tensor([-2.0, -2.0])) + if span_trim: + self.trim = nn.Linear(hidden_size, 2 * TRIM_CLASSES) + nn.init.zeros_(self.trim.weight) + with torch.no_grad(): + self.trim.bias.copy_( + torch.tensor(([TRIM_INIT] + [0.0] * (TRIM_CLASSES - 1)) * 2) + ) + if readout == "linear": + self.number = nn.Linear(hidden_size, NUMBER_WIDTH) + self.date = nn.Linear(hidden_size, DATE_WIDTH) + self.time = nn.Linear(hidden_size, TIME_WIDTH) + else: + self.route_in = nn.Linear(hidden_size, route_width, bias=False) + self.kind_embedding = nn.Embedding( + len(ROUTE_KINDS) + len(LIST_KINDS), route_width + ) + self.memory_in = nn.Linear(hidden_size, route_width, bias=False) + self.route_layers = nn.ModuleList( + [ + nn.TransformerDecoderLayer( + d_model=route_width, + nhead=route_heads, + dim_feedforward=4 * route_width, + dropout=0.0, + activation="gelu", + batch_first=True, + norm_first=True, + ) + for _ in range(route_layers) + ] + ) + self.route_norm = nn.LayerNorm(route_width) + self.route_out = nn.Linear(route_width, hidden_size) + nn.init.zeros_(self.route_out.weight) + nn.init.zeros_(self.route_out.bias) + self.route_gate = nn.Parameter(torch.tensor(-2.0)) + self.register_buffer( + "digit_rows", torch.randn(10, hidden_size) / math.sqrt(hidden_size) + ) + self.register_buffer( + "month_rows", torch.randn(12, hidden_size) / math.sqrt(hidden_size) + ) + self.register_buffer( + "sign_rows", torch.randn(2, hidden_size) / math.sqrt(hidden_size) + ) + self.lexical_scale = nn.Parameter(torch.full((3,), math.log(10.0))) + self.slot_down = nn.Linear( + hidden_size, (DIGIT_SLOTS + 2) * slot_rank, bias=False + ) + self.slot_up = nn.Parameter(torch.zeros(DIGIT_SLOTS, slot_rank, 10)) + self.month_up = nn.Parameter(torch.zeros(slot_rank, 12)) + self.sign_up = nn.Parameter(torch.zeros(slot_rank, 2)) + self.slot_residual = nn.Linear(route_width, DIGIT_SLOTS * 10 + 12 + 2) + nn.init.zeros_(self.slot_residual.weight) + nn.init.zeros_(self.slot_residual.bias) + self.residual_gate = nn.Parameter(torch.tensor(-2.0)) + self.exponent = nn.Linear(hidden_size, EXPONENT_CLASSES) + self.slot_rank = slot_rank + self.register_buffer( + "year_index", + torch.tensor( + [ + [ + int(f"{year:04d}"[position]) + for year in range(YEAR_LOW, YEAR_HIGH + 1) + ] + for position in range(YEAR_DIGITS) + ], + dtype=torch.long, + ), + persistent=False, + ) + self.register_buffer( + "day_index", two_digit_index(range(1, 32)), persistent=False + ) + self.register_buffer( + "hour_index", two_digit_index(range(24)), persistent=False + ) + self.register_buffer( + "sixty_index", two_digit_index(range(60)), persistent=False + ) + self.item_end = nn.Linear(hidden_size, 1) + self.bins = nn.Linear(hidden_size, bins) + self.offset = nn.Linear(hidden_size, bins) + self.overflow = nn.Linear(hidden_size, 2) + nn.init.zeros_(self.overflow.weight) + nn.init.constant_(self.overflow.bias, OVERFLOW_INIT) + nn.init.zeros_(self.offset.weight) + nn.init.zeros_(self.offset.bias) + if estimate_lexical: + self.estimate_gate = nn.Linear(hidden_size, 1) + nn.init.zeros_(self.estimate_gate.weight) + nn.init.constant_(self.estimate_gate.bias, ESTIMATE_GATE_INIT) + self.estimate_spread = nn.Linear(hidden_size, 1 + LEXICAL_DIGITS) + nn.init.zeros_(self.estimate_spread.weight) + nn.init.zeros_(self.estimate_spread.bias) + if loop_feedback: + self.loop_norm = nn.LayerNorm(hidden_size) + self.loop_down = nn.Linear(hidden_size, LOOP_RANK, bias=False) + self.loop_up = nn.Linear(LOOP_RANK, hidden_size, bias=False) + nn.init.zeros_(self.loop_up.weight) + if number_groups: + self.enable_number_groups() + if number_pointer: + self.enable_number_pointer() + edges = torch.linspace(0.0, 1.0, bins + 1) + self.register_buffer("bin_edges", edges, persistent=False) + + def enable_number_groups(self) -> None: + if self.readout != "lexical": + raise ValueError("number_groups needs the lexical readout") + if self.number_groups: + return + hidden_size = self.config["hidden_size"] + device = self.norm.weight.device + self.number_group_down = nn.Linear(hidden_size, GROUP_RANK).to(device) + self.number_group_up = nn.Linear(GROUP_RANK, NUMBER_GROUPS * GROUP_CLASSES).to( + device + ) + nn.init.zeros_(self.number_group_up.weight) + nn.init.zeros_(self.number_group_up.bias) + (self.number_groups,) = (True,) + self.config.update(number_groups=True) + + def enable_number_pointer(self) -> None: + if self.readout != "lexical": + raise ValueError("number_pointer needs the lexical readout") + if self.number_pointer: + return + device = self.norm.weight.device + self.copy_query = nn.Linear( + self.config["hidden_size"], 2 * self.pointer_width + ).to(device) + self.copy_gate = nn.Linear(self.config["hidden_size"], 1).to(device) + with torch.no_grad(): + self.copy_query.weight.copy_(self.span_query.weight) + self.copy_query.bias.copy_(self.span_query.bias) + self.copy_gate.weight.zero_() + self.copy_gate.bias.fill_(COPY_GATE_INIT) + (self.number_pointer,) = (True,) + self.config.update(number_pointer=True) + + def group_scores(self, states: torch.Tensor) -> torch.Tensor: + return self.number_group_up( + functional.gelu(self.number_group_down(states.float())) + ).view(-1, NUMBER_GROUPS, GROUP_CLASSES) + + def loop_input(self, states: torch.Tensor) -> torch.Tensor: + with torch.autocast(device_type=states.device.type, enabled=False): + return self.loop_scale * self.loop_up( + self.loop_down(self.loop_norm(states.float())) + ) + + @torch.no_grad() + def set_lexical_rows(self, tokenizer: Any, output_embeddings: torch.Tensor) -> None: + if self.readout != "lexical": + return + for name, words in ( + ("digit_rows", [str(digit) for digit in range(10)]), + ("month_rows", MONTH_WORDS), + ("sign_rows", SIGN_WORDS), + ): + rows = [] + for word in words: + ids = tokenizer(word, add_special_tokens=False)["input_ids"] + if not ids: + raise ValueError(f"tokenizer has no tokens for {word!r}") + rows.append( + output_embeddings[ + torch.tensor(ids, device=output_embeddings.device) + ] + .float() + .mean(dim=0) + ) + getattr(self, name).copy_(torch.stack(rows).to(getattr(self, name).device)) + + def route( + self, + hidden: torch.Tensor, + positions: torch.Tensor, + kinds: torch.Tensor, + stop: int, + ) -> tuple[torch.Tensor, torch.Tensor]: + queries = ( + self.route_in(self.norm(hidden[positions].float())) + + self.kind_embedding(kinds) + ).unsqueeze(0) + memory = self.memory_in(self.token_norm(hidden[:stop].float())).unsqueeze(0) + for layer in self.route_layers: + queries = layer(queries, memory) + routed = self.route_norm(queries[0]) + return (torch.sigmoid(self.route_gate) * self.route_out(routed), routed) + + def answer_slot_states( + self, + hidden: torch.Tensor, + field_queries: torch.Tensor, + kinds: torch.Tensor, + positions: torch.Tensor, + lengths: torch.Tensor, + stop: int, + chunk: int = 256, + ) -> torch.Tensor: + if not self.answer_slot_routing: + return hidden[positions] + if ( + chunk < 1 + or hidden.ndim != 2 + or field_queries.ndim != 1 + or (positions.ndim != 2) + or (positions.shape[0] != field_queries.numel()) + ): + raise ValueError("invalid answer-slot routing layout") + width = positions.shape[1] + if kinds.shape != field_queries.shape or lengths.shape != field_queries.shape: + raise ValueError("answer-slot fields, kinds and lengths must align") + if ( + width < 1 + or bool(((lengths < 1) | (lengths > width)).any()) + or (not 0 < stop <= hidden.shape[0]) + ): + raise ValueError("invalid answer-slot lengths or evidence boundary") + if ( + bool(((positions < 0) | (positions >= hidden.shape[0])).any()) + or bool(((field_queries < 0) | (field_queries >= hidden.shape[0])).any()) + or bool(((kinds < 0) | (kinds >= len(ROUTE_KINDS))).any()) + ): + raise ValueError("answer-slot query, token or kind out of range") + raw = hidden[positions] + if not positions.shape[0]: + return raw + output = [] + with torch.autocast(device_type=hidden.device.type, enabled=False): + memory = self.memory_in(self.token_norm(hidden[:stop].float())).unsqueeze(0) + + def route_chunk( + query_positions: torch.Tensor, + query_kinds: torch.Tensor, + padding: torch.Tensor, + ) -> torch.Tensor: + queries = self.route_in(self.norm(hidden[query_positions].float())) + queries = queries + self.kind_embedding(query_kinds)[:, None, :] + for layer in self.route_layers: + queries = layer( + queries, + memory.expand(queries.shape[0], -1, -1), + tgt_key_padding_mask=padding, + ) + return self.route_out(self.route_norm(queries[:, 1:])) + + field_batch = max(1, min(chunk // (width + 1), 262144 // stop)) + for first in range(0, positions.shape[0], field_batch): + last = first + field_batch + rows = positions[first:last] + query_positions = torch.cat( + [field_queries[first:last, None], rows], dim=1 + ) + padding = ( + torch.arange(width + 1, device=hidden.device)[None, :] + > lengths[first:last, None] + ) + evidence = route_chunk(query_positions, kinds[first:last], padding) + scale = self.slot_route_scale[kinds[first:last]].tanh()[:, None, None] + corrected = ( + raw[first:last].float() + + scale * torch.sigmoid(self.route_gate) * evidence + ) + indices = torch.minimum( + torch.arange(width, device=hidden.device)[None, :], + lengths[first:last, None] - 1, + ) + output.append( + corrected.gather( + 1, indices[..., None].expand(-1, -1, corrected.shape[-1]) + ) + ) + return torch.cat(output).to(hidden.dtype) + + def typed_logits( + self, raw: torch.Tensor, routed: torch.Tensor + ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + count = raw.shape[0] + scale = self.lexical_scale.clamp(max=math.log(100.0)).exp() + unit = functional.normalize(raw, dim=-1) + down = self.slot_down(raw).view(count, DIGIT_SLOTS + 2, self.slot_rank) + digit_prior = ( + unit @ functional.normalize(self.digit_rows, dim=-1).t() + ).unsqueeze(1) + month_prior = unit @ functional.normalize(self.month_rows, dim=-1).t() + sign_prior = unit @ functional.normalize(self.sign_rows, dim=-1).t() + residual = torch.sigmoid(self.residual_gate) * self.slot_residual(routed) + (digit_residual, month_residual, sign_residual) = residual.split( + (DIGIT_SLOTS * 10, 12, 2), dim=-1 + ) + return ( + scale[0] * digit_prior + + torch.einsum("nsr,srd->nsd", down[:, :DIGIT_SLOTS], self.slot_up) + + digit_residual.view(count, DIGIT_SLOTS, 10), + scale[1] * month_prior + + down[:, DIGIT_SLOTS] @ self.month_up + + month_residual, + scale[2] * sign_prior + + down[:, DIGIT_SLOTS + 1] @ self.sign_up + + sign_residual, + ) + + def typed_outputs( + self, raw: torch.Tensor, states: torch.Tensor, routed: torch.Tensor + ) -> dict[str, torch.Tensor]: + (digits, months, signs) = self.typed_logits(raw, routed) + year_slots = digits[:, YEAR_SLOT : YEAR_SLOT + YEAR_DIGITS] + return { + "number": torch.cat( + [signs, self.exponent(states), digits[:, :NUMBER_DIGITS].flatten(1)], + dim=-1, + ), + "date": torch.cat( + [ + year_slots.flatten(1), + compose_years(year_slots, self.year_index), + months, + compose_two_digits( + digits[:, DAY_SLOT : DAY_SLOT + 2], self.day_index + ), + ], + dim=-1, + ), + "time": torch.cat( + [ + compose_two_digits( + digits[:, HOUR_SLOT : HOUR_SLOT + 2], self.hour_index + ), + compose_two_digits( + digits[:, MINUTE_SLOT : MINUTE_SLOT + 2], self.sixty_index + ), + compose_two_digits( + digits[:, SECOND_SLOT : SECOND_SLOT + 2], self.sixty_index + ), + ], + dim=-1, + ), + } + + def forward( + self, hidden: torch.Tensor, batch: dict[str, Any] + ) -> list[dict[str, Any]]: + return [ + self.forward_record(hidden[index], record, batch["views"][index]) + for (index, record) in enumerate(batch["records"]) + ] + + def forward_record( + self, + hidden: torch.Tensor, + record: EncodedRecord, + views: dict[str, torch.Tensor], + ) -> dict[str, Any]: + return self.forward_states( + hidden, record.input_start, record.input_start + len(record.offsets), views + ) + + def forward_states( + self, + hidden: torch.Tensor, + start: int, + stop: int, + views: dict[str, torch.Tensor], + ) -> dict[str, Any]: + if self.readout == "lexical": + return self.forward_lexical(hidden, start, stop, views) + with torch.autocast(device_type=hidden.device.type, enabled=False): + states = self.norm(hidden[views["query_positions"]].float()) + output: dict[str, Any] = {"null": self.null(states).squeeze(-1)} + tokens = None + if views["text_rows"].shape[0] or views["tags_rows"].shape[0]: + tokens = self.token_norm(hidden[start:stop].float()) + if self.span_trim: + output["trim"] = self.trim(tokens).view(-1, 2, TRIM_CLASSES) + scale = 1.0 / math.sqrt(self.pointer_width) + if views["text_rows"].shape[0]: + (query_start, query_end) = self.span_query( + states[views["text_rows"]] + ).split(self.pointer_width, dim=-1) + (key_start, key_end) = self.span_key(tokens).split( + self.pointer_width, dim=-1 + ) + output["text"] = span_scores( + query_start @ key_start.transpose(0, 1) * scale, + query_end @ key_end.transpose(0, 1) * scale, + self.length_bias, + ) + if views["tags_rows"].shape[0]: + (query_begin, query_inside) = self.tag_query( + states[views["tags_rows"]] + ).split(self.pointer_width, dim=-1) + keys = self.tag_key(tokens) + begin = query_begin @ keys.transpose(0, 1) * scale + self.tag_bias[0] + inside = query_inside @ keys.transpose(0, 1) * scale + self.tag_bias[1] + output["tags"] = torch.stack( + [torch.zeros_like(begin), begin, inside], dim=-1 + ) + for group in ("number", "date", "time"): + if views[f"{group}_rows"].shape[0]: + output[group] = getattr(self, group)(states[views[f"{group}_rows"]]) + if "number" in output: + output["number"] = apply_prefix(output["number"], views, "number") + if views["estimate_rows"].shape[0]: + estimate_states = states[views["estimate_rows"]] + output["estimate"] = self.bins(estimate_states) + output["estimate_offset"] = self.offset(estimate_states) + output["estimate_overflow"] = self.overflow(estimate_states) + for kind, readout in ( + ("number_list", self.number), + ("date_list", self.date), + ): + if views[f"{kind}_rows"].shape[0]: + items = self.norm(hidden[views[f"{kind}_items"]].float()) + output[f"{kind}_end"] = self.item_end(items).squeeze(-1) + output[kind] = readout(items) + return output + + def forward_lexical( + self, + hidden: torch.Tensor, + start: int, + stop: int, + views: dict[str, torch.Tensor], + ) -> dict[str, Any]: + with torch.autocast(device_type=hidden.device.type, enabled=False): + queries = views["query_positions"] + item_kinds = [kind for kind in LIST_KINDS if views[f"{kind}_rows"].shape[0]] + positions = torch.cat( + [queries] + [views[f"{kind}_items"].flatten() for kind in item_kinds] + ) + kinds = torch.zeros( + positions.shape[0], dtype=torch.long, device=positions.device + ) + for index, group in enumerate(ROUTE_KINDS): + kinds[views[f"{group}_rows"]] = index + offset = queries.shape[0] + for kind in item_kinds: + size = views[f"{kind}_items"].numel() + kinds[offset : offset + size] = len(ROUTE_KINDS) + LIST_KINDS.index( + kind + ) + offset += size + (gated, routed) = self.route(hidden, positions, kinds, stop) + raw = hidden[positions].float() + states = self.norm(raw) + gated + count = queries.shape[0] + output: dict[str, Any] = {"null": self.null(states[:count]).squeeze(-1)} + (field_states, field_raw, field_routed) = ( + states[:count], + raw[:count], + routed[:count], + ) + tokens = None + pointed = self.number_pointer and views["number_rows"].shape[0] > 0 + if views["text_rows"].shape[0] or views["tags_rows"].shape[0] or pointed: + tokens = self.token_norm(hidden[start:stop].float()) + if self.span_trim and ( + views["text_rows"].shape[0] or views["tags_rows"].shape[0] + ): + output["trim"] = self.trim(tokens).view(-1, 2, TRIM_CLASSES) + scale = 1.0 / math.sqrt(self.pointer_width) + if pointed: + number_states = field_states[views["number_rows"]] + (query_start, query_end) = self.copy_query(number_states).split( + self.pointer_width, dim=-1 + ) + (key_start, key_end) = self.span_key(tokens).split( + self.pointer_width, dim=-1 + ) + output["number_copy"] = span_scores( + query_start @ key_start.transpose(0, 1) * scale, + query_end @ key_end.transpose(0, 1) * scale, + self.length_bias, + ) + output["number_copy_gate"] = self.copy_gate(number_states).squeeze(-1) + if views["text_rows"].shape[0]: + (query_start, query_end) = self.span_query( + field_states[views["text_rows"]] + ).split(self.pointer_width, dim=-1) + (key_start, key_end) = self.span_key(tokens).split( + self.pointer_width, dim=-1 + ) + output["text"] = span_scores( + query_start @ key_start.transpose(0, 1) * scale, + query_end @ key_end.transpose(0, 1) * scale, + self.length_bias, + ) + if views["tags_rows"].shape[0]: + (query_begin, query_inside) = self.tag_query( + field_states[views["tags_rows"]] + ).split(self.pointer_width, dim=-1) + keys = self.tag_key(tokens) + begin = query_begin @ keys.transpose(0, 1) * scale + self.tag_bias[0] + inside = query_inside @ keys.transpose(0, 1) * scale + self.tag_bias[1] + output["tags"] = torch.stack( + [torch.zeros_like(begin), begin, inside], dim=-1 + ) + typed_rows = [ + views[f"{group}_rows"] for group in ("number", "date", "time") + ] + if any((rows.shape[0] for rows in typed_rows)): + selected = torch.cat(typed_rows) + typed = self.typed_outputs( + field_raw[selected], field_states[selected], field_routed[selected] + ) + offset = 0 + for group, rows in zip(("number", "date", "time"), typed_rows): + if rows.shape[0]: + output[group] = typed[group][offset : offset + rows.shape[0]] + offset += rows.shape[0] + if "number" in output: + output["number"] = apply_prefix(output["number"], views, "number") + if self.number_groups: + output["number_groups"] = self.group_scores( + field_states[views["number_rows"]] + ) + if views["estimate_rows"].shape[0]: + estimate_rows = views["estimate_rows"] + estimate_states = field_states[estimate_rows] + output["estimate"] = self.bins(estimate_states) + output["estimate_offset"] = self.offset(estimate_states) + output["estimate_overflow"] = self.overflow(estimate_states) + if self.estimate_lexical: + number_logits = self.typed_outputs( + field_raw[estimate_rows], + estimate_states, + field_routed[estimate_rows], + )["number"] + output["estimate_number_raw"] = apply_prefix( + number_logits, views, "estimate" + ) + output["estimate_number"] = apply_prefix( + temper_number( + number_logits, self.estimate_spread(estimate_states) + ), + views, + "estimate", + ) + output["estimate_gate"] = self.estimate_gate( + estimate_states + ).squeeze(-1) + offset = count + for kind in item_kinds: + shape = views[f"{kind}_items"].shape + size = shape[0] * shape[1] + (item_states, item_raw, item_routed) = ( + states[offset : offset + size], + raw[offset : offset + size], + routed[offset : offset + size], + ) + offset += size + output[f"{kind}_end"] = ( + self.item_end(item_states).squeeze(-1).view(shape) + ) + group = "number" if kind == "number_list" else "date" + output[kind] = self.typed_outputs(item_raw, item_states, item_routed)[ + group + ].view(*shape, -1) + if kind == "number_list" and self.number_groups: + output["number_list_groups"] = self.group_scores(item_states).view( + *shape, NUMBER_GROUPS, GROUP_CLASSES + ) + return output + + +SPELL_END = "\n" +SPELL_SEPARATOR = " | " +SPELL_MIN_CONFIDENCE = 0.3 + + +def spelled_answer( + ids: list[int], log_probs: list[float], tokenizer: Any, end_id: int +) -> tuple[str, float, list[tuple[str, float]]]: + stop = ids.index(end_id) if end_id in ids else len(ids) + pieces = [tokenizer.decode([token]) for token in ids[:stop]] + items: list[tuple[list[int], list[float]]] = [] + current: list[int] = [] + scores: list[float] = [] + for token, piece, score in zip(ids[:stop], pieces, log_probs[:stop]): + if piece.strip() == SPELL_SEPARATOR.strip(): + items.append((current, scores)) + (current, scores) = ([], []) + else: + current.append(token) + scores.append(score) + items.append((current, scores)) + parts = [] + for item_ids, item_scores in items: + words = tokenizer.decode(item_ids).split() + if words: + parts.append( + (" ".join(words), math.exp(sum(item_scores) / len(item_scores))) + ) + text = SPELL_SEPARATOR.join((part for (part, _) in parts)) + return (text, math.exp(sum(log_probs[: min(stop + 1, len(ids))])), parts) + + +def evidence_text_answer( + prediction: dict[str, Any], + ids: list[int], + log_probs: list[float], + tokenizer: Any, + end_id: int, + text: str, + spec: dict[str, Any], + has_images: bool = False, +) -> dict[str, Any]: + if end_id not in ids: + return prediction + stop = ids.index(end_id) + value = tokenizer.decode(ids[:stop]).strip() + confidence = math.exp(sum(log_probs[: stop + 1])) + exact = ( + [ + span + for span in zs.find_spans(text, value) + if text[span[0] : span[1]] == value + ] + if value + else [] + ) + present = prediction["p_null"] < zs.null_threshold(spec) + return { + **prediction, + "guess": value or None, + "value": value if value and present else None, + "span": list(exact[0]) if exact else None, + "confidence": confidence, + "source": "input" if exact else "image" if has_images else "readout", + } + + +def _simple(value: Any) -> str: + return "".join( + (character for character in str(value).lower() if character.isalnum()) + ) + + +def _similarity(left: Any, right: Any) -> float: + import difflib + + (left, right) = (_simple(left), _simple(right)) + return difflib.SequenceMatcher(None, left, right).ratio() if left and right else 0.0 + + +def _state_match( + text: str, value: str, threshold: float = 0.8, max_words: int = 4000 +) -> tuple[int, int] | None: + if not text or not value: + return None + spans = zs.find_spans(text, value) + if spans: + return spans[0] + import re + + words = [match.span() for match in re.finditer("\\S+", text)][:max_words] + size = len(value.split()) + (best, best_span) = (threshold, None) + for width in range(max(1, size - 1), size + 2): + for first in range(max(0, len(words) - width + 1)): + (start, end) = (words[first][0], words[first + width - 1][1]) + score = _similarity(text[start:end].strip(".,;:()[]\"'"), value) + if score >= best: + (best, best_span) = (score, (start, end)) + return best_span + + +def _stutters(value: Any) -> bool: + import re + + simple = _simple(value) + return bool(re.search("(.)\\1\\1", simple) or re.search("([a-z]{4,})\\1", simple)) + + +def _digit_code(value: Any) -> bool: + simple = _simple(value) + return bool(simple) and 2 * sum( + (character.isdigit() for character in simple) + ) >= len(simple) + + +PLACEHOLDER_WORDS = frozenset( + ( + "a", + "above", + "an", + "app", + "attach", + "attached", + "attachment", + "below", + "doc", + "document", + "documents", + "file", + "heic", + "here", + "image", + "images", + "is", + "jpeg", + "jpg", + "mobile", + "page", + "pdf", + "photo", + "photos", + "pic", + "picture", + "please", + "png", + "scan", + "scanned", + "screenshot", + "see", + "the", + "this", + "upload", + "uploaded", + "via", + ) +) + + +def _placeholder_text(value: Any) -> bool: + import re + + words = re.findall("[a-z0-9]+", str(value or "").lower()) + return bool(words) and all((word in PLACEHOLDER_WORDS for word in words)) + + +def _defers(pointer_value: Any, spelled: str) -> bool: + if pointer_value is None: + return False + return _stutters(spelled) or ( + _digit_code(spelled) + and any((character.isdigit() for character in str(pointer_value))) + ) + + +def route_spelled( + prediction: dict[str, Any], + kind: str, + spelled: str, + items: list[tuple[str, float]], + text: str, + defer_lists: bool = False, +) -> dict[str, Any]: + if kind == "text": + if _placeholder_text(prediction.get("guess")): + prediction = { + **prediction, + "value": None, + "guess": None, + "span": None, + "confidence": 0.0, + } + items = [ + (item, score) for (item, score) in items if not _placeholder_text(item) + ] + if not items: + return prediction + (value, confidence) = ( + items[0][0] if len(items) == 1 else spelled, + min((score for (_, score) in items)), + ) + routed = {**prediction, "spell_confidence": confidence} + if _similarity(prediction.get("guess"), value) >= 0.6: + if ( + routed.get("value") is None + and prediction.get("guess") + and (confidence >= SPELL_MIN_CONFIDENCE) + ): + return {**routed, "value": prediction["guess"]} + return routed + span = _state_match(text, value) + if span is not None: + return { + **routed, + "value": text[span[0] : span[1]], + "guess": text[span[0] : span[1]], + "span": list(span), + } + if confidence >= SPELL_MIN_CONFIDENCE and ( + not _defers(prediction.get("value"), value) + ): + return { + **routed, + "value": value, + "guess": value, + "span": None, + "confidence": confidence, + "source": "image", + } + return routed + (pointer_values, pointer_spans) = ( + list(prediction.get("value") or []), + list(prediction.get("spans") or []), + ) + if any((_placeholder_text(value) for value in pointer_values)): + kept = [ + (value, pointer_spans[index] if index < len(pointer_spans) else None) + for (index, value) in enumerate(pointer_values) + if not _placeholder_text(value) + ] + (pointer_values, pointer_spans) = ( + [value for (value, _) in kept], + [span for (_, span) in kept], + ) + items = [(item, score) for (item, score) in items if not _placeholder_text(item)] + (values, spans, used) = ([], [], set()) + for item, confidence in items: + match = max( + range(len(pointer_values)), + key=lambda index: _similarity(pointer_values[index], item), + default=None, + ) + if ( + match is not None + and match not in used + and (_similarity(pointer_values[match], item) >= 0.6) + ): + used.add(match) + values.append(pointer_values[match]) + spans.append(pointer_spans[match] if match < len(pointer_spans) else None) + continue + span = _state_match(text, item) + if span is not None: + value = text[span[0] : span[1]] + if not any((_simple(value) == _simple(other) for other in values)): + values.append(value) + spans.append(list(span)) + elif ( + confidence >= SPELL_MIN_CONFIDENCE + and ( + not ( + pointer_values and (defer_lists or _defers(pointer_values[0], item)) + ) + ) + and (not any((_simple(item) == _simple(other) for other in values))) + ): + values.append(item) + spans.append(None) + for index, (value, span) in enumerate(zip(pointer_values, pointer_spans)): + if index not in used and ( + not any((_simple(value) == _simple(other) for other in values)) + ): + values.append(value) + spans.append(span) + routed = {**prediction, "value": values, "spans": spans} + if any((span is None for span in spans)): + routed["source"] = "image" + return routed + + +def number_parts( + logits: torch.Tensor, +) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + (sign, exponent, digits) = logits.split( + (2, EXPONENT_CLASSES, NUMBER_DIGITS * 10), dim=-1 + ) + return (sign, exponent, digits.unflatten(-1, (NUMBER_DIGITS, 10))) + + +def group_class_digits(device: Any = None) -> torch.Tensor: + classes = torch.arange(GROUP_CLASSES, device=device) + return torch.stack( + [ + classes // 10 ** (GROUP_DIGITS - 1 - place) % 10 + for place in range(GROUP_DIGITS) + ], + dim=-1, + ) + + +def group_slots(exponent_class: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]: + exponent_class = exponent_class.long() + device = exponent_class.device + power = (exponent_class - 1 + MIN_EXPONENT).view(-1, 1, 1) + groups = torch.arange(GROUP_LOW, GROUP_HIGH + 1, device=device).view(1, -1, 1) + places = torch.arange(GROUP_DIGITS - 1, -1, -1, device=device).view(1, 1, -1) + slots = power - (groups * GROUP_DIGITS + places) + inside = ( + (slots >= 0) & (slots < NUMBER_DIGITS) & (exponent_class > 0).view(-1, 1, 1) + ) + return ( + torch.where(inside, slots, torch.full_like(slots, NUMBER_DIGITS)), + inside.any(dim=-1), + ) + + +def number_group_logits( + digit_logits: torch.Tensor, + exponent_class: torch.Tensor, + learned: torch.Tensor | None = None, +) -> tuple[torch.Tensor, torch.Tensor]: + (count, device) = (digit_logits.shape[0], digit_logits.device) + leading_zero = torch.zeros(NUMBER_DIGITS, 10, dtype=torch.bool, device=device) + leading_zero[0, 0] = True + log = torch.log_softmax(digit_logits.float(), dim=-1).masked_fill( + leading_zero, MASKED + ) + held = torch.full((count, 1, 10), MASKED, device=device, dtype=log.dtype) + held[..., 0] = 0.0 + padded = torch.cat([log, held], dim=1) + (slots, groups) = group_slots(exponent_class.to(device)) + picked = padded.gather(1, slots.flatten(1).unsqueeze(-1).expand(-1, -1, 10)).view( + count, NUMBER_GROUPS, GROUP_DIGITS, 10 + ) + table = group_class_digits(device) + scores = cast( + torch.Tensor, + sum( + ( + picked[:, :, place, :].index_select(-1, table[:, place]) + for place in range(GROUP_DIGITS) + ) + ), + ) + if learned is not None: + scores = scores + learned.float() + return (scores, groups) + + +def temper_number(logits: torch.Tensor, spread: torch.Tensor) -> torch.Tensor: + (sign, exponent, digits) = number_parts(logits) + temperature = torch.exp( + ESTIMATE_SPREAD_LIMIT * torch.tanh(spread.float() / ESTIMATE_SPREAD_LIMIT) + ) + exponent = exponent / temperature[:, :1] + digits = torch.cat( + [ + digits[:, :LEXICAL_DIGITS] / temperature[:, 1:].unsqueeze(-1), + digits[:, LEXICAL_DIGITS:], + ], + dim=1, + ) + return torch.cat([sign, exponent, digits.flatten(1)], dim=-1) + + +def apply_prefix( + logits: torch.Tensor, views: dict[str, torch.Tensor], group: str +) -> torch.Tensor: + length = views.get(f"{group}_prefix_length") + if length is None or not logits.shape[0]: + return logits + device = logits.device + (sign_logits, exponent_logits, digit_logits) = number_parts(logits) + slots = torch.arange(NUMBER_DIGITS, device=device) + length = length.long() + relative = (slots.unsqueeze(0) - length.unsqueeze(1)).clamp_min(0) + shifted = digit_logits.gather(1, relative.unsqueeze(-1).expand(-1, -1, 10)) + forced_digits = torch.where( + torch.arange(10, device=device) + == views[f"{group}_prefix_digits"].long().unsqueeze(-1), + 0.0, + MASKED, + ).to(logits.dtype) + digit_logits = torch.where( + (slots.unsqueeze(0) < length.unsqueeze(1)).unsqueeze(-1), forced_digits, shifted + ) + written = (length > 0).unsqueeze(1) + forced_sign = torch.where( + torch.arange(2, device=device) + == views[f"{group}_prefix_sign"].long().unsqueeze(1), + 0.0, + MASKED, + ).to(logits.dtype) + forced_exponent = torch.where( + torch.arange(EXPONENT_CLASSES, device=device) + == views[f"{group}_prefix_exponent"].long().unsqueeze(1), + 0.0, + MASKED, + ).to(logits.dtype) + return torch.cat( + [ + torch.where(written, forced_sign, sign_logits), + torch.where(written, forced_exponent, exponent_logits), + digit_logits.flatten(1), + ], + dim=-1, + ) + + +def window_log_probs( + log_probs: torch.Tensor, window: torch.Tensor +) -> tuple[torch.Tensor, torch.Tensor]: + bins = torch.arange(log_probs.shape[-1], device=log_probs.device) + window = window.to(log_probs.device).long() + inside = (bins >= window[:, :1]) & (bins <= window[:, 1:]) + return (torch.log_softmax(log_probs.masked_fill(~inside, MASKED), dim=-1), inside) + + +def date_parts( + logits: torch.Tensor, +) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: + (digits, year, month, day) = logits.split( + (YEAR_DIGITS * 10, YEAR_CLASSES, 12, 31), dim=-1 + ) + return (digits.unflatten(-1, (YEAR_DIGITS, 10)), year, month, day) + + +def time_parts(logits: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + return logits.split((24, 60, 60), dim=-1) + + +def lexical_bin_log_probs( + number_logits: torch.Tensor, + edge_sign: torch.Tensor, + edge_class: torch.Tensor, + edge_digits: torch.Tensor, + edge_frac: torch.Tensor, +) -> torch.Tensor: + (sign, exponent, digits) = number_parts(number_logits.double()) + sign_probs = torch.softmax(sign, dim=-1) + exponent_probs = torch.softmax(exponent, dim=-1) + zero = exponent_probs[:, :1] + nonzero = exponent_probs[:, 1:] / (1.0 - zero).clamp_min(1e-300) + below_class = torch.cumsum(nonzero, dim=-1) - nonzero + leading = digits[:, :1].masked_fill( + torch.arange(10, device=digits.device) == 0, float("-inf") + ) + digit_probs = torch.softmax( + torch.cat([leading, digits[:, 1:LEXICAL_DIGITS]], dim=1), dim=-1 + ) + below_digit = torch.cumsum(digit_probs, dim=-1) - digit_probs + classes = (edge_class - 1).clamp(0, nonzero.shape[1] - 1) + within = edge_frac.double() + for position in reversed(range(LEXICAL_DIGITS)): + index = edge_digits[..., position] + within = ( + below_digit[:, position].gather(1, index) + + digit_probs[:, position].gather(1, index) * within + ) + magnitude = below_class.gather(1, classes) + nonzero.gather(1, classes) * within + negative = sign_probs[:, 1:] * (1.0 - zero) + positive = sign_probs[:, :1] * (1.0 - zero) + cdf = torch.where( + edge_sign == EDGE_NEGATIVE, + negative * (1.0 - magnitude), + torch.where( + edge_sign == EDGE_ZERO, + negative.expand_as(magnitude), + negative + zero + positive * magnitude, + ), + ) + mass = (cdf[:, 1:] - cdf[:, :-1]).clamp_min(0.0) + 1e-30 + return (mass.log() - mass.sum(dim=-1, keepdim=True).log()).float() + + +def estimate_log_probs( + range_logits: torch.Tensor, + number_logits: torch.Tensor | None = None, + gate: torch.Tensor | None = None, + lexical: torch.Tensor | None = None, + edge_sign: torch.Tensor | None = None, + edge_class: torch.Tensor | None = None, + edge_digits: torch.Tensor | None = None, + edge_frac: torch.Tensor | None = None, +) -> torch.Tensor: + range_part = torch.log_softmax(range_logits.float(), dim=-1) + if number_logits is None or not number_logits.shape[0]: + return range_part + lexical_part = lexical_bin_log_probs( + number_logits, + cast(torch.Tensor, edge_sign), + cast(torch.Tensor, edge_class), + cast(torch.Tensor, edge_digits), + cast(torch.Tensor, edge_frac), + ) + use = cast(torch.Tensor, lexical).bool().unsqueeze(1) + gate = cast(torch.Tensor, gate).float().unsqueeze(1) + mixed = torch.logaddexp( + functional.logsigmoid(gate) + lexical_part, + functional.logsigmoid(-gate) + range_part, + ) + return torch.where(use, mixed, range_part) + + +def decode_number( + logits: torch.Tensor, spec: dict[str, Any], groups: torch.Tensor | None = None +) -> tuple[float | int, float]: + (sign_logits, exponent_logits, digit_logits) = number_parts(logits.float().cpu()) + decimals = zs.allowed_decimals(spec) + exponent_log = torch.log_softmax(exponent_logits, dim=-1) + exponent_class = int(exponent_log.argmax()) + log_probability = float(exponent_log[exponent_class]) + if exponent_class == 0: + return (0, math.exp(log_probability)) + power = exponent_class - 1 + MIN_EXPONENT + kept = NUMBER_DIGITS if decimals is None else power + decimals + 1 + if kept < 0: + return (0, math.exp(log_probability)) + leading_zero = torch.zeros(NUMBER_DIGITS, 10, dtype=torch.bool) + leading_zero[0, 0] = True + digit_log = torch.log_softmax(digit_logits, dim=-1).masked_fill( + leading_zero, MASKED + ) + digits = digit_log.argmax(dim=-1).tolist() + sign_log = torch.log_softmax(sign_logits, dim=-1) + sign = int(sign_log.argmax()) + kept = min(kept, NUMBER_DIGITS) + log_probability += float(sign_log[sign]) + if groups is None: + log_probability += sum( + (float(digit_log[position, digits[position]]) for position in range(kept)) + ) + if kept < NUMBER_DIGITS: + rounding = digit_log[kept, 5:] if digits[kept] >= 5 else digit_log[kept, :5] + log_probability += float(torch.logsumexp(rounding, dim=0)) + else: + exponent_tensor = torch.tensor([exponent_class]) + (scores, present) = number_group_logits( + digit_logits.unsqueeze(0), + exponent_tensor, + groups.float().cpu().unsqueeze(0), + ) + group_log = torch.log_softmax(scores[0], dim=-1) + slots = group_slots(exponent_tensor)[0][0].tolist() + table = group_class_digits() + counted = set() + for group in range(NUMBER_GROUPS): + if not bool(present[0, group]): + continue + best = int(group_log[group].argmax()) + for place, slot in enumerate(slots[group]): + if slot < NUMBER_DIGITS: + digits[slot] = int(table[best, place]) + if any((slot < kept for slot in slots[group])): + counted.add(group) + log_probability += float(group_log[group, best]) + if kept < NUMBER_DIGITS: + for group in range(NUMBER_GROUPS): + if kept in slots[group]: + if group not in counted: + place = slots[group].index(kept) + same = ( + table[:, place] >= 5 + if digits[kept] >= 5 + else table[:, place] < 5 + ) + log_probability += float( + torch.logsumexp(group_log[group][same], dim=0) + ) + break + else: + rounding = ( + digit_log[kept, 5:] if digits[kept] >= 5 else digit_log[kept, :5] + ) + log_probability += float(torch.logsumexp(rounding, dim=0)) + magnitude = Decimal(int("".join((str(digit) for digit in digits)))).scaleb( + power - NUMBER_DIGITS + 1 + ) + if decimals is not None: + magnitude = magnitude.quantize( + Decimal(1).scaleb(-decimals), rounding=ROUND_HALF_UP + ) + if magnitude == 0: + return (0, math.exp(log_probability)) + value = -magnitude if sign else magnitude + if decimals == 0 or (value == value.to_integral_value() and abs(value) < 2**53): + return (int(value), math.exp(log_probability)) + return (float(value), math.exp(log_probability)) + + +def number_from_numeral( + magnitude: Decimal, + logits: torch.Tensor, + spec: dict[str, Any], + check_exponent: bool = True, +) -> float | int | None: + (sign_logits, exponent_logits, _) = number_parts(logits.float().cpu()) + if check_exponent: + target = number_components(float(magnitude)) + if target is None or int(exponent_logits.argmax()) != target[1]: + return None + decimals = zs.allowed_decimals(spec) + if decimals is not None: + magnitude = magnitude.quantize( + Decimal(1).scaleb(-decimals), rounding=ROUND_HALF_UP + ) + if magnitude == 0: + return 0 + value = -magnitude if int(sign_logits.argmax()) else magnitude + if decimals == 0 or (value == value.to_integral_value() and abs(value) < 2**53): + return int(value) + return float(value) + + +def number_from_numerals( + values: list[Decimal], + logits: torch.Tensor, + spec: dict[str, Any], + readout: float | None, + confident: bool = False, +) -> float | int | None: + agreeing = [ + value + for value in ( + number_from_numeral(magnitude, logits, spec) for magnitude in values + ) + if value is not None + ] + if not agreeing and confident and values: + + def distance(magnitude: Decimal) -> float: + if readout is None or not readout or (not magnitude): + return 0.0 + return abs(math.log10(float(magnitude)) - math.log10(abs(float(readout)))) + + agreeing = [ + value + for value in ( + number_from_numeral(min(values, key=distance), logits, spec, False), + ) + if value is not None + ] + if not agreeing: + return None + if readout is None: + return agreeing[0] + return min(agreeing, key=lambda value: abs(float(value) - float(readout))) + + +def copied_number( + output: dict[str, Any], + row: int, + record: EncodedRecord, + spec: dict[str, Any], + readout: float | None = None, +) -> tuple[float | int, float, list[int]] | None: + if ( + "number_copy_gate" not in output + or record.record.get("images") + or (not record.text) + ): + return None + if os.environ.get("SZ_DISABLE_NUMBER_COPY") == "1": + return None + gate = float( + torch.sigmoid(torch.as_tensor(output["number_copy_gate"][row]).float()) + ) + if gate < 0.5: + return None + if "number_copy_best" in output: + (best, probability) = ( + output["number_copy_best"][row], + output["number_copy_probability"][row], + ) + else: + scores = output["number_copy"][row].float().flatten() + (best, probability) = ( + int(scores.argmax()), + float(torch.softmax(scores, dim=0).max()), + ) + (start, length) = divmod(int(best), MAX_SPAN) + last = min(start + length, len(record.offsets) - 1) + if start >= len(record.offsets): + return None + (char_start, char_end) = (record.offsets[start][0], record.offsets[last][1]) + while char_start < char_end and (not record.text[char_start].isalnum()): + char_start += 1 + around = [ + found + for found in read_numerals(record.text, every=True) + if found[1] < char_end and found[2] > char_start + ] + if not around: + return None + nearest = min( + around, + key=lambda found: ( + abs(found[1] - char_start) + abs(found[2] - char_end), + -(found[2] - found[1]), + ), + ) + whole = [ + found + for found in read_numerals(record.text) + if found[1] <= char_start + and found[2] >= char_end + and (found[2] - found[1] > nearest[2] - nearest[1]) + ] + choices = whole[:1] + [nearest] + if gate * probability < COPY_MIN_CONFIDENCE: + return None + sure = gate >= COPY_CONFIDENT + (value, numeral_start, numeral_end) = (None, nearest[1], nearest[2]) + for values, start, end in choices: + run = NUMERAL_RUN.match(record.text, start) + printed = record.text[start : min(end, run.end())] if run else "" + settled = ( + numeral_convention(record.text) is not None + and len(numeral_values(printed)) > 1 + ) + enclosing = bool(whole) and (start, end) == (whole[0][1], whole[0][2]) + value = number_from_numerals( + values, + output["number"][row], + spec, + readout, + confident=settled or (sure and enclosing and (len(values) == 1)), + ) + if value is not None: + (numeral_start, numeral_end) = (start, end) + break + if value is None and sure: + (values, numeral_start, numeral_end) = choices[0] + value = number_from_numerals( + values, output["number"][row], spec, readout, confident=True + ) + if value is None or (readout is not None and rescaled(readout, value)): + return None + return (value, gate * probability, [numeral_start, numeral_end]) + + +def rescaled(readout: float, copied: float) -> bool: + try: + (first, second) = ( + abs(Decimal(str(readout))).normalize(), + abs(Decimal(str(copied))).normalize(), + ) + except ArithmeticError: + return False + if not first or not second or first == second: + return False + return first.as_tuple().digits == second.as_tuple().digits + + +def decode_date(logits: torch.Tensor) -> tuple[str, float]: + (digit_logits, year_logits, month_logits, day_logits) = date_parts( + logits.float().cpu() + ) + class_log = torch.log_softmax(year_logits, dim=-1) + year_class = int(class_log.argmax()) + log_probability = float(class_log[year_class]) + if year_class < YEAR_OUTSIDE: + year = YEAR_LOW + year_class + else: + digit_log = torch.log_softmax(digit_logits, dim=-1) + digits = digit_log.argmax(dim=-1).tolist() + if digits == [0, 0, 0, 0]: + masked = digit_log[3].clone() + masked[0] = MASKED + digits[3] = int(masked.argmax()) + year = int("".join((str(digit) for digit in digits))) + log_probability += float( + sum((digit_log[position, digit] for (position, digit) in enumerate(digits))) + ) + (month_log, day_log) = ( + torch.log_softmax(month_logits, dim=-1), + torch.log_softmax(day_logits, dim=-1), + ) + joint = month_log.unsqueeze(1) + day_log.unsqueeze(0) + lengths = torch.tensor( + [calendar.monthrange(year, month)[1] for month in range(1, 13)] + ) + joint = joint.masked_fill( + torch.arange(31).unsqueeze(0) >= lengths.unsqueeze(1), MASKED + ) + best = int(joint.flatten().argmax()) + (month, day) = divmod(best, 31) + log_probability += float(joint.flatten()[best]) + return ( + datetime.date(year, month + 1, day + 1).isoformat(), + math.exp(log_probability), + ) + + +def decode_time(logits: torch.Tensor) -> tuple[str, float]: + parts = [ + torch.log_softmax(part, dim=-1) for part in time_parts(logits.float().cpu()) + ] + values = [int(part.argmax()) for part in parts] + return ( + zs.format_time(*values), + math.exp(sum((float(part[value]) for (part, value) in zip(parts, values)))), + ) + + +def decode_spans( + labels: list[int], + offsets: list[tuple[int, int]], + text: str, + trims: list[list[int]] | None = None, +) -> tuple[list[str], list[list[int]]]: + spans: list[list[int]] = [] + current: list[int] | None = None + for position, label in enumerate(labels): + if label == TAG_BEGIN: + if current is not None: + spans.append(current) + current = [position, position] + elif label == TAG_INSIDE and current is not None: + current[1] = position + elif current is not None: + spans.append(current) + current = None + if current is not None: + spans.append(current) + values: list[str] = [] + char_spans: list[list[int]] = [] + for first, last in spans: + (char_start, char_end) = trimmed_span(text, offsets, first, last, trims) + if not text[char_start:char_end].strip(): + continue + char_spans.append([char_start, char_end]) + if text[char_start:char_end] not in values: + values.append(text[char_start:char_end]) + return (values, char_spans) + + +def quantile_from_cdf( + cdf: list[float], probabilities: list[float], fraction: float, bins: int +) -> float: + index = min(bisect_left(cdf, fraction), bins - 1) + previous = cdf[index - 1] if index > 0 else 0.0 + inside = ( + (fraction - previous) / probabilities[index] + if probabilities[index] > 0 + else 0.5 + ) + return (index + min(1.0, max(0.0, inside))) / bins + + +def decode_estimate( + logits: torch.Tensor, + offsets: torch.Tensor, + overflow: torch.Tensor, + spec: dict[str, Any], +) -> dict[str, Any]: + probabilities = torch.softmax(logits.float(), dim=-1) + cdf = torch.cumsum(probabilities, dim=-1).tolist() + row = probabilities.tolist() + bins = len(row) + shift = torch.sigmoid(offsets.float()).tolist() + (low, high, scale) = zs.estimate_range(spec) + centers = [(index + shift[index]) / bins for index in range(bins)] + mean = sum( + ( + probability * zs.from_unit(center, low, high, scale) + for (probability, center) in zip(row, centers) + ) + ) + quantiles: list[Any] = [ + zs.from_unit( + quantile_from_cdf(cdf, row, fraction / 100.0, bins), low, high, scale + ) + for fraction in range(1, 100) + ] + value = ( + zs.from_unit( + sum((probability * center for (probability, center) in zip(row, centers))), + low, + high, + scale, + ) + if scale == "log" + else mean + ) + step = zs.estimate_step(spec) + if step > 0: + quantiles = [ + zs.snap_quantile(quantile, step, low, high) for quantile in quantiles + ] + value = zs.snap(mean, step, low, high) + if zs.is_date_estimate(spec): + (value, mean) = (zs.estimate_value(spec, value), zs.estimate_value(spec, mean)) + quantiles = [zs.estimate_value(spec, quantile) for quantile in quantiles] + prediction = { + "value": value, + "mean": mean, + "median": quantiles[49], + "p10": quantiles[9], + "p90": quantiles[89], + "quantiles": quantiles, + } + if not zs.is_date_estimate(spec): + sides = torch.softmax( + torch.cat([overflow.new_zeros(1), overflow.float()]), dim=0 + ).tolist() + (prediction["p_below"], prediction["p_above"]) = (sides[BELOW], sides[ABOVE]) + return prediction + + +def estimate_row_log_probs( + output: dict[str, Any], + row: int, + spec: dict[str, Any], + window: tuple[int, int] | None = None, +) -> torch.Tensor: + log_probs = _estimate_row_log_probs(output, row, spec) + if window is None: + return log_probs + return window_log_probs(log_probs.unsqueeze(0), torch.tensor([window]))[0][0] + + +def _estimate_row_log_probs( + output: dict[str, Any], row: int, spec: dict[str, Any] +) -> torch.Tensor: + logits = output["estimate"][row].float().cpu() + parts = ( + estimate_edge_parts(spec, logits.shape[-1]) + if "estimate_number" in output + else None + ) + if parts is None: + return torch.log_softmax(logits, dim=-1) + (sign, classes, digits, fractions) = parts + count = len(sign) + return estimate_log_probs( + logits.unsqueeze(0), + output["estimate_number"][row].float().cpu().unsqueeze(0), + output["estimate_gate"][row].float().cpu().reshape(1), + torch.ones(1), + torch.tensor([sign]), + torch.tensor([classes]), + torch.tensor(digits).view(1, count, LEXICAL_DIGITS), + torch.tensor([fractions], dtype=torch.float64), + )[0] + + +def decode_record( + output: dict[str, Any], + record: EncodedRecord, + tokenizer: Any = None, + end_id: int | None = None, +) -> dict[str, dict[str, Any]]: + fields = record.fields + specs = {item.name: record.record["fields"][item.name] for item in fields} + p_null = torch.sigmoid(output["null"].float()).tolist() + predictions: dict[str, dict[str, Any]] = {} + value: Any + rows_of = { + group: [ + position for (position, item) in enumerate(fields) if item.kind in kinds + ] + for (group, kinds) in ( + ("text", ("text",)), + ("tags", ("text_list",)), + ("number", ("number",)), + ("date", ("date", "datetime")), + ("time", ("time", "datetime")), + ("estimate", ("estimate",)), + ("number_list", ("number_list",)), + ("date_list", ("date_list",)), + ) + } + index_of = { + group: {position: row for (row, position) in enumerate(rows)} + for (group, rows) in rows_of.items() + } + if rows_of["text"] and "text_best" in output: + (best, probabilities) = (output["text_best"], output["text_probability"]) + elif rows_of["text"]: + scores = output["text"].float() + best = scores.flatten(1).argmax(dim=1).tolist() + probabilities = torch.softmax(scores.flatten(1), dim=1).amax(dim=1).tolist() + if rows_of["tags"]: + labels = ( + output["tags_labels"] + if "tags_labels" in output + else output["tags"].argmax(dim=-1).tolist() + ) + trims = output.get("trim_labels") + if trims is None and "trim" in output: + trims = output["trim"].argmax(dim=-1).tolist() + for position, item in enumerate(fields): + spec = specs[item.name] + kind = item.kind + if kind == "estimate": + row = index_of["estimate"][position] + predictions[item.name] = decode_estimate( + estimate_row_log_probs(output, row, spec, item.window), + output["estimate_offset"][row], + output["estimate_overflow"][row], + spec, + ) + continue + if kind == "text_list": + row = index_of["tags"][position] + (values, spans) = decode_spans( + labels[row], record.offsets, record.text, trims + ) + predictions[item.name] = {"value": values, "spans": spans} + if "tags_log_probability" in output: + predictions[item.name]["confidence"] = math.exp( + output["tags_log_probability"][row] + ) + elif "tags" in output: + label_log = torch.log_softmax(output["tags"][row].float().cpu(), dim=-1) + predictions[item.name]["confidence"] = math.exp( + float(label_log.amax(dim=-1).sum()) + ) + continue + if kind in LIST_KINDS: + row = index_of[kind][position] + ends = torch.sigmoid(output[f"{kind}_end"][row].float()).tolist() + (values, confidences) = ([], []) + log_probability = 0.0 + for slot in range(LIST_SLOTS): + if ends[slot] >= 0.5: + log_probability += math.log(max(ends[slot], 1e-12)) + break + log_probability += math.log(max(1.0 - ends[slot], 1e-12)) + if kind == "number_list": + (value, confidence) = decode_number( + output[kind][row, slot], + spec, + output["number_list_groups"][row, slot] + if "number_list_groups" in output + else None, + ) + else: + (value, confidence) = decode_date(output[kind][row, slot]) + values.append(value) + confidences.append(confidence) + log_probability += math.log(max(confidence, 1e-12)) + predictions[item.name] = { + "value": values, + "confidences": confidences, + "confidence": math.exp(log_probability), + } + continue + prediction: dict[str, Any] = {"p_null": p_null[position]} + if kind == "text": + row = index_of["text"][position] + (start, length) = divmod(best[row], MAX_SPAN) + last = min(start + length, len(record.offsets) - 1) + (char_start, char_end) = trimmed_span( + record.text, record.offsets, start, last, trims + ) + (value, confidence) = (record.text[char_start:char_end], probabilities[row]) + prediction["span"] = [char_start, char_end] + if not record.text: + (value, prediction["span"]) = (None, None) + elif kind == "number": + number_row = index_of["number"][position] + (value, confidence) = decode_number( + output["number"][number_row], + spec, + output["number_groups"][number_row] + if "number_groups" in output + else None, + ) + copied = copied_number(output, number_row, record, spec, value) + if ( + copied is not None + and value is not None + and (os.environ.get("SZ_NUMBER_COPY_VETO", "1") == "1") + and numeral_spans(record.text, value, every=False) + ): + copied = None + if copied is not None: + (value, confidence, prediction["span"]) = copied + prediction["source"] = "input" + if spec["type"] == "duration": + prediction["unit"] = zs.duration_unit(spec) + elif kind == "date": + (value, confidence) = decode_date( + output["date"][index_of["date"][position]] + ) + elif kind == "time": + (value, confidence) = decode_time( + output["time"][index_of["time"][position]] + ) + else: + (day, day_confidence) = decode_date( + output["date"][index_of["date"][position]] + ) + (clock, clock_confidence) = decode_time( + output["time"][index_of["time"][position]] + ) + (value, confidence) = (f"{day}T{clock}", day_confidence * clock_confidence) + prediction["guess"] = value + prediction["confidence"] = confidence + prediction["value"] = ( + None if p_null[position] >= zs.null_threshold(spec) else value + ) + predictions[item.name] = prediction + if tokenizer is not None and end_id is not None and ("spell_ids" in output): + (ids, log_probs) = ( + output["spell_ids"].tolist(), + output["spell_log_probs"].float().tolist(), + ) + for position, item in enumerate(fields): + if item.kind == "text": + row = index_of["text"][position] + elif item.kind == "text_list": + row = len(rows_of["text"]) + index_of["tags"][position] + else: + continue + (spelled, _, parts) = spelled_answer( + ids[row], log_probs[row], tokenizer, end_id + ) + predictions[item.name] = route_spelled( + predictions[item.name], item.kind, spelled, parts, record.text + ) + if tokenizer is not None and "slot_ids" in output: + slot_end = _tokens(tokenizer, SPELL_END)[0] + (slot_ids, slot_log_probs) = ( + output["slot_ids"].tolist(), + output["slot_log_probs"].float().tolist(), + ) + slot_rows = [ + position for (position, item) in enumerate(fields) if item.answer_slots + ] + for row, position in enumerate(slot_rows): + item = fields[position] + if item.kind not in ("text", "text_list"): + continue + count = len(item.answer_slots) + (spelled, _probability, parts) = spelled_answer( + slot_ids[row][:count], slot_log_probs[row][:count], tokenizer, slot_end + ) + prediction = predictions[item.name] + if (record.record.get("_meta") or {}).get( + "construction" + ) == zs.EVIDENCE_CONSTRUCTION and item.kind == "text": + predictions[item.name] = evidence_text_answer( + prediction, + slot_ids[row][:count], + slot_log_probs[row][:count], + tokenizer, + slot_end, + record.text, + specs[item.name], + bool(record.record.get("images")), + ) + continue + if item.kind in ("text", "text_list"): + predictions[item.name] = route_spelled( + prediction, item.kind, spelled, parts, record.text, defer_lists=True + ) + continue + return predictions + + +FIELD_KINDS = SCALAR_KINDS + ("estimate", "text_list") + LIST_KINDS +VIEW_GROUPS = ( + ("text", ("text",)), + ("tags", ("text_list",)), + ("number", ("number",)), + ("date", ("date", "datetime")), + ("time", ("time", "datetime")), + ("estimate", ("estimate",)), +) +COMPACT_KEYS = ( + "number", + "date", + "time", + "estimate", + "estimate_offset", + "estimate_overflow", + "number_list_end", + "number_list", + "date_list_end", + "date_list", + "estimate_number", + "estimate_gate", + "number_groups", + "number_list_groups", + "number_copy_gate", +) +COMPACT_GROUPS = { + "number": "number", + "date": "date", + "time": "time", + "estimate": "estimate", + "estimate_offset": "estimate", + "estimate_overflow": "estimate", + "number_list_end": "number_list", + "number_list": "number_list", + "date_list_end": "date_list", + "date_list": "date_list", + "estimate_number": "estimate", + "estimate_gate": "estimate", + "number_groups": "number", + "number_list_groups": "number_list", + "number_copy_gate": "number", +} + + +def prediction_layout(record: EncodedRecord) -> list[int]: + layout = [ + len(record.input_ids), + record.input_start, + len(record.offsets), + len(record.fields), + ] + for item in record.fields: + layout += [ + FIELD_KINDS.index(item.kind), + item.query, + len(item.items), + *item.items, + ] + slotted = [ + (index, item.answer_slots) + for (index, item) in enumerate(record.fields) + if item.answer_slots + ] + if slotted: + layout.append(len(slotted)) + for index, slots in slotted: + layout += [index, len(slots), *slots] + return layout + + +def parse_prediction_layout( + layout: list[int], num_tokens: int, with_slots: bool = False +) -> tuple[Any, ...]: + if len(layout) < 4 or any((type(value) is not int for value in layout)): + raise ValueError("prediction_layout must be a list of ints") + (prompt_length, input_start, input_length, count) = layout[:4] + shift = num_tokens - prompt_length + if ( + shift < 0 + or count < 1 + or input_length < 1 + or (not 0 <= input_start < input_start + input_length <= prompt_length) + ): + raise ValueError("prediction_layout does not match the prompt") + kinds: list[str] = [] + queries: list[int] = [] + items: list[list[int]] = [] + cursor = 4 + for _ in range(count): + if cursor + 3 > len(layout): + raise ValueError("prediction_layout is truncated") + (kind, query, size) = layout[cursor : cursor + 3] + cursor += 3 + if ( + not 0 <= kind < len(FIELD_KINDS) + or not input_start + input_length <= query < prompt_length + ): + raise ValueError("prediction_layout has an invalid field") + expected = LIST_SLOTS if FIELD_KINDS[kind] in LIST_KINDS else 0 + slots = layout[cursor : cursor + size] + cursor += size + if ( + size != expected + or len(slots) != size + or any((not query < slot < prompt_length for slot in slots)) + ): + raise ValueError("prediction_layout has invalid item positions") + kinds.append(FIELD_KINDS[kind]) + queries.append(query + shift) + items.append([slot + shift for slot in slots]) + slot_fields: list[int] = [] + answer_slots: list[list[int]] = [] + if cursor < len(layout): + slotted = layout[cursor] + cursor += 1 + if not 0 < slotted <= count: + raise ValueError("prediction_layout has invalid answer slots") + for _ in range(slotted): + if cursor + 2 > len(layout): + raise ValueError("prediction_layout is truncated") + (index, size) = layout[cursor : cursor + 2] + cursor += 2 + positions = layout[cursor : cursor + size] + cursor += size + if ( + not 0 <= index < count + or (slot_fields and index <= slot_fields[-1]) + or size < 1 + or (len(positions) != size) + or any( + ( + not queries[index] - shift < slot < prompt_length + for slot in positions + ) + ) + ): + raise ValueError("prediction_layout has invalid answer slots") + slot_fields.append(index) + answer_slots.append([slot + shift for slot in positions]) + if cursor != len(layout): + raise ValueError("prediction_layout has trailing values") + parsed = (input_start + shift, input_length, kinds, queries, items) + return (*parsed, slot_fields, answer_slots) if with_slots else parsed + + +def prediction_views( + kinds: list[str], queries: list[int], items: list[list[int]], device: Any = None +) -> dict[str, torch.Tensor]: + views = {"query_positions": torch.tensor(queries, dtype=torch.long, device=device)} + for group, members in VIEW_GROUPS: + views[f"{group}_rows"] = torch.tensor( + [row for (row, kind) in enumerate(kinds) if kind in members], + dtype=torch.long, + device=device, + ) + for kind in LIST_KINDS: + rows = [row for (row, field_kind) in enumerate(kinds) if field_kind == kind] + views[f"{kind}_rows"] = torch.tensor(rows, dtype=torch.long, device=device) + views[f"{kind}_items"] = torch.tensor( + [slot for row in rows for slot in items[row]], + dtype=torch.long, + device=device, + ).view(len(rows), LIST_SLOTS) + return views + + +def compact_output(output: dict[str, Any]) -> torch.Tensor: + parts = [output["null"].float().flatten()] + if "text" in output: + scores = output["text"].float().flatten(1) + parts += [ + scores.argmax(dim=1).float(), + torch.softmax(scores, dim=1).amax(dim=1), + ] + if "tags" in output: + tag_log = torch.log_softmax(output["tags"].float(), dim=-1) + parts.append(tag_log.argmax(dim=-1).float().flatten()) + parts.append(tag_log.amax(dim=-1).sum(dim=-1)) + if "trim" in output: + parts.append(output["trim"].argmax(dim=-1).float().flatten()) + if "number_copy" in output: + scores = output["number_copy"].float().flatten(1) + parts += [ + scores.argmax(dim=1).float(), + torch.softmax(scores, dim=1).amax(dim=1), + ] + parts += [output[key].float().flatten() for key in COMPACT_KEYS if key in output] + if "slot_ids" in output: + parts += [ + output["slot_ids"].float().flatten(), + output["slot_log_probs"].float().flatten(), + ] + return torch.cat(parts) + + +def expand_output( + flat: torch.Tensor, + record: EncodedRecord, + bins: int = BINS, + estimate_lexical: bool = False, + span_trim: bool = False, + number_groups: bool = False, + number_pointer: bool = False, +) -> dict[str, Any]: + flat = flat.float().cpu() + kinds = [item.kind for item in record.fields] + rows = { + group: sum((kind in members for kind in kinds)) + for (group, members) in VIEW_GROUPS + } + rows.update({kind: kinds.count(kind) for kind in LIST_KINDS}) + cursor = 0 + + def take(*shape: int) -> torch.Tensor: + nonlocal cursor + size = math.prod(shape) + if cursor + size > flat.numel(): + raise ValueError("compact output is shorter than the layout needs") + value = flat[cursor : cursor + size].view(*shape) + cursor += size + return value + + output: dict[str, Any] = {"null": take(len(kinds))} + if rows["text"]: + output["text_best"] = take(rows["text"]).round().long().tolist() + output["text_probability"] = take(rows["text"]).tolist() + if rows["tags"]: + output["tags_labels"] = ( + take(rows["tags"], len(record.offsets)).round().long().tolist() + ) + output["tags_log_probability"] = take(rows["tags"]).tolist() + if span_trim and (rows["text"] or rows["tags"]): + output["trim_labels"] = take(len(record.offsets), 2).round().long().tolist() + if number_pointer and rows["number"]: + output["number_copy_best"] = take(rows["number"]).round().long().tolist() + output["number_copy_probability"] = take(rows["number"]).tolist() + widths = { + "number": (NUMBER_WIDTH,), + "date": (DATE_WIDTH,), + "time": (TIME_WIDTH,), + "estimate": (bins,), + "estimate_offset": (bins,), + "estimate_overflow": (2,), + "number_list_end": (LIST_SLOTS,), + "number_list": (LIST_SLOTS, NUMBER_WIDTH), + "date_list_end": (LIST_SLOTS,), + "date_list": (LIST_SLOTS, DATE_WIDTH), + "estimate_number": (NUMBER_WIDTH,), + "estimate_gate": (), + "number_groups": (NUMBER_GROUPS, GROUP_CLASSES), + "number_list_groups": (LIST_SLOTS, NUMBER_GROUPS, GROUP_CLASSES), + "number_copy_gate": (), + } + for key in COMPACT_KEYS: + if key in ("estimate_number", "estimate_gate") and (not estimate_lexical): + continue + if key in ("number_groups", "number_list_groups") and (not number_groups): + continue + if key == "number_copy_gate" and (not number_pointer): + continue + count = rows[COMPACT_GROUPS[key]] + if count: + output[key] = take(count, *widths[key]) + slotted = [len(item.answer_slots) for item in record.fields if item.answer_slots] + if slotted: + output["slot_ids"] = take(len(slotted), max(slotted)).round().long() + output["slot_log_probs"] = take(len(slotted), max(slotted)) + if cursor != flat.numel(): + raise ValueError("compact output is longer than the layout needs") + return output + + +def merge_forecasts( + predictions: dict[str, dict[str, Any]], groups: dict[str, list[str]] +) -> dict[str, dict[str, Any]]: + merged = { + name: prediction + for (name, prediction) in predictions.items() + if not any((name in names for names in groups.values())) + } + for name, names in groups.items(): + steps = [predictions[step] for step in names] + merged[name] = { + key: [step[key] for step in steps] + for key in ("value", "median", "p10", "p90") + } + merged[name]["steps"] = steps + return merged