from __future__ import annotations import calendar import datetime import functools import math import os import re from bisect import bisect_left, bisect_right from dataclasses import dataclass, field from decimal import ROUND_HALF_UP, Decimal from typing import Any, cast import torch from torch import nn from torch.nn import functional from . import schema as zs SYSTEM_PREFIX = "<|im_start|>system\n" + zs.SYSTEM_PREAMBLE + "\nFields:\n" EVIDENCE_SYSTEM_PREFIX = ( "<|im_start|>system\n" + zs.EVIDENCE_SYSTEM_PREAMBLE + "\nFields:\n" ) USER_PREFIX = "<|im_end|>\n<|im_start|>user\n" ASSISTANT_SUFFIX = "<|im_end|>\n<|im_start|>assistant\n\n\n\n" ANSWER_CUE = "\nAnswer:" @dataclass(frozen=True) class PromptLayout: schema_header: str query_header: str answer_cue: str slot_header: str PROMPT_LAYOUTS = ( PromptLayout("", "", ANSWER_CUE, ""), PromptLayout( "Field definition:\n", "\nField query:", "\nValue:", "\nAnswer field:" ), PromptLayout( "Requested field:\n", "\nRequested field:", "\nResult:", "\nResult for field:" ), ) PROMPT_VARIANTS = (0, 1, 2) def prompt_layout( template_variant: int = 0, complete_field: bool = True ) -> PromptLayout: if ( not isinstance(template_variant, int) or isinstance(template_variant, bool) or template_variant not in PROMPT_VARIANTS ): raise ValueError("template_variant must be 0, 1, or 2") if template_variant and (not complete_field): raise ValueError("alternative layouts require the evidence construction") return PROMPT_LAYOUTS[template_variant] QUERY_TOKENS = 256 MAX_SPAN = 64 TRIM_CLASSES = 8 TRIM_INIT = 2.0 MIN_INPUT_TOKENS = 16 BINS = 256 POINTER_WIDTH = 256 NUMBER_DIGITS = 12 (MIN_EXPONENT, MAX_EXPONENT) = (-12, 15) EXPONENT_CLASSES = MAX_EXPONENT - MIN_EXPONENT + 2 YEAR_DIGITS = 4 (YEAR_LOW, YEAR_HIGH) = (1800, 2199) YEAR_OUTSIDE = YEAR_HIGH - YEAR_LOW + 1 YEAR_CLASSES = YEAR_OUTSIDE + 1 LIST_SLOTS = zs.MAX_LIST_ITEMS ITEM_CUES = tuple((f"\n{index}." for index in range(1, LIST_SLOTS + 1))) (INSIDE, BELOW, ABOVE) = (0, 1, 2) OVERFLOW_INIT = -5.0 QUERY_ORDER_RATE = 0.8 LOOP_RANK = 256 LEXICAL_DIGITS = 4 ESTIMATE_GATE_INIT = 0.0 ESTIMATE_SPREAD_LIMIT = 2.0 (EDGE_NEGATIVE, EDGE_ZERO, EDGE_POSITIVE) = (0, 1, 2) (TAG_OUTSIDE, TAG_BEGIN, TAG_INSIDE) = (0, 1, 2) NUMBER_WIDTH = 2 + EXPONENT_CLASSES + NUMBER_DIGITS * 10 GROUP_DIGITS = 3 GROUP_CLASSES = 10**GROUP_DIGITS (GROUP_LOW, GROUP_HIGH) = (MIN_EXPONENT // GROUP_DIGITS, MAX_EXPONENT // GROUP_DIGITS) NUMBER_GROUPS = GROUP_HIGH - GROUP_LOW + 1 GROUP_RANK = 256 COPY_CATEGORIES: tuple[str | None, ...] | None = ("extraction", None) COPY_GATE_INIT = -3.0 NUMERAL_RUN = re.compile( "(?:(? 2 or word in ("mn", "bn", "tn", "cr", "Cr", "Mn", "Bn", "MM") ), key=len, reverse=True, ) ) + ")|(k|K|M|B))(?![^\\W\\d_])" ) NUMERAL_SPAN = 48 CONVENTION_LANGUAGE = re.compile( "\\b(?:Rechnung|Gesamtbetrag|Betrag|Lieferschein|Factuur|Bedrag|Totaal|Pakbon|Fattura|Importo|Totale|Fatura|Nota fiscal|Factura|Importe|Albar\\u00e1n|Facture|Montant)\\b" ) CONVENTION_LANGUAGE_US = re.compile("\\b(?:M\\u00e9xico|Mexico|MXN|CFDI|RFC)\\b") CONVENTION_STATED = { "eu": re.compile( "european (?:number )?(?:notation|format)|(?:dot|period|point)s? (?:separates?|for|marks?) (?:the )?thousands|comma (?:separates?|for|marks?|as) (?:the )?decimal|comma decimals?|decimal comma|dot thousands|1\\.234,56|1 234,56", re.IGNORECASE, ), "us": re.compile( "comma (?:separates?|for|marks?) (?:the )?thousands|(?:dot|period|point) (?:separates?|for|marks?|as) (?:the )?decimal|comma thousands|dot decimals?|1,234\\.56", re.IGNORECASE, ), } COPY_CONFIDENT = 0.9 COPY_MIN_CONFIDENCE = 0.5 SLOT_TEXT = "_" SLOT_COUNTS = {"text": 80, "text_list": 64, "number": 32} SLOT_CUE_TOKENS = 32 DATE_WIDTH = YEAR_DIGITS * 10 + YEAR_CLASSES + 12 + 31 TIME_WIDTH = 24 + 60 + 60 MASKED = -1000000000.0 SCALAR_KINDS = ("text", "number", "date", "time", "datetime") IMAGE_PLACEHOLDER = "<|vision_start|><|image_pad|><|vision_end|>" MEDIA_BATCH_KEYS = ("pixel_values", "image_grid_thw") MEDIA_TOKEN_KEYS = ("mm_token_type_ids",) LIST_KINDS = ("number_list", "date_list") def head_kind(spec: dict[str, Any]) -> str: kind = spec["type"] if kind == "duration": return "number" if kind == "list": return f"{spec['items']}_list" return kind def number_components(value: float) -> tuple[int, int, list[int]] | None: value = float(value) if not math.isfinite(value): return None if value == 0: return (0, 0, [0] * NUMBER_DIGITS) (mantissa, exponent) = f"{abs(value):.{NUMBER_DIGITS - 1}e}".split("e") power = int(exponent) if not MIN_EXPONENT <= power <= MAX_EXPONENT: return None return ( int(value < 0), power - MIN_EXPONENT + 1, [int(digit) for digit in mantissa.replace(".", "")], ) def edge_number(value: float) -> tuple[int, int, list[int], float]: if value == 0 or not math.isfinite(value): return (EDGE_ZERO, 1, [0] * LEXICAL_DIGITS, 0.0) (mantissa, exponent) = f"{abs(value):.{LEXICAL_DIGITS + 8}e}".split("e") (power, digits) = (int(exponent), mantissa.replace(".", "")) sign = EDGE_NEGATIVE if value < 0 else EDGE_POSITIVE if power < MIN_EXPONENT: return (sign, 1, [0] * LEXICAL_DIGITS, 0.0) if power > MAX_EXPONENT: return (sign, EXPONENT_CLASSES - 1, [9] * LEXICAL_DIGITS, 1.0) return ( sign, power - MIN_EXPONENT + 1, [int(digit) for digit in digits[:LEXICAL_DIGITS]], float("0." + digits[LEXICAL_DIGITS:]), ) @functools.lru_cache(maxsize=8192) def _edge_parts( low: float, high: float, scale: str, bins: int ) -> tuple[tuple[int, ...], tuple[int, ...], tuple[int, ...], tuple[float, ...]]: parts = [ edge_number(zs.from_unit(index / bins, low, high, scale)) for index in range(bins + 1) ] return ( tuple((part[0] for part in parts)), tuple((part[1] for part in parts)), tuple((digit for part in parts for digit in part[2])), tuple((part[3] for part in parts)), ) def lexical_estimate(spec: dict[str, Any]) -> bool: return not zs.is_date_estimate(spec) or zs.date_granularity(spec) == "year" def estimate_edge_parts( spec: dict[str, Any], bins: int = BINS ) -> tuple[tuple[int, ...], tuple[int, ...], tuple[int, ...], tuple[float, ...]] | None: if not lexical_estimate(spec): return None (low, high, scale) = zs.estimate_range(spec) return _edge_parts(float(low), float(high), scale, bins) @dataclass class EncodedField: name: str kind: str query: int items: list[int] = field(default_factory=list) prefix: tuple[int, int, tuple[int, ...]] | None = None window: tuple[int, int] | None = None answer_slots: list[int] = field(default_factory=list) @dataclass(frozen=True) class Commit: text: str done: bool = False prefix: tuple[int, int, tuple[int, ...]] | None = None probability: float = 1.0 @dataclass class EncodedRecord: input_ids: list[int] input_start: int offsets: list[tuple[int, int]] fields: list[EncodedField] text: str record: dict[str, Any] media: dict[str, Any] | None = None image_expansion: int = 0 @dataclass class EncodedMedia: ids: list[int] media: dict[str, Any] | None = None expansion: int = 0 @property def length(self) -> int: return len(self.ids) + self.expansion def configure_image_processor(processor: Any, max_pixels: int) -> dict[str, Any]: image_processor = getattr(processor, "image_processor", None) if image_processor is None or max_pixels <= 0: raise ValueError("an image processor and positive pixel budget are required") legacy = hasattr(image_processor, "max_pixels") size = getattr(image_processor, "size", {}) sized = "longest_edge" in size if not legacy and (not sized): raise ValueError( f"unsupported image pixel configuration: {type(image_processor).__name__}" ) if legacy: image_processor.max_pixels = max_pixels if sized: size["longest_edge"] = max_pixels actual = size["longest_edge"] if sized else image_processor.max_pixels if actual != max_pixels or (legacy and image_processor.max_pixels != max_pixels): raise ValueError( "the image processor did not retain its requested pixel budget" ) return {"class": type(image_processor).__name__, "max_pixels": actual} def encode_media( tokenizer: Any, images: list[Any], processor: Any = None, image_token_counts: list[int] | None = None, ) -> EncodedMedia: if not images: return EncodedMedia([]) text = IMAGE_PLACEHOLDER * len(images) + "\n" if image_token_counts is not None: if len(image_token_counts) != len(images) or any( (count < 1 for count in image_token_counts) ): raise ValueError("every image needs a positive token count") return EncodedMedia( _tokens(tokenizer, text), None, sum((count - 1 for count in image_token_counts)), ) if processor is None: raise ValueError("records with images need the model processor") encoded = processor(text=[text], images=list(images), return_tensors="pt") media = {key: encoded[key] for key in MEDIA_BATCH_KEYS if key in encoded} for key in MEDIA_TOKEN_KEYS: if key in encoded: media[key] = encoded[key][0].tolist() return EncodedMedia(encoded["input_ids"][0].tolist(), media) def _tokens(tokenizer: Any, text: str) -> list[int]: return tokenizer(text, add_special_tokens=False).input_ids def _input_tokens(tokenizer: Any, text: str) -> tuple[list[int], list[tuple[int, int]]]: try: encoding = tokenizer( text, add_special_tokens=False, return_offsets_mapping=True, split_special_tokens=True, ) except TypeError: encoding = tokenizer( text, add_special_tokens=False, return_offsets_mapping=True ) return (list(encoding.input_ids), [tuple(pair) for pair in encoding.offset_mapping]) def schema_line(name: str, spec: dict[str, Any], complete_field: bool = False) -> str: return "- " + zs.field_line(name, spec, preserve_instructions=complete_field) + "\n" def reference_line(record: dict[str, Any]) -> str: reference = record.get("reference_date") return f"Reference date: {reference}\n" if reference else "" def query_block( tokenizer: Any, name: str, spec: dict[str, Any], commit: Commit | None = None, complete_field: bool = False, template_variant: int = 0, ) -> tuple[list[int], int, list[int]]: layout = prompt_layout(template_variant, complete_field) line = _tokens( tokenizer, "\n" + zs.field_line(name, spec, preserve_instructions=complete_field), ) ids = ( _tokens(tokenizer, layout.query_header) + (line if complete_field else line[:QUERY_TOKENS]) + _tokens(tokenizer, layout.answer_cue) ) if commit is not None: ids.extend(_tokens(tokenizer, " " + commit.text)) query = len(ids) - 1 items: list[int] = [] if head_kind(spec) in LIST_KINDS and commit is None: for cue in ITEM_CUES: ids.extend(_tokens(tokenizer, cue)) items.append(len(ids) - 1) return (ids, query, items) def clean_cut( text: str, offsets: list[tuple[int, int]], budget: int, lookback: int = 32 ) -> int: lowest = max(0, budget - lookback) for keep in range(budget, lowest, -1): end = offsets[keep - 1][1] if end >= len(text) or ( end > 0 and text[end].isspace() and (not text[end - 1].isspace()) ): return keep for keep in range(budget, lowest, -1): end = offsets[keep - 1][1] if end > 0 and (text[end].isspace() or text[end - 1].isspace()): return keep return budget def char_span_to_tokens( starts: list[int], ends: list[int], char_start: int, char_end: int ) -> tuple[int, int] | None: first = bisect_right(ends, char_start) last = bisect_left(starts, char_end) - 1 if first >= len(starts) or last < first: return None return (first, last) def numeral_values(printed: str, convention: str | None = None) -> list[Decimal]: values: list[Decimal] = [] both = convention is not None and all( (form.fullmatch(printed) for (form, _, _) in NUMERAL_READINGS[:2]) ) for index, (form, separators, mark) in enumerate(NUMERAL_READINGS): if both and index == (1 if convention == "us" else 0): continue if form.fullmatch(printed): plain = printed.translate( {ord(character): None for character in separators} ).replace(mark, ".") try: value = Decimal(plain) except ArithmeticError: continue if value not in values: values.append(value) return values @functools.lru_cache(maxsize=256) def numeral_convention(text: str) -> str | None: stated = { name for (name, pattern) in CONVENTION_STATED.items() if pattern.search(text or "") } if len(stated) == 1: return stated.pop() shown: list[str] = [] for run in NUMERAL_RUN.finditer(text or ""): printed = run.group() if not any((mark in printed for mark in ",.")): continue (us, eu) = ( bool(form.fullmatch(printed)) for (form, _, _) in NUMERAL_READINGS[:2] ) if us != eu: shown.append("us" if us else "eu") if len(shown) >= 2 and len(set(shown)) == 1: return shown[0] if ( not shown and CONVENTION_LANGUAGE.search(text or "") and (not CONVENTION_LANGUAGE_US.search(text or "")) ): return "eu" return None @functools.lru_cache(maxsize=256) def _numeral_candidates(text: str) -> tuple[tuple[list[Decimal], int, int, bool], ...]: found: list[tuple[list[Decimal], int, int, bool]] = [] for run in NUMERAL_RUN.finditer(text or ""): (begin, finish) = (run.start(), run.end()) cuts = [ index for index in range(begin, finish) if text[index] in NUMERAL_SEPARATORS and index > begin ] starts = [begin] + [index + 1 for index in cuts] ends = cuts + [finish] convention = numeral_convention(text) scale = NUMERAL_SCALE.match(text, finish) exponent = ( None if scale is None else NUMERAL_SCALES[scale.group(1) or scale.group(2)] ) run_found = [] for start in starts: for end in ends[bisect_left(ends, start + 1) :]: if end - start > NUMERAL_SPAN: break values = numeral_values(text[start:end], convention) if not values: continue if end == finish and exponent is not None: scaled = [value.scaleb(exponent) for value in values] run_found.append( ( scaled + [value for value in values if value not in scaled], start, cast(re.Match[str], scale).end(), ) ) else: run_found.append((values, start, end)) by_start: dict[int, tuple[list[Decimal], int, int]] = {} for item in run_found: if item[1] not in by_start or item[2] > by_start[item[1]][2]: by_start[item[1]] = item (position, whole) = (begin, set()) while True: longest = by_start.get(position) following = bisect_left( starts, position + 1 if longest is None else longest[2] ) if longest is not None: whole.add((longest[1], longest[2])) if following >= len(starts): break position = starts[following] found.extend( ( (values, start, end, (start, end) in whole) for (values, start, end) in run_found ) ) return tuple(found) def read_numerals( text: str, every: bool = False ) -> list[tuple[list[Decimal], int, int]]: found = [ (values, start, end) for (values, start, end, whole) in _numeral_candidates(text) if every or whole ] return sorted(found, key=lambda item: (item[1], -item[2])) def numeral_spans(text: str, value: Any, every: bool = True) -> list[tuple[int, int]]: try: target = abs(Decimal(str(value))) except ArithmeticError: return [] candidates = _numeral_candidates(text) whole = [ (start, end) for (values, start, end, longest) in candidates if longest and target in values ] if whole or not every: return sorted(whole) return sorted( {(start, end) for (values, start, end, _) in candidates if target in values}, key=lambda item: (item[0], -item[1]), ) def slot_token_id(tokenizer: Any) -> int: ids = _tokens(tokenizer, SLOT_TEXT) if len(ids) != 1: raise ValueError(f"answer slots need {SLOT_TEXT!r} to be one token") return ids[0] def slot_block( tokenizer: Any, name: str, spec: dict[str, Any], complete_field: bool = False, template_variant: int = 0, ) -> list[int]: count = SLOT_COUNTS.get(head_kind(spec)) layout = prompt_layout(template_variant, complete_field) if count is None: return [] cue = _tokens(tokenizer, "\n" + name + ":") return ( _tokens(tokenizer, layout.slot_header) + (cue if complete_field else cue[:SLOT_CUE_TOKENS]) + [slot_token_id(tokenizer)] * count ) def system_prefix(record: dict[str, Any], text_answer_slots: bool = False) -> str: return ( EVIDENCE_SYSTEM_PREFIX if evidence_construction(record, text_answer_slots) else SYSTEM_PREFIX ) def evidence_construction( record: dict[str, Any], text_answer_slots: bool = False ) -> bool: return ( text_answer_slots or (record.get("_meta") or {}).get("construction") == zs.EVIDENCE_CONSTRUCTION ) def encode_record( tokenizer: Any, record: dict[str, Any], max_length: int = 8192, media: EncodedMedia | None = None, commits: dict[str, Commit] | None = None, answer_slots: bool = False, text_answer_slots: bool = False, template_variant: int = 0, ) -> EncodedRecord | None: if text_answer_slots and (not answer_slots): raise ValueError("text_answer_slots needs answer_slots") commits = commits or {} if record.get("images") and media is None: raise ValueError("records with images need their encoded media") media = media or EncodedMedia([]) items = list(record["fields"].items()) complete_field = evidence_construction(record, text_answer_slots) layout = prompt_layout(template_variant, complete_field) ids = _tokens(tokenizer, system_prefix(record, text_answer_slots)) for name, spec in items: ids.extend(_tokens(tokenizer, layout.schema_header)) ids.extend(_tokens(tokenizer, schema_line(name, spec, complete_field))) ids.extend(_tokens(tokenizer, reference_line(record))) ids.extend(_tokens(tokenizer, USER_PREFIX)) media_start = len(ids) ids.extend(media.ids) input_start = len(ids) suffix = _tokens(tokenizer, ASSISTANT_SUFFIX) blocks = [ query_block( tokenizer, name, spec, commits.get(name), complete_field, template_variant ) for (name, spec) in items ] slot_blocks: dict[int, list[int]] = {} if answer_slots and (media.ids or text_answer_slots): for index, (name, spec) in enumerate(items): if not (name in commits and commits[name].done): block = slot_block( tokenizer, name, spec, complete_field, template_variant ) if block: slot_blocks[index] = block budget = ( max_length - len(ids) - media.expansion - len(suffix) - sum((len(block[0]) for block in blocks)) - sum((len(block) for block in slot_blocks.values())) ) if budget < MIN_INPUT_TOKENS: return None text = record["input"] if media.ids and (not text.strip()): text = "" (input_ids, offsets) = (_tokens(tokenizer, "\n"), [(0, 0)]) else: (input_ids, offsets) = _input_tokens(tokenizer, text) if len(input_ids) > budget: keep = clean_cut(text, offsets, budget) text = text[: offsets[keep - 1][1]] (input_ids, offsets) = _input_tokens(tokenizer, text) if len(input_ids) > budget: (input_ids, offsets) = (input_ids[:budget], offsets[:budget]) text = text[: offsets[-1][1]] if not input_ids: return None ids.extend(input_ids) ids.extend(suffix) order = list(range(len(items))) done = { index for (index, (name, _)) in enumerate(items) if name in commits and commits[name].done } if done: order = [index for index in order if index in done] + [ index for index in order if index not in done ] queries: dict[int, int] = {} slots: dict[int, list[int]] = {} for index in order: (block, query, item_offsets) = blocks[index] start = len(ids) ids.extend(block) queries[index] = start + query slots[index] = [start + offset for offset in item_offsets] answer_positions: dict[int, list[int]] = {} for index in order: if index in slot_blocks: ids.extend(slot_blocks[index]) count = SLOT_COUNTS[head_kind(items[index][1])] answer_positions[index] = list(range(len(ids) - count, len(ids))) fields: list[EncodedField] = [] for index, (name, spec) in enumerate(items): if index in done: continue encoded = EncodedField(name, head_kind(spec), queries[index], slots[index]) if index in answer_positions: encoded.answer_slots = answer_positions[index] commit = commits.get(name) if ( commit is not None and commit.prefix is not None and (encoded.kind in ("number", "estimate")) ): encoded.prefix = commit.prefix if encoded.kind == "estimate": encoded.window = prefix_window(spec, commit.prefix) fields.append(encoded) if not fields: return None record_media = None if media.media is not None: record_media = {**media.media, "token_offset": media_start} return EncodedRecord( ids, input_start, offsets, fields, text, record, record_media, media.expansion ) def span_base(text: str, starts: list[int], ends: list[int], first: int) -> int: base = starts[first] while base < ends[first] - 1 and text[base].isspace(): base += 1 return base def trimmed_span( text: str, offsets: list[tuple[int, int]], first: int, last: int, trims: list[list[int]] | None, ) -> tuple[int, int]: (char_start, char_end) = (offsets[first][0], offsets[last][1]) while char_start < offsets[first][1] - 1 and text[char_start].isspace(): char_start += 1 if trims is not None: start = max( char_start, min(char_start + int(trims[first][0]), offsets[first][1] - 1) ) end = min(char_end, max(char_end - int(trims[last][1]), offsets[last][0] + 1)) if start < end: (char_start, char_end) = (start, end) while char_start < char_end - 1 and text[char_start].isspace(): char_start += 1 return (char_start, char_end) def plain_number(value: Any) -> str: number = Decimal(str(value)) if not number.is_finite(): raise ValueError("A plain number must be finite.") text = format(number, "f") return text.rstrip("0").rstrip(".") if "." in text else text def prefix_window( spec: dict[str, Any], prefix: tuple[int, int, tuple[int, ...]], bins: int = BINS ) -> tuple[int, int] | None: (sign, exponent_class, digits) = prefix power = exponent_class - 1 + MIN_EXPONENT unit = Decimal(1).scaleb(power - len(digits) + 1) start = Decimal(int("".join((str(digit) for digit in digits)))) * unit (first, last) = (float(start), float(start + unit)) if sign: (first, last) = (-last, -first) (low, high, scale) = zs.estimate_range(spec) (lower, upper) = sorted( (zs.raw_unit(first, low, high, scale), zs.raw_unit(last, low, high, scale)) ) if upper <= 0.0 or lower >= 1.0: return None first_bin = max(0, min(bins - 1, math.floor(lower * bins))) last_bin = max(first_bin, min(bins - 1, math.ceil(upper * bins) - 1)) return (first_bin, last_bin) def span_scores( start: torch.Tensor, end: torch.Tensor, length_bias: torch.Tensor ) -> torch.Tensor: width = length_bias.shape[0] padded = functional.pad(end, (0, width - 1), value=float("-inf")) return start.unsqueeze(-1) + padded.unfold(-1, width, 1) + length_bias DIGIT_SLOTS = NUMBER_DIGITS + YEAR_DIGITS + 2 + 2 + 2 + 2 (YEAR_SLOT, DAY_SLOT, HOUR_SLOT, MINUTE_SLOT, SECOND_SLOT) = ( NUMBER_DIGITS, NUMBER_DIGITS + YEAR_DIGITS, NUMBER_DIGITS + YEAR_DIGITS + 2, NUMBER_DIGITS + YEAR_DIGITS + 4, NUMBER_DIGITS + YEAR_DIGITS + 6, ) MONTH_WORDS = tuple((" " + name for name in calendar.month_name[1:])) SIGN_WORDS = (" ", " -") ROUTE_KINDS = ( "text", "tags", "number", "date", "time", "estimate", "number_list", "date_list", ) def two_digit_index(values: range) -> torch.Tensor: return torch.tensor( [[value // 10 for value in values], [value % 10 for value in values]], dtype=torch.long, ) def compose_two_digits(slots: torch.Tensor, index: torch.Tensor) -> torch.Tensor: return slots[..., 0, :].index_select(-1, index[0]) + slots[..., 1, :].index_select( -1, index[1] ) def compose_years(slots: torch.Tensor, index: torch.Tensor) -> torch.Tensor: inside = cast( torch.Tensor, sum( ( slots[..., position, :].index_select(-1, index[position]) for position in range(YEAR_DIGITS) ) ), ) total = torch.logsumexp(slots, dim=-1).sum(dim=-1) gap = (torch.logsumexp(inside, dim=-1) - total).clamp(max=-1e-06) outside = total + torch.log(-torch.expm1(gap)) return torch.cat([inside, outside.unsqueeze(-1)], dim=-1) class ClairicHead(nn.Module): digit_rows: torch.Tensor month_rows: torch.Tensor sign_rows: torch.Tensor year_index: torch.Tensor day_index: torch.Tensor hour_index: torch.Tensor sixty_index: torch.Tensor def __init__( self, hidden_size: int, bins: int = BINS, pointer_width: int = POINTER_WIDTH, readout: str = "linear", route_width: int = 1024, route_layers: int = 2, route_heads: int = 8, slot_rank: int = 32, estimate_lexical: bool = False, span_trim: bool = False, loop_feedback: bool = False, loop_scale: float = 1.0, number_groups: bool = False, number_pointer: bool = False, answer_slots: bool = False, answer_slot_routing: bool = False, ) -> None: super().__init__() if readout not in ("linear", "lexical"): raise ValueError(f"unknown readout {readout!r}") if (number_groups or number_pointer) and readout != "lexical": raise ValueError( "number_groups and number_pointer need the lexical readout" ) if estimate_lexical and readout != "lexical": raise ValueError("estimate_lexical needs the lexical readout") if answer_slot_routing and (readout != "lexical" or not answer_slots): raise ValueError( "answer_slot_routing needs lexical readout and answer_slots" ) self.config: dict[str, Any] = { "hidden_size": hidden_size, "bins": bins, "pointer_width": pointer_width, } if readout == "lexical": self.config.update( readout=readout, route_width=route_width, route_layers=route_layers, route_heads=route_heads, slot_rank=slot_rank, ) if estimate_lexical: self.config.update(estimate_lexical=True) if span_trim: self.config.update(span_trim=True) if loop_feedback: self.config.update(loop_feedback=True, loop_scale=float(loop_scale)) self.number_groups = False self.number_pointer = False self.answer_slots = bool(answer_slots) if answer_slots: self.config["answer_slots"] = True self.answer_slot_routing = bool(answer_slot_routing) if answer_slot_routing: self.config["answer_slot_routing"] = True self.slot_route_scale = nn.Parameter(torch.ones(len(ROUTE_KINDS))) self.readout = readout self.loop_feedback = loop_feedback self.loop_scale = float(loop_scale) self.span_trim = span_trim (self.estimate_lexical,) = (estimate_lexical,) (self.pointer_width,) = (pointer_width,) self.norm = nn.LayerNorm(hidden_size) self.token_norm = nn.LayerNorm(hidden_size) self.null = nn.Linear(hidden_size, 1) self.span_query = nn.Linear(hidden_size, 2 * pointer_width) self.span_key = nn.Linear(hidden_size, 2 * pointer_width) self.length_bias = nn.Parameter( -0.05 * torch.arange(MAX_SPAN, dtype=torch.float32) ) self.tag_query = nn.Linear(hidden_size, 2 * pointer_width) self.tag_key = nn.Linear(hidden_size, pointer_width) self.tag_bias = nn.Parameter(torch.tensor([-2.0, -2.0])) if span_trim: self.trim = nn.Linear(hidden_size, 2 * TRIM_CLASSES) nn.init.zeros_(self.trim.weight) with torch.no_grad(): self.trim.bias.copy_( torch.tensor(([TRIM_INIT] + [0.0] * (TRIM_CLASSES - 1)) * 2) ) if readout == "linear": self.number = nn.Linear(hidden_size, NUMBER_WIDTH) self.date = nn.Linear(hidden_size, DATE_WIDTH) self.time = nn.Linear(hidden_size, TIME_WIDTH) else: self.route_in = nn.Linear(hidden_size, route_width, bias=False) self.kind_embedding = nn.Embedding( len(ROUTE_KINDS) + len(LIST_KINDS), route_width ) self.memory_in = nn.Linear(hidden_size, route_width, bias=False) self.route_layers = nn.ModuleList( [ nn.TransformerDecoderLayer( d_model=route_width, nhead=route_heads, dim_feedforward=4 * route_width, dropout=0.0, activation="gelu", batch_first=True, norm_first=True, ) for _ in range(route_layers) ] ) self.route_norm = nn.LayerNorm(route_width) self.route_out = nn.Linear(route_width, hidden_size) nn.init.zeros_(self.route_out.weight) nn.init.zeros_(self.route_out.bias) self.route_gate = nn.Parameter(torch.tensor(-2.0)) self.register_buffer( "digit_rows", torch.randn(10, hidden_size) / math.sqrt(hidden_size) ) self.register_buffer( "month_rows", torch.randn(12, hidden_size) / math.sqrt(hidden_size) ) self.register_buffer( "sign_rows", torch.randn(2, hidden_size) / math.sqrt(hidden_size) ) self.lexical_scale = nn.Parameter(torch.full((3,), math.log(10.0))) self.slot_down = nn.Linear( hidden_size, (DIGIT_SLOTS + 2) * slot_rank, bias=False ) self.slot_up = nn.Parameter(torch.zeros(DIGIT_SLOTS, slot_rank, 10)) self.month_up = nn.Parameter(torch.zeros(slot_rank, 12)) self.sign_up = nn.Parameter(torch.zeros(slot_rank, 2)) self.slot_residual = nn.Linear(route_width, DIGIT_SLOTS * 10 + 12 + 2) nn.init.zeros_(self.slot_residual.weight) nn.init.zeros_(self.slot_residual.bias) self.residual_gate = nn.Parameter(torch.tensor(-2.0)) self.exponent = nn.Linear(hidden_size, EXPONENT_CLASSES) self.slot_rank = slot_rank self.register_buffer( "year_index", torch.tensor( [ [ int(f"{year:04d}"[position]) for year in range(YEAR_LOW, YEAR_HIGH + 1) ] for position in range(YEAR_DIGITS) ], dtype=torch.long, ), persistent=False, ) self.register_buffer( "day_index", two_digit_index(range(1, 32)), persistent=False ) self.register_buffer( "hour_index", two_digit_index(range(24)), persistent=False ) self.register_buffer( "sixty_index", two_digit_index(range(60)), persistent=False ) self.item_end = nn.Linear(hidden_size, 1) self.bins = nn.Linear(hidden_size, bins) self.offset = nn.Linear(hidden_size, bins) self.overflow = nn.Linear(hidden_size, 2) nn.init.zeros_(self.overflow.weight) nn.init.constant_(self.overflow.bias, OVERFLOW_INIT) nn.init.zeros_(self.offset.weight) nn.init.zeros_(self.offset.bias) if estimate_lexical: self.estimate_gate = nn.Linear(hidden_size, 1) nn.init.zeros_(self.estimate_gate.weight) nn.init.constant_(self.estimate_gate.bias, ESTIMATE_GATE_INIT) self.estimate_spread = nn.Linear(hidden_size, 1 + LEXICAL_DIGITS) nn.init.zeros_(self.estimate_spread.weight) nn.init.zeros_(self.estimate_spread.bias) if loop_feedback: self.loop_norm = nn.LayerNorm(hidden_size) self.loop_down = nn.Linear(hidden_size, LOOP_RANK, bias=False) self.loop_up = nn.Linear(LOOP_RANK, hidden_size, bias=False) nn.init.zeros_(self.loop_up.weight) if number_groups: self.enable_number_groups() if number_pointer: self.enable_number_pointer() edges = torch.linspace(0.0, 1.0, bins + 1) self.register_buffer("bin_edges", edges, persistent=False) def enable_number_groups(self) -> None: if self.readout != "lexical": raise ValueError("number_groups needs the lexical readout") if self.number_groups: return hidden_size = self.config["hidden_size"] device = self.norm.weight.device self.number_group_down = nn.Linear(hidden_size, GROUP_RANK).to(device) self.number_group_up = nn.Linear(GROUP_RANK, NUMBER_GROUPS * GROUP_CLASSES).to( device ) nn.init.zeros_(self.number_group_up.weight) nn.init.zeros_(self.number_group_up.bias) (self.number_groups,) = (True,) self.config.update(number_groups=True) def enable_number_pointer(self) -> None: if self.readout != "lexical": raise ValueError("number_pointer needs the lexical readout") if self.number_pointer: return device = self.norm.weight.device self.copy_query = nn.Linear( self.config["hidden_size"], 2 * self.pointer_width ).to(device) self.copy_gate = nn.Linear(self.config["hidden_size"], 1).to(device) with torch.no_grad(): self.copy_query.weight.copy_(self.span_query.weight) self.copy_query.bias.copy_(self.span_query.bias) self.copy_gate.weight.zero_() self.copy_gate.bias.fill_(COPY_GATE_INIT) (self.number_pointer,) = (True,) self.config.update(number_pointer=True) def group_scores(self, states: torch.Tensor) -> torch.Tensor: return self.number_group_up( functional.gelu(self.number_group_down(states.float())) ).view(-1, NUMBER_GROUPS, GROUP_CLASSES) def loop_input(self, states: torch.Tensor) -> torch.Tensor: with torch.autocast(device_type=states.device.type, enabled=False): return self.loop_scale * self.loop_up( self.loop_down(self.loop_norm(states.float())) ) @torch.no_grad() def set_lexical_rows(self, tokenizer: Any, output_embeddings: torch.Tensor) -> None: if self.readout != "lexical": return for name, words in ( ("digit_rows", [str(digit) for digit in range(10)]), ("month_rows", MONTH_WORDS), ("sign_rows", SIGN_WORDS), ): rows = [] for word in words: ids = tokenizer(word, add_special_tokens=False)["input_ids"] if not ids: raise ValueError(f"tokenizer has no tokens for {word!r}") rows.append( output_embeddings[ torch.tensor(ids, device=output_embeddings.device) ] .float() .mean(dim=0) ) getattr(self, name).copy_(torch.stack(rows).to(getattr(self, name).device)) def route( self, hidden: torch.Tensor, positions: torch.Tensor, kinds: torch.Tensor, stop: int, ) -> tuple[torch.Tensor, torch.Tensor]: queries = ( self.route_in(self.norm(hidden[positions].float())) + self.kind_embedding(kinds) ).unsqueeze(0) memory = self.memory_in(self.token_norm(hidden[:stop].float())).unsqueeze(0) for layer in self.route_layers: queries = layer(queries, memory) routed = self.route_norm(queries[0]) return (torch.sigmoid(self.route_gate) * self.route_out(routed), routed) def answer_slot_states( self, hidden: torch.Tensor, field_queries: torch.Tensor, kinds: torch.Tensor, positions: torch.Tensor, lengths: torch.Tensor, stop: int, chunk: int = 256, ) -> torch.Tensor: if not self.answer_slot_routing: return hidden[positions] if ( chunk < 1 or hidden.ndim != 2 or field_queries.ndim != 1 or (positions.ndim != 2) or (positions.shape[0] != field_queries.numel()) ): raise ValueError("invalid answer-slot routing layout") width = positions.shape[1] if kinds.shape != field_queries.shape or lengths.shape != field_queries.shape: raise ValueError("answer-slot fields, kinds and lengths must align") if ( width < 1 or bool(((lengths < 1) | (lengths > width)).any()) or (not 0 < stop <= hidden.shape[0]) ): raise ValueError("invalid answer-slot lengths or evidence boundary") if ( bool(((positions < 0) | (positions >= hidden.shape[0])).any()) or bool(((field_queries < 0) | (field_queries >= hidden.shape[0])).any()) or bool(((kinds < 0) | (kinds >= len(ROUTE_KINDS))).any()) ): raise ValueError("answer-slot query, token or kind out of range") raw = hidden[positions] if not positions.shape[0]: return raw output = [] with torch.autocast(device_type=hidden.device.type, enabled=False): memory = self.memory_in(self.token_norm(hidden[:stop].float())).unsqueeze(0) def route_chunk( query_positions: torch.Tensor, query_kinds: torch.Tensor, padding: torch.Tensor, ) -> torch.Tensor: queries = self.route_in(self.norm(hidden[query_positions].float())) queries = queries + self.kind_embedding(query_kinds)[:, None, :] for layer in self.route_layers: queries = layer( queries, memory.expand(queries.shape[0], -1, -1), tgt_key_padding_mask=padding, ) return self.route_out(self.route_norm(queries[:, 1:])) field_batch = max(1, min(chunk // (width + 1), 262144 // stop)) for first in range(0, positions.shape[0], field_batch): last = first + field_batch rows = positions[first:last] query_positions = torch.cat( [field_queries[first:last, None], rows], dim=1 ) padding = ( torch.arange(width + 1, device=hidden.device)[None, :] > lengths[first:last, None] ) evidence = route_chunk(query_positions, kinds[first:last], padding) scale = self.slot_route_scale[kinds[first:last]].tanh()[:, None, None] corrected = ( raw[first:last].float() + scale * torch.sigmoid(self.route_gate) * evidence ) indices = torch.minimum( torch.arange(width, device=hidden.device)[None, :], lengths[first:last, None] - 1, ) output.append( corrected.gather( 1, indices[..., None].expand(-1, -1, corrected.shape[-1]) ) ) return torch.cat(output).to(hidden.dtype) def typed_logits( self, raw: torch.Tensor, routed: torch.Tensor ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: count = raw.shape[0] scale = self.lexical_scale.clamp(max=math.log(100.0)).exp() unit = functional.normalize(raw, dim=-1) down = self.slot_down(raw).view(count, DIGIT_SLOTS + 2, self.slot_rank) digit_prior = ( unit @ functional.normalize(self.digit_rows, dim=-1).t() ).unsqueeze(1) month_prior = unit @ functional.normalize(self.month_rows, dim=-1).t() sign_prior = unit @ functional.normalize(self.sign_rows, dim=-1).t() residual = torch.sigmoid(self.residual_gate) * self.slot_residual(routed) (digit_residual, month_residual, sign_residual) = residual.split( (DIGIT_SLOTS * 10, 12, 2), dim=-1 ) return ( scale[0] * digit_prior + torch.einsum("nsr,srd->nsd", down[:, :DIGIT_SLOTS], self.slot_up) + digit_residual.view(count, DIGIT_SLOTS, 10), scale[1] * month_prior + down[:, DIGIT_SLOTS] @ self.month_up + month_residual, scale[2] * sign_prior + down[:, DIGIT_SLOTS + 1] @ self.sign_up + sign_residual, ) def typed_outputs( self, raw: torch.Tensor, states: torch.Tensor, routed: torch.Tensor ) -> dict[str, torch.Tensor]: (digits, months, signs) = self.typed_logits(raw, routed) year_slots = digits[:, YEAR_SLOT : YEAR_SLOT + YEAR_DIGITS] return { "number": torch.cat( [signs, self.exponent(states), digits[:, :NUMBER_DIGITS].flatten(1)], dim=-1, ), "date": torch.cat( [ year_slots.flatten(1), compose_years(year_slots, self.year_index), months, compose_two_digits( digits[:, DAY_SLOT : DAY_SLOT + 2], self.day_index ), ], dim=-1, ), "time": torch.cat( [ compose_two_digits( digits[:, HOUR_SLOT : HOUR_SLOT + 2], self.hour_index ), compose_two_digits( digits[:, MINUTE_SLOT : MINUTE_SLOT + 2], self.sixty_index ), compose_two_digits( digits[:, SECOND_SLOT : SECOND_SLOT + 2], self.sixty_index ), ], dim=-1, ), } def forward( self, hidden: torch.Tensor, batch: dict[str, Any] ) -> list[dict[str, Any]]: return [ self.forward_record(hidden[index], record, batch["views"][index]) for (index, record) in enumerate(batch["records"]) ] def forward_record( self, hidden: torch.Tensor, record: EncodedRecord, views: dict[str, torch.Tensor], ) -> dict[str, Any]: return self.forward_states( hidden, record.input_start, record.input_start + len(record.offsets), views ) def forward_states( self, hidden: torch.Tensor, start: int, stop: int, views: dict[str, torch.Tensor], ) -> dict[str, Any]: if self.readout == "lexical": return self.forward_lexical(hidden, start, stop, views) with torch.autocast(device_type=hidden.device.type, enabled=False): states = self.norm(hidden[views["query_positions"]].float()) output: dict[str, Any] = {"null": self.null(states).squeeze(-1)} tokens = None if views["text_rows"].shape[0] or views["tags_rows"].shape[0]: tokens = self.token_norm(hidden[start:stop].float()) if self.span_trim: output["trim"] = self.trim(tokens).view(-1, 2, TRIM_CLASSES) scale = 1.0 / math.sqrt(self.pointer_width) if views["text_rows"].shape[0]: (query_start, query_end) = self.span_query( states[views["text_rows"]] ).split(self.pointer_width, dim=-1) (key_start, key_end) = self.span_key(tokens).split( self.pointer_width, dim=-1 ) output["text"] = span_scores( query_start @ key_start.transpose(0, 1) * scale, query_end @ key_end.transpose(0, 1) * scale, self.length_bias, ) if views["tags_rows"].shape[0]: (query_begin, query_inside) = self.tag_query( states[views["tags_rows"]] ).split(self.pointer_width, dim=-1) keys = self.tag_key(tokens) begin = query_begin @ keys.transpose(0, 1) * scale + self.tag_bias[0] inside = query_inside @ keys.transpose(0, 1) * scale + self.tag_bias[1] output["tags"] = torch.stack( [torch.zeros_like(begin), begin, inside], dim=-1 ) for group in ("number", "date", "time"): if views[f"{group}_rows"].shape[0]: output[group] = getattr(self, group)(states[views[f"{group}_rows"]]) if "number" in output: output["number"] = apply_prefix(output["number"], views, "number") if views["estimate_rows"].shape[0]: estimate_states = states[views["estimate_rows"]] output["estimate"] = self.bins(estimate_states) output["estimate_offset"] = self.offset(estimate_states) output["estimate_overflow"] = self.overflow(estimate_states) for kind, readout in ( ("number_list", self.number), ("date_list", self.date), ): if views[f"{kind}_rows"].shape[0]: items = self.norm(hidden[views[f"{kind}_items"]].float()) output[f"{kind}_end"] = self.item_end(items).squeeze(-1) output[kind] = readout(items) return output def forward_lexical( self, hidden: torch.Tensor, start: int, stop: int, views: dict[str, torch.Tensor], ) -> dict[str, Any]: with torch.autocast(device_type=hidden.device.type, enabled=False): queries = views["query_positions"] item_kinds = [kind for kind in LIST_KINDS if views[f"{kind}_rows"].shape[0]] positions = torch.cat( [queries] + [views[f"{kind}_items"].flatten() for kind in item_kinds] ) kinds = torch.zeros( positions.shape[0], dtype=torch.long, device=positions.device ) for index, group in enumerate(ROUTE_KINDS): kinds[views[f"{group}_rows"]] = index offset = queries.shape[0] for kind in item_kinds: size = views[f"{kind}_items"].numel() kinds[offset : offset + size] = len(ROUTE_KINDS) + LIST_KINDS.index( kind ) offset += size (gated, routed) = self.route(hidden, positions, kinds, stop) raw = hidden[positions].float() states = self.norm(raw) + gated count = queries.shape[0] output: dict[str, Any] = {"null": self.null(states[:count]).squeeze(-1)} (field_states, field_raw, field_routed) = ( states[:count], raw[:count], routed[:count], ) tokens = None pointed = self.number_pointer and views["number_rows"].shape[0] > 0 if views["text_rows"].shape[0] or views["tags_rows"].shape[0] or pointed: tokens = self.token_norm(hidden[start:stop].float()) if self.span_trim and ( views["text_rows"].shape[0] or views["tags_rows"].shape[0] ): output["trim"] = self.trim(tokens).view(-1, 2, TRIM_CLASSES) scale = 1.0 / math.sqrt(self.pointer_width) if pointed: number_states = field_states[views["number_rows"]] (query_start, query_end) = self.copy_query(number_states).split( self.pointer_width, dim=-1 ) (key_start, key_end) = self.span_key(tokens).split( self.pointer_width, dim=-1 ) output["number_copy"] = span_scores( query_start @ key_start.transpose(0, 1) * scale, query_end @ key_end.transpose(0, 1) * scale, self.length_bias, ) output["number_copy_gate"] = self.copy_gate(number_states).squeeze(-1) if views["text_rows"].shape[0]: (query_start, query_end) = self.span_query( field_states[views["text_rows"]] ).split(self.pointer_width, dim=-1) (key_start, key_end) = self.span_key(tokens).split( self.pointer_width, dim=-1 ) output["text"] = span_scores( query_start @ key_start.transpose(0, 1) * scale, query_end @ key_end.transpose(0, 1) * scale, self.length_bias, ) if views["tags_rows"].shape[0]: (query_begin, query_inside) = self.tag_query( field_states[views["tags_rows"]] ).split(self.pointer_width, dim=-1) keys = self.tag_key(tokens) begin = query_begin @ keys.transpose(0, 1) * scale + self.tag_bias[0] inside = query_inside @ keys.transpose(0, 1) * scale + self.tag_bias[1] output["tags"] = torch.stack( [torch.zeros_like(begin), begin, inside], dim=-1 ) typed_rows = [ views[f"{group}_rows"] for group in ("number", "date", "time") ] if any((rows.shape[0] for rows in typed_rows)): selected = torch.cat(typed_rows) typed = self.typed_outputs( field_raw[selected], field_states[selected], field_routed[selected] ) offset = 0 for group, rows in zip(("number", "date", "time"), typed_rows): if rows.shape[0]: output[group] = typed[group][offset : offset + rows.shape[0]] offset += rows.shape[0] if "number" in output: output["number"] = apply_prefix(output["number"], views, "number") if self.number_groups: output["number_groups"] = self.group_scores( field_states[views["number_rows"]] ) if views["estimate_rows"].shape[0]: estimate_rows = views["estimate_rows"] estimate_states = field_states[estimate_rows] output["estimate"] = self.bins(estimate_states) output["estimate_offset"] = self.offset(estimate_states) output["estimate_overflow"] = self.overflow(estimate_states) if self.estimate_lexical: number_logits = self.typed_outputs( field_raw[estimate_rows], estimate_states, field_routed[estimate_rows], )["number"] output["estimate_number_raw"] = apply_prefix( number_logits, views, "estimate" ) output["estimate_number"] = apply_prefix( temper_number( number_logits, self.estimate_spread(estimate_states) ), views, "estimate", ) output["estimate_gate"] = self.estimate_gate( estimate_states ).squeeze(-1) offset = count for kind in item_kinds: shape = views[f"{kind}_items"].shape size = shape[0] * shape[1] (item_states, item_raw, item_routed) = ( states[offset : offset + size], raw[offset : offset + size], routed[offset : offset + size], ) offset += size output[f"{kind}_end"] = ( self.item_end(item_states).squeeze(-1).view(shape) ) group = "number" if kind == "number_list" else "date" output[kind] = self.typed_outputs(item_raw, item_states, item_routed)[ group ].view(*shape, -1) if kind == "number_list" and self.number_groups: output["number_list_groups"] = self.group_scores(item_states).view( *shape, NUMBER_GROUPS, GROUP_CLASSES ) return output SPELL_END = "\n" SPELL_SEPARATOR = " | " SPELL_MIN_CONFIDENCE = 0.3 def spelled_answer( ids: list[int], log_probs: list[float], tokenizer: Any, end_id: int ) -> tuple[str, float, list[tuple[str, float]]]: stop = ids.index(end_id) if end_id in ids else len(ids) pieces = [tokenizer.decode([token]) for token in ids[:stop]] items: list[tuple[list[int], list[float]]] = [] current: list[int] = [] scores: list[float] = [] for token, piece, score in zip(ids[:stop], pieces, log_probs[:stop]): if piece.strip() == SPELL_SEPARATOR.strip(): items.append((current, scores)) (current, scores) = ([], []) else: current.append(token) scores.append(score) items.append((current, scores)) parts = [] for item_ids, item_scores in items: words = tokenizer.decode(item_ids).split() if words: parts.append( (" ".join(words), math.exp(sum(item_scores) / len(item_scores))) ) text = SPELL_SEPARATOR.join((part for (part, _) in parts)) return (text, math.exp(sum(log_probs[: min(stop + 1, len(ids))])), parts) def evidence_text_answer( prediction: dict[str, Any], ids: list[int], log_probs: list[float], tokenizer: Any, end_id: int, text: str, spec: dict[str, Any], has_images: bool = False, ) -> dict[str, Any]: if end_id not in ids: return prediction stop = ids.index(end_id) value = tokenizer.decode(ids[:stop]).strip() confidence = math.exp(sum(log_probs[: stop + 1])) exact = ( [ span for span in zs.find_spans(text, value) if text[span[0] : span[1]] == value ] if value else [] ) present = prediction["p_null"] < zs.null_threshold(spec) return { **prediction, "guess": value or None, "value": value if value and present else None, "span": list(exact[0]) if exact else None, "confidence": confidence, "source": "input" if exact else "image" if has_images else "readout", } def _simple(value: Any) -> str: return "".join( (character for character in str(value).lower() if character.isalnum()) ) def _similarity(left: Any, right: Any) -> float: import difflib (left, right) = (_simple(left), _simple(right)) return difflib.SequenceMatcher(None, left, right).ratio() if left and right else 0.0 def _state_match( text: str, value: str, threshold: float = 0.8, max_words: int = 4000 ) -> tuple[int, int] | None: if not text or not value: return None spans = zs.find_spans(text, value) if spans: return spans[0] import re words = [match.span() for match in re.finditer("\\S+", text)][:max_words] size = len(value.split()) (best, best_span) = (threshold, None) for width in range(max(1, size - 1), size + 2): for first in range(max(0, len(words) - width + 1)): (start, end) = (words[first][0], words[first + width - 1][1]) score = _similarity(text[start:end].strip(".,;:()[]\"'"), value) if score >= best: (best, best_span) = (score, (start, end)) return best_span def _stutters(value: Any) -> bool: import re simple = _simple(value) return bool(re.search("(.)\\1\\1", simple) or re.search("([a-z]{4,})\\1", simple)) def _digit_code(value: Any) -> bool: simple = _simple(value) return bool(simple) and 2 * sum( (character.isdigit() for character in simple) ) >= len(simple) PLACEHOLDER_WORDS = frozenset( ( "a", "above", "an", "app", "attach", "attached", "attachment", "below", "doc", "document", "documents", "file", "heic", "here", "image", "images", "is", "jpeg", "jpg", "mobile", "page", "pdf", "photo", "photos", "pic", "picture", "please", "png", "scan", "scanned", "screenshot", "see", "the", "this", "upload", "uploaded", "via", ) ) def _placeholder_text(value: Any) -> bool: import re words = re.findall("[a-z0-9]+", str(value or "").lower()) return bool(words) and all((word in PLACEHOLDER_WORDS for word in words)) def _defers(pointer_value: Any, spelled: str) -> bool: if pointer_value is None: return False return _stutters(spelled) or ( _digit_code(spelled) and any((character.isdigit() for character in str(pointer_value))) ) def route_spelled( prediction: dict[str, Any], kind: str, spelled: str, items: list[tuple[str, float]], text: str, defer_lists: bool = False, ) -> dict[str, Any]: if kind == "text": if _placeholder_text(prediction.get("guess")): prediction = { **prediction, "value": None, "guess": None, "span": None, "confidence": 0.0, } items = [ (item, score) for (item, score) in items if not _placeholder_text(item) ] if not items: return prediction (value, confidence) = ( items[0][0] if len(items) == 1 else spelled, min((score for (_, score) in items)), ) routed = {**prediction, "spell_confidence": confidence} if _similarity(prediction.get("guess"), value) >= 0.6: if ( routed.get("value") is None and prediction.get("guess") and (confidence >= SPELL_MIN_CONFIDENCE) ): return {**routed, "value": prediction["guess"]} return routed span = _state_match(text, value) if span is not None: return { **routed, "value": text[span[0] : span[1]], "guess": text[span[0] : span[1]], "span": list(span), } if confidence >= SPELL_MIN_CONFIDENCE and ( not _defers(prediction.get("value"), value) ): return { **routed, "value": value, "guess": value, "span": None, "confidence": confidence, "source": "image", } return routed (pointer_values, pointer_spans) = ( list(prediction.get("value") or []), list(prediction.get("spans") or []), ) if any((_placeholder_text(value) for value in pointer_values)): kept = [ (value, pointer_spans[index] if index < len(pointer_spans) else None) for (index, value) in enumerate(pointer_values) if not _placeholder_text(value) ] (pointer_values, pointer_spans) = ( [value for (value, _) in kept], [span for (_, span) in kept], ) items = [(item, score) for (item, score) in items if not _placeholder_text(item)] (values, spans, used) = ([], [], set()) for item, confidence in items: match = max( range(len(pointer_values)), key=lambda index: _similarity(pointer_values[index], item), default=None, ) if ( match is not None and match not in used and (_similarity(pointer_values[match], item) >= 0.6) ): used.add(match) values.append(pointer_values[match]) spans.append(pointer_spans[match] if match < len(pointer_spans) else None) continue span = _state_match(text, item) if span is not None: value = text[span[0] : span[1]] if not any((_simple(value) == _simple(other) for other in values)): values.append(value) spans.append(list(span)) elif ( confidence >= SPELL_MIN_CONFIDENCE and ( not ( pointer_values and (defer_lists or _defers(pointer_values[0], item)) ) ) and (not any((_simple(item) == _simple(other) for other in values))) ): values.append(item) spans.append(None) for index, (value, span) in enumerate(zip(pointer_values, pointer_spans)): if index not in used and ( not any((_simple(value) == _simple(other) for other in values)) ): values.append(value) spans.append(span) routed = {**prediction, "value": values, "spans": spans} if any((span is None for span in spans)): routed["source"] = "image" return routed def number_parts( logits: torch.Tensor, ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: (sign, exponent, digits) = logits.split( (2, EXPONENT_CLASSES, NUMBER_DIGITS * 10), dim=-1 ) return (sign, exponent, digits.unflatten(-1, (NUMBER_DIGITS, 10))) def group_class_digits(device: Any = None) -> torch.Tensor: classes = torch.arange(GROUP_CLASSES, device=device) return torch.stack( [ classes // 10 ** (GROUP_DIGITS - 1 - place) % 10 for place in range(GROUP_DIGITS) ], dim=-1, ) def group_slots(exponent_class: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]: exponent_class = exponent_class.long() device = exponent_class.device power = (exponent_class - 1 + MIN_EXPONENT).view(-1, 1, 1) groups = torch.arange(GROUP_LOW, GROUP_HIGH + 1, device=device).view(1, -1, 1) places = torch.arange(GROUP_DIGITS - 1, -1, -1, device=device).view(1, 1, -1) slots = power - (groups * GROUP_DIGITS + places) inside = ( (slots >= 0) & (slots < NUMBER_DIGITS) & (exponent_class > 0).view(-1, 1, 1) ) return ( torch.where(inside, slots, torch.full_like(slots, NUMBER_DIGITS)), inside.any(dim=-1), ) def number_group_logits( digit_logits: torch.Tensor, exponent_class: torch.Tensor, learned: torch.Tensor | None = None, ) -> tuple[torch.Tensor, torch.Tensor]: (count, device) = (digit_logits.shape[0], digit_logits.device) leading_zero = torch.zeros(NUMBER_DIGITS, 10, dtype=torch.bool, device=device) leading_zero[0, 0] = True log = torch.log_softmax(digit_logits.float(), dim=-1).masked_fill( leading_zero, MASKED ) held = torch.full((count, 1, 10), MASKED, device=device, dtype=log.dtype) held[..., 0] = 0.0 padded = torch.cat([log, held], dim=1) (slots, groups) = group_slots(exponent_class.to(device)) picked = padded.gather(1, slots.flatten(1).unsqueeze(-1).expand(-1, -1, 10)).view( count, NUMBER_GROUPS, GROUP_DIGITS, 10 ) table = group_class_digits(device) scores = cast( torch.Tensor, sum( ( picked[:, :, place, :].index_select(-1, table[:, place]) for place in range(GROUP_DIGITS) ) ), ) if learned is not None: scores = scores + learned.float() return (scores, groups) def temper_number(logits: torch.Tensor, spread: torch.Tensor) -> torch.Tensor: (sign, exponent, digits) = number_parts(logits) temperature = torch.exp( ESTIMATE_SPREAD_LIMIT * torch.tanh(spread.float() / ESTIMATE_SPREAD_LIMIT) ) exponent = exponent / temperature[:, :1] digits = torch.cat( [ digits[:, :LEXICAL_DIGITS] / temperature[:, 1:].unsqueeze(-1), digits[:, LEXICAL_DIGITS:], ], dim=1, ) return torch.cat([sign, exponent, digits.flatten(1)], dim=-1) def apply_prefix( logits: torch.Tensor, views: dict[str, torch.Tensor], group: str ) -> torch.Tensor: length = views.get(f"{group}_prefix_length") if length is None or not logits.shape[0]: return logits device = logits.device (sign_logits, exponent_logits, digit_logits) = number_parts(logits) slots = torch.arange(NUMBER_DIGITS, device=device) length = length.long() relative = (slots.unsqueeze(0) - length.unsqueeze(1)).clamp_min(0) shifted = digit_logits.gather(1, relative.unsqueeze(-1).expand(-1, -1, 10)) forced_digits = torch.where( torch.arange(10, device=device) == views[f"{group}_prefix_digits"].long().unsqueeze(-1), 0.0, MASKED, ).to(logits.dtype) digit_logits = torch.where( (slots.unsqueeze(0) < length.unsqueeze(1)).unsqueeze(-1), forced_digits, shifted ) written = (length > 0).unsqueeze(1) forced_sign = torch.where( torch.arange(2, device=device) == views[f"{group}_prefix_sign"].long().unsqueeze(1), 0.0, MASKED, ).to(logits.dtype) forced_exponent = torch.where( torch.arange(EXPONENT_CLASSES, device=device) == views[f"{group}_prefix_exponent"].long().unsqueeze(1), 0.0, MASKED, ).to(logits.dtype) return torch.cat( [ torch.where(written, forced_sign, sign_logits), torch.where(written, forced_exponent, exponent_logits), digit_logits.flatten(1), ], dim=-1, ) def window_log_probs( log_probs: torch.Tensor, window: torch.Tensor ) -> tuple[torch.Tensor, torch.Tensor]: bins = torch.arange(log_probs.shape[-1], device=log_probs.device) window = window.to(log_probs.device).long() inside = (bins >= window[:, :1]) & (bins <= window[:, 1:]) return (torch.log_softmax(log_probs.masked_fill(~inside, MASKED), dim=-1), inside) def date_parts( logits: torch.Tensor, ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: (digits, year, month, day) = logits.split( (YEAR_DIGITS * 10, YEAR_CLASSES, 12, 31), dim=-1 ) return (digits.unflatten(-1, (YEAR_DIGITS, 10)), year, month, day) def time_parts(logits: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: return logits.split((24, 60, 60), dim=-1) def lexical_bin_log_probs( number_logits: torch.Tensor, edge_sign: torch.Tensor, edge_class: torch.Tensor, edge_digits: torch.Tensor, edge_frac: torch.Tensor, ) -> torch.Tensor: (sign, exponent, digits) = number_parts(number_logits.double()) sign_probs = torch.softmax(sign, dim=-1) exponent_probs = torch.softmax(exponent, dim=-1) zero = exponent_probs[:, :1] nonzero = exponent_probs[:, 1:] / (1.0 - zero).clamp_min(1e-300) below_class = torch.cumsum(nonzero, dim=-1) - nonzero leading = digits[:, :1].masked_fill( torch.arange(10, device=digits.device) == 0, float("-inf") ) digit_probs = torch.softmax( torch.cat([leading, digits[:, 1:LEXICAL_DIGITS]], dim=1), dim=-1 ) below_digit = torch.cumsum(digit_probs, dim=-1) - digit_probs classes = (edge_class - 1).clamp(0, nonzero.shape[1] - 1) within = edge_frac.double() for position in reversed(range(LEXICAL_DIGITS)): index = edge_digits[..., position] within = ( below_digit[:, position].gather(1, index) + digit_probs[:, position].gather(1, index) * within ) magnitude = below_class.gather(1, classes) + nonzero.gather(1, classes) * within negative = sign_probs[:, 1:] * (1.0 - zero) positive = sign_probs[:, :1] * (1.0 - zero) cdf = torch.where( edge_sign == EDGE_NEGATIVE, negative * (1.0 - magnitude), torch.where( edge_sign == EDGE_ZERO, negative.expand_as(magnitude), negative + zero + positive * magnitude, ), ) mass = (cdf[:, 1:] - cdf[:, :-1]).clamp_min(0.0) + 1e-30 return (mass.log() - mass.sum(dim=-1, keepdim=True).log()).float() def estimate_log_probs( range_logits: torch.Tensor, number_logits: torch.Tensor | None = None, gate: torch.Tensor | None = None, lexical: torch.Tensor | None = None, edge_sign: torch.Tensor | None = None, edge_class: torch.Tensor | None = None, edge_digits: torch.Tensor | None = None, edge_frac: torch.Tensor | None = None, ) -> torch.Tensor: range_part = torch.log_softmax(range_logits.float(), dim=-1) if number_logits is None or not number_logits.shape[0]: return range_part lexical_part = lexical_bin_log_probs( number_logits, cast(torch.Tensor, edge_sign), cast(torch.Tensor, edge_class), cast(torch.Tensor, edge_digits), cast(torch.Tensor, edge_frac), ) use = cast(torch.Tensor, lexical).bool().unsqueeze(1) gate = cast(torch.Tensor, gate).float().unsqueeze(1) mixed = torch.logaddexp( functional.logsigmoid(gate) + lexical_part, functional.logsigmoid(-gate) + range_part, ) return torch.where(use, mixed, range_part) def decode_number( logits: torch.Tensor, spec: dict[str, Any], groups: torch.Tensor | None = None ) -> tuple[float | int, float]: (sign_logits, exponent_logits, digit_logits) = number_parts(logits.float().cpu()) decimals = zs.allowed_decimals(spec) exponent_log = torch.log_softmax(exponent_logits, dim=-1) exponent_class = int(exponent_log.argmax()) log_probability = float(exponent_log[exponent_class]) if exponent_class == 0: return (0, math.exp(log_probability)) power = exponent_class - 1 + MIN_EXPONENT kept = NUMBER_DIGITS if decimals is None else power + decimals + 1 if kept < 0: return (0, math.exp(log_probability)) leading_zero = torch.zeros(NUMBER_DIGITS, 10, dtype=torch.bool) leading_zero[0, 0] = True digit_log = torch.log_softmax(digit_logits, dim=-1).masked_fill( leading_zero, MASKED ) digits = digit_log.argmax(dim=-1).tolist() sign_log = torch.log_softmax(sign_logits, dim=-1) sign = int(sign_log.argmax()) kept = min(kept, NUMBER_DIGITS) log_probability += float(sign_log[sign]) if groups is None: log_probability += sum( (float(digit_log[position, digits[position]]) for position in range(kept)) ) if kept < NUMBER_DIGITS: rounding = digit_log[kept, 5:] if digits[kept] >= 5 else digit_log[kept, :5] log_probability += float(torch.logsumexp(rounding, dim=0)) else: exponent_tensor = torch.tensor([exponent_class]) (scores, present) = number_group_logits( digit_logits.unsqueeze(0), exponent_tensor, groups.float().cpu().unsqueeze(0), ) group_log = torch.log_softmax(scores[0], dim=-1) slots = group_slots(exponent_tensor)[0][0].tolist() table = group_class_digits() counted = set() for group in range(NUMBER_GROUPS): if not bool(present[0, group]): continue best = int(group_log[group].argmax()) for place, slot in enumerate(slots[group]): if slot < NUMBER_DIGITS: digits[slot] = int(table[best, place]) if any((slot < kept for slot in slots[group])): counted.add(group) log_probability += float(group_log[group, best]) if kept < NUMBER_DIGITS: for group in range(NUMBER_GROUPS): if kept in slots[group]: if group not in counted: place = slots[group].index(kept) same = ( table[:, place] >= 5 if digits[kept] >= 5 else table[:, place] < 5 ) log_probability += float( torch.logsumexp(group_log[group][same], dim=0) ) break else: rounding = ( digit_log[kept, 5:] if digits[kept] >= 5 else digit_log[kept, :5] ) log_probability += float(torch.logsumexp(rounding, dim=0)) magnitude = Decimal(int("".join((str(digit) for digit in digits)))).scaleb( power - NUMBER_DIGITS + 1 ) if decimals is not None: magnitude = magnitude.quantize( Decimal(1).scaleb(-decimals), rounding=ROUND_HALF_UP ) if magnitude == 0: return (0, math.exp(log_probability)) value = -magnitude if sign else magnitude if decimals == 0 or (value == value.to_integral_value() and abs(value) < 2**53): return (int(value), math.exp(log_probability)) return (float(value), math.exp(log_probability)) def number_from_numeral( magnitude: Decimal, logits: torch.Tensor, spec: dict[str, Any], check_exponent: bool = True, ) -> float | int | None: (sign_logits, exponent_logits, _) = number_parts(logits.float().cpu()) if check_exponent: target = number_components(float(magnitude)) if target is None or int(exponent_logits.argmax()) != target[1]: return None decimals = zs.allowed_decimals(spec) if decimals is not None: magnitude = magnitude.quantize( Decimal(1).scaleb(-decimals), rounding=ROUND_HALF_UP ) if magnitude == 0: return 0 value = -magnitude if int(sign_logits.argmax()) else magnitude if decimals == 0 or (value == value.to_integral_value() and abs(value) < 2**53): return int(value) return float(value) def number_from_numerals( values: list[Decimal], logits: torch.Tensor, spec: dict[str, Any], readout: float | None, confident: bool = False, ) -> float | int | None: agreeing = [ value for value in ( number_from_numeral(magnitude, logits, spec) for magnitude in values ) if value is not None ] if not agreeing and confident and values: def distance(magnitude: Decimal) -> float: if readout is None or not readout or (not magnitude): return 0.0 return abs(math.log10(float(magnitude)) - math.log10(abs(float(readout)))) agreeing = [ value for value in ( number_from_numeral(min(values, key=distance), logits, spec, False), ) if value is not None ] if not agreeing: return None if readout is None: return agreeing[0] return min(agreeing, key=lambda value: abs(float(value) - float(readout))) def copied_number( output: dict[str, Any], row: int, record: EncodedRecord, spec: dict[str, Any], readout: float | None = None, ) -> tuple[float | int, float, list[int]] | None: if ( "number_copy_gate" not in output or record.record.get("images") or (not record.text) ): return None if os.environ.get("SZ_DISABLE_NUMBER_COPY") == "1": return None gate = float( torch.sigmoid(torch.as_tensor(output["number_copy_gate"][row]).float()) ) if gate < 0.5: return None if "number_copy_best" in output: (best, probability) = ( output["number_copy_best"][row], output["number_copy_probability"][row], ) else: scores = output["number_copy"][row].float().flatten() (best, probability) = ( int(scores.argmax()), float(torch.softmax(scores, dim=0).max()), ) (start, length) = divmod(int(best), MAX_SPAN) last = min(start + length, len(record.offsets) - 1) if start >= len(record.offsets): return None (char_start, char_end) = (record.offsets[start][0], record.offsets[last][1]) while char_start < char_end and (not record.text[char_start].isalnum()): char_start += 1 around = [ found for found in read_numerals(record.text, every=True) if found[1] < char_end and found[2] > char_start ] if not around: return None nearest = min( around, key=lambda found: ( abs(found[1] - char_start) + abs(found[2] - char_end), -(found[2] - found[1]), ), ) whole = [ found for found in read_numerals(record.text) if found[1] <= char_start and found[2] >= char_end and (found[2] - found[1] > nearest[2] - nearest[1]) ] choices = whole[:1] + [nearest] if gate * probability < COPY_MIN_CONFIDENCE: return None sure = gate >= COPY_CONFIDENT (value, numeral_start, numeral_end) = (None, nearest[1], nearest[2]) for values, start, end in choices: run = NUMERAL_RUN.match(record.text, start) printed = record.text[start : min(end, run.end())] if run else "" settled = ( numeral_convention(record.text) is not None and len(numeral_values(printed)) > 1 ) enclosing = bool(whole) and (start, end) == (whole[0][1], whole[0][2]) value = number_from_numerals( values, output["number"][row], spec, readout, confident=settled or (sure and enclosing and (len(values) == 1)), ) if value is not None: (numeral_start, numeral_end) = (start, end) break if value is None and sure: (values, numeral_start, numeral_end) = choices[0] value = number_from_numerals( values, output["number"][row], spec, readout, confident=True ) if value is None or (readout is not None and rescaled(readout, value)): return None return (value, gate * probability, [numeral_start, numeral_end]) def rescaled(readout: float, copied: float) -> bool: try: (first, second) = ( abs(Decimal(str(readout))).normalize(), abs(Decimal(str(copied))).normalize(), ) except ArithmeticError: return False if not first or not second or first == second: return False return first.as_tuple().digits == second.as_tuple().digits def decode_date(logits: torch.Tensor) -> tuple[str, float]: (digit_logits, year_logits, month_logits, day_logits) = date_parts( logits.float().cpu() ) class_log = torch.log_softmax(year_logits, dim=-1) year_class = int(class_log.argmax()) log_probability = float(class_log[year_class]) if year_class < YEAR_OUTSIDE: year = YEAR_LOW + year_class else: digit_log = torch.log_softmax(digit_logits, dim=-1) digits = digit_log.argmax(dim=-1).tolist() if digits == [0, 0, 0, 0]: masked = digit_log[3].clone() masked[0] = MASKED digits[3] = int(masked.argmax()) year = int("".join((str(digit) for digit in digits))) log_probability += float( sum((digit_log[position, digit] for (position, digit) in enumerate(digits))) ) (month_log, day_log) = ( torch.log_softmax(month_logits, dim=-1), torch.log_softmax(day_logits, dim=-1), ) joint = month_log.unsqueeze(1) + day_log.unsqueeze(0) lengths = torch.tensor( [calendar.monthrange(year, month)[1] for month in range(1, 13)] ) joint = joint.masked_fill( torch.arange(31).unsqueeze(0) >= lengths.unsqueeze(1), MASKED ) best = int(joint.flatten().argmax()) (month, day) = divmod(best, 31) log_probability += float(joint.flatten()[best]) return ( datetime.date(year, month + 1, day + 1).isoformat(), math.exp(log_probability), ) def decode_time(logits: torch.Tensor) -> tuple[str, float]: parts = [ torch.log_softmax(part, dim=-1) for part in time_parts(logits.float().cpu()) ] values = [int(part.argmax()) for part in parts] return ( zs.format_time(*values), math.exp(sum((float(part[value]) for (part, value) in zip(parts, values)))), ) def decode_spans( labels: list[int], offsets: list[tuple[int, int]], text: str, trims: list[list[int]] | None = None, ) -> tuple[list[str], list[list[int]]]: spans: list[list[int]] = [] current: list[int] | None = None for position, label in enumerate(labels): if label == TAG_BEGIN: if current is not None: spans.append(current) current = [position, position] elif label == TAG_INSIDE and current is not None: current[1] = position elif current is not None: spans.append(current) current = None if current is not None: spans.append(current) values: list[str] = [] char_spans: list[list[int]] = [] for first, last in spans: (char_start, char_end) = trimmed_span(text, offsets, first, last, trims) if not text[char_start:char_end].strip(): continue char_spans.append([char_start, char_end]) if text[char_start:char_end] not in values: values.append(text[char_start:char_end]) return (values, char_spans) def quantile_from_cdf( cdf: list[float], probabilities: list[float], fraction: float, bins: int ) -> float: index = min(bisect_left(cdf, fraction), bins - 1) previous = cdf[index - 1] if index > 0 else 0.0 inside = ( (fraction - previous) / probabilities[index] if probabilities[index] > 0 else 0.5 ) return (index + min(1.0, max(0.0, inside))) / bins def decode_estimate( logits: torch.Tensor, offsets: torch.Tensor, overflow: torch.Tensor, spec: dict[str, Any], ) -> dict[str, Any]: probabilities = torch.softmax(logits.float(), dim=-1) cdf = torch.cumsum(probabilities, dim=-1).tolist() row = probabilities.tolist() bins = len(row) shift = torch.sigmoid(offsets.float()).tolist() (low, high, scale) = zs.estimate_range(spec) centers = [(index + shift[index]) / bins for index in range(bins)] mean = sum( ( probability * zs.from_unit(center, low, high, scale) for (probability, center) in zip(row, centers) ) ) quantiles: list[Any] = [ zs.from_unit( quantile_from_cdf(cdf, row, fraction / 100.0, bins), low, high, scale ) for fraction in range(1, 100) ] value = ( zs.from_unit( sum((probability * center for (probability, center) in zip(row, centers))), low, high, scale, ) if scale == "log" else mean ) step = zs.estimate_step(spec) if step > 0: quantiles = [ zs.snap_quantile(quantile, step, low, high) for quantile in quantiles ] value = zs.snap(mean, step, low, high) if zs.is_date_estimate(spec): (value, mean) = (zs.estimate_value(spec, value), zs.estimate_value(spec, mean)) quantiles = [zs.estimate_value(spec, quantile) for quantile in quantiles] prediction = { "value": value, "mean": mean, "median": quantiles[49], "p10": quantiles[9], "p90": quantiles[89], "quantiles": quantiles, } if not zs.is_date_estimate(spec): sides = torch.softmax( torch.cat([overflow.new_zeros(1), overflow.float()]), dim=0 ).tolist() (prediction["p_below"], prediction["p_above"]) = (sides[BELOW], sides[ABOVE]) return prediction def estimate_row_log_probs( output: dict[str, Any], row: int, spec: dict[str, Any], window: tuple[int, int] | None = None, ) -> torch.Tensor: log_probs = _estimate_row_log_probs(output, row, spec) if window is None: return log_probs return window_log_probs(log_probs.unsqueeze(0), torch.tensor([window]))[0][0] def _estimate_row_log_probs( output: dict[str, Any], row: int, spec: dict[str, Any] ) -> torch.Tensor: logits = output["estimate"][row].float().cpu() parts = ( estimate_edge_parts(spec, logits.shape[-1]) if "estimate_number" in output else None ) if parts is None: return torch.log_softmax(logits, dim=-1) (sign, classes, digits, fractions) = parts count = len(sign) return estimate_log_probs( logits.unsqueeze(0), output["estimate_number"][row].float().cpu().unsqueeze(0), output["estimate_gate"][row].float().cpu().reshape(1), torch.ones(1), torch.tensor([sign]), torch.tensor([classes]), torch.tensor(digits).view(1, count, LEXICAL_DIGITS), torch.tensor([fractions], dtype=torch.float64), )[0] def decode_record( output: dict[str, Any], record: EncodedRecord, tokenizer: Any = None, end_id: int | None = None, ) -> dict[str, dict[str, Any]]: fields = record.fields specs = {item.name: record.record["fields"][item.name] for item in fields} p_null = torch.sigmoid(output["null"].float()).tolist() predictions: dict[str, dict[str, Any]] = {} value: Any rows_of = { group: [ position for (position, item) in enumerate(fields) if item.kind in kinds ] for (group, kinds) in ( ("text", ("text",)), ("tags", ("text_list",)), ("number", ("number",)), ("date", ("date", "datetime")), ("time", ("time", "datetime")), ("estimate", ("estimate",)), ("number_list", ("number_list",)), ("date_list", ("date_list",)), ) } index_of = { group: {position: row for (row, position) in enumerate(rows)} for (group, rows) in rows_of.items() } if rows_of["text"] and "text_best" in output: (best, probabilities) = (output["text_best"], output["text_probability"]) elif rows_of["text"]: scores = output["text"].float() best = scores.flatten(1).argmax(dim=1).tolist() probabilities = torch.softmax(scores.flatten(1), dim=1).amax(dim=1).tolist() if rows_of["tags"]: labels = ( output["tags_labels"] if "tags_labels" in output else output["tags"].argmax(dim=-1).tolist() ) trims = output.get("trim_labels") if trims is None and "trim" in output: trims = output["trim"].argmax(dim=-1).tolist() for position, item in enumerate(fields): spec = specs[item.name] kind = item.kind if kind == "estimate": row = index_of["estimate"][position] predictions[item.name] = decode_estimate( estimate_row_log_probs(output, row, spec, item.window), output["estimate_offset"][row], output["estimate_overflow"][row], spec, ) continue if kind == "text_list": row = index_of["tags"][position] (values, spans) = decode_spans( labels[row], record.offsets, record.text, trims ) predictions[item.name] = {"value": values, "spans": spans} if "tags_log_probability" in output: predictions[item.name]["confidence"] = math.exp( output["tags_log_probability"][row] ) elif "tags" in output: label_log = torch.log_softmax(output["tags"][row].float().cpu(), dim=-1) predictions[item.name]["confidence"] = math.exp( float(label_log.amax(dim=-1).sum()) ) continue if kind in LIST_KINDS: row = index_of[kind][position] ends = torch.sigmoid(output[f"{kind}_end"][row].float()).tolist() (values, confidences) = ([], []) log_probability = 0.0 for slot in range(LIST_SLOTS): if ends[slot] >= 0.5: log_probability += math.log(max(ends[slot], 1e-12)) break log_probability += math.log(max(1.0 - ends[slot], 1e-12)) if kind == "number_list": (value, confidence) = decode_number( output[kind][row, slot], spec, output["number_list_groups"][row, slot] if "number_list_groups" in output else None, ) else: (value, confidence) = decode_date(output[kind][row, slot]) values.append(value) confidences.append(confidence) log_probability += math.log(max(confidence, 1e-12)) predictions[item.name] = { "value": values, "confidences": confidences, "confidence": math.exp(log_probability), } continue prediction: dict[str, Any] = {"p_null": p_null[position]} if kind == "text": row = index_of["text"][position] (start, length) = divmod(best[row], MAX_SPAN) last = min(start + length, len(record.offsets) - 1) (char_start, char_end) = trimmed_span( record.text, record.offsets, start, last, trims ) (value, confidence) = (record.text[char_start:char_end], probabilities[row]) prediction["span"] = [char_start, char_end] if not record.text: (value, prediction["span"]) = (None, None) elif kind == "number": number_row = index_of["number"][position] (value, confidence) = decode_number( output["number"][number_row], spec, output["number_groups"][number_row] if "number_groups" in output else None, ) copied = copied_number(output, number_row, record, spec, value) if ( copied is not None and value is not None and (os.environ.get("SZ_NUMBER_COPY_VETO", "1") == "1") and numeral_spans(record.text, value, every=False) ): copied = None if copied is not None: (value, confidence, prediction["span"]) = copied prediction["source"] = "input" if spec["type"] == "duration": prediction["unit"] = zs.duration_unit(spec) elif kind == "date": (value, confidence) = decode_date( output["date"][index_of["date"][position]] ) elif kind == "time": (value, confidence) = decode_time( output["time"][index_of["time"][position]] ) else: (day, day_confidence) = decode_date( output["date"][index_of["date"][position]] ) (clock, clock_confidence) = decode_time( output["time"][index_of["time"][position]] ) (value, confidence) = (f"{day}T{clock}", day_confidence * clock_confidence) prediction["guess"] = value prediction["confidence"] = confidence prediction["value"] = ( None if p_null[position] >= zs.null_threshold(spec) else value ) predictions[item.name] = prediction if tokenizer is not None and end_id is not None and ("spell_ids" in output): (ids, log_probs) = ( output["spell_ids"].tolist(), output["spell_log_probs"].float().tolist(), ) for position, item in enumerate(fields): if item.kind == "text": row = index_of["text"][position] elif item.kind == "text_list": row = len(rows_of["text"]) + index_of["tags"][position] else: continue (spelled, _, parts) = spelled_answer( ids[row], log_probs[row], tokenizer, end_id ) predictions[item.name] = route_spelled( predictions[item.name], item.kind, spelled, parts, record.text ) if tokenizer is not None and "slot_ids" in output: slot_end = _tokens(tokenizer, SPELL_END)[0] (slot_ids, slot_log_probs) = ( output["slot_ids"].tolist(), output["slot_log_probs"].float().tolist(), ) slot_rows = [ position for (position, item) in enumerate(fields) if item.answer_slots ] for row, position in enumerate(slot_rows): item = fields[position] if item.kind not in ("text", "text_list"): continue count = len(item.answer_slots) (spelled, _probability, parts) = spelled_answer( slot_ids[row][:count], slot_log_probs[row][:count], tokenizer, slot_end ) prediction = predictions[item.name] if (record.record.get("_meta") or {}).get( "construction" ) == zs.EVIDENCE_CONSTRUCTION and item.kind == "text": predictions[item.name] = evidence_text_answer( prediction, slot_ids[row][:count], slot_log_probs[row][:count], tokenizer, slot_end, record.text, specs[item.name], bool(record.record.get("images")), ) continue if item.kind in ("text", "text_list"): predictions[item.name] = route_spelled( prediction, item.kind, spelled, parts, record.text, defer_lists=True ) continue return predictions FIELD_KINDS = SCALAR_KINDS + ("estimate", "text_list") + LIST_KINDS VIEW_GROUPS = ( ("text", ("text",)), ("tags", ("text_list",)), ("number", ("number",)), ("date", ("date", "datetime")), ("time", ("time", "datetime")), ("estimate", ("estimate",)), ) COMPACT_KEYS = ( "number", "date", "time", "estimate", "estimate_offset", "estimate_overflow", "number_list_end", "number_list", "date_list_end", "date_list", "estimate_number", "estimate_gate", "number_groups", "number_list_groups", "number_copy_gate", ) COMPACT_GROUPS = { "number": "number", "date": "date", "time": "time", "estimate": "estimate", "estimate_offset": "estimate", "estimate_overflow": "estimate", "number_list_end": "number_list", "number_list": "number_list", "date_list_end": "date_list", "date_list": "date_list", "estimate_number": "estimate", "estimate_gate": "estimate", "number_groups": "number", "number_list_groups": "number_list", "number_copy_gate": "number", } def prediction_layout(record: EncodedRecord) -> list[int]: layout = [ len(record.input_ids), record.input_start, len(record.offsets), len(record.fields), ] for item in record.fields: layout += [ FIELD_KINDS.index(item.kind), item.query, len(item.items), *item.items, ] slotted = [ (index, item.answer_slots) for (index, item) in enumerate(record.fields) if item.answer_slots ] if slotted: layout.append(len(slotted)) for index, slots in slotted: layout += [index, len(slots), *slots] return layout def parse_prediction_layout( layout: list[int], num_tokens: int, with_slots: bool = False ) -> tuple[Any, ...]: if len(layout) < 4 or any((type(value) is not int for value in layout)): raise ValueError("prediction_layout must be a list of ints") (prompt_length, input_start, input_length, count) = layout[:4] shift = num_tokens - prompt_length if ( shift < 0 or count < 1 or input_length < 1 or (not 0 <= input_start < input_start + input_length <= prompt_length) ): raise ValueError("prediction_layout does not match the prompt") kinds: list[str] = [] queries: list[int] = [] items: list[list[int]] = [] cursor = 4 for _ in range(count): if cursor + 3 > len(layout): raise ValueError("prediction_layout is truncated") (kind, query, size) = layout[cursor : cursor + 3] cursor += 3 if ( not 0 <= kind < len(FIELD_KINDS) or not input_start + input_length <= query < prompt_length ): raise ValueError("prediction_layout has an invalid field") expected = LIST_SLOTS if FIELD_KINDS[kind] in LIST_KINDS else 0 slots = layout[cursor : cursor + size] cursor += size if ( size != expected or len(slots) != size or any((not query < slot < prompt_length for slot in slots)) ): raise ValueError("prediction_layout has invalid item positions") kinds.append(FIELD_KINDS[kind]) queries.append(query + shift) items.append([slot + shift for slot in slots]) slot_fields: list[int] = [] answer_slots: list[list[int]] = [] if cursor < len(layout): slotted = layout[cursor] cursor += 1 if not 0 < slotted <= count: raise ValueError("prediction_layout has invalid answer slots") for _ in range(slotted): if cursor + 2 > len(layout): raise ValueError("prediction_layout is truncated") (index, size) = layout[cursor : cursor + 2] cursor += 2 positions = layout[cursor : cursor + size] cursor += size if ( not 0 <= index < count or (slot_fields and index <= slot_fields[-1]) or size < 1 or (len(positions) != size) or any( ( not queries[index] - shift < slot < prompt_length for slot in positions ) ) ): raise ValueError("prediction_layout has invalid answer slots") slot_fields.append(index) answer_slots.append([slot + shift for slot in positions]) if cursor != len(layout): raise ValueError("prediction_layout has trailing values") parsed = (input_start + shift, input_length, kinds, queries, items) return (*parsed, slot_fields, answer_slots) if with_slots else parsed def prediction_views( kinds: list[str], queries: list[int], items: list[list[int]], device: Any = None ) -> dict[str, torch.Tensor]: views = {"query_positions": torch.tensor(queries, dtype=torch.long, device=device)} for group, members in VIEW_GROUPS: views[f"{group}_rows"] = torch.tensor( [row for (row, kind) in enumerate(kinds) if kind in members], dtype=torch.long, device=device, ) for kind in LIST_KINDS: rows = [row for (row, field_kind) in enumerate(kinds) if field_kind == kind] views[f"{kind}_rows"] = torch.tensor(rows, dtype=torch.long, device=device) views[f"{kind}_items"] = torch.tensor( [slot for row in rows for slot in items[row]], dtype=torch.long, device=device, ).view(len(rows), LIST_SLOTS) return views def compact_output(output: dict[str, Any]) -> torch.Tensor: parts = [output["null"].float().flatten()] if "text" in output: scores = output["text"].float().flatten(1) parts += [ scores.argmax(dim=1).float(), torch.softmax(scores, dim=1).amax(dim=1), ] if "tags" in output: tag_log = torch.log_softmax(output["tags"].float(), dim=-1) parts.append(tag_log.argmax(dim=-1).float().flatten()) parts.append(tag_log.amax(dim=-1).sum(dim=-1)) if "trim" in output: parts.append(output["trim"].argmax(dim=-1).float().flatten()) if "number_copy" in output: scores = output["number_copy"].float().flatten(1) parts += [ scores.argmax(dim=1).float(), torch.softmax(scores, dim=1).amax(dim=1), ] parts += [output[key].float().flatten() for key in COMPACT_KEYS if key in output] if "slot_ids" in output: parts += [ output["slot_ids"].float().flatten(), output["slot_log_probs"].float().flatten(), ] return torch.cat(parts) def expand_output( flat: torch.Tensor, record: EncodedRecord, bins: int = BINS, estimate_lexical: bool = False, span_trim: bool = False, number_groups: bool = False, number_pointer: bool = False, ) -> dict[str, Any]: flat = flat.float().cpu() kinds = [item.kind for item in record.fields] rows = { group: sum((kind in members for kind in kinds)) for (group, members) in VIEW_GROUPS } rows.update({kind: kinds.count(kind) for kind in LIST_KINDS}) cursor = 0 def take(*shape: int) -> torch.Tensor: nonlocal cursor size = math.prod(shape) if cursor + size > flat.numel(): raise ValueError("compact output is shorter than the layout needs") value = flat[cursor : cursor + size].view(*shape) cursor += size return value output: dict[str, Any] = {"null": take(len(kinds))} if rows["text"]: output["text_best"] = take(rows["text"]).round().long().tolist() output["text_probability"] = take(rows["text"]).tolist() if rows["tags"]: output["tags_labels"] = ( take(rows["tags"], len(record.offsets)).round().long().tolist() ) output["tags_log_probability"] = take(rows["tags"]).tolist() if span_trim and (rows["text"] or rows["tags"]): output["trim_labels"] = take(len(record.offsets), 2).round().long().tolist() if number_pointer and rows["number"]: output["number_copy_best"] = take(rows["number"]).round().long().tolist() output["number_copy_probability"] = take(rows["number"]).tolist() widths = { "number": (NUMBER_WIDTH,), "date": (DATE_WIDTH,), "time": (TIME_WIDTH,), "estimate": (bins,), "estimate_offset": (bins,), "estimate_overflow": (2,), "number_list_end": (LIST_SLOTS,), "number_list": (LIST_SLOTS, NUMBER_WIDTH), "date_list_end": (LIST_SLOTS,), "date_list": (LIST_SLOTS, DATE_WIDTH), "estimate_number": (NUMBER_WIDTH,), "estimate_gate": (), "number_groups": (NUMBER_GROUPS, GROUP_CLASSES), "number_list_groups": (LIST_SLOTS, NUMBER_GROUPS, GROUP_CLASSES), "number_copy_gate": (), } for key in COMPACT_KEYS: if key in ("estimate_number", "estimate_gate") and (not estimate_lexical): continue if key in ("number_groups", "number_list_groups") and (not number_groups): continue if key == "number_copy_gate" and (not number_pointer): continue count = rows[COMPACT_GROUPS[key]] if count: output[key] = take(count, *widths[key]) slotted = [len(item.answer_slots) for item in record.fields if item.answer_slots] if slotted: output["slot_ids"] = take(len(slotted), max(slotted)).round().long() output["slot_log_probs"] = take(len(slotted), max(slotted)) if cursor != flat.numel(): raise ValueError("compact output is longer than the layout needs") return output def merge_forecasts( predictions: dict[str, dict[str, Any]], groups: dict[str, list[str]] ) -> dict[str, dict[str, Any]]: merged = { name: prediction for (name, prediction) in predictions.items() if not any((name in names for names in groups.values())) } for name, names in groups.items(): steps = [predictions[step] for step in names] merged[name] = { key: [step[key] for step in steps] for key in ("value", "median", "p10", "p90") } merged[name]["steps"] = steps return merged