from __future__ import annotations import json import re import unicodedata from decimal import Decimal, InvalidOperation, localcontext from typing import Any from . import ar MAX_GRAMMAR_CHARACTERS = 4096 _STATE_ONLY = re.compile( "\\b(?:from (?:the )?(?:accompanying (?:state )?|state )text only|supplied in the accompanying state text)\\b", re.IGNORECASE, ) _NULL_ABSENCE_GUARD = re.compile( "(?:^|(?<=[.;\\n]))\\s*return\\s+null\\s+(?:only\\s+)?if\\s+(?:not\\s+(?:explicitly\\s+)?printed|absent|missing)\\s*(?:[.;]|$)", re.IGNORECASE, ) _EXCLUDED_SOURCE = re.compile( "\\b(?:not|ignore|exclude|excluding)\\b[^.;:\\n]*$", re.IGNORECASE ) _DOCUMENT = "(?:(?:this|the)(?:\\s+(?:supplied|provided))?|supplied|provided)\\s+(?:note|document|text)" _DOCUMENT_ONLY = re.compile( f"\\b(?:use\\s+(?:only\\s+{_DOCUMENT}|{_DOCUMENT}\\s+only)|from\\s+{_DOCUMENT}\\s+only)\\b", re.IGNORECASE, ) _DOCUMENT_EXCLUDED = re.compile( "\\b(?:not|never|don't|ignore|exclude|excluding)\\b[^.;:\\n]*$", re.IGNORECASE ) _DERIVED_NUMBER = re.compile( "\\b(?:calculat\\w*|comput\\w*|deriv\\w*|convert\\w*|conversion\\w*|count\\w*|sum\\w*|averag\\w*|mean|median|difference|ratio|growth|product|multipli\\w*|multiply|subtract\\w*|minus|divid\\w*|division|quotient|remainder|add|adding|addition|plus|times|percent\\w*|proportion|fraction)\\b|\\b(?:how many|how much|total number of|number of (?:line )?(?:items|rows|entries|records))\\b|[+*/\\u00d7\\u00f7=]|\\s[-\\u2212]\\s", re.IGNORECASE, ) _CURRENCY_CODES = frozenset( { "USD", "EUR", "GBP", "JPY", "CNY", "INR", "AUD", "CAD", "CHF", "HKD", "NZD", "SGD", "SEK", "NOK", "DKK", "KRW", "BRL", "MXN", "ZAR", } ) _EXACT_TEXT = re.compile( "\\b(?:verbatim|raw|literal\\w*|print\\w*|exact\\w*|characters?|digits?|whitespace|spacing|newlines?)\\b|\\b(?:line\\s+breaks?|white\\s+space|do\\s+not\\s+normaliz\\w*)\\b", re.IGNORECASE, ) def allows_whitespace_normalization(spec: dict[str, Any]) -> bool: return ( spec.get("type") == "text" and (not spec.get("knowledge")) and ("source_only" not in spec) and (not spec.get("_physical_row_scope")) and ( not any( (key in spec for key in ("enum", "pattern", "minLength", "maxLength")) ) ) and (not _EXACT_TEXT.search(spec.get("instructions", ""))) ) def collapse_whitespace(source: str) -> tuple[str, list[tuple[int, int]]]: (characters, offsets) = ([], []) for match in re.finditer("\\s+|\\S", source): characters.append(" " if match[0].isspace() else match[0]) offsets.append(match.span()) return ("".join(characters), offsets) def state_only(spec: dict[str, Any]) -> bool: instructions = spec.get("instructions", "") scope = _NULL_ABSENCE_GUARD.sub("", instructions) return ( spec["type"] in ("text", "number") and (not spec.get("knowledge")) and (not re.search("\\b(?:if|unless|compare|combine)\\b", scope, re.IGNORECASE)) and any( ( not _EXCLUDED_SOURCE.search(instructions[: match.start()]) for match in _STATE_ONLY.finditer(instructions) ) ) ) def explicit_media_scope(spec: dict[str, Any]) -> bool: return ( spec.get("type") in ("text", "number") and (not spec.get("knowledge")) and bool( re.search( "\\b(?:page|image)\\s+\\d+\\b", spec.get("instructions", ""), re.IGNORECASE, ) ) ) def document_only(spec: dict[str, Any]) -> bool: instructions = spec.get("instructions", "") scope = _NULL_ABSENCE_GUARD.sub("", instructions) return ( spec["type"] in ("text", "number") and (not spec.get("knowledge")) and (not re.search("\\b(?:if|unless|compare|combine)\\b", scope, re.IGNORECASE)) and (not (spec["type"] == "number" and _DERIVED_NUMBER.search(scope))) and any( ( not _DOCUMENT_EXCLUDED.search(instructions[: match.start()]) for match in _DOCUMENT_ONLY.finditer(instructions) ) ) ) def applies(spec: dict[str, Any], media: object | None) -> bool: return ( spec["type"] in ("text", "number") and media is None and (not spec.get("knowledge", False)) and ( spec["type"] == "text" or spec.get("source_only", False) or state_only(spec) or document_only(spec) ) ) def _document_number_span(text: str, span: tuple[int, int]) -> bool: (start, _) = span left = start while left and (text[left - 1].isalnum() or text[left - 1] in "_./-"): left -= 1 prefix = text[left:start].lstrip("_./-") return not prefix or not prefix[0].isalpha() or prefix.upper() in _CURRENCY_CODES def _text_states(source: str, prefix: str) -> list[str]: transitions: list[dict[str, int]] (transitions, links, lengths) = ([{}], [-1], [0]) last = 0 for char in source: current = len(transitions) transitions.append({}) lengths.append(lengths[last] + 1) links.append(0) parent = last while parent >= 0 and char not in transitions[parent]: transitions[parent][char] = current parent = links[parent] if parent >= 0: target = transitions[parent][char] if lengths[parent] + 1 == lengths[target]: links[current] = target else: clone = len(transitions) transitions.append(dict(transitions[target])) lengths.append(lengths[parent] + 1) links.append(links[target]) while parent >= 0 and transitions[parent].get(char) == target: transitions[parent][char] = clone parent = links[parent] links[target] = links[current] = clone last = current quote = json.dumps('"') rules = [] for index, edges in enumerate(transitions): choices = [quote] if index else [] for char, target in edges.items(): escaped = json.dumps(char, ensure_ascii=False)[1:-1] choices.append( json.dumps(escaped, ensure_ascii=False) + f" {prefix}{target}" ) rules.append(f"{prefix}{index} ::= " + " | ".join(choices)) return rules def text_grammar( source: str, *, nullable: bool = True, normalized_source: str | None = None ) -> str: if not source: if not nullable: raise ValueError("Empty source permits no non-null text value") return 'root ::= "null"' quote = json.dumps('"') starts = ['"null"'] if nullable else [] starts.append(quote + " s0") states = _text_states(source, "s") if normalized_source and normalized_source != source: starts.append(quote + " n0") states.extend(_text_states(normalized_source, "n")) return "\n".join(["root ::= " + " | ".join(starts), *states]) def numbers(source: str) -> dict[Decimal, list[tuple[int, int]]]: from .model import NUMERAL_RUN, numeral_convention, numeral_values convention = numeral_convention(source) result: dict[Decimal, list[tuple[int, int]]] = {} consumed_end = 0 for run in NUMERAL_RUN.finditer(source): if run.start() < consumed_end: continue readings = [(run.group(), run.start(), run.end())] if not numeral_values(run.group(), convention): readings = [ (m.group(), run.start() + m.start(), run.start() + m.end()) for m in re.finditer("[^, \\u00a0\\u202f\\u2009\\u2007]+", run.group()) ] for printed, left, end in readings: values = numeral_values(printed, convention) if not values: continue exponent = re.match("[eE]([-+]?\\d+)", source[end:]) if exponent: consumed_end = end + len(exponent[0]) if len(exponent[1]) > 4 or abs(int(exponent[1])) > 308: continue with localcontext() as context: context.prec = max(context.prec, len(printed) + 310) values = [ value * Decimal(10) ** int(exponent[1]) for value in values ] end += len(exponent[0]) start = left while start > max(0, left - 32) and ( source[start - 1] in " \t-+−" or unicodedata.category(source[start - 1]) == "Sc" ): start -= 1 signed = source[start:left].strip() negative = any((c in signed for c in "-−")) and ( start == 0 or not source[start - 1].isalnum() ) left = start + len(source[start:left]) - len(source[start:left].lstrip()) before = source[max(0, left - 32) : left] after = source[end : end + 64] (prefix, suffix) = (before.rstrip(), after.lstrip()) accounting = ( any( ( c in ".," or unicodedata.category(c) == "Sc" for c in printed + signed ) ) or not suffix[1:].split("\n")[0].strip() ) if prefix.endswith("(") and suffix.startswith(")") and accounting: negative = True left -= len(before) - len(prefix) + 1 end += len(after) - len(suffix) + 1 for value in values: if negative: value = value.copy_negate() numeric = ( int(value) if value == value.to_integral_value() else float(value) ) if Decimal(str(numeric)) != value: continue result.setdefault(value, []).append((left, end)) return result class Source: def __init__(self, text: str, *, normalize_whitespace: bool = False) -> None: self.text = text self.numerals = numbers(text) self.normalize_whitespace = normalize_whitespace self._normalized_source: tuple[str, list[tuple[int, int]]] | None = None self._grammars: dict[tuple[bool, bool], str] = {} self._constraints: dict[tuple[str, bool, bool], dict[str, str]] = {} def whitespace_normalization(self, spec: dict[str, Any]) -> bool: return self.normalize_whitespace and allows_whitespace_normalization(spec) def _whitespace_source(self) -> tuple[str, list[tuple[int, int]]]: if self._normalized_source is None: self._normalized_source = collapse_whitespace(self.text) return self._normalized_source def normalized_span(self, value: object) -> tuple[bool, list[int] | None]: if not isinstance(value, str) or not value: return (False, None) matches = set() start = self.text.find(value) while start >= 0: matches.add((start, start + len(value))) start = self.text.find(value, start + 1) (normalized, offsets) = self._whitespace_source() start = normalized.find(value) while start >= 0: matches.add((offsets[start][0], offsets[start + len(value) - 1][1])) start = normalized.find(value, start + 1) return (bool(matches), list(next(iter(matches))) if len(matches) == 1 else None) def span( self, value: object, *, spec: dict[str, Any] | None = None ) -> tuple[bool, list[int] | None]: if isinstance(value, str): if spec is not None and self.whitespace_normalization(spec): return self.normalized_span(value) if not value: return (False, None) start = self.text.find(value) if start < 0: return (False, None) unique = bool(value) and self.text.find(value, start + 1) < 0 return (True, [start, start + len(value)] if unique else None) try: number = Decimal(str(value)) except InvalidOperation: return (False, None) if not number.is_finite(): return (False, None) matches = self.numerals.get(number, []) return (bool(matches), list(matches[0]) if len(matches) == 1 else None) def constraint(self, spec: dict[str, Any]) -> dict[str, str]: document_numbers = ( spec["type"] == "number" and document_only(spec) and (not spec.get("source_only")) and (not state_only(spec)) ) key = ( json.dumps(ar.schema(spec), ensure_ascii=False), bool(document_numbers), self.whitespace_normalization(spec), ) if key not in self._constraints: self._constraints[key] = self._constraint( spec, document_numbers=document_numbers ) return self._constraints[key] def _constraint( self, spec: dict[str, Any], *, document_numbers: bool = False ) -> dict[str, str]: from jsonschema import Draft202012Validator schema = ar.schema(spec) if spec["type"] == "number" or "enum" in schema: numerals = self.numerals if document_numbers: numerals = { value: spans for (value, spans) in numerals.items() if any((_document_number_span(self.text, span) for span in spans)) } candidates = schema.get("enum", list(numerals)) validator = Draft202012Validator(schema) values = [] for candidate in candidates: value = candidate if isinstance(value, Decimal): value = ( int(value) if value == value.to_integral_value() else float(value) ) if ( value is not None and validator.is_valid(value) and (not document_numbers or Decimal(str(value)) in numerals) and self.span(value)[0] ): values.append(value) if validator.is_valid(None): values.append(None) if not values: raise ValueError( "Field constraints permit no literal source value or null" ) if spec["type"] == "number": literals = [ "null" if value is None else str(Decimal(str(value))) for value in values ] return { "ebnf": "root ::= " + " | ".join((json.dumps(value) for value in literals)) } return { "json_schema": json.dumps( {**schema, "enum": values}, ensure_ascii=False ) } if len(self.text) > MAX_GRAMMAR_CHARACTERS or any( (key in schema for key in ("minLength", "maxLength", "pattern")) ): return {"json_schema": json.dumps(schema, ensure_ascii=False)} nullable = spec.get("nullable", True) normalize = self.whitespace_normalization(spec) key = (nullable, normalize) if key not in self._grammars: normalized = self._whitespace_source()[0] if normalize else None self._grammars[key] = text_grammar( self.text, nullable=nullable, normalized_source=normalized ) return {"ebnf": self._grammars[key]}