diff --git "a/api/formula_runtime.py" "b/api/formula_runtime.py" --- "a/api/formula_runtime.py" +++ "b/api/formula_runtime.py" @@ -11,8 +11,14 @@ import datetime as dt import hashlib import json import math +from functools import lru_cache +import random import re import warnings +import unicodedata +from bisect import bisect_left, bisect_right, insort +from collections import OrderedDict, deque +from threading import RLock from decimal import Decimal, ROUND_FLOOR, ROUND_HALF_UP @@ -31,6 +37,14 @@ AUTHORITY = "server" #: republished verbatim in `contract_dto()`, which is what the browser reads. CONSUMERS = ("preview", "grid", "filter", "export", "automation", "query", "timeseries", "sheet") FN_ARITY = { + "FV": (3, 5), + "DATEDIF": (3, 3), + "IRR": (1, 2), + "XIRR": (2, 3), + "NORM.DIST": (4, 4), + "BAHTTEXT": (1, 1), + "ADD": (2, 2), + "MULTIPLY": (2, 2), "ABS": (1, 1), "ROUND": (1, 2), "ROUNDUP": (1, 2), "ROUNDDOWN": (1, 2), "INT": (1, 1), "MOD": (2, 2), "SQRT": (1, 1), "POWER": (2, 2), "EXP": (1, 1), "LN": (1, 1), "SUM": (1, MANY), "AVERAGE": (1, MANY), "COUNT": (1, MANY), @@ -67,6 +81,8 @@ FN_ARITY = { # Text and Information. "TEXTJOIN": (3, MANY), "REPT": (2, 2), "EXACT": (2, 2), "ISNUMBER": (1, 1), "ISTEXT": (1, 1), "ISERROR": (1, 1), + "OFFSET": (3, 5), "INDIRECT": (1, 2), "RAND": (0, 0), + "FILTER": (2, 3), "SORT": (1, 4), } ALIASES = {"CONCAT": "CONCATENATE"} @@ -85,8 +101,8 @@ ALIASES = {"CONCAT": "CONCATENATE"} #: ⛔ AND THIS IS NOT `ALIASES`. `catalogue_rows()` publishes `ALIASES` to consumers as the #: alternative spellings a PERSON may type; `STDEV_P` is a transport form of this parser's own #: making and offering it to a user would advertise a spelling Excel does not have. -DOTTED_SPELLINGS = {"STDEV_P": "STDEV.P", "VAR_P": "VAR.P"} -_DOTTED_RE = re.compile(r"(STDEV|VAR)\.P(?=\s*\()", re.IGNORECASE) +DOTTED_SPELLINGS = {"STDEV_P": "STDEV.P", "VAR_P": "VAR.P", "NORM_DIST": "NORM.DIST"} +_DOTTED_RE = re.compile(r"(?:STDEV\.P|VAR\.P|NORM\.DIST)(?=\s*\()", re.IGNORECASE) #: ⭐⭐ WAVE 50 · W50-T15 (ruling R5, contract C2) — THE FORMULA CATALOGUE. #: @@ -122,6 +138,24 @@ _BOTH = ("grid", "sheet") _SHEET = ("sheet",) FN_CATALOGUE = { + "FV": {"category": "Math", "signature": "FV(rate, nper, pmt, [pv], [type])", "description": "Returns the future value of periodic payments.", "contexts": _BOTH}, + "DATEDIF": {"category": "Date", "signature": "DATEDIF(start_date, end_date, unit)", "description": "Returns elapsed calendar years, months or days.", "contexts": _BOTH}, + "IRR": {"category": "Math", "signature": "IRR(values, [guess])", "description": "Returns the periodic internal rate of return near a starting guess.", "contexts": _SHEET}, + "XIRR": {"category": "Math", "signature": "XIRR(values, dates, [guess])", "description": "Returns the annual internal rate of return for dated cash flows.", "contexts": _SHEET}, + "NORM.DIST": {"category": "Statistical", "signature": "NORM.DIST(x, mean, standard_dev, cumulative)", "description": "Returns a normal probability or density.", "contexts": _BOTH}, + "BAHTTEXT": {"category": "Text", "signature": "BAHTTEXT(number)", "description": "Spells a monetary amount in Thai baht and satang.", "contexts": _BOTH}, + "ADD": {"category": "Math", "signature": "ADD(value1, value2)", "description": "Adds two values.", "contexts": _BOTH}, + "MULTIPLY": {"category": "Math", "signature": "MULTIPLY(value1, value2)", "description": "Multiplies two values.", "contexts": _BOTH}, + "FILTER": {"category": "Lookup", "signature": "FILTER(array, include, [if_empty])", + "description": "Returns the rows or columns that meet a condition.", "contexts": _SHEET}, + "SORT": {"category": "Lookup", "signature": "SORT(array, [sort_index], [sort_order], [by_col])", + "description": "Returns an array sorted by a row or column.", "contexts": _SHEET}, + "OFFSET": {"category": "Lookup", "signature": "OFFSET(reference, rows, cols, [height], [width])", + "description": "Returns a reference shifted by a number of rows and columns.", "contexts": _SHEET}, + "INDIRECT": {"category": "Lookup", "signature": "INDIRECT(ref_text, [a1])", + "description": "Returns the reference named by a text value.", "contexts": _SHEET}, + "RAND": {"category": "Math", "signature": "RAND()", + "description": "Returns a random number between zero and one.", "contexts": _BOTH}, # ── Math ───────────────────────────────────────────────────────────────────────────────── "ABS": {"category": "Math", "signature": "ABS(number)", "description": "Returns the absolute value of a number.", "contexts": _BOTH}, @@ -491,6 +525,19 @@ EVALUATION_ERROR_FIXTURES = { # Shared contract fixture corpus. The API publishes this exact data so the browser preview gate # can evaluate the same source/row/today triples without maintaining a second handwritten grammar. PARITY_FIXTURES = { + "FV": ("FV(0,2,10,100)", -120), + "DATEDIF": ('DATEDIF("2001-06-01","2002-08-15","D")', 440), + "IRR": ("IRR(B2:B3)", 0.1), + "XIRR": ("XIRR(B2:B3,C2:C3)", 0.1), + "NORM.DIST": ("NORM.DIST(0,0,1,TRUE)", 0.5), + "BAHTTEXT": ("BAHTTEXT(21)", "ยี่สิบเอ็ดบาทถ้วน"), + "ADD": ("ADD(4,7)", 11), + "MULTIPLY": ("MULTIPLY(4,7)", 28), + "FILTER": ("SUM(FILTER(3,TRUE))", 3), + "SORT": ("SUM(SORT(3))", 3), + "OFFSET": ("OFFSET(B2, 1, 1)", 20), + "INDIRECT": ('INDIRECT("B2")', 1), + "RAND": ("RAND()>=0", "TRUE"), "ABS": ("ABS(-2)", 2.0), "ROUND": ("ROUND(2.345, 2)", 2.35), "ROUNDUP": ("ROUNDUP(2.341, 2)", 2.35), "ROUNDDOWN": ("ROUNDDOWN(2.349, 2)", 2.34), "INT": ("INT(2.9)", 2), "MOD": ("MOD(7, 4)", 3.0), "SQRT": ("SQRT(9)", 3.0), @@ -596,6 +643,9 @@ _CELLS_BC = {"B2": 1, "B3": 2, "B4": 3, "B5": 4, "B6": 5, #: `"row": {"empty": None}` for every fixture. A consumer that runs the PUBLISHED corpus therefore #: cannot reproduce a range fixture from the DTO alone and must read this table. PARITY_ROWS = { + "IRR": {"B2": -100, "B3": 110}, + "XIRR": {"B2": -100, "B3": 110, "C2": "2021-01-01", "C3": "2022-01-01"}, + "OFFSET": _CELLS_BC, "INDIRECT": _CELLS_BC, "SUMIF": _CELLS_BC, "SUMIFS": _CELLS_BC, "COUNTIF": _CELLS_BC, "COUNTIFS": _CELLS_BC, "AVERAGEIF": _CELLS_BC, "AVERAGEIFS": _CELLS_BC, "MAXIFS": _CELLS_BC, "MINIFS": _CELLS_BC, "SUMPRODUCT": _CELLS_BC, "VLOOKUP": _CELLS_BC, "XLOOKUP": _CELLS_BC, "INDEX": _CELLS_BC, @@ -839,6 +889,8 @@ ALL_ARGS = "*" #: ⛔ SUMIF PUTS THE SUM RANGE LAST AND SUMIFS PUTS IT FIRST. Reading this table backwards is the #: classic Excel bug, so both spellings carry a fixture over the same cells. RANGE_ARGS = { + "IRR": ((0,), None), "XIRR": ((0, 1), None), + "FILTER": ((0, 1), None), "SORT": ((0,), None), "SUMIF": ((0, 2), None), # SUMIF(range, criteria, [sum_range]) "AVERAGEIF": ((0, 2), None), # AVERAGEIF(range, criteria, [average_range]) "COUNTIF": ((0,), None), # COUNTIF(range, criteria) @@ -954,7 +1006,7 @@ def _rewrite(source, sheet=""): # never touched. See `DOTTED_SPELLINGS` for why the pattern names only two functions. dotted = _DOTTED_RE.match(source, i) if dotted: - out.append(dotted.group(1).upper() + "_P") + out.append(dotted.group(0).upper().replace(".", "_")) i = dotted.end() continue got = _a1_at(source, i, sheet) @@ -989,7 +1041,7 @@ def _rewrite(source, sheet=""): return "".join(out) -def _normal(node, depth=0): +def _normal(node, depth=0, compact_ranges=False): if depth > MAX_DEPTH: raise FormulaError("formula is nested too deeply") if isinstance(node, ast.Constant) and isinstance(node.value, (int, float, str, bool)): @@ -999,15 +1051,15 @@ def _normal(node, depth=0): if isinstance(node, ast.Name) and node.id.upper() in ("TRUE", "FALSE"): return {"t": "const", "v": node.id.upper() == "TRUE"} if isinstance(node, ast.UnaryOp) and isinstance(node.op, ast.USub): - return {"t": "neg", "e": _normal(node.operand, depth + 1)} + return {"t": "neg", "e": _normal(node.operand, depth + 1, compact_ranges)} if isinstance(node, ast.BinOp): ops = {ast.Add: "+", ast.Sub: "-", ast.Mult: "*", ast.Div: "/", ast.Pow: "^", ast.BitAnd: "&"} op = ops.get(type(node.op)) if not op: raise FormulaError("that operator is not supported") - return {"t": "bin", "op": op, "l": _normal(node.left, depth + 1), - "r": _normal(node.right, depth + 1)} + return {"t": "bin", "op": op, "l": _normal(node.left, depth + 1, compact_ranges), + "r": _normal(node.right, depth + 1, compact_ranges)} if isinstance(node, ast.Compare): if len(node.ops) != 1 or len(node.comparators) != 1: raise FormulaError("comparisons cannot be chained") @@ -1016,8 +1068,8 @@ def _normal(node, depth=0): op = ops.get(type(node.ops[0])) if not op: raise FormulaError("that comparison is not supported") - return {"t": "cmp", "op": op, "l": _normal(node.left, depth + 1), - "r": _normal(node.comparators[0], depth + 1)} + return {"t": "cmp", "op": op, "l": _normal(node.left, depth + 1, compact_ranges), + "r": _normal(node.comparators[0], depth + 1, compact_ranges)} if isinstance(node, ast.Call) and isinstance(node.func, ast.Name): fn = ALIASES.get(node.func.id.upper(), node.func.id.upper()) # The transport spelling folds back to Excel's own, so `FN_ARITY` and the catalogue are @@ -1034,6 +1086,11 @@ def _normal(node, depth=0): vals = [str(x.value) for x in node.args] if fn == "A1REF": return {"t": "a1", "k": a1_key(vals[0], vals[1])} + if compact_ranges: + bounds = _range_bounds(vals[0], vals[1], vals[2]) + return {"t": "a1range", "range": bounds, + "rows": bounds[4] - bounds[3] + 1, + "cols": bounds[2] - bounds[1] + 1} # The range is expanded HERE, at normalize time, so a malformed or oversized range is a # PARSE error the author sees immediately rather than a surprise at evaluation. # ⭐ `rows` and `cols` RIDE ON THE NODE (W50-T16) because they are known here and are @@ -1048,11 +1105,198 @@ def _normal(node, depth=0): lo, hi = FN_ARITY[fn] if not lo <= len(node.args) <= hi: raise FormulaError(f"{fn} takes {lo} to {hi} arguments") - return {"t": "call", "fn": fn, "args": [_normal(x, depth + 1) for x in node.args]} + return {"t": "call", "fn": fn, + "args": [_normal(x, depth + 1, compact_ranges) for x in node.args]} raise FormulaError("formula contains unsupported syntax") -def parse_formula(source, *, sheet=""): +_SHEET_TOKEN = re.compile( + r'(?P\s+)|(?P"(?:[^"]|"")*")|' + r"(?P(?:'[^']*\[[^']*'|\[[^\]]*\][A-Za-z0-9_ ]*)![^ ,()+*/]+)|" + r"(?P(?:'(?:[^']|'')+'|[A-Za-z_][A-Za-z0-9_.]*)!(?:\$?[A-Za-z]{1,3}\$?[1-9][0-9]*|[A-Za-z_][A-Za-z0-9_.]*))|" + r'(?P\$?[A-Za-z]{1,3}\$?[1-9][0-9]*(?![A-Za-z0-9_.]))|' + r'(?P(?:[0-9]+(?:\.[0-9]*)?|\.[0-9]+)(?:[eE][+-]?[0-9]+)?)|' + r'(?P[A-Za-z_\\][A-Za-z0-9_.\\]*)|' + r'(?P\#[A-Z0-9/]+[!?]?)|(?P<=|>=|<>|!=|[+*/^&=<>():,%#-])') +REFERENCE_ERRORS = {"null_intersection": "#NULL!", "unknown_name": "#NAME?", + "external_reference": "#REF!", "invalid_reference": "#REF!", + "spill_obstructed": "#SPILL!", "empty_array": "#CALC!"} +_ERROR_LITERALS = {"#NULL!": "null_intersection", "#REF!": "invalid_reference", + "#NAME?": "unknown_name", "#DIV/0!": "div_zero", "#VALUE!": "bad_value", + "#N/A": "no_match", "#NUM!": "bad_number"} +_VOLATILE_FUNCTIONS = frozenset(("NOW", "TODAY", "RAND", "OFFSET", "INDIRECT")) + + +_COPY_REFERENCE = re.compile( + r'"(?:[^"]|"")*"|(?(?:(?:'(?:[^']|'')+'|[A-Za-z_][A-Za-z0-9_.]*)!)?)" + r'(?P\$?[A-Za-z]{1,3}\$?[1-9][0-9]*(?![A-Za-z0-9_.]|\s*\()|' + r'\$?[A-Za-z]{1,3}:\$?[A-Za-z]{1,3}(?![A-Za-z0-9_.$])|' + r'\$?[1-9][0-9]*:\$?[1-9][0-9]*(?![A-Za-z0-9_.$]))') + + +def transform_formula_references(source, drow=0, dcol=0, mode="copy"): + """The client transformSheetFormula contract: cells/bands shift; names/text stay literal.""" + if mode == "cut" or not (drow or dcol): return source + def replace(match): + ref = match.group("ref") + if not ref: return match.group() + shifted = [] + for end in ref.split(":"): + cell = re.fullmatch(r"(\$?)([A-Za-z]+)(\$?)([0-9]+)", end) + if cell: + c = col_to_index(cell[2]) + (0 if cell[1] else dcol) + r = int(cell[4]) + (0 if cell[3] else drow) + if not (1 <= c <= 16384 and 1 <= r <= 1048576): return "#REF!" + shifted.append(cell[1] + index_to_col(c) + cell[3] + str(r)) + else: + fixed = end.startswith("$"); raw = end.lstrip("$"); column = raw.isalpha() + value = (col_to_index(raw) if column else int(raw)) + (0 if fixed else dcol if column else drow) + if not 1 <= value <= (16384 if column else 1048576): return "#REF!" + shifted.append(("$" if fixed else "") + (index_to_col(value) if column else str(value))) + return match.group("prefix") + ":".join(shifted) + return _COPY_REFERENCE.sub(replace, source) + + +def _normalize_named_ranges(value): + if isinstance(value, dict): + value = [{"name": name, **(entry if isinstance(entry, dict) else {"ref": entry})} + for name, entry in value.items()] + return tuple(dict(entry) for entry in (value or ()) + if isinstance(entry, dict) and isinstance(entry.get("name"), str) + and isinstance(entry.get("ref"), str)) + + +def _sheet_syntax(source, sheet, named_ranges=(), active_names=(), address="", budget=None): + """Reference operators share one AST with arithmetic; names bind without string eval. + + The original token rides on each address/name node. Names are local then global and + remain names when a formula is copied. Dynamic text is parsed as a reference, never code. + """ + if len(source) > MAX_SOURCE: raise FormulaError("formula source is too long") + if budget is None: named_ranges = _normalize_named_ranges(named_ranges) + budget = [MAX_NODES] if budget is None else budget + tokens, pos = [], 0 + text = _expand_bands(re.sub(r'"(?:[^"]|"")*"|(?:_xlfn\.|_xlws\.|_xludf\.)+(?=[A-Za-z_][A-Za-z0-9_.]*\s*\()', + lambda match: match[0] if match[0].startswith('"') else '', source, flags=re.I)) + while pos < len(text): + match = _SHEET_TOKEN.match(text, pos) + if not match: + raise FormulaError("the formula could not be parsed") + tokens.append((match.lastgroup, match.group())) + if len(tokens) > MAX_NODES * 2: raise FormulaError("formula has too many tokens") + pos = match.end() + # Whitespace is intersection only between operands, not around punctuation/functions. + clean = [] + for index, token in enumerate(tokens): + if token[0] != "space": + clean.append(token) + continue + prev = clean[-1] if clean else None + nxt = next((t for t in tokens[index + 1:] if t[0] != "space"), None) + if (prev and nxt and (prev[0] in ("cell", "qualified", "name") or prev[1] == ")") + and (nxt[0] in ("cell", "qualified", "name") or nxt[1] == "(") + and not (prev[0] == "name" and nxt[1] == "(")): + clean.append(("op", " ")) + index = 0 + def peek(): return clean[index][1] if index < len(clean) else "" + def take(): + nonlocal index + if index >= len(clean): raise FormulaError("the formula is incomplete") + token = clean[index]; index += 1 + return token + def reference(token, qualifier=""): + own = qualifier or sheet + if re.fullmatch(_A1_CELL, token): + key = a1_key(token, own) + _, c, r = _address_parts(key) + if c > 16384 or r > 1048576: + return {"t": "error", "code": "invalid_reference"} + return {"t": "a1", "k": key, "token": token} + name = token.upper() + candidates = [n for n in named_ranges if str(n.get("name", "")).upper() == name] + local = next((n for n in candidates if str(n.get("sheet", "")).upper() == own.upper()), None) + target = local or next((n for n in candidates if not n.get("sheet")), None) + identity = (own.upper(), name) + if target is None: bound = {"t": "error", "code": "unknown_name"} + elif identity in active_names: bound = {"t": "error", "code": "circular_reference"} + elif len(active_names) >= MAX_DEPTH: raise FormulaError("names are nested too deeply") + else: + # A relative defined name is a template anchored at base (A1 for legacy names). + # Its dependency is bound at the actual caller address, including mixed-$ axes. + _, bc, br = _address_parts(str(target.get("base") or "A1").replace("$", "").upper()) + _, cc, cr = _address_parts(address or "A1") + shifted = transform_formula_references(str(target.get("ref", "")).lstrip("="), cr-br, cc-bc) + bound = _sheet_syntax(shifted, + str(target.get("sheet") or sheet), named_ranges, + (*active_names, identity), address, budget) + return {"t": "name", "name": name, "sheet": own, "token": token, "e": bound} + def atom(depth): + budget[0] -= 1 + if budget[0] < 0: raise FormulaError("formula has too many expanded nodes") + kind, token = take() + if kind == "num": return {"t": "const", "v": float(token)} + if kind == "str": return {"t": "const", "v": token[1:-1].replace('""', '"')} + if kind == "external": return {"t": "error", "code": "external_reference"} + if kind == "err": return {"t": "error", "code": _ERROR_LITERALS.get(token, "bad_value")} + if kind in ("cell", "name"): + if peek() == "(": + take(); fn = ALIASES.get(token.upper(), token.upper()); args = [] + if peek() != ")": + while True: + args.append({"t": "const", "v": None} if peek() in (",", ")") else expr(1, depth + 1)) + if peek() != ",": break + take() + if take()[1] != ")": raise FormulaError("function arguments are incomplete") + if fn not in FN_ARITY: return {"t": "error", "code": "unknown_name"} + lo, hi = FN_ARITY[fn] + if not lo <= len(args) <= hi: raise FormulaError("function arity is invalid") + return {"t": "call", "fn": fn, "args": args} + if token.upper() in ("TRUE", "FALSE"): return {"t": "const", "v": token.upper() == "TRUE"} + return reference(token) + if kind == "qualified": + qualifier, token = token.rsplit("!", 1) + bound = reference(token, qualifier.strip("'").replace("''", "'")) + bound["token"] = qualifier + "!" + token + return bound + if token == "(": + node = expr(0, depth + 1) + if take()[1] != ")": raise FormulaError("parenthesis is incomplete") + return node + if token in ("+", "-"): + node = expr(6, depth + 1) + return node if token == "+" else {"t": "neg", "e": node} + raise FormulaError("the formula could not be parsed") + precedence = {",": 0, "=": 1, "<>": 1, "!=": 1, "<": 1, ">": 1, "<=": 1, ">=": 1, + "&": 2, "+": 3, "-": 3, "*": 4, "/": 4, "^": 5, " ": 7, ":": 8} + def expr(minimum, depth): + if depth > MAX_DEPTH: raise FormulaError("formula is nested too deeply") + node = atom(depth) + if peek() == "#": + take(); node = {"t": "spill", "e": node} + while peek() in precedence and precedence[peek()] >= minimum: + budget[0] -= 1 + if budget[0] < 0: raise FormulaError("formula has too many expanded nodes") + op = take()[1]; right = expr(precedence[op] + 1, depth + 1) + if op == ":" and node["t"] == right["t"] == "a1": + a, b = node["k"].rpartition("!"), right["k"].rpartition("!") + # An unqualified end inherits the explicit start's tab. + end_sheet = b[0] if "!" in right.get("token", "") else a[0] + bounds = _range_bounds(a[2], b[2], a[0]) + node = {"t": "a1range", "range": bounds, "rows": bounds[4]-bounds[3]+1, + "cols": bounds[2]-bounds[1]+1} if end_sheet == a[0] else {"t": "error", "code": "invalid_reference"} + elif op in (":", " ", ","): + node = {"t": "reference_op", "op": op, "l": node, "r": right} + else: + node = {"t": "cmp" if precedence[op] == 1 else "bin", "op": "!=" if op == "<>" else op, + "l": node, "r": right} + return node + root = expr(0, 0) + if index != len(clean): raise FormulaError("the formula has trailing tokens") + return root + + +def parse_formula(source, *, sheet="", compact_ranges=False, named_ranges=(), address=""): """One formula's stored definition. `sheet` is the tab it lives on, or "" for a grid field. ⭐ KEYWORD ONLY, AND DEFAULTED TO "", so every caller that has no notion of a tab keeps the @@ -1066,30 +1310,38 @@ def parse_formula(source, *, sheet=""): raise FormulaError("the formula is empty") if len(source) > MAX_SOURCE: raise FormulaError(f"formula longer than {MAX_SOURCE} characters") + if compact_ranges: + root = _sheet_syntax(source, sheet, named_ranges, address=address) + else: + root = None try: # Regex patterns such as ``"\\s"`` are data for REGEX_* rather than Python escapes. # Python 3.12 warns while parsing them even though the string is valid for this grammar. with warnings.catch_warnings(): warnings.simplefilter("ignore", SyntaxWarning) - parsed = ast.parse(_rewrite(source, sheet), mode="eval") + parsed = ast.parse(_rewrite(source, sheet), mode="eval") if root is None else None except (SyntaxError, ValueError) as exc: raise FormulaError("the formula could not be parsed") from exc - if sum(1 for _ in ast.walk(parsed)) > MAX_NODES: + if parsed is not None and sum(1 for _ in ast.walk(parsed)) > MAX_NODES: raise FormulaError(f"formula has more than {MAX_NODES} nodes") - root = _normal(parsed.body) + root = root if root is not None else _normal(parsed.body, compact_ranges=compact_ranges) refs = [] # ⭐ `cells` IS A SECOND, SEPARATE LIST AND NOT A MERGE INTO `refs`. `refs` names FIELD keys and # is already persisted in every stored definition, read by `evaluate_fields`, `evaluate_batch` # and the contract DTO; folding cell addresses into it would hand every one of those consumers # a key that is not a field. `cells` is additive, so an existing definition simply carries an # empty one. - cells = [] + cells, ranges, precedents = [], [], [] def walk(row): if row["t"] == "ref" and row["k"] not in refs: refs.append(row["k"]) if row["t"] == "a1" and row["k"] not in cells: cells.append(row["k"]) + precedents.append({"cell": row["k"]}) if row["t"] == "a1range": + if "range" in row: + ranges.append(row["range"]) + precedents.append({"range": row["range"]}) for key in row.get("keys") or (): if key not in cells: cells.append(key) @@ -1097,7 +1349,14 @@ def parse_formula(source, *, sheet=""): if isinstance(row.get(key), dict): walk(row[key]) for child in row.get("args") or (): walk(child) walk(root) - return {"version": AST_VERSION, "source": source, "ast": root, "refs": refs, "cells": cells} + result = {"version": AST_VERSION, "source": source, "ast": root, "refs": refs, "cells": cells} + if compact_ranges: + def volatile(node): + return (node.get("fn") in _VOLATILE_FUNCTIONS + or any(volatile(node[k]) for k in ("e", "l", "r") if isinstance(node.get(k), dict)) + or any(volatile(child) for child in node.get("args", ()))) + result.update(ranges=ranges, precedents=precedents, volatile=volatile(root), sheet=sheet) + return result def _num(v): @@ -1266,10 +1525,12 @@ def evaluate_result(definition, row, *, today=None, strict=False): today or dt.date.today().isoformat(), diagnostics) if diagnostics: value = None - if isinstance(value, bool): + if isinstance(value, bool) and not isinstance(row, _SheetInputs): value = "TRUE" if value else "FALSE" elif isinstance(value, float) and not math.isfinite(value): value = None + elif isinstance(value, Range) and isinstance(row, _SheetInputs): + pass elif value is not None and not isinstance(value, (int, float, str)): value = None diagnostics.append("evaluation_error") @@ -1293,9 +1554,173 @@ def _diagnose(diagnostics, code): diagnostics.append(code) +def _reference_node(node): + if node.get("t") == "name": return _reference_node(node["e"]) + return (node.get("t") in ("a1", "a1range", "reference_op", "spill") + or (node.get("t") == "call" and node.get("fn") in ("OFFSET", "INDIRECT"))) + + +def _reference_areas(node, row, today, diagnostics): + """Resolve rectangles, not values. Unions retain each area and overlapping occurrences.""" + kind = node.get("t") + if kind == "error": + _diagnose(diagnostics, node["code"]); return [] + if kind == "name": return _reference_areas(node["e"], row, today, diagnostics) + if kind == "spill": + child = node["e"] + if child.get("t") != "a1" or not isinstance(row, _SheetInputs): + _diagnose(diagnostics, "invalid_reference"); return [] + key = child["k"] + row.get(key) + owned = row.engine.spills.get(key) + if not owned: _diagnose(diagnostics, "invalid_reference"); return [] + return [owned["range"]] + if kind == "a1": + sheet, col, r = _address_parts(node["k"]) + return [(sheet, col, col, r, r)] + if kind == "a1range": + if "range" in node: return [node["range"]] + keys = node["keys"] + return [_range_bounds(keys[0].rsplit("!", 1)[-1], keys[-1].rsplit("!", 1)[-1], _sheet_half(keys[0]))] + if kind == "reference_op": + left = _reference_areas(node["l"], row, today, diagnostics) + right = _reference_areas(node["r"], row, today, diagnostics) + if diagnostics: return [] + if node["op"] == ",": return left + right + if node["op"] == ":": + if len(left) != 1 or len(right) != 1 or left[0][0] != right[0][0]: + _diagnose(diagnostics, "invalid_reference"); return [] + a, b = left[0], right[0] + return [(a[0], min(a[1], b[1]), max(a[2], b[2]), min(a[3], b[3]), max(a[4], b[4]))] + result = [] + for a in left: + for b in right: + area = (a[0], max(a[1], b[1]), min(a[2], b[2]), max(a[3], b[3]), min(a[4], b[4])) + if a[0] == b[0] and area[1] <= area[2] and area[3] <= area[4]: result.append(area) + if not result: _diagnose(diagnostics, "null_intersection") + return result + if kind == "call" and node["fn"] == "OFFSET": + base = _reference_areas(node["args"][0], row, today, diagnostics) + args = [_num(_eval(arg, row, today, diagnostics)) for arg in node["args"][1:]] + if len(base) != 1 or any(v is None for v in args): + _diagnose(diagnostics, "bad_value"); return [] + s, c0, c1, r0, r1 = base[0] + dr, dc = (int(args[0]), int(args[1])) + height = int(args[2]) if len(args) > 2 else r1-r0+1 + width = int(args[3]) if len(args) > 3 else c1-c0+1 + area = (s, c0+dc, c0+dc+width-1, r0+dr, r0+dr+height-1) + if height < 1 or width < 1: + _diagnose(diagnostics, "invalid_reference"); return [] + return _checked_areas([area], row, diagnostics) + if kind == "call" and node["fn"] == "INDIRECT": + text = _eval(node["args"][0], row, today, diagnostics) + if not isinstance(text, str): _diagnose(diagnostics, "invalid_reference"); return [] + if "[" in text or "]" in text: + # R1C1 bracket offsets are allowed only under the explicit second FALSE argument. + r1c1 = len(node["args"]) == 2 and _eval(node["args"][1], row, today, diagnostics) is False + if not r1c1 or not re.fullmatch(r"R(?:\[-?\d+\]|\d*)C(?:\[-?\d+\]|\d*)", text, re.I): + _diagnose(diagnostics, "external_reference"); return [] + here = row.get(SHEET_ADDR_KEY, "") + sheet = _sheet_half(here) + if len(node["args"]) == 2 and _eval(node["args"][1], row, today, diagnostics) is False: + m = re.fullmatch(r"R(\[-?\d+\]|\d*)C(\[-?\d+\]|\d*)", text, re.I) + if not m or not here: _diagnose(diagnostics, "invalid_reference"); return [] + _, col, r = _address_parts(here) + def coord(raw, origin): return origin + int(raw[1:-1]) if raw.startswith("[") else int(raw) if raw else origin + r, col = coord(m[1], r), coord(m[2], col) + if min(r, col) < 1: _diagnose(diagnostics, "invalid_reference"); return [] + text = index_to_col(col) + str(r) + try: + target = row.engine.reference_syntax(text, sheet, here) if isinstance(row, _SheetInputs) else _sheet_syntax(text, sheet, address=here) + except FormulaError: + _diagnose(diagnostics, "invalid_reference"); return [] + if (not _reference_node(target) and target.get("t") != "error") or target.get("t") == "call": + _diagnose(diagnostics, "invalid_reference"); return [] + return _checked_areas(_reference_areas(target, row, today, diagnostics), row, diagnostics) + _diagnose(diagnostics, "invalid_reference") + return [] + + +def _checked_areas(areas, row, diagnostics): + for s, c0, c1, r0, r1 in areas: + if min(c0, r0) < 1 or c1 > 16384 or r1 > 1048576: + _diagnose(diagnostics, "invalid_reference"); return [] + if isinstance(row, _SheetInputs) and row.engine.sheets is not None and s and s not in row.engine.sheets: + _diagnose(diagnostics, "unknown_sheet"); return [] + return areas + + +def _area_values(areas, row, sparse=True): + for bounds in areas: + if sparse and isinstance(row, _SheetInputs): + yield from row.range_values(bounds) + else: + s, c0, c1, r0, r1 = bounds + if (c1-c0+1)*(r1-r0+1) > MAX_RANGE_CELLS: raise FormulaError("range is too large") + for r in range(r0, r1+1): + for c in range(c0, c1+1): yield row.get(a1_key(index_to_col(c)+str(r), s)) + + +def _excel_number(value): + return float(format(value, ".15g")) if isinstance(value, (int, float)) and not isinstance(value, bool) else value + + +def _logical(value, diagnostics): + if isinstance(value, (bool, int, float)) or value is None: return bool(value) + if isinstance(value, str) and value.upper() in ("TRUE", "FALSE"): return value.upper() == "TRUE" + _diagnose(diagnostics, "bad_value") + return False + + +def _format_rule(node, formats): + formats = [item for item in formats if item] + kind, fn = node.get("t"), node.get("fn") + if kind == "cmp" or (kind == "bin" and node["op"] in ("&", "^")): return None + if kind == "call": + if fn in ("DATE", "DATEVALUE", "TODAY", "NOW", "EDATE", "EOMONTH", "WORKDAY", "DATEADD"): + return {"kind": "date"} + if fn in ("IRR", "XIRR"): return {"kind": "percent", "decimals": 2} + if fn not in ("IF", "IFS", "IFERROR", "IFNA", "SWITCH", "CHOOSE", "SUM", "AVERAGE", "MIN", "MAX", "SUMIF", "SUMIFS", "AVERAGEIF", "AVERAGEIFS", "INDEX", "XLOOKUP", "VLOOKUP", "HLOOKUP", "OFFSET", "INDIRECT", "FILTER", "SORT", "ROUND", "ROUNDUP", "ROUNDDOWN", "ABS", "ADD", "MULTIPLY", "FV"): + return None + if kind == "bin" and len(formats) > 1: + kinds = [f.get("kind") if isinstance(f, dict) else f for f in formats] + if node["op"] == "-" and kinds[:2] == ["date", "date"]: return None + if node["op"] == "/" and kinds[0] == kinds[1]: return {"kind": "percent", "decimals": 2} + return next((f for f in formats if isinstance(f, dict) and f.get("kind") in ("currency", "date")), formats[0] if formats else None) + + def _eval(node, row, today, diagnostics=None): + if not isinstance(row, _SheetInputs): return _eval_value(node, row, today, diagnostics) + row.format_stack.append([]) + try: + value = _eval_value(node, row, today, diagnostics) + fmt = _format_rule(node, row.format_stack[-1]) + row.last_format = fmt if ((isinstance(value, (int, float, Range)) and not isinstance(value, bool)) + or (isinstance(value, str) and isinstance(fmt, dict) and fmt.get("kind") == "date")) else None + value = _excel_number(value) + finally: + row.format_stack.pop() + if row.format_stack: row.format_stack[-1].append(row.last_format) + return value + + +def _eval_value(node, row, today, diagnostics=None): kind = node.get("t") if isinstance(node, dict) else None if kind == "const": return node.get("v") + if kind == "error": + _diagnose(diagnostics, node["code"]); return None + if kind == "name" and not _reference_node(node["e"]): + return _eval(node["e"], row, today, diagnostics) + if kind in ("name", "reference_op", "spill") or (kind == "call" and node["fn"] in ("OFFSET", "INDIRECT")): + areas = _reference_areas(node, row, today, diagnostics) + if len(areas) == 1 and areas[0][1] == areas[0][2] and areas[0][3] == areas[0][4]: + area = areas[0] + return row.get(a1_key(index_to_col(area[1])+str(area[3]), area[0])) + if isinstance(row, _SheetInputs) and len(areas) == 1: + area = areas[0] + return Range(_area_values(areas, row, sparse=False), area[4]-area[3]+1, area[2]-area[1]+1) + if not diagnostics: _diagnose(diagnostics, "range_outside_function") + return None if kind == "ref": key = node.get("k") if key not in row: @@ -1310,17 +1735,26 @@ def _eval(node, row, today, diagnostics=None): _diagnose(diagnostics, "unknown_ref") return row.get(key) if kind == "a1range": + if isinstance(row, _SheetInputs): return _range_arg(node, row, today, diagnostics) # ⛔ A BARE RANGE IS NOT A VALUE. `=B4:B8` alone has no meaning; a range is only ever an # ARGUMENT to an aggregate, and the call dispatch below is what splices it in. Returning # the list here would let `=B4:B8*2` silently produce nonsense. _diagnose(diagnostics, "range_outside_function") return None if kind == "neg": - n = _num(_eval(node["e"], row, today, diagnostics)); return None if n is None else -n + value = _eval(node["e"], row, today, diagnostics) + if isinstance(value, Range): return _array_binary(value, -1, "*", diagnostics) + n = _num(value) + return None if n is None else -n if kind == "bin": left = _eval(node["l"], row, today, diagnostics) right = _eval(node["r"], row, today, diagnostics) + if isinstance(left, Range) or isinstance(right, Range): + return _array_binary(left, right, node["op"], diagnostics) if node["op"] == "&": return _text(left) + _text(right) + if isinstance(row, _SheetInputs): + left, right = 0 if left is None else left, 0 if right is None else right + left, right = int(left) if isinstance(left, bool) else left, int(right) if isinstance(right, bool) else right a, b = _num(left), _num(right) if a is None or b is None: return None if node["op"] == "/" and b == 0: @@ -1331,38 +1765,55 @@ def _eval(node, row, today, diagnostics=None): if kind == "cmp": a = _eval(node["l"], row, today, diagnostics) b = _eval(node["r"], row, today, diagnostics) - if a is None or b is None: return None - na, nb = _num(a), _num(b) - if (na is None) != (nb is None): return None - x, y = (na, nb) if na is not None else (str(a).casefold(), str(b).casefold()) + if isinstance(a, Range) or isinstance(b, Range): + return _array_binary(a, b, node["op"], diagnostics) + if isinstance(row, _SheetInputs): + def comparison(value): + if isinstance(value, bool): return (2, value) + if isinstance(value, str): return (1, value.casefold()) + return (0, _excel_number(value or 0)) + x, y = comparison(a), comparison(b) + else: + if a is None or b is None: return None + na, nb = _num(a), _num(b) + if (na is None) != (nb is None): return None + x, y = (na, nb) if na is not None else (str(a).casefold(), str(b).casefold()) return {"<": x < y, "<=": x <= y, ">": x > y, ">=": x >= y, "=": x == y, "!=": x != y}[node["op"]] if kind == "call": fn = node["fn"] + if fn in ("RAND", "NOW", "TODAY") and isinstance(row, _SheetInputs): + return row.engine.volatile_value(node, fn) + if fn == "RAND": return random.random() # IF/IFERROR are lazy, so an unselected bad branch cannot blank the result. if fn == "IF": cond = _eval(node["args"][0], row, today, diagnostics) - if not isinstance(cond, bool): return None + if isinstance(row, _SheetInputs): cond = _logical(cond, diagnostics) + elif not isinstance(cond, bool): return None pick = 1 if cond else 2 return (_eval(node["args"][pick], row, today, diagnostics) if pick < len(node["args"]) else None) if fn == "IFERROR": # A handled failure is not an error of the whole formula, so diagnostics from the # losing branch remain local. + local = [] try: - first = _eval(node["args"][0], row, today, []) + first = _eval(node["args"][0], row, today, local) except (ArithmeticError, TypeError, ValueError, OverflowError, KeyError, re.error, - FormulaError): - first = None - return first if first is not None else _eval( - node["args"][1], row, today, diagnostics) + FormulaError, _ReferenceFault): + first = None; local.append("evaluation_error") + if isinstance(row, _SheetInputs): + answer = _eval(node["args"][1], row, today, diagnostics) if local else first + return "" if answer is None and not diagnostics else answer + return first if first is not None and not local else _eval(node["args"][1], row, today, diagnostics) if fn == "IFS": # Conditions are read in order and STOP at the first TRUE one, so a later branch that # would refuse never runs. Excel answers a no-match error when none of them is TRUE. pairs = node.get("args") or () for index in range(0, len(pairs) - 1, 2): condition = _eval(pairs[index], row, today, diagnostics) - if not isinstance(condition, bool): + if isinstance(row, _SheetInputs): condition = _logical(condition, diagnostics) + elif not isinstance(condition, bool): return None if condition: return _eval(pairs[index + 1], row, today, diagnostics) @@ -1373,19 +1824,24 @@ def _eval(node, row, today, diagnostics=None): # IFERROR. A lookup that found nothing is answered; a division by zero is still a # division by zero and its diagnostic is handed back to the caller unchanged. local = [] - first = _eval(node["args"][0], row, today, local) + try: + first = _eval(node["args"][0], row, today, local) + except _ReferenceFault as exc: + if row.engine.errors[str(exc)] != "no_match": raise + first = None; local.append("no_match") if "no_match" in local: - return _eval(node["args"][1], row, today, diagnostics) + first = _eval(node["args"][1], row, today, diagnostics) + return "" if first is None and isinstance(row, _SheetInputs) and not diagnostics else first for code in local: _diagnose(diagnostics, code) - return first + return "" if first is None and isinstance(row, _SheetInputs) and not local else first if fn == "ISERROR": # The whole job is to ask whether a refusal happened, so the refusal must stay LOCAL. local = [] try: _eval(node["args"][0], row, today, local) except (ArithmeticError, TypeError, ValueError, OverflowError, KeyError, re.error, - FormulaError): + FormulaError, _ReferenceFault): return True return bool(local) # ⭐ A POSITIONAL FUNCTION TAKES ITS RANGES BY POSITION (W50-T16). See `RANGE_ARGS` for why @@ -1415,14 +1871,31 @@ def _eval(node, row, today, diagnostics=None): # so a range must become MORE ARGUMENTS rather than one list-shaped argument. Doing it here # instead of inside `_call` is what keeps all 46 functions unchanged. args = [] + if fn in ("COUNT", "COUNTA") and isinstance(row, _SheetInputs): + row.ignore_reference_errors += 1 + try: + for child in node.get("args") or (): + if _reference_node(child): args.extend(_area_values(_reference_areas(child, row, today, diagnostics), row)) + else: args.append(_eval(child, row, today, diagnostics)) + finally: row.ignore_reference_errors -= 1 + return _call(fn, args, today, diagnostics) for x in node.get("args") or (): + if _reference_node(x) and x.get("t") not in ("a1", "a1range"): + args.extend(_area_values(_reference_areas(x, row, today, diagnostics), row)) + continue if isinstance(x, dict) and x.get("t") == "a1range": + if "range" in x and isinstance(row, _SheetInputs): + args.extend(row.range_values(x["range"])) + continue for key in x.get("keys") or (): if key not in row: _diagnose(diagnostics, "unknown_ref") args.append(row.get(key)) else: - args.append(_eval(x, row, today, diagnostics)) + value = _eval(x, row, today, diagnostics) + if isinstance(value, Range) and fn in ("SUM", "AVERAGE", "COUNT", "COUNTA", "MIN", "MAX", "SUMPRODUCT"): + args.extend(value.values) + else: args.append(value) return _call(fn, args, today, diagnostics) raise FormulaError("formula AST node is not supported") @@ -1437,7 +1910,23 @@ def _addr_arg(node, diagnostics): def _range_arg(node, row, today, diagnostics): """One argument as a shaped `Range`. A scalar standing there becomes a 1 by 1 rectangle.""" + if _reference_node(node) and node.get("t") not in ("a1", "a1range"): + areas = _reference_areas(node, row, today, diagnostics) + if len(areas) != 1: _diagnose(diagnostics, "bad_value"); return Range.scalar(None) + s, c0, c1, r0, r1 = areas[0] + return Range(_area_values(areas, row, sparse=False), r1-r0+1, c1-c0+1) if isinstance(node, dict) and node.get("t") == "a1range": + if "range" in node and isinstance(row, _SheetInputs): + bounds = node["range"] + size = node["rows"] * node["cols"] + if size > MAX_RANGE_CELLS: + # Shaped array/lookup execution is extended by the array engine. Aggregates + # already read sparse ranges without enumerating blanks. + _diagnose(diagnostics, "range_too_large") + return Range.scalar(None) + return Range([row.get(a1_key(index_to_col(c) + str(r), bounds[0])) + for r in range(bounds[3], bounds[4] + 1) + for c in range(bounds[1], bounds[2] + 1)], node["rows"], node["cols"]) keys = list(node.get("keys") or ()) rows, cols = node.get("rows"), node.get("cols") if (isinstance(rows, bool) or isinstance(cols, bool) @@ -1449,7 +1938,47 @@ def _range_arg(node, row, today, diagnostics): # always contains empty cells; that is what COUNTBLANK counts and what SUM skips. Calling # it `unknown_ref` would refuse the commonest formula on any real sheet. return Range([row.get(key) for key in keys], rows, cols) - return Range.scalar(_eval(node, row, today, diagnostics)) + value = _eval(node, row, today, diagnostics) + return value if isinstance(value, Range) else Range.scalar(value) + + +def _array_binary(left, right, op, diagnostics): + a, b = _as_range(left), _as_range(right) + rows, cols = max(a.rows, b.rows), max(a.cols, b.cols) + if any(x.rows not in (1, rows) or x.cols not in (1, cols) for x in (a, b)): + _diagnose(diagnostics, "bad_value"); return None + values = [] + for r in range(rows): + for c in range(cols): + x, y = a.cell(r % a.rows+1, c % a.cols+1), b.cell(r % b.rows+1, c % b.cols+1) + if op == "&": value = _text(x) + _text(y) + elif op in ("=", "!=", "<", ">", "<=", ">="): + nx, ny = _num(x), _num(y) + x, y = (nx, ny) if nx is not None and ny is not None else (_text(x).casefold(), _text(y).casefold()) + value = {"=": lambda: x == y, "!=": lambda: x != y, "<": lambda: x < y, + ">": lambda: x > y, "<=": lambda: x <= y, ">=": lambda: x >= y}[op]() + else: + x, y = int(x) if isinstance(x, bool) else _num(x), int(y) if isinstance(y, bool) else _num(y) + if x is None or y is None: _diagnose(diagnostics, "bad_value"); return None + if op == "/" and y == 0: _diagnose(diagnostics, "div_zero"); return None + value = {"+": lambda: x+y, "-": lambda: x-y, "*": lambda: x*y, + "/": lambda: x/y, "^": lambda: x**y}[op]() + values.append(value) + return Range(values, rows, cols) if isinstance(left, Range) or isinstance(right, Range) else values[0] + + +def _english_sort_key(value): + """English collation: letters before their accents, canonical accent ties stay stable. + + Case is ignored, matching Intl.Collator('en', {sensitivity:'accent'}). NFD makes + composed/decomposed Latin accents identical without depending on the host locale. + """ + if isinstance(value, bool): return (2, int(value), ()) + if isinstance(value, (int, float)): return (0, value, ()) + if value is None: return (3, "", ()) + text = unicodedata.normalize("NFD", str(value).casefold()) + return (1, "".join(ch for ch in text if not unicodedata.combining(ch)), + tuple((i, ord(ch)) for i, ch in enumerate(text) if unicodedata.combining(ch))) def _strict_num(v, diagnostics): @@ -1461,6 +1990,9 @@ def _strict_num(v, diagnostics): different question. `CEILING("x", 2)` is not a request to round nothing, it is a mistake, and answering a number to it would hide the mistake inside a plausible total. """ + if isinstance(v, Range): + if len(v) != 1: _diagnose(diagnostics, "bad_value"); return None + v = v.values[0] if v is None or (isinstance(v, str) and v == ""): return None n = _num(v) @@ -1590,8 +2122,127 @@ def _percentile(values, k): return float(ordered[low] + (ordered[high] - ordered[low]) * (position - low)) +def _cashflow_rate(values, times, guess, diagnostics): + if (len(values) < 2 or not any(v < 0 for v in values) or not any(v > 0 for v in values) + or guess <= -1 or any(t < 0 for t in times)): + _diagnose(diagnostics, "numeric_error"); return None + # This bounded Newton solver is a function algorithm, never workbook iteration. + scale = max(abs(v) for v in values) + rate = guess + for _ in range(100): + try: + discount = [math.exp(-t * math.log1p(rate)) for t in times] + value = math.fsum(v/scale*d for v, d in zip(values, discount)) + derivative = math.fsum(-t*v/scale*d/(1+rate) for v, t, d in zip(values, times, discount)) + except (ValueError, OverflowError): break + if abs(value) < 1e-13: return rate + if not derivative or not math.isfinite(derivative): break + candidate = rate-value/derivative + if candidate <= -1: candidate = (rate-1)/2 + if not math.isfinite(candidate): break + if abs(candidate-rate) < 1e-12 and abs(value) < 1e-9: return candidate + rate = candidate + _diagnose(diagnostics, "numeric_error") + return None + + +def _baht_text(number): + digits = ("ศูนย์", "หนึ่ง", "สอง", "สาม", "สี่", "ห้า", "หก", "เจ็ด", "แปด", "เก้า") + places = ("", "สิบ", "ร้อย", "พัน", "หมื่น", "แสน") + def spell(integer): + if integer >= 1_000_000: + high, low = divmod(integer, 1_000_000) + return spell(high) + "ล้าน" + (spell(low) if low else "") + text = "" + for position in range(5, -1, -1): + digit = integer // 10**position % 10 + if not digit: continue + word = "ย���่" if position == 1 and digit == 2 else "" if position == 1 and digit == 1 else "เอ็ด" if position == 0 and digit == 1 and integer > 10 else digits[digit] + text += word + places[position] + return text or digits[0] + cents = int(Decimal(str(abs(number))).quantize(Decimal(".01"), rounding=ROUND_HALF_UP)*100) + whole, fraction = divmod(cents, 100) + return ("ลบ" if number < 0 else "") + spell(whole) + "บาท" + (spell(fraction) + "สตางค์" if fraction else "ถ้วน") + + def _call(fn, a, today, diagnostics=None): nums = [_num(x) for x in a]; present = [x for x in nums if x is not None] + if fn in ("ADD", "MULTIPLY", "FV", "NORM.DIST", "BAHTTEXT"): + numeric = [float(v) if isinstance(v, bool) else 0.0 if v is None else _num(v) for v in a] + if any(v is None for v in numeric): _diagnose(diagnostics, "bad_value"); return None + if fn == "ADD": return numeric[0]+numeric[1] + if fn == "MULTIPLY": return numeric[0]*numeric[1] + if fn == "BAHTTEXT": return _baht_text(numeric[0]) + if fn == "FV": + rate, nper, payment = numeric[:3] + pv, when = numeric[3] if len(numeric) > 3 else 0, numeric[4] if len(numeric) > 4 else 0 + if when not in (0, 1): _diagnose(diagnostics, "bad_value"); return None + if rate == 0: return -(pv+payment*nper) + growth = (1+rate)**nper + if isinstance(growth, complex): _diagnose(diagnostics, "numeric_error"); return None + return -(pv*growth+payment*(1+rate*when)*(growth-1)/rate) + x, mean, sigma = numeric[:3] + if sigma <= 0: _diagnose(diagnostics, "numeric_error"); return None + z = (x-mean)/sigma + return (math.erfc(-z/math.sqrt(2))/2 if _logical(a[3], diagnostics) + else math.exp(-z*z/2)/(sigma*math.sqrt(2*math.pi))) + if fn == "DATEDIF": + start, end = _date(a[0]), _date(a[1]); unit = _text(a[2]).upper() + if start is None or end is None: _diagnose(diagnostics, "bad_value"); return None + if start > end or unit not in ("Y", "M", "D", "YM", "YD", "MD"): + _diagnose(diagnostics, "numeric_error"); return None + months = (end.year-start.year)*12+end.month-start.month-(end.day < start.day) + if unit == "Y": return months//12 + if unit == "M": return months + if unit == "YM": return months % 12 + if unit == "D": return (end-start).days + if unit == "MD": + # Excel's documented legacy MD algorithm can return a negative day count. + return end.day-start.day+(calendar.monthrange(end.year if end.month > 1 else end.year-1, end.month-1 if end.month > 1 else 12)[1] if end.day < start.day else 0) + year = end.year - ((end.month, end.day) < (start.month, start.day)) + anniversary = dt.date(year, start.month, 1)+dt.timedelta(days=start.day-1) + return (end-anniversary).days + if fn in ("IRR", "XIRR"): + raw = _as_range(a[0]).values + values = [float(v) for v in raw if isinstance(v, (int, float)) and not isinstance(v, bool)] + guess_at = 1 if fn == "IRR" else 2 + guess = _num(a[guess_at]) if len(a) > guess_at else .1 + if guess is None: _diagnose(diagnostics, "bad_value"); return None + if fn == "IRR": return _cashflow_rate(values, range(len(values)), guess, diagnostics) + dates = [_date(v) for v in _as_range(a[1]).values] + if len(values) != len(dates) or any(v is None for v in dates): + _diagnose(diagnostics, "numeric_error"); return None + return _cashflow_rate(values, [(v-dates[0]).days/365 for v in dates], guess, diagnostics) + if fn == "FILTER": + array, include = _as_range(a[0]), _as_range(a[1]) + vertical = include.cols == 1 and include.rows == array.rows + horizontal = include.rows == 1 and include.cols == array.cols + if not vertical and not horizontal: + _diagnose(diagnostics, "bad_value"); return None + if any(not isinstance(v, (bool, int, float)) and v is not None for v in include.values): + _diagnose(diagnostics, "bad_value"); return None + selected = [i+1 for i, value in enumerate(include.values) if value] + if not selected: + if len(a) == 3: return _as_range(a[2]) + _diagnose(diagnostics, "empty_array"); return None + rows = selected if vertical else range(1, array.rows+1) + cols = range(1, array.cols+1) if vertical else selected + return Range([array.cell(r, c) for r in rows for c in cols], len(rows), len(cols)) + if fn == "SORT": + array = _as_range(a[0]) + index = _strict_num(a[1], diagnostics) if len(a) > 1 else 1 + order = _strict_num(a[2], diagnostics) if len(a) > 2 else 1 + by_col = a[3] if len(a) > 3 else False + if not isinstance(by_col, bool) or index is None or int(index) != index or order not in (-1, 1): + _diagnose(diagnostics, "bad_value"); return None + span = array.rows if by_col else array.cols + if not 1 <= index <= span: + _diagnose(diagnostics, "bad_value"); return None + indices = sorted(range(1, (array.cols if by_col else array.rows)+1), + key=lambda i: _english_sort_key(array.cell(int(index), i) if by_col else array.cell(i, int(index))), + reverse=order == -1) + return Range([array.cell(r, c) for r in (range(1, array.rows+1) if by_col else indices) + for c in (indices if by_col else range(1, array.cols+1))], array.rows, array.cols) if fn == "ABS": return abs(nums[0]) if nums[0] is not None else None if fn == "ROUND": return _round(a[0], a[1] if len(a) > 1 else 0) if fn == "ROUNDUP": @@ -1773,18 +2424,30 @@ def _call(fn, a, today, diagnostics=None): return table.cell(hit, offset) if vertical else table.cell(offset, hit) if fn == "XLOOKUP": lookup_array, return_array = _as_range(a[1]), _as_range(a[2]) - if len(lookup_array) != len(return_array): + vertical = lookup_array.cols == 1 + if ((not vertical and lookup_array.rows != 1) + or (lookup_array.rows != return_array.rows if vertical else lookup_array.cols != return_array.cols)): _diagnose(diagnostics, "bad_value") return None + if isinstance(a[0], Range): + wanted = a[0] + pieces = [_call(fn, [value, *a[1:]], today, diagnostics) for value in wanted.values] + if diagnostics: return None + pieces = [_as_range(piece) for piece in pieces] + if any((p.rows, p.cols) != (1, 1) for p in pieces): + _diagnose(diagnostics, "bad_value"); return None + return Range([p.values[0] for p in pieces], wanted.rows, wanted.cols) mode = 0 if len(a) > 4: mode = _strict_num(a[4], diagnostics) if mode is None: return None mode = int(mode) + if mode not in (-1, 0, 1, 2): _diagnose(diagnostics, "bad_value"); return None hit = None for i, value in enumerate(lookup_array.values): - if _values_equal(value, a[0]): + if (bool(re.fullmatch(_wildcard_pattern(_text(a[0])), _text(value), re.I)) + if mode == 2 else _values_equal(value, a[0])): hit = i break if hit is None and mode in (-1, 1): @@ -1802,7 +2465,10 @@ def _call(fn, a, today, diagnostics=None): return a[3] _diagnose(diagnostics, "no_match") return None - return return_array.values[hit] + values = ([return_array.cell(hit+1, c) for c in range(1, return_array.cols+1)] if vertical + else [return_array.cell(r, hit+1) for r in range(1, return_array.rows+1)]) + return values[0] if len(values) == 1 else Range(values, 1 if vertical else return_array.rows, + return_array.cols if vertical else 1) if fn == "INDEX": array = _as_range(a[0]) first = _strict_num(a[1], diagnostics) @@ -1818,13 +2484,14 @@ def _call(fn, a, today, diagnostics=None): elif array.cols == 1: wanted_row, wanted_col = int(first), 1 else: - # A single index into a rectangle is ambiguous, and Excel answers a whole row or - # column there. This engine has no array values, so it refuses rather than guess. - _diagnose(diagnostics, "bad_value") - return None - if not (1 <= wanted_row <= array.rows and 1 <= wanted_col <= array.cols): + wanted_row, wanted_col = int(first), 0 + if not (0 <= wanted_row <= array.rows and 0 <= wanted_col <= array.cols): _diagnose(diagnostics, "bad_value") return None + if wanted_row == 0 or wanted_col == 0: + rows = [wanted_row] if wanted_row else range(1, array.rows+1) + cols = [wanted_col] if wanted_col else range(1, array.cols+1) + return Range([array.cell(r, c) for r in rows for c in cols], len(rows), len(cols)) return array.cell(wanted_row, wanted_col) if fn == "MATCH": array = _as_range(a[1]) @@ -2216,130 +2883,863 @@ def _unknown_sheets(definition, known): return out -def evaluate_sheet(cells, *, today=None, sheets=None): - """Evaluate a whole WORKBOOK of A1 cells in dependency order (wave 49 W49-T03, W51-T23). - - `cells` is `{addr_key: {"formula": "=..."} or {"value": v}}` where `addr_key` is what - `a1_key` produces: `B4` on a single unnamed tab, `SHEET2!C3` when the caller keys a workbook. - Returns `{"values": {addr: value}, "errors": {addr: code}}`. - - ⭐⭐ A SPEC ENTRY MAY CARRY `"sheet"`, AND THAT IS HOW A WORKBOOK GETS ONE MEMOIZED PASS - (W51-T23). It names the tab that cell lives on, and it is handed to `parse_formula` so this - formula's own unqualified references key as `TAB!B4` rather than `B4`. Every tab in the book - therefore goes into ONE `cells` dict with one `resolve` memo over it, so a cross-tab reference - is an ordinary lookup and a chain that crosses tabs twice is still computed once. - ⛔ QUALIFYING THE KEYS WITHOUT `"sheet"` IS THE PROTOTYPE'S FAILURE MODE 3, MEASURED: cross-tab - references resolve and every LOCAL one breaks, 3,770 blanks against a true 785. - - `sheets` is the list of tab names the workbook actually has. When it is supplied, a formula - naming a tab outside it is refused `unknown_sheet` rather than quietly reading as blank; when - it is None nothing is checked, so every wave-49 caller keeps its behaviour exactly. - - ⚠ `SHEET_ADDR_KEY` IS STILL HANDED THE FULL KEY, and `ROW()` / `COLUMN()` still answer, - because `_call` splits an address at its last `!` before `_split_addr` reads it. That split - lands the same way for `G4` and for `TAB05!G4`, so the bare and the argument spelling both - survive a workbook pass. It has its own gate leg because it does not look like it would. - - ⛔⛔ **THE REFUSAL IS `circular_reference`, AND IT IS DELIBERATELY NOT THE `"cycle"` THAT - `evaluate_fields` ALREADY USES.** Renaming that one was considered and rejected on evidence: - `"cycle"` is a SHIPPED WIRE CODE. It is published in `contract_dto()`'s `errorCodes`, so it is - on the wire to the browser; `routes_query.py` branches on it behaviourally; and - `verify_w45_semantic_apps.py` pins the string in two fixtures. Both of those files belong to - another lane's fence. - - ⭐ AND THE TWO ARE GENUINELY DIFFERENT GRAPHS, which is what makes keeping both honest rather - than a dodge. `evaluate_fields`'s cycle is over FIELD MEMBERS within one grid row: every node is - a column of the same record. A sheet cycle is over CELL ADDRESSES and spans tabs, so its nodes - are not fields, its edges come from `parse_formula`'s `cells` list rather than `refs`, and a - range edge expands to many nodes at once. Unifying them would mean one detector walking two - unrelated graphs. If you are here to merge these names, read this paragraph first: the merge - breaks `routes_query.py` and a lane C gate in a single edit. - """ - values, errors, active, resolved = {}, {}, [], set() - parsed = {} - known = None if sheets is None else {str(n).upper() for n in sheets if str(n or "")} +def _range_bounds(first, last, sheet=""): + a, ar = _split_addr(first) + b, br = _split_addr(last) + ac, bc = col_to_index(a), col_to_index(b) + return (str(sheet).upper(), min(ac, bc), max(ac, bc), min(ar, br), max(ar, br)) + + +_BAND_SCAN = re.compile( + r'"(?:[^"]|"")*"|\'(?:[^\']|\'\')*\'(?!\!)|' + r"(?(?:(?:'[^']{1,64}'|[A-Za-z_][A-Za-z0-9_]{0,63})!)?)" + r"\$?(?P[A-Za-z]{1,3}|[1-9][0-9]{0,6}):\$?(?P[A-Za-z]{1,3}|[1-9][0-9]{0,6})" + r"(?![A-Za-z0-9_$!])") + + +def _expand_bands(source): + """Normalize whole bands to bounded Excel rectangles, without expanding any cells.""" + def replace(match): + first, last = match.group("first"), match.group("last") + if not first or first.isalpha() != last.isalpha(): + return match.group(0) + ends = (first + "1:" + last + "1048576" if first.isalpha() + else "A" + first + ":XFD" + last) + return match.group("prefix") + ends + return _BAND_SCAN.sub(replace, source) + + +def _address_parts(key): + sheet, _, addr = str(key).rpartition("!") + col, row = _split_addr(addr) + return sheet.upper(), col_to_index(col), row + + +def _segments(lo, hi): + """Canonical dyadic cover. A million-row band takes one subscription, not a million.""" + lo -= 1 + while lo < hi: + width = (lo & -lo) if lo else 1 << (hi.bit_length() - 1) + while width > hi - lo: + width >>= 1 + yield width.bit_length() - 1, lo // width + lo += width + + +class _RangeIndex: + def __init__(self): + self.buckets, self.owned = {}, {} + self.levels = {} + + def add(self, owner, bounds): + sheet, c0, c1, r0, r1 = bounds + for cl, ci in _segments(c0, c1): + for rl, ri in _segments(r0, r1): + token = (sheet, cl, ci, rl, ri) + if token not in self.buckets: + levels = self.levels.setdefault(sheet, {}) + levels[cl, rl] = levels.get((cl, rl), 0) + 1 + self.buckets.setdefault(token, set()).add(owner) + self.owned.setdefault(owner, set()).add(token) + + def remove(self, owner): + for token in self.owned.pop(owner, ()): + self.buckets[token].discard(owner) + if not self.buckets[token]: + del self.buckets[token] + sheet, cl, _, rl, _ = token + levels = self.levels[sheet] + levels[cl, rl] -= 1 + if not levels[cl, rl]: del levels[cl, rl] + if not levels: del self.levels[sheet] + + def at(self, key, stats): + sheet, col, row = _address_parts(key) + out = set() + for cl, rl in self.levels.get(sheet, ()): + stats["range_probes"] += 1 + hits = self.buckets.get((sheet, cl, (col - 1) >> cl, rl, (row - 1) >> rl), ()) + stats["range_checks"] += len(hits) + out.update(hits) + return out + + +class _SparseCells: + """Sorted populated coordinates; range queries never inspect unrelated cells or formulas.""" + def __init__(self): + self.columns, self.rows = {}, {} + + def add(self, key): + sheet, col, row = _address_parts(key) + cols = self.columns.setdefault(sheet, []) + rows = self.rows.get((sheet, col)) + if rows is None: + insort(cols, col) + rows = self.rows[(sheet, col)] = [] + at = bisect_left(rows, row) + if at == len(rows) or rows[at] != row: + rows.insert(at, row) + + def remove(self, key): + sheet, col, row = _address_parts(key) + rows = self.rows.get((sheet, col), []) + at = bisect_left(rows, row) + if at < len(rows) and rows[at] == row: + rows.pop(at) + if not rows and (sheet, col) in self.rows: + del self.rows[sheet, col] + self.columns[sheet].remove(col) + + def within(self, bounds): + sheet, c0, c1, r0, r1 = bounds + cols = self.columns.get(sheet, []) + found = [] + for col in cols[bisect_left(cols, c0):bisect_right(cols, c1)]: + rows = self.rows[sheet, col] + found.extend((row, col) for row in rows[bisect_left(rows, r0):bisect_right(rows, r1)]) + for row, col in sorted(found): + yield a1_key(index_to_col(col) + str(row), sheet) + + +class _SheetInputs(dict): + def __init__(self, engine, address): + super().__init__({SHEET_ADDR_KEY: address}) + self.engine = engine + self.ignore_reference_errors = 0 + self.format_stack = [] + self.last_format = None + + def __contains__(self, key): + return True # Missing Sheet cells are blanks, as in the existing evaluate_sheet DTO. + + def get(self, key, default=None): + if key == SHEET_ADDR_KEY: + return super().get(key, default) + self.engine._consume(super().get(SHEET_ADDR_KEY), key) + owner = self.engine.spill_parents.get(key) + if owner in self.engine._pending: raise _NeedReference(owner) + fault = key if key in self.engine.errors else None + if not _occupied(self.engine.specs.get(key)): + producers = self.engine.prospective.at(key, self.engine.stats) + pending = producers & self.engine._pending + if pending: raise _NeedReference(sorted(pending)[0]) + fault = next((ref for ref in sorted(producers) if self.engine.errors.get(ref) == "circular_reference"), fault) + if key in self.engine._pending: + raise _NeedReference(key) + if fault: + if self.ignore_reference_errors: return "#ERROR!" + raise _ReferenceFault(fault) + self.engine.stats["reads"] += 1 + if self.format_stack: self.format_stack[-1].append(self.engine.formats.get(key)) + return self.engine.values.get(key, default) + + def range_values(self, bounds): + # A sparse range must settle potential producers before deciding which cells exist. + for key in self.engine.array_positions.within(bounds): + if key in self.engine._pending: raise _NeedReference(key) + for key in self.engine.array_roots: + if key not in self.engine._pending: continue + area = self.engine.prospective_bounds.get(key) + if area and area[0] == bounds[0] and max(area[1], bounds[1]) <= min(area[2], bounds[2]) and max(area[3], bounds[3]) <= min(area[4], bounds[4]): + raise _NeedReference(key) + return [self.get(key) for key in self.engine.positions.within(bounds)] + + +def _spill_marker(spec, key): + marker = (spec.get("formulaMeta") or {}).get("spillChild") if isinstance(spec, dict) else None + return (marker if isinstance(marker, dict) and marker.get("version") == 1 + and marker.get("address") == key and isinstance(marker.get("origin"), str) + and _is_a1_key(marker["origin"]) and _sheet_half(marker["origin"]) == _sheet_half(key) + and marker["origin"] != key and spec.get("kind") != "formula" else None) + + +def _occupied(spec): + if not isinstance(spec, dict): return spec is not None + if spec.get("kind") == "empty": return False + return (spec.get("kind") in ("formula", "bound", "binding") or bool(spec.get("formula")) + or spec.get("binding") is not None or spec.get("value") is not None + or spec.get("text") is not None or bool(spec.get("refused"))) + + +@lru_cache(maxsize=8192) +def _static_reference_shape(text): + return _array_shape(_sheet_syntax(text, "")) + + +def _array_shape(node, lookup=None, seen=()): + """Conservative output extent, independent of values; scalars never reserve children.""" + kind, args = node.get("t"), node.get("args", []) + if kind == "a1range": return (node["rows"], node["cols"]) + shape = lambda child: _array_shape(child, lookup, seen) + if kind in ("name", "neg"): return shape(node["e"]) + if kind == "spill" and lookup and node["e"].get("t") == "a1": + key = node["e"]["k"] + target = lookup(key) + if target and key not in seen: return _array_shape(target, lookup, (*seen, key)) + if kind in ("bin", "cmp"): + a, b = shape(node["l"]), shape(node["r"]) + return max(a[0], b[0]), max(a[1], b[1]) + if kind == "call": + fn = node["fn"] + if fn in ("SORT", "FILTER"): return shape(args[0]) + if fn == "INDIRECT" and args[0].get("t") == "const" and isinstance(args[0].get("v"), str): + try: return _static_reference_shape(args[0]["v"]) + except FormulaError: return 1, 1 + if fn == "OFFSET": + rows, cols = shape(args[0]) + def dimension(index, default): + value = args[index].get("v") if len(args) > index else None + return max(1, int(value)) if isinstance(value, (int, float)) and math.isfinite(value) else default + return dimension(3, rows), dimension(4, cols) + if fn == "IF": + shapes = [shape(arg) for arg in args[1:]] + return max(s[0] for s in shapes), max(s[1] for s in shapes) + if fn in ("IFERROR", "IFNA"): + a, b = [shape(arg) for arg in args]; return max(a[0], b[0]), max(a[1], b[1]) + if fn == "XLOOKUP": + source, result = shape(args[1]), shape(args[2]) + wanted = shape(args[0]) + return wanted if wanted[0]*wanted[1] > 1 else (1, result[1]) if source[1] == 1 else (result[0], 1) + if fn == "INDEX": + rows, cols = shape(args[0]) + r = args[1].get("v") + c = args[2].get("v") if len(args) > 2 else 0 if rows > 1 and cols > 1 else 1 + return (rows if r == 0 else 1, cols if c == 0 else 1) + return 1, 1 + + +def _dynamic_array_shape(node): + if node.get("t") == "call": + args = node.get("args", []) + if node["fn"] == "INDIRECT" and args[0].get("t") != "const": return True + if node["fn"] == "OFFSET" and any(arg.get("t") != "const" for arg in args[3:]): return True + return any(_dynamic_array_shape(arg) for arg in args) + return any(_dynamic_array_shape(node[k]) for k in ("e", "l", "r") if isinstance(node.get(k), dict)) + + +def _calculation_stats(): + return dict(compiled=0, evaluated=0, visited=0, scanned=0, edges=0, + range_checks=0, range_probes=0, reads=0) + + +class _NeedReference(Exception): + pass - for addr, spec in (cells or {}).items(): - if not isinstance(spec, dict): - continue - # ⛔⛔ A CELL THAT ALREADY REFUSED IS CARRIED AS A REFUSAL, NOT AS AN ABSENCE (W49-T09, C2). - # This is the whole of "refusals propagate". If a refused cell were simply left out of the - # inputs, a formula over it would answer `unknown_ref` - "this points at a cell that is not - # filled in" - which is a DIFFERENT and misleading sentence: the cell IS filled in, and the - # reason it has no value is one the reader needs to see. Worse, `SUM` over an absent cell - # answers a NUMBER, so the refusal would vanish into a smaller total that looks fine. - if spec.get("refused"): - errors[addr] = str(spec["refused"]) - values[addr] = None - resolved.add(addr) - continue - raw = spec.get("formula") - if isinstance(raw, str) and raw.strip().startswith("="): - try: - definition = parse_formula(raw.strip()[1:], sheet=str(spec.get("sheet") or "")) - except FormulaError: - errors[addr] = "bad_formula" - values[addr] = None - continue - # ⛔ A MISSING TAB IS ITS OWN REFUSAL, NOT A BLANK. Left to the lookup it would read as - # `unknown_ref` at best and as a confident 0 through a range at worst, and both send - # the reader to inspect a formula that is correct except for a tab that is not there. - if known is not None and _unknown_sheets(definition, known): - errors[addr] = "unknown_sheet" - values[addr] = None + +class _ReferenceFault(Exception): + pass + + +def _cyclic_components(nodes, reads): + """Iterative Kosaraju: cycle members only, linear in the affected graph.""" + seen, finished = set(), [] + reverse = {} + for key in nodes: + for ref in reads.get(key, ()): + if ref in nodes: reverse.setdefault(ref, []).append(key) + for root in sorted(nodes): + if root in seen: continue + seen.add(root); stack = [(root, iter(sorted(set(reads.get(root, ())) & nodes)))] + while stack: + key, children = stack[-1] + ref = next(children, None) + if ref is None: finished.append(key); stack.pop() + elif ref not in seen: + seen.add(ref); stack.append((ref, iter(sorted(set(reads.get(ref, ())) & nodes)))) + seen, components = set(), [] + for root in reversed(finished): + if root in seen: continue + stack, members = [root], []; seen.add(root) + while stack: + key = stack.pop(); members.append(key) + for ref in reverse.get(key, ()): + if ref not in seen: seen.add(ref); stack.append(ref) + if len(members) > 1 or root in reads.get(root, ()): components.append(sorted(members)) + return sorted(components) + + +class SheetCalculation: + """C8 graph contract: ordered scalar/rectangle precedents, reverse subscriptions, revision. + + A reset visits the sparse snapshot once. An update visits only supplied cells and the + reachable subgraph, including range consumers of newly populated or deleted addresses. + Result dictionaries are owned by this instance; callers must not mutate them. + """ + def __init__(self, cells, *, revision=0, sheets=None, today=None, named_ranges=(), date_system="1900"): + self.revision = revision + self.sheets = None if sheets is None else {str(s).upper() for s in sheets} + self.today = today + self.named_ranges = _normalize_named_ranges(named_ranges) + self._reference_syntax_cache = OrderedDict() + self.date_system = date_system + self.volatile_roots, self._pending = set(), set() + self._volatile_values = {} + self._clock = dt.datetime.now(dt.timezone.utc) + self.stats = _calculation_stats() + self.specs, self.compiled, self.values, self.errors, self.provenance = {}, {}, {}, {}, {} + self.formats = {} + self.reads, self.read_by = {}, {} + self.consumed, self.consumed_by = {}, {} + self.ranges, self.positions, self.formula_positions = _RangeIndex(), _SparseCells(), _SparseCells() + self.spills, self.spill_parents, self.cleared_spills = {}, {}, {} + self.prospective, self.prospective_bounds = _RangeIndex(), {} + self.array_roots, self.array_positions = set(), _SparseCells() + for key, spec in (cells or {}).items(): + marker = _spill_marker(spec, str(key).upper()) + if marker: + self.cleared_spills[str(key).upper()] = marker["origin"] + base = marker.get("base") + if isinstance(base, dict) and not _occupied(base): self._put(str(key).upper(), base) continue - parsed[addr] = definition - else: - values[addr] = spec.get("value") - - def resolve(addr): - if addr in resolved: - return values.get(addr) - if addr not in parsed: - return values.get(addr) - if addr in active: - # Every member of the loop is refused, not just the one that closed it: a person - # looking at any cell in the cycle needs to be told it is part of one. - for member in active[active.index(addr):]: - errors[member] = "circular_reference" - values[member] = None - resolved.add(member) - return None - active.append(addr) - definition = parsed[addr] - # ⭐ THE CELL TELLS THE EVALUATOR WHERE IT IS (W50-T16). `ROW()` and `COLUMN()` with no - # argument have no other way to know, and this is the only caller that does. See - # `SHEET_ADDR_KEY` for why the key cannot collide with a field or a cell. - inputs = {SHEET_ADDR_KEY: addr} - inherited = None - for key in definition.get("cells") or (): - inputs[key] = resolve(key) if key in parsed else values.get(key) - # ⭐ THE UPSTREAM REFUSAL WINS, AND THE FIRST ONE FOUND IS THE ONE REPORTED. A formula - # over two refused cells has two reasons and one place to say them; naming the first is - # honest and actionable, while inventing a combined sentence is neither. The cell is - # refused either way, so no number is served. - if inherited is None and errors.get(key): - inherited = errors[key] - if errors.get(addr) == "circular_reference": - active.pop() - return None - if inherited is not None: - values[addr] = None - errors[addr] = inherited - active.pop() - resolved.add(addr) - return None - result = evaluate_result(definition, inputs, today=today) - active.pop() - values[addr] = result["value"] - if result["error"]: - errors[addr] = result["error"] - resolved.add(addr) - return values.get(addr) + self._put(str(key).upper(), spec) + for key in self.compiled: + self._connect(key) + self._calculate(set(self.compiled)) + + def _put(self, key, spec): + self.stats["scanned"] += 1 + self.errors.pop(key, None) + self.provenance.pop(key, None) + self.formats.pop(key, None) + if spec is None: + self.specs.pop(key, None) + self.values.pop(key, None) + self.positions.remove(key) + return + spec = dict(spec) if isinstance(spec, dict) else {"value": spec} + self.specs[key] = spec + fmt = (spec.get("style") or {}).get("format") + inferred = (spec.get("formulaMeta") or {}).get("inferredFormat") + if fmt and not (isinstance(inferred, dict) and inferred.get("version") == 1 and inferred.get("format") == fmt): + self.formats[key] = fmt + self.positions.add(key) + self.values[key] = spec.get("value") + if spec.get("refused"): + self._error(key, str(spec["refused"])) + return + source = spec.get("formula") + if not isinstance(source, str) or not source.strip().startswith("="): + return + self.stats["compiled"] += 1 + try: + definition = parse_formula(source.strip()[1:], sheet=spec.get("sheet") or "", compact_ranges=True, + named_ranges=self.named_ranges, address=key) + except FormulaError: + self._error(key, "bad_formula") + return + names = [_sheet_half(ref) for ref in definition["cells"]] + names.extend(bounds[0] for bounds in definition["ranges"]) + if self.sheets is not None and any(name and name not in self.sheets for name in names): + self._error(key, "unknown_sheet") + return + self.compiled[key] = definition + definition["dynamic_array_shape"] = _dynamic_array_shape(definition["ast"]) + rows, cols = _array_shape(definition["ast"]) + if rows*cols > 1: + s, c, r = _address_parts(key) + bounds = (s, c, min(16384, c+cols-1), r, min(1048576, r+rows-1)) + self.prospective.add(key, bounds); self.prospective_bounds[key] = bounds + self.array_roots.add(key); self.array_positions.add(key) + if definition["volatile"]: self.volatile_roots.add(key) + self.formula_positions.add(key) + + def reference_syntax(self, text, sheet, address): + # Relative named definitions depend on the calling cell. Ordinary A1 reference + # text depends only on its sheet; literal R1C1 offsets were resolved by the caller. + key = (text, sheet, address if self.named_ranges else "") + if key not in self._reference_syntax_cache: + self._reference_syntax_cache[key] = _sheet_syntax(text, sheet, self.named_ranges, address=address) + if len(self._reference_syntax_cache) > 8192: self._reference_syntax_cache.popitem(last=False) + else: self._reference_syntax_cache.move_to_end(key) + return self._reference_syntax_cache[key] + + def volatile_value(self, node, fn): + token = id(node) + if token not in self._volatile_values: + now = dt.datetime.fromisoformat(str(self.today)) if self.today else self._clock.replace(tzinfo=None) + epoch = dt.datetime(1904, 1, 1) if self.date_system == "1904" else dt.datetime(1899, 12, 30) + self._volatile_values[token] = (random.random() if fn == "RAND" else + (now - epoch).total_seconds() / 86400 if fn == "NOW" else (now - epoch).days) + return self._volatile_values[token] + + def _connect(self, key): + definition = self.compiled[key] + self._index_array(key) + deps = set(definition["cells"]) + for bounds in definition["ranges"]: + self.ranges.add(key, bounds) + deps.update(self.formula_positions.within(bounds)) + self.reads[key] = deps + for ref in deps: + self.read_by.setdefault(ref, set()).add(key) + self.stats["edges"] += 1 + + def _index_array(self, key): + rows, cols = _array_shape(self.compiled[key]["ast"], lambda ref: self.compiled.get(ref, {}).get("ast"), (key,)) + self.prospective.remove(key); self.prospective_bounds.pop(key, None); self.array_roots.discard(key) + if rows*cols > 1: + s, c, r = _address_parts(key) + area = (s, c, min(16384, c+cols-1), r, min(1048576, r+rows-1)) + self.prospective.add(key, area); self.prospective_bounds[key] = area; self.array_roots.add(key) + if self.compiled[key].get("dynamic_array_shape"): + self.array_roots.add(key); self.array_positions.add(key) + + def _disconnect(self, key): + self._forget_consumed(key) + self.prospective.remove(key); self.prospective_bounds.pop(key, None) + self.array_roots.discard(key); self.array_positions.remove(key) + self.volatile_roots.discard(key) + for ref in self.reads.pop(key, ()): + self.read_by[ref].discard(key) + if not self.read_by[ref]: + del self.read_by[ref] + self.ranges.remove(key) + self.compiled.pop(key, None) + self.formula_positions.remove(key) + + def _forget_consumed(self, key): + for ref in self.consumed.pop(key, ()): + owners = self.consumed_by[ref]; owners.discard(key) + if not owners: del self.consumed_by[ref] + + def _consume(self, key, ref): + refs = self.consumed.setdefault(key, set()) + if ref not in refs: + refs.add(ref); self.consumed_by.setdefault(ref, set()).add(key) + + def _spill_cycle(self, key, children): + # A newly discovered child closes a dependency through its producer. Only + # consumed references participate: an unused branch cannot create this edge. + # Walk the producer's precedents, never the workbook or all range subscribers. + if not any(self.consumed_by.get(child) for child in children): return False + targets, reachable, edges, queue = set(children), {key}, {}, [key] + for current in queue: + deps = set() + for ref in self.consumed.get(current, ()): + owner = key if ref in targets else self.spill_parents.get(ref, ref) + if owner in self.compiled: + deps.add(owner) + if owner not in reachable: reachable.add(owner); queue.append(owner) + edges[current] = deps + component = next((members for members in _cyclic_components(reachable, edges) if key in members), None) + if not component: return False + self._pass_cyclic.update(component) + for member in component: + self._error(member, "circular_reference"); self._pending.discard(member) + self.provenance[member].update(origin=component[0], via=None, path=component) + self.formats.pop(member, None) + # Earlier consumers can already hold scalars. A graph refusal invalidates + # those descendants once; they evaluate normally so IFERROR still catches. + queue, seen = [*component, *children], set(component) | set(children) + for ref in queue: + for owner in self.read_by.get(ref, set()) | self.consumed_by.get(ref, set()) | self.ranges.at(ref, self.stats): + if owner in seen or owner in self._pass_cyclic: continue + seen.add(owner); queue.append(owner) + if owner in self.compiled: + if owner not in self._pass_dirty: self.stats["visited"] += 1 + self._pass_dirty.add(owner); self._pending.add(owner) + return True - for addr in list(parsed): - resolve(addr) - return {"values": values, "errors": errors} + def _error(self, key, code, upstream=None): + self._clear_spill(key) + self.values[key] = None + self.errors[key] = code + trace = self.provenance.get(upstream) if upstream else None + self.provenance[key] = {"code": code, "origin": trace["origin"] if trace else key, + "via": upstream} + + def _clear_spill(self, key): + previous = self.spills.pop(key, None) + if not previous: return + for child in previous["children"]: + if self.spill_parents.get(child) != key: continue + self.spill_parents.pop(child, None) + self.formats.pop(child, None) + # Replacing an owned output with an authored literal retires ownership, + # but must not send a clear for that newly authored cell. + if not _occupied(self.specs.get(child)): + self.cleared_spills[child] = key + if child not in self.specs: + self.values.pop(child, None); self.positions.remove(child) + else: + self.values[child] = self.specs[child].get("value") + fmt = (self.specs[child].get("style") or {}).get("format") + if fmt: self.formats[child] = fmt + + def _publish(self, key, value): + self._clear_spill(key) + if not isinstance(value, Range): self.values[key] = value; return + s, c, r = _address_parts(key) + if r+value.rows-1 > 1048576 or c+value.cols-1 > 16384 or len(value) > MAX_RANGE_CELLS: + self._error(key, "spill_obstructed"); return + children = [a1_key(index_to_col(c+dc)+str(r+dr), s) + for dr in range(value.rows) for dc in range(value.cols)] + if any(_occupied(self.specs.get(child)) or child in self.spill_parents for child in children[1:]): + self._error(key, "spill_obstructed"); return + if self._spill_cycle(key, children[1:]): + bounds = (s, c, c+value.cols-1, r, r+value.rows-1) + self.prospective.add(key, bounds); self.prospective_bounds[key] = bounds + return + self.spills[key] = {"origin": key, "rows": value.rows, "cols": value.cols, + "range": (s, c, c+value.cols-1, r, r+value.rows-1), "children": children[1:]} + for child, item in zip(children, value.values): + self.values[child] = item + fmt = (self.specs.get(child, {}).get("style") or {}).get("format") if child != key else None + if fmt or self.formats.get(key): self.formats[child] = fmt or self.formats[key] + self.positions.add(child) + if child != key: + self.spill_parents[child] = key; self.cleared_spills.pop(child, None) + # Runtime shape discovery is a dependency event. Newly created targets can have + # subscribers that were clean when the input delta was scheduled. + queue = [] + for child in children[1:]: + if child not in self._output_notifications: + self._output_notifications.add(child); queue.append(child) + seen = set(queue) + for child in queue: + for owner in self.read_by.get(child, set()) | self.consumed_by.get(child, set()) | self.ranges.at(child, self.stats): + if owner in seen: continue + # An already evaluated formula needs the newly discovered edge only + # if it consumed that address. Static unused branches stay untouched. + if owner in self._pass_dirty and (owner in self._pending or owner not in self.consumed_by.get(child, ())): continue + seen.add(owner); queue.append(owner) + if owner in self.compiled: + if owner not in self._pass_dirty: self.stats["visited"] += 1 + self._pass_dirty.add(owner); self._pending.add(owner) + + def _calculate(self, dirty): + self._pass_dirty, self._output_notifications = dirty, set() + for key in dirty: self._forget_consumed(key) + for key in dirty: self._index_array(key) + for key in sorted(dirty): self._clear_spill(key) + indegree, dependents = {}, {} + for key in dirty: + self.stats["visited"] += 1 + deps = self.reads.get(key, set()) & dirty + indegree[key] = len(deps) + for ref in deps: + dependents.setdefault(ref, []).append(key) + self.stats["edges"] += 1 + ready = deque(sorted(k for k, degree in indegree.items() if not degree)) + ordered = [] + while ready: + key = ready.popleft() + ordered.append(key) + for dependent in sorted(dependents.get(key, ())): + indegree[dependent] -= 1 + if indegree[dependent] == 0: + ready.append(dependent) + blocked = sorted(dirty - set(ordered)) + components = _cyclic_components(set(blocked), self.reads) + cyclic = {member for component in components for member in component} + self._pass_cyclic = cyclic + for component in components: + for key in component: + self._error(key, "circular_reference") + self.provenance[key].update(origin=component[0], via=None, path=component) + self.formats.pop(key, None) + ordered += [key for key in blocked if key not in cyclic] + self._pending = set(ordered) + # Dynamic references can discover a dirty target after topological scheduling. + # Suspend the current AST, settle its target iteratively, then retry with the same + # per-pass volatile values. No recursive dependency walk and no whole-sheet scan. + def settle(key): + stack, active = [key], {key} + while stack: + current = stack[-1] + if current not in self._pending: + stack.pop(); active.discard(current); continue + needed = next((ref for ref in self.reads.get(current, ()) if ref in self._pending), None) + try: + if needed: raise _NeedReference(needed) + calculate_one(current) + except _NeedReference as exc: + ref = str(exc) + if ref in active: + component = sorted(stack[stack.index(ref):]) + for member in component: + self._error(member, "circular_reference"); self._pending.discard(member) + self.provenance[member].update(origin=component[0], via=None, path=component) + cyclic.add(member) + else: + stack.append(ref); active.add(ref) + continue + except _ReferenceFault as exc: + ref = str(exc); self._error(current, self.errors[ref], ref) + self._pending.discard(current) + stack.pop(); active.discard(current) + def calculate_one(key): + self.stats["evaluated"] += 1 + self._forget_consumed(key) + self.errors.pop(key, None) + self.provenance.pop(key, None) + inputs = _SheetInputs(self, key) + self.formats.pop(key, None) + result = evaluate_result(self.compiled[key], inputs, today=self.today) + spec = self.specs[key] + fmt = (spec.get("style") or {}).get("format") + inferred = (spec.get("formulaMeta") or {}).get("inferredFormat") + authored = fmt and not (isinstance(inferred, dict) and inferred.get("version") == 1 and inferred.get("format") == fmt) + if authored or inputs.last_format: self.formats[key] = fmt if authored else inputs.last_format + if result["error"]: + self._error(key, result["error"]) + else: self._publish(key, result["value"]) + for key in sorted(dirty & self.array_roots) + ordered: + if key in self._pending: settle(key) + while self._pending: + settle(min(self._pending)) + self.dirty, self.order, self.cyclic = dirty, ordered + sorted(dirty-set(ordered)-cyclic) + sorted(cyclic), sorted(cyclic) + + def update(self, changes, *, revision, today=None): + if revision != self.revision + 1: + raise ValueError("a calculation delta requires the next revision; rebuild from the snapshot") + self.stats = _calculation_stats() + self._volatile_values = {} + self._clock = dt.datetime.now(dt.timezone.utc) + self.today = today if today is not None else self.today + changed = {str(key).upper(): spec for key, spec in changes.items()} + self.cleared_spills = {} + spill_roots, old_children = set(), set() + for key in changed: + owner = self.spill_parents.get(key) + if owner: spill_roots.add(owner) + spill_roots.update(self.prospective.at(key, self.stats)) + old_children.update(self.spills.get(key, {}).get("children", ())) + for key in changed: self._clear_spill(key) + # Disconnect all edited formulas before reconnecting; a multi-cell paste sees one + # final graph. Existing range formulas gain/lose formula edges at the changed point. + consumers = {key: self.ranges.at(key, self.stats) for key in changed} + for key in changed: + self._disconnect(key) + for key, spec in changed.items(): + self._put(key, spec) + for key in changed: + if key in self.compiled: + self._connect(key) + for owner in consumers[key] - changed.keys(): + if key in self.compiled: + self.reads[owner].add(key) + self.read_by.setdefault(key, set()).add(owner) + elif key not in self.compiled[owner]["cells"]: + self.reads[owner].discard(key) + self.read_by.get(key, set()).discard(owner) + roots = set(changed) | self.volatile_roots | spill_roots | old_children + seen, dirty, queue = set(roots), set(), deque(roots) + while queue: + key = queue.popleft() + if key in self.compiled: + dirty.add(key) + for child in self.spills.get(key, {}).get("children", ()): + if child not in seen: seen.add(child); queue.append(child) + area = self.prospective_bounds.get(key) + if area and (area[2]-area[1]+1)*(area[4]-area[3]+1) <= MAX_RANGE_CELLS: + for r in range(area[3], area[4]+1): + for c in range(area[1], area[2]+1): + child = a1_key(index_to_col(c)+str(r), area[0]) + if child not in seen: seen.add(child); queue.append(child) + for owner in self.read_by.get(key, set()) | self.consumed_by.get(key, set()) | self.ranges.at(key, self.stats): + self.stats["edges"] += 1 + if owner not in seen: + seen.add(owner) + queue.append(owner) + self.revision = revision + self._calculate(dirty) + return self.result() + + def result(self): + return {"values": self.values, "errors": self.errors, "provenance": self.provenance, "formats": self.formats, + "spills": self.spills, "spillParents": self.spill_parents, "clearedSpills": self.cleared_spills, + "order": self.order, "cyclic": self.cyclic, "dirty": sorted(self.dirty), + "revision": self.revision, "stats": dict(self.stats)} + + +def evaluate_sheet(cells, *, today=None, sheets=None): + """Compatibility entry point over the same compiled Sheet graph as document refresh. + + Grid fields retain their separate row evaluator and legacy Boolean DTO. This wrapper + preserves its two-key response; document consumers get graph traces and formats directly. + """ + result = SheetCalculation(cells, today=today, sheets=sheets).result() + return {"values": result["values"], "errors": result["errors"]} + + +_DOCUMENT_CACHE_LIMIT = 16 +_DOCUMENT_CACHE_CELL_LIMIT = 200_000 +_document_calculations = OrderedDict() +_document_calculations_lock = RLock() +_UNADDRESSABLE_SHEET = "\x00 a tab that cannot be named inside a formula, which is the point \x00" + + +def _document_scopes(tabs): + names, seen = [], set() + for index, tab in enumerate(tabs): + name = str(tab.get("name") or "") + if not name or name.upper() in seen: + name = _UNADDRESSABLE_SHEET + str(index) + names.append(name) + seen.add(name.upper()) + return names + + +def _document_spec(cell, sheet): + # Calculation state/errors on formulas are previous OUTPUT, never trusted input. + if cell.get("kind") == "formula" and cell.get("formula"): + return {**cell, "formula": cell["formula"], "sheet": sheet} + if cell.get("state") == "refused" or cell.get("error"): + code = (cell.get("refusal") or cell.get("error") or {}).get("code") or "refused" + return {**cell, "refused": code, "sheet": sheet} + return {**cell, "sheet": sheet} + + +def _sheet_error_message(code): + from modules.sheet import REFUSALS + if code in REFUSALS: + return REFUSALS[code].format(subject="that database") + return { + "circular_reference": "This formula refers back to itself, so it has no value.", + "bad_formula": "This formula could not be read.", + "unknown_ref": "This formula points at a cell that is not filled in.", + "unknown_sheet": "This formula points at a tab that is not in this workbook.", + "div_zero": "This formula divides by zero.", + "no_match": "This lookup found nothing to match.", + "bad_value": "This function cannot use one of the values it was given.", + "null_intersection": "#NULL! These references do not intersect.", + "unknown_name": "#NAME? This name is not defined in this workbook.", + "external_reference": "#REF! External workbook references are not supported.", + "invalid_reference": "#REF! This reference is outside the worksheet.", + "spill_obstructed": "#SPILL! This array needs an empty range inside the worksheet.", + "empty_array": "#CALC! This filter has no matching values.", + }.get(code, "This formula could not be calculated.") + + +def recompute_document(book, *, cache_key=None, changes=None): + """Recalculate a C7 document in place; return C8 values/errors/provenance/work counters. + + Only a trusted (tenant, user, workbook) tuple can opt into this process-local bounded LRU. + Consecutive revisions plus an explicit accepted delta are required. Unkeyed reads, + refreshes, imports, missing revisions, gaps, renames, names and date-system edits rebuild. + The output projection visits the cached formula index because the route returns all + formula values; it never scans a sheet's cell map on a valid delta. + """ + with _document_calculations_lock: + tabs = [tab for tab in book.get("sheets", ()) if isinstance(tab, dict)] + names = _document_scopes(tabs) + tab_by_scope = {name.upper(): tab for tab, name in zip(tabs, names)} + signature = json.dumps([[(tab.get("id"), name) for tab, name in zip(tabs, names)], + book.get("namedRanges"), book.get("dateSystem"), + dt.date.today().isoformat()], sort_keys=True) + revision = book.get("rev") + valid_revision = type(revision) is int and revision >= 0 + trusted = (isinstance(cache_key, tuple) and len(cache_key) == 3 + and all(isinstance(part, str) and part for part in cache_key)) + entry = _document_calculations.get(cache_key) if trusted else None + incremental = (entry is not None and changes is not None and valid_revision + and entry[0] == signature and entry[1].revision + 1 == revision) + if incremental: + _, engine, formula_keys = entry + delta = {} + for raw, cell in changes.items(): + key = str(raw).upper() + scope = _sheet_half(key) + if scope not in tab_by_scope: + incremental = False + break + delta[key] = _document_spec(cell, scope) if isinstance(cell, dict) else None + if isinstance(cell, dict) and cell.get("kind") == "formula": + formula_keys.add(key) + else: + formula_keys.discard(key) + if incremental: + engine.update(delta, revision=revision) + if not incremental: + specs, formula_keys = {}, set() + for tab, name in zip(tabs, names): + for addr, cell in (tab.get("cells") or {}).items(): + if not isinstance(cell, dict): + continue + key = a1_key(addr, name) + specs[key] = _document_spec(cell, name) + if cell.get("kind") == "formula": + formula_keys.add(key) + engine = SheetCalculation(specs, revision=revision if valid_revision else 0, sheets=names, + named_ranges=book.get("namedRanges"), date_system=book.get("dateSystem", "1900")) + projected = 0 + for key in formula_keys: + scope, _, addr = key.rpartition("!") + cell = (tab_by_scope[scope].get("cells") or {}).get(addr) + if not isinstance(cell, dict): + continue + projected += 1 + code = engine.errors.get(key) + cell["value"] = engine.values.get(key) + if code: + message = _sheet_error_message(code) + cell.update(state="refused", display="", refusal={"code": code, "message": message}, + error={"code": code, "message": message, + **engine.provenance.get(key, {})}) + excel = {**REFERENCE_ERRORS, "no_match": "#N/A", "bad_value": "#VALUE!", "div_zero": "#DIV/0!", "numeric_error": "#NUM!"}.get(code) + if excel: cell["error"]["excelCode"] = excel + else: + cell.update(state="fresh", display="" if cell["value"] is None else str(cell["value"])) + cell.pop("refusal", None) + cell.pop("error", None) + cell.pop("spill", None) + meta = dict(cell.get("formulaMeta") or {}) + meta.pop("spill", None) + style = dict(cell.get("style") or {}) + inherited = meta.pop("inferredFormat", None) + if isinstance(inherited, dict) and inherited.get("version") == 1 and inherited.get("format") == style.get("format"): + style.pop("format", None) + if "format" not in style and engine.formats.get(key) and not code: + style["format"] = engine.formats[key] + meta["inferredFormat"] = {"version": 1, "format": engine.formats[key]} + if style or "style" in cell: cell["style"] = style + if key in engine.spills: + spill = engine.spills[key] + details = {"version": 1, "origin": key, "rows": spill["rows"], "cols": spill["cols"], + "ref": addr + ":" + index_to_col(spill["range"][2]) + str(spill["range"][4])} + cell["spill"] = details + meta["spill"] = details + if meta: cell["formulaMeta"] = meta + elif "formulaMeta" in cell: cell["formulaMeta"] = {} + # These are output records, never authored literals. The formulaMeta marker survives + # C7 migration and the existing route's spillParent predicate forwards clears as well. + for child, origin in engine.cleared_spills.items(): + scope, _, addr = child.rpartition("!") + if scope not in tab_by_scope: continue + cells = tab_by_scope[scope].setdefault("cells", {}) + existing = cells.get(addr) + if _occupied(engine.specs.get(child)) or (existing is not None and not _spill_marker(existing, child)): continue + base = engine.specs.get(child, {}) + cells[addr] = {"kind": "literal", "value": None, "display": "", "state": "typed", + **({"style": base["style"]} if base.get("style") else {}), + "spillParent": origin, "formulaMeta": {"spillClear": True, + "spillChild": {"version": 1, "origin": origin, "address": child, + **({"base": base} if base else {})}}} + for child, origin in engine.spill_parents.items(): + scope, _, addr = child.rpartition("!") + owner_scope, _, owner_addr = origin.rpartition("!") + owner = tab_by_scope[owner_scope]["cells"][owner_addr] + base = engine.specs.get(child, {}) + style = {**(owner.get("style") or {}), **(base.get("style") or {})} + record = {"kind": "literal", "value": engine.values.get(child), "state": "fresh", + "display": "" if engine.values.get(child) is None else str(engine.values[child]), + "spillParent": origin, "style": style, + "formulaMeta": {"spillChild": {"version": 1, "origin": origin, "address": child, + **({"base": base} if base else {})}}} + tab_by_scope[scope].setdefault("cells", {})[addr] = record + if trusted and valid_revision: + _document_calculations[cache_key] = (signature, engine, formula_keys) + _document_calculations.move_to_end(cache_key) + while (len(_document_calculations) > _DOCUMENT_CACHE_LIMIT + or sum(len(item[1].specs) for item in _document_calculations.values()) + > _DOCUMENT_CACHE_CELL_LIMIT): + _document_calculations.popitem(last=False) + elif trusted: + _document_calculations.pop(cache_key, None) + result = engine.result() + result["stats"].update(rebuilt=not incremental, projected=projected) + return result def evaluate_batch(definitions, rows, *, consumer, today=None, field_revisions=None):