"""S72-T68: freeze what a turn made or answered into one `snapshot` artifact (C7, R96, A3).""" import copy import datetime import importlib import json import logging import re import time import assistant_artifacts LOG = logging.getLogger(__name__) ROW_LIMIT = 200 MAX_CONTENT_BYTES = 256 * 1024 MAX_SUMMED_COLUMNS = 15 MAX_SHEET_COLUMNS = 52 RECORD_TOOLS = frozenset({"query_database", "database_object"}) SUMMED_TYPES = frozenset({"currency", "int"}) OPEN_SENTENCE = "Open in Loopable to see every row." _SHEET_READS = {"sheet_read_cells", "sheet_delete_tab"} _OBJECT_READS = {"workbook_object": {"list", "get", "export"}, "worksheet_object": {"list", "get"}, "range_object": {"read"}, "calculation_object": set()} _ADDR = re.compile(r"([A-Z]{1,3})([0-9]{1,7})") def _module(name): """Import a live reader by name at call time; a fixture swaps it in `sys.modules`.""" return importlib.import_module(name) def _query(session, database, payload): return _module("routes_external")._query_database(session, database, payload) def _visible_keys(session, database): """The field keys the maker may read, from the Assistant read scope.""" scoped = _module("deps").assistant_read_scope(session, database, mode="schema") return {field["key"] for field in scoped.get("fields") or () if isinstance(field, dict) and isinstance(field.get("key"), str)} def _authorization_scope(session, database): """Resolve current field and row authority without reading source values.""" from core import perm_scope scoped = _module("deps").assistant_read_scope(session, database, mode="schema") grant = perm_scope.assistant_entry(session.user, database) or {} fields = scoped.get("fields") or () audience = False if database.startswith("ut_"): definitions = session.runtime.get_projection("user_tables", drop=("rows",)) definition = definitions.get(database) if isinstance(definitions, dict) else None if not isinstance(definition, dict): raise ValueError("The source definition is unavailable.") audience = perm_scope.audience_declared(database, st=session.runtime, defn=definition) fields = definition.get("fields") or () pool = perm_scope.derive_pool_scope(session.user, database) return {"database": database, "fields": sorted(field["key"] for field in scoped.get("fields") or () if isinstance(field, dict) and isinstance(field.get("key"), str)), "field_types": {field["key"]: field.get("type") for field in fields if isinstance(field, dict) and isinstance(field.get("key"), str)}, "hidden": sorted(str(key) for key in grant.get("hiddenFields") or ()), "filter": copy.deepcopy(grant.get("filter")) or None, "pool": list(pool), "audience": audience} def _authorization_proof(session, sources): """Bind a server-captured document to its maker, tenant and permission scopes.""" return {"version": 1, "tenant": session.tenant, "owner": session.uname, "scopes": [_authorization_scope(session, source) for source in sources]} def require_permission(session, sources, content): """Refuse frozen values whose current authorization cannot be proved without live rows.""" def refuse(): raise _module("deps").err(403, "assistant_forbidden", "Your current permissions do not allow this snapshot.") if sources is None or content.get("sources") != sources: refuse() proof = content.get("_authorization") try: current = _authorization_proof(session, sources) except Exception: refuse() # Recipient membership can change independently of the rows-free permission contract. if any(scope["audience"] for scope in current["scopes"]): refuse() if proof is not None: if proof != current: refuse() return # Legacy sheets and unknown origins cannot prove which fields supplied their displayed values. if len(sources) != 1 or content.get("kind_of") not in ("records", "view", "app"): refuse() scope = current["scopes"][0] if scope["hidden"] or scope["filter"] or any(value is not None for value in scope["pool"]): refuse() columns, sums = content.get("columns"), (content.get("totals") or {}).get("sum") if not isinstance(columns, list) or not isinstance(sums, dict): refuse() visible = set(scope["fields"]) if (any(not isinstance(column, dict) or column.get("key") not in visible for column in columns) or not set(sums).issubset(visible)): refuse() def _load_view(session, database, view_id): found = _module("routes_external")._view_context(session, database, {}).table.find_view( view_id, prefer_owner=session.uname) return found[1] if found else None def _load_workbook(session, workbook_id): book = _module("core.sheets").get_workbook(session.runtime, workbook_id) routes = _module("routes_sheet") routes._require_reader(session, book) return routes._read_for(session, book) def _binding_readable(session, binding, book, tab): """True only when the maker can resolve this binding now.""" try: _module("routes_sheet")._resolve_cell_binding(session, binding, book, tab.get("id")) return True except Exception: return False def _now(): return datetime.datetime.fromtimestamp(time.time(), datetime.timezone.utc).isoformat() def _text(value): return value.strip() if isinstance(value, str) and value.strip() else None def _kind_of(name, args): action = args.get("action") if name == "query_database" or (name == "database_object" and action == "query"): return "records" if name.startswith("sheet_") and name not in _SHEET_READS: return "sheet" if name.startswith("workbook_") and name != "workbook_object": return "sheet" if name in _OBJECT_READS and action not in _OBJECT_READS[name] and action != "delete": return "sheet" if name == "view_upsert" or (name == "view_object" and action == "upsert"): return "view" if name == "app_upsert": return "app" return None def _rows_sentence(shown, total, cause): return f"Showing {shown:,} of {total:,} rows because {cause}. {OPEN_SENTENCE}" def _totals(session, database, filters, columns): """Count every permitted row and sum the number columns, server side, over ALL rows.""" numeric = [col["key"] for col in columns if col.get("type") in SUMMED_TYPES] notes = [] if len(numeric) > MAX_SUMMED_COLUMNS: notes.append(f"Totals cover the first {MAX_SUMMED_COLUMNS} number columns.") numeric = numeric[:MAX_SUMMED_COLUMNS] base = {"limit": 1} if filters: base["filters"] = copy.deepcopy(filters) specs = [{"fn": "count"}] + [{"fn": "sum", "field": key} for key in numeric] try: answer = _query(session, database, {**base, "aggregates": specs}) except Exception: if not numeric: raise notes.append("Totals show the row count only because a number column holds text.") answer = _query(session, database, {**base, "aggregates": [{"fn": "count"}]}) totals = {"count": None, "sum": {}} for agg in answer.get("aggregates") or (): if agg.get("fn") == "count" and agg.get("field") is None: totals["count"] = agg.get("value") elif agg.get("fn") == "sum": totals["sum"][agg["field"]] = agg.get("value") if not isinstance(totals["count"], int): raise ValueError("the reader returned no row count") return totals, notes def _columns(fields, keys, visible): """Column order follows `keys`; a field hidden from the maker never becomes a column.""" by_key = {f["key"]: f for f in fields or () if isinstance(f, dict) and isinstance(f.get("key"), str)} out = [] for key in keys: if key in by_key and key in visible: field = by_key[key] out.append({"key": key, "label": _text(field.get("label")) or key, "type": field.get("type")}) return out def _table_content(session, database, title, kind_of, result, filters, page_read, column_keys=None): """Build records, view or app content from one reader result.""" visible = _visible_keys(session, database) fields = result.get("fields") or [] keys = column_keys if column_keys is not None else [f.get("key") for f in fields if isinstance(f, dict)] columns = _columns(fields, keys, visible) records = result.get("records") or [] rows = [[copy.deepcopy(rec.get(col["key"])) for col in columns] for rec in records[:ROW_LIMIT]] totals, notes = _totals(session, database, filters, columns) total = totals["count"] if len(rows) < total: cause = ("Loopy read only those rows" if page_read and len(records) <= ROW_LIMIT else f"a snapshot keeps the first {ROW_LIMIT}") notes.insert(0, _rows_sentence(len(rows), total, cause)) return {"title": title, "kind_of": kind_of, "columns": [{"key": c["key"], "label": c["label"]} for c in columns], "rows": rows, "totals": totals, "row_count": total, "shown": len(rows), "notes": notes, "sources": [database]} def _record_list(session, name, args, result): if (not isinstance(result, dict) or not isinstance(result.get("records"), list) or result.get("groups")): return None database = _text(args.get("database")) or _text(result.get("database")) if database is None: return None query = args.get("query") if name == "database_object" else args filters = (query or {}).get("filters") title = f"{_text(result.get('label')) or database} records" return _table_content(session, database, title, "records", result, filters, True) def _view_payload(config, visible): keys = config.get("visible") keys = list(keys) if isinstance(keys, list) else None order = [k for k in config.get("order") or () if isinstance(k, str)] if keys is not None and order: keys = [k for k in order if k in keys] + [k for k in keys if k not in order] payload = {"limit": ROW_LIMIT} if keys is not None: payload["fields"] = [k for k in dict.fromkeys(keys) if k in visible] sorts = [{"field": rule.get("colId"), "direction": rule.get("dir")} for rule in config.get("sorts") or () if isinstance(rule, dict)] if sorts: payload["sorts"] = sorts[:8] filters = config.get("filters") if isinstance(filters, list) and filters: payload["filters"] = {"conj": config.get("filterConj") or "and", "nodes": filters} elif isinstance(filters, dict) and filters.get("nodes"): payload["filters"] = filters return payload def _view(session, args, result): database = _text(args.get("database")) or _text((result or {}).get("database")) draft = args.get("view") if isinstance(args.get("view"), dict) else {} view_id = _text((result or {}).get("view_id")) or _text(draft.get("id")) if database is None or view_id is None: return None view = _load_view(session, database, view_id) or draft config = view.get("config") if isinstance(view.get("config"), dict) else {} payload = _view_payload(config, _visible_keys(session, database)) answer = _query(session, database, payload) title = _text(view.get("name")) or "Saved View" return _table_content(session, database, title, "view", answer, payload.get("filters"), False, payload.get("fields")) def _app(session, args, result): app = (result or {}).get("application") app = app if isinstance(app, dict) else {} title = _text(app.get("name")) or _text(args.get("name")) or "App" compatible = app.get("compatibleDatabases") or args.get("compatibleDatabases") or [] database = (_text(app.get("sourceDatabase")) or _text(args.get("sourceDatabase")) or (_text(compatible[0]) if isinstance(compatible, list) and compatible else None)) if database is None: return {"title": title, "kind_of": "app", "columns": [], "rows": [], "totals": {"count": 0, "sum": {}}, "row_count": 0, "shown": 0, "notes": ["This App reads no database."], "sources": None} payload = {"limit": ROW_LIMIT} field_map = app.get("fieldMap") or args.get("fieldMap") if isinstance(field_map, dict) and field_map: visible = _visible_keys(session, database) payload["fields"] = [k for k in dict.fromkeys(field_map.values()) if k in visible] answer = _query(session, database, payload) return _table_content(session, database, title, "app", answer, None, False, payload.get("fields")) def _col_index(letters): n = 0 for ch in letters: n = n * 26 + ord(ch) - 64 return n def _col_name(n): out = "" while n: n, rem = divmod(n - 1, 26) out = chr(65 + rem) + out return out def _cell_shown(cell): for key in ("display", "text"): if isinstance(cell.get(key), str) and cell[key] != "": return cell[key] value = cell.get("value") return "" if value is None else value def _sheet(session, name, args, result): result = result if isinstance(result, dict) else {} nested = result.get("workbook") if isinstance(result.get("workbook"), dict) else {} workbook_id = (_text(args.get("workbook_id")) or _text(result.get("workbook_id")) or _text(nested.get("id")) or _text(result.get("id"))) if workbook_id is None: return None book = _load_workbook(session, workbook_id) tabs = [tab for tab in book.get("sheets") or () if isinstance(tab, dict)] if not tabs: return None wanted = args.get("tab") or args.get("worksheet") or result.get("tab") wanted = wanted.get("id") if isinstance(wanted, dict) else wanted tab = next((t for t in tabs if wanted is not None and wanted in (t.get("id"), t.get("name"))), tabs[0]) cells = {} for addr, cell in (tab.get("cells") or {}).items(): match = _ADDR.fullmatch(str(addr).strip().upper()) if match and isinstance(cell, dict): cells[(int(match.group(2)), _col_index(match.group(1)))] = cell sources, unnamed, checked = set(), False, {} # A displayed formula can read a binding on another tab. for source_tab in tabs: for cell in (source_tab.get("cells") or {}).values(): if not isinstance(cell, dict) or cell.get("binding") is None: continue binding = cell["binding"] database = _text(binding.get("database")) if isinstance(binding, dict) else None if database is None: unnamed = True else: sources.add(database) shown = {} for at, cell in cells.items(): binding = cell.get("binding") if binding is not None: database = _text(binding.get("database")) if isinstance(binding, dict) else None if database is None: continue memo = json.dumps(binding, sort_keys=True, default=str) if memo not in checked: checked[memo] = _binding_readable(session, binding, book, tab) if not checked[memo]: continue shown[at] = copy.deepcopy(_cell_shown(cell)) last_row = max((r for r, _ in cells), default=0) last_col = max((c for _, c in cells), default=0) notes = [] if last_col > MAX_SHEET_COLUMNS: notes.append(f"Showing the first {MAX_SHEET_COLUMNS} columns because a snapshot keeps " f"{MAX_SHEET_COLUMNS} columns. Open in Loopable to see every column.") width = min(last_col, MAX_SHEET_COLUMNS) rows = [[shown.get((r, c), "") for c in range(1, width + 1)] for r in range(1, min(last_row, ROW_LIMIT) + 1)] if last_row > ROW_LIMIT: notes.insert(0, _rows_sentence(ROW_LIMIT, last_row, f"a snapshot keeps the first {ROW_LIMIT}")) book_name = _text(book.get("name")) or "Sheet" tab_name = _text(tab.get("name")) return {"title": f"{book_name}, {tab_name}" if tab_name and len(tabs) > 1 else book_name, "kind_of": "sheet", "columns": [{"key": _col_name(c), "label": _col_name(c)} for c in range(1, width + 1)], "rows": rows, "totals": {"count": last_row, "sum": {}}, "row_count": last_row, "shown": len(rows), "notes": notes, "sources": None if unnamed else sorted(sources)} def _size(content): return len(json.dumps(content, sort_keys=True, default=str).encode("utf-8")) def _finish(content, frozen_at): """Apply the byte bound, join the notes and stamp the time, keeping totals over all rows.""" content = json.loads(json.dumps(content, default=str)) notes = content.pop("notes") content.update(frozen_at=frozen_at, note=" ".join(notes) or None) if _size(content) > MAX_CONTENT_BYTES: lo, hi, rows = 0, len(content["rows"]), content["rows"] notes = [n for n in notes if not re.match(r"Showing [\d,]+ of [\d,]+ rows because ", n)] def bounded(count): content["rows"] = rows[:count] content["shown"] = count content["note"] = " ".join([ _rows_sentence(count, content["row_count"], f"a snapshot keeps at most {MAX_CONTENT_BYTES // 1024} KB"), *notes]) return _size(content) <= MAX_CONTENT_BYTES if not bounded(0): raise ValueError("Snapshot labels and totals exceed the content bound.") while lo < hi: mid = (lo + hi + 1) // 2 lo, hi = (mid, hi) if bounded(mid) else (lo, mid - 1) bounded(lo) return {key: content[key] for key in ("title", "kind_of", "columns", "rows", "totals", "row_count", "shown", "note", "sources", "frozen_at", "_authorization") if key in content} def _retire_earlier(rt, username, turn): """Keep one active record-list snapshot per turn.""" for item_id in list(turn.get("artifacts") or ()): row = assistant_artifacts.get(rt, username, item_id) if (row and row.get("kind") == "snapshot" and row.get("status") == "active" and row.get("toolCall", {}).get("name") in RECORD_TOOLS): assistant_artifacts.set_status(rt, username, item_id, "deleted") def capture(session, door, name, args, result, turn): """Store a frozen snapshot of this tool call when it qualifies; return its id or None. Never raises.""" try: if (door not in assistant_artifacts.DOORS or not isinstance(name, str) or not isinstance(turn, dict) or not isinstance(turn.get("artifacts"), list)): return None args = args if isinstance(args, dict) else {} kind_of = _kind_of(name, args) if kind_of is None: return None build = {"records": lambda: _record_list(session, name, args, result), "sheet": lambda: _sheet(session, name, args, result), "view": lambda: _view(session, args, result), "app": lambda: _app(session, args, result)}[kind_of] content = build() if content is None: return None if content["sources"] is not None: if kind_of == "sheet": try: content["_authorization"] = _authorization_proof(session, content["sources"]) except Exception: pass # A blanked unreadable binding retains protected provenance. else: content["_authorization"] = _authorization_proof(session, content["sources"]) rt, username = session.runtime, session.uname frozen_at = _now() content = _finish(content, frozen_at) ref = assistant_artifacts.store_snapshot(rt, username, content) if kind_of == "records": _retire_earlier(rt, username, turn) item_id = assistant_artifacts.record(rt, username, { "kind": "snapshot", "ref": ref, "title": content["title"], "door": door, "toolCall": {"name": name, "args": copy.deepcopy(args)}, "createdAt": time.time(), "sources": content["sources"], "frozen_at": frozen_at}) turn["artifacts"].append(item_id) return item_id except Exception as exc: LOG.warning("snapshot not captured for %s: %s: %s", name, type(exc).__name__, exc) return None def read(rt, artifact_id): """Return the stored artifact and its frozen content; read only those two documents.""" row = assistant_artifacts.get_by_id(rt, artifact_id) if row is None or row.get("kind") != "snapshot" or row.get("status") == "deleted": return None content = assistant_artifacts.read_snapshot(rt, row) if content is None: return None return {"artifact": assistant_artifacts.c7(row), "snapshot": {key: value for key, value in content.items() if key != "_authorization"}}