runloopable / api /artifact_snapshot.py
fsanyoto's picture
Deploy AIOS web (React glide grid + FastAPI slice)
816fa8a verified
Raw History Blame Contribute Delete
21.2 kB
"""S72-T68: freeze what a turn made or answered into one `snapshot` artifact (C7, R96, A3)."""
import copy
import datetime
import importlib
import json
import logging
import re
import time
import assistant_artifacts
LOG = logging.getLogger(__name__)
ROW_LIMIT = 200
MAX_CONTENT_BYTES = 256 * 1024
MAX_SUMMED_COLUMNS = 15
MAX_SHEET_COLUMNS = 52
RECORD_TOOLS = frozenset({"query_database", "database_object"})
SUMMED_TYPES = frozenset({"currency", "int"})
OPEN_SENTENCE = "Open in Loopable to see every row."
_SHEET_READS = {"sheet_read_cells", "sheet_delete_tab"}
_OBJECT_READS = {"workbook_object": {"list", "get", "export"}, "worksheet_object": {"list", "get"},
"range_object": {"read"}, "calculation_object": set()}
_ADDR = re.compile(r"([A-Z]{1,3})([0-9]{1,7})")
def _module(name):
"""Import a live reader by name at call time; a fixture swaps it in `sys.modules`."""
return importlib.import_module(name)
def _query(session, database, payload):
return _module("routes_external")._query_database(session, database, payload)
def _visible_keys(session, database):
"""The field keys the maker may read, from the Assistant read scope."""
scoped = _module("deps").assistant_read_scope(session, database, mode="schema")
return {field["key"] for field in scoped.get("fields") or ()
if isinstance(field, dict) and isinstance(field.get("key"), str)}
def _authorization_scope(session, database):
"""Resolve current field and row authority without reading source values."""
from core import perm_scope
scoped = _module("deps").assistant_read_scope(session, database, mode="schema")
grant = perm_scope.assistant_entry(session.user, database) or {}
fields = scoped.get("fields") or ()
audience = False
if database.startswith("ut_"):
definitions = session.runtime.get_projection("user_tables", drop=("rows",))
definition = definitions.get(database) if isinstance(definitions, dict) else None
if not isinstance(definition, dict):
raise ValueError("The source definition is unavailable.")
audience = perm_scope.audience_declared(database, st=session.runtime, defn=definition)
fields = definition.get("fields") or ()
pool = perm_scope.derive_pool_scope(session.user, database)
return {"database": database,
"fields": sorted(field["key"] for field in scoped.get("fields") or ()
if isinstance(field, dict) and isinstance(field.get("key"), str)),
"field_types": {field["key"]: field.get("type") for field in fields
if isinstance(field, dict) and isinstance(field.get("key"), str)},
"hidden": sorted(str(key) for key in grant.get("hiddenFields") or ()),
"filter": copy.deepcopy(grant.get("filter")) or None,
"pool": list(pool), "audience": audience}
def _authorization_proof(session, sources):
"""Bind a server-captured document to its maker, tenant and permission scopes."""
return {"version": 1, "tenant": session.tenant, "owner": session.uname,
"scopes": [_authorization_scope(session, source) for source in sources]}
def require_permission(session, sources, content):
"""Refuse frozen values whose current authorization cannot be proved without live rows."""
def refuse():
raise _module("deps").err(403, "assistant_forbidden",
"Your current permissions do not allow this snapshot.")
if sources is None or content.get("sources") != sources:
refuse()
proof = content.get("_authorization")
try:
current = _authorization_proof(session, sources)
except Exception:
refuse()
# Recipient membership can change independently of the rows-free permission contract.
if any(scope["audience"] for scope in current["scopes"]):
refuse()
if proof is not None:
if proof != current:
refuse()
return
# Legacy sheets and unknown origins cannot prove which fields supplied their displayed values.
if len(sources) != 1 or content.get("kind_of") not in ("records", "view", "app"):
refuse()
scope = current["scopes"][0]
if scope["hidden"] or scope["filter"] or any(value is not None for value in scope["pool"]):
refuse()
columns, sums = content.get("columns"), (content.get("totals") or {}).get("sum")
if not isinstance(columns, list) or not isinstance(sums, dict):
refuse()
visible = set(scope["fields"])
if (any(not isinstance(column, dict) or column.get("key") not in visible for column in columns)
or not set(sums).issubset(visible)):
refuse()
def _load_view(session, database, view_id):
found = _module("routes_external")._view_context(session, database, {}).table.find_view(
view_id, prefer_owner=session.uname)
return found[1] if found else None
def _load_workbook(session, workbook_id):
book = _module("core.sheets").get_workbook(session.runtime, workbook_id)
routes = _module("routes_sheet")
routes._require_reader(session, book)
return routes._read_for(session, book)
def _binding_readable(session, binding, book, tab):
"""True only when the maker can resolve this binding now."""
try:
_module("routes_sheet")._resolve_cell_binding(session, binding, book, tab.get("id"))
return True
except Exception:
return False
def _now():
return datetime.datetime.fromtimestamp(time.time(), datetime.timezone.utc).isoformat()
def _text(value):
return value.strip() if isinstance(value, str) and value.strip() else None
def _kind_of(name, args):
action = args.get("action")
if name == "query_database" or (name == "database_object" and action == "query"):
return "records"
if name.startswith("sheet_") and name not in _SHEET_READS:
return "sheet"
if name.startswith("workbook_") and name != "workbook_object":
return "sheet"
if name in _OBJECT_READS and action not in _OBJECT_READS[name] and action != "delete":
return "sheet"
if name == "view_upsert" or (name == "view_object" and action == "upsert"):
return "view"
if name == "app_upsert":
return "app"
return None
def _rows_sentence(shown, total, cause):
return f"Showing {shown:,} of {total:,} rows because {cause}. {OPEN_SENTENCE}"
def _totals(session, database, filters, columns):
"""Count every permitted row and sum the number columns, server side, over ALL rows."""
numeric = [col["key"] for col in columns if col.get("type") in SUMMED_TYPES]
notes = []
if len(numeric) > MAX_SUMMED_COLUMNS:
notes.append(f"Totals cover the first {MAX_SUMMED_COLUMNS} number columns.")
numeric = numeric[:MAX_SUMMED_COLUMNS]
base = {"limit": 1}
if filters:
base["filters"] = copy.deepcopy(filters)
specs = [{"fn": "count"}] + [{"fn": "sum", "field": key} for key in numeric]
try:
answer = _query(session, database, {**base, "aggregates": specs})
except Exception:
if not numeric:
raise
notes.append("Totals show the row count only because a number column holds text.")
answer = _query(session, database, {**base, "aggregates": [{"fn": "count"}]})
totals = {"count": None, "sum": {}}
for agg in answer.get("aggregates") or ():
if agg.get("fn") == "count" and agg.get("field") is None:
totals["count"] = agg.get("value")
elif agg.get("fn") == "sum":
totals["sum"][agg["field"]] = agg.get("value")
if not isinstance(totals["count"], int):
raise ValueError("the reader returned no row count")
return totals, notes
def _columns(fields, keys, visible):
"""Column order follows `keys`; a field hidden from the maker never becomes a column."""
by_key = {f["key"]: f for f in fields or () if isinstance(f, dict) and isinstance(f.get("key"), str)}
out = []
for key in keys:
if key in by_key and key in visible:
field = by_key[key]
out.append({"key": key, "label": _text(field.get("label")) or key, "type": field.get("type")})
return out
def _table_content(session, database, title, kind_of, result, filters, page_read, column_keys=None):
"""Build records, view or app content from one reader result."""
visible = _visible_keys(session, database)
fields = result.get("fields") or []
keys = column_keys if column_keys is not None else [f.get("key") for f in fields if isinstance(f, dict)]
columns = _columns(fields, keys, visible)
records = result.get("records") or []
rows = [[copy.deepcopy(rec.get(col["key"])) for col in columns] for rec in records[:ROW_LIMIT]]
totals, notes = _totals(session, database, filters, columns)
total = totals["count"]
if len(rows) < total:
cause = ("Loopy read only those rows" if page_read and len(records) <= ROW_LIMIT
else f"a snapshot keeps the first {ROW_LIMIT}")
notes.insert(0, _rows_sentence(len(rows), total, cause))
return {"title": title, "kind_of": kind_of,
"columns": [{"key": c["key"], "label": c["label"]} for c in columns],
"rows": rows, "totals": totals, "row_count": total, "shown": len(rows),
"notes": notes, "sources": [database]}
def _record_list(session, name, args, result):
if (not isinstance(result, dict) or not isinstance(result.get("records"), list)
or result.get("groups")):
return None
database = _text(args.get("database")) or _text(result.get("database"))
if database is None:
return None
query = args.get("query") if name == "database_object" else args
filters = (query or {}).get("filters")
title = f"{_text(result.get('label')) or database} records"
return _table_content(session, database, title, "records", result, filters, True)
def _view_payload(config, visible):
keys = config.get("visible")
keys = list(keys) if isinstance(keys, list) else None
order = [k for k in config.get("order") or () if isinstance(k, str)]
if keys is not None and order:
keys = [k for k in order if k in keys] + [k for k in keys if k not in order]
payload = {"limit": ROW_LIMIT}
if keys is not None:
payload["fields"] = [k for k in dict.fromkeys(keys) if k in visible]
sorts = [{"field": rule.get("colId"), "direction": rule.get("dir")}
for rule in config.get("sorts") or () if isinstance(rule, dict)]
if sorts:
payload["sorts"] = sorts[:8]
filters = config.get("filters")
if isinstance(filters, list) and filters:
payload["filters"] = {"conj": config.get("filterConj") or "and", "nodes": filters}
elif isinstance(filters, dict) and filters.get("nodes"):
payload["filters"] = filters
return payload
def _view(session, args, result):
database = _text(args.get("database")) or _text((result or {}).get("database"))
draft = args.get("view") if isinstance(args.get("view"), dict) else {}
view_id = _text((result or {}).get("view_id")) or _text(draft.get("id"))
if database is None or view_id is None:
return None
view = _load_view(session, database, view_id) or draft
config = view.get("config") if isinstance(view.get("config"), dict) else {}
payload = _view_payload(config, _visible_keys(session, database))
answer = _query(session, database, payload)
title = _text(view.get("name")) or "Saved View"
return _table_content(session, database, title, "view", answer, payload.get("filters"), False,
payload.get("fields"))
def _app(session, args, result):
app = (result or {}).get("application")
app = app if isinstance(app, dict) else {}
title = _text(app.get("name")) or _text(args.get("name")) or "App"
compatible = app.get("compatibleDatabases") or args.get("compatibleDatabases") or []
database = (_text(app.get("sourceDatabase")) or _text(args.get("sourceDatabase"))
or (_text(compatible[0]) if isinstance(compatible, list) and compatible else None))
if database is None:
return {"title": title, "kind_of": "app", "columns": [], "rows": [], "totals": {"count": 0, "sum": {}},
"row_count": 0, "shown": 0, "notes": ["This App reads no database."], "sources": None}
payload = {"limit": ROW_LIMIT}
field_map = app.get("fieldMap") or args.get("fieldMap")
if isinstance(field_map, dict) and field_map:
visible = _visible_keys(session, database)
payload["fields"] = [k for k in dict.fromkeys(field_map.values()) if k in visible]
answer = _query(session, database, payload)
return _table_content(session, database, title, "app", answer, None, False, payload.get("fields"))
def _col_index(letters):
n = 0
for ch in letters:
n = n * 26 + ord(ch) - 64
return n
def _col_name(n):
out = ""
while n:
n, rem = divmod(n - 1, 26)
out = chr(65 + rem) + out
return out
def _cell_shown(cell):
for key in ("display", "text"):
if isinstance(cell.get(key), str) and cell[key] != "":
return cell[key]
value = cell.get("value")
return "" if value is None else value
def _sheet(session, name, args, result):
result = result if isinstance(result, dict) else {}
nested = result.get("workbook") if isinstance(result.get("workbook"), dict) else {}
workbook_id = (_text(args.get("workbook_id")) or _text(result.get("workbook_id"))
or _text(nested.get("id")) or _text(result.get("id")))
if workbook_id is None:
return None
book = _load_workbook(session, workbook_id)
tabs = [tab for tab in book.get("sheets") or () if isinstance(tab, dict)]
if not tabs:
return None
wanted = args.get("tab") or args.get("worksheet") or result.get("tab")
wanted = wanted.get("id") if isinstance(wanted, dict) else wanted
tab = next((t for t in tabs if wanted is not None and wanted in (t.get("id"), t.get("name"))), tabs[0])
cells = {}
for addr, cell in (tab.get("cells") or {}).items():
match = _ADDR.fullmatch(str(addr).strip().upper())
if match and isinstance(cell, dict):
cells[(int(match.group(2)), _col_index(match.group(1)))] = cell
sources, unnamed, checked = set(), False, {}
# A displayed formula can read a binding on another tab.
for source_tab in tabs:
for cell in (source_tab.get("cells") or {}).values():
if not isinstance(cell, dict) or cell.get("binding") is None:
continue
binding = cell["binding"]
database = _text(binding.get("database")) if isinstance(binding, dict) else None
if database is None:
unnamed = True
else:
sources.add(database)
shown = {}
for at, cell in cells.items():
binding = cell.get("binding")
if binding is not None:
database = _text(binding.get("database")) if isinstance(binding, dict) else None
if database is None:
continue
memo = json.dumps(binding, sort_keys=True, default=str)
if memo not in checked:
checked[memo] = _binding_readable(session, binding, book, tab)
if not checked[memo]:
continue
shown[at] = copy.deepcopy(_cell_shown(cell))
last_row = max((r for r, _ in cells), default=0)
last_col = max((c for _, c in cells), default=0)
notes = []
if last_col > MAX_SHEET_COLUMNS:
notes.append(f"Showing the first {MAX_SHEET_COLUMNS} columns because a snapshot keeps "
f"{MAX_SHEET_COLUMNS} columns. Open in Loopable to see every column.")
width = min(last_col, MAX_SHEET_COLUMNS)
rows = [[shown.get((r, c), "") for c in range(1, width + 1)] for r in range(1, min(last_row, ROW_LIMIT) + 1)]
if last_row > ROW_LIMIT:
notes.insert(0, _rows_sentence(ROW_LIMIT, last_row, f"a snapshot keeps the first {ROW_LIMIT}"))
book_name = _text(book.get("name")) or "Sheet"
tab_name = _text(tab.get("name"))
return {"title": f"{book_name}, {tab_name}" if tab_name and len(tabs) > 1 else book_name,
"kind_of": "sheet",
"columns": [{"key": _col_name(c), "label": _col_name(c)} for c in range(1, width + 1)],
"rows": rows, "totals": {"count": last_row, "sum": {}}, "row_count": last_row,
"shown": len(rows), "notes": notes, "sources": None if unnamed else sorted(sources)}
def _size(content):
return len(json.dumps(content, sort_keys=True, default=str).encode("utf-8"))
def _finish(content, frozen_at):
"""Apply the byte bound, join the notes and stamp the time, keeping totals over all rows."""
content = json.loads(json.dumps(content, default=str))
notes = content.pop("notes")
content.update(frozen_at=frozen_at, note=" ".join(notes) or None)
if _size(content) > MAX_CONTENT_BYTES:
lo, hi, rows = 0, len(content["rows"]), content["rows"]
notes = [n for n in notes if not re.match(r"Showing [\d,]+ of [\d,]+ rows because ", n)]
def bounded(count):
content["rows"] = rows[:count]
content["shown"] = count
content["note"] = " ".join([
_rows_sentence(count, content["row_count"],
f"a snapshot keeps at most {MAX_CONTENT_BYTES // 1024} KB"), *notes])
return _size(content) <= MAX_CONTENT_BYTES
if not bounded(0):
raise ValueError("Snapshot labels and totals exceed the content bound.")
while lo < hi:
mid = (lo + hi + 1) // 2
lo, hi = (mid, hi) if bounded(mid) else (lo, mid - 1)
bounded(lo)
return {key: content[key] for key in ("title", "kind_of", "columns", "rows", "totals", "row_count",
"shown", "note", "sources", "frozen_at", "_authorization")
if key in content}
def _retire_earlier(rt, username, turn):
"""Keep one active record-list snapshot per turn."""
for item_id in list(turn.get("artifacts") or ()):
row = assistant_artifacts.get(rt, username, item_id)
if (row and row.get("kind") == "snapshot" and row.get("status") == "active"
and row.get("toolCall", {}).get("name") in RECORD_TOOLS):
assistant_artifacts.set_status(rt, username, item_id, "deleted")
def capture(session, door, name, args, result, turn):
"""Store a frozen snapshot of this tool call when it qualifies; return its id or None. Never raises."""
try:
if (door not in assistant_artifacts.DOORS or not isinstance(name, str)
or not isinstance(turn, dict) or not isinstance(turn.get("artifacts"), list)):
return None
args = args if isinstance(args, dict) else {}
kind_of = _kind_of(name, args)
if kind_of is None:
return None
build = {"records": lambda: _record_list(session, name, args, result),
"sheet": lambda: _sheet(session, name, args, result),
"view": lambda: _view(session, args, result),
"app": lambda: _app(session, args, result)}[kind_of]
content = build()
if content is None:
return None
if content["sources"] is not None:
if kind_of == "sheet":
try:
content["_authorization"] = _authorization_proof(session, content["sources"])
except Exception:
pass # A blanked unreadable binding retains protected provenance.
else:
content["_authorization"] = _authorization_proof(session, content["sources"])
rt, username = session.runtime, session.uname
frozen_at = _now()
content = _finish(content, frozen_at)
ref = assistant_artifacts.store_snapshot(rt, username, content)
if kind_of == "records":
_retire_earlier(rt, username, turn)
item_id = assistant_artifacts.record(rt, username, {
"kind": "snapshot", "ref": ref, "title": content["title"], "door": door,
"toolCall": {"name": name, "args": copy.deepcopy(args)}, "createdAt": time.time(),
"sources": content["sources"], "frozen_at": frozen_at})
turn["artifacts"].append(item_id)
return item_id
except Exception as exc:
LOG.warning("snapshot not captured for %s: %s: %s", name, type(exc).__name__, exc)
return None
def read(rt, artifact_id):
"""Return the stored artifact and its frozen content; read only those two documents."""
row = assistant_artifacts.get_by_id(rt, artifact_id)
if row is None or row.get("kind") != "snapshot" or row.get("status") == "deleted":
return None
content = assistant_artifacts.read_snapshot(rt, row)
if content is None:
return None
return {"artifact": assistant_artifacts.c7(row),
"snapshot": {key: value for key, value in content.items() if key != "_authorization"}}