Download api/artifact_snapshot.py from fsanyoto/loopable: direct link, hf CLI and curl.
- Browser
- Download file 21.2 kB
-
https://huggingface.co/spaces/fsanyoto/loopable/resolve/main/api/artifact_snapshot.py
- Command line
-
hf download hf://spaces/fsanyoto/loopable/api/artifact_snapshot.py
-
curl -L -o artifact_snapshot.py https://huggingface.co/spaces/fsanyoto/loopable/resolve/main/api/artifact_snapshot.py
21.2 kB
| """S72-T68: freeze what a turn made or answered into one `snapshot` artifact (C7, R96, A3).""" | |
| import copy | |
| import datetime | |
| import importlib | |
| import json | |
| import logging | |
| import re | |
| import time | |
| import assistant_artifacts | |
| LOG = logging.getLogger(__name__) | |
| ROW_LIMIT = 200 | |
| MAX_CONTENT_BYTES = 256 * 1024 | |
| MAX_SUMMED_COLUMNS = 15 | |
| MAX_SHEET_COLUMNS = 52 | |
| RECORD_TOOLS = frozenset({"query_database", "database_object"}) | |
| SUMMED_TYPES = frozenset({"currency", "int"}) | |
| OPEN_SENTENCE = "Open in Loopable to see every row." | |
| _SHEET_READS = {"sheet_read_cells", "sheet_delete_tab"} | |
| _OBJECT_READS = {"workbook_object": {"list", "get", "export"}, "worksheet_object": {"list", "get"}, | |
| "range_object": {"read"}, "calculation_object": set()} | |
| _ADDR = re.compile(r"([A-Z]{1,3})([0-9]{1,7})") | |
| def _module(name): | |
| """Import a live reader by name at call time; a fixture swaps it in `sys.modules`.""" | |
| return importlib.import_module(name) | |
| def _query(session, database, payload): | |
| return _module("routes_external")._query_database(session, database, payload) | |
| def _visible_keys(session, database): | |
| """The field keys the maker may read, from the Assistant read scope.""" | |
| scoped = _module("deps").assistant_read_scope(session, database, mode="schema") | |
| return {field["key"] for field in scoped.get("fields") or () | |
| if isinstance(field, dict) and isinstance(field.get("key"), str)} | |
| def _authorization_scope(session, database): | |
| """Resolve current field and row authority without reading source values.""" | |
| from core import perm_scope | |
| scoped = _module("deps").assistant_read_scope(session, database, mode="schema") | |
| grant = perm_scope.assistant_entry(session.user, database) or {} | |
| fields = scoped.get("fields") or () | |
| audience = False | |
| if database.startswith("ut_"): | |
| definitions = session.runtime.get_projection("user_tables", drop=("rows",)) | |
| definition = definitions.get(database) if isinstance(definitions, dict) else None | |
| if not isinstance(definition, dict): | |
| raise ValueError("The source definition is unavailable.") | |
| audience = perm_scope.audience_declared(database, st=session.runtime, defn=definition) | |
| fields = definition.get("fields") or () | |
| pool = perm_scope.derive_pool_scope(session.user, database) | |
| return {"database": database, | |
| "fields": sorted(field["key"] for field in scoped.get("fields") or () | |
| if isinstance(field, dict) and isinstance(field.get("key"), str)), | |
| "field_types": {field["key"]: field.get("type") for field in fields | |
| if isinstance(field, dict) and isinstance(field.get("key"), str)}, | |
| "hidden": sorted(str(key) for key in grant.get("hiddenFields") or ()), | |
| "filter": copy.deepcopy(grant.get("filter")) or None, | |
| "pool": list(pool), "audience": audience} | |
| def _authorization_proof(session, sources): | |
| """Bind a server-captured document to its maker, tenant and permission scopes.""" | |
| return {"version": 1, "tenant": session.tenant, "owner": session.uname, | |
| "scopes": [_authorization_scope(session, source) for source in sources]} | |
| def require_permission(session, sources, content): | |
| """Refuse frozen values whose current authorization cannot be proved without live rows.""" | |
| def refuse(): | |
| raise _module("deps").err(403, "assistant_forbidden", | |
| "Your current permissions do not allow this snapshot.") | |
| if sources is None or content.get("sources") != sources: | |
| refuse() | |
| proof = content.get("_authorization") | |
| try: | |
| current = _authorization_proof(session, sources) | |
| except Exception: | |
| refuse() | |
| # Recipient membership can change independently of the rows-free permission contract. | |
| if any(scope["audience"] for scope in current["scopes"]): | |
| refuse() | |
| if proof is not None: | |
| if proof != current: | |
| refuse() | |
| return | |
| # Legacy sheets and unknown origins cannot prove which fields supplied their displayed values. | |
| if len(sources) != 1 or content.get("kind_of") not in ("records", "view", "app"): | |
| refuse() | |
| scope = current["scopes"][0] | |
| if scope["hidden"] or scope["filter"] or any(value is not None for value in scope["pool"]): | |
| refuse() | |
| columns, sums = content.get("columns"), (content.get("totals") or {}).get("sum") | |
| if not isinstance(columns, list) or not isinstance(sums, dict): | |
| refuse() | |
| visible = set(scope["fields"]) | |
| if (any(not isinstance(column, dict) or column.get("key") not in visible for column in columns) | |
| or not set(sums).issubset(visible)): | |
| refuse() | |
| def _load_view(session, database, view_id): | |
| found = _module("routes_external")._view_context(session, database, {}).table.find_view( | |
| view_id, prefer_owner=session.uname) | |
| return found[1] if found else None | |
| def _load_workbook(session, workbook_id): | |
| book = _module("core.sheets").get_workbook(session.runtime, workbook_id) | |
| routes = _module("routes_sheet") | |
| routes._require_reader(session, book) | |
| return routes._read_for(session, book) | |
| def _binding_readable(session, binding, book, tab): | |
| """True only when the maker can resolve this binding now.""" | |
| try: | |
| _module("routes_sheet")._resolve_cell_binding(session, binding, book, tab.get("id")) | |
| return True | |
| except Exception: | |
| return False | |
| def _now(): | |
| return datetime.datetime.fromtimestamp(time.time(), datetime.timezone.utc).isoformat() | |
| def _text(value): | |
| return value.strip() if isinstance(value, str) and value.strip() else None | |
| def _kind_of(name, args): | |
| action = args.get("action") | |
| if name == "query_database" or (name == "database_object" and action == "query"): | |
| return "records" | |
| if name.startswith("sheet_") and name not in _SHEET_READS: | |
| return "sheet" | |
| if name.startswith("workbook_") and name != "workbook_object": | |
| return "sheet" | |
| if name in _OBJECT_READS and action not in _OBJECT_READS[name] and action != "delete": | |
| return "sheet" | |
| if name == "view_upsert" or (name == "view_object" and action == "upsert"): | |
| return "view" | |
| if name == "app_upsert": | |
| return "app" | |
| return None | |
| def _rows_sentence(shown, total, cause): | |
| return f"Showing {shown:,} of {total:,} rows because {cause}. {OPEN_SENTENCE}" | |
| def _totals(session, database, filters, columns): | |
| """Count every permitted row and sum the number columns, server side, over ALL rows.""" | |
| numeric = [col["key"] for col in columns if col.get("type") in SUMMED_TYPES] | |
| notes = [] | |
| if len(numeric) > MAX_SUMMED_COLUMNS: | |
| notes.append(f"Totals cover the first {MAX_SUMMED_COLUMNS} number columns.") | |
| numeric = numeric[:MAX_SUMMED_COLUMNS] | |
| base = {"limit": 1} | |
| if filters: | |
| base["filters"] = copy.deepcopy(filters) | |
| specs = [{"fn": "count"}] + [{"fn": "sum", "field": key} for key in numeric] | |
| try: | |
| answer = _query(session, database, {**base, "aggregates": specs}) | |
| except Exception: | |
| if not numeric: | |
| raise | |
| notes.append("Totals show the row count only because a number column holds text.") | |
| answer = _query(session, database, {**base, "aggregates": [{"fn": "count"}]}) | |
| totals = {"count": None, "sum": {}} | |
| for agg in answer.get("aggregates") or (): | |
| if agg.get("fn") == "count" and agg.get("field") is None: | |
| totals["count"] = agg.get("value") | |
| elif agg.get("fn") == "sum": | |
| totals["sum"][agg["field"]] = agg.get("value") | |
| if not isinstance(totals["count"], int): | |
| raise ValueError("the reader returned no row count") | |
| return totals, notes | |
| def _columns(fields, keys, visible): | |
| """Column order follows `keys`; a field hidden from the maker never becomes a column.""" | |
| by_key = {f["key"]: f for f in fields or () if isinstance(f, dict) and isinstance(f.get("key"), str)} | |
| out = [] | |
| for key in keys: | |
| if key in by_key and key in visible: | |
| field = by_key[key] | |
| out.append({"key": key, "label": _text(field.get("label")) or key, "type": field.get("type")}) | |
| return out | |
| def _table_content(session, database, title, kind_of, result, filters, page_read, column_keys=None): | |
| """Build records, view or app content from one reader result.""" | |
| visible = _visible_keys(session, database) | |
| fields = result.get("fields") or [] | |
| keys = column_keys if column_keys is not None else [f.get("key") for f in fields if isinstance(f, dict)] | |
| columns = _columns(fields, keys, visible) | |
| records = result.get("records") or [] | |
| rows = [[copy.deepcopy(rec.get(col["key"])) for col in columns] for rec in records[:ROW_LIMIT]] | |
| totals, notes = _totals(session, database, filters, columns) | |
| total = totals["count"] | |
| if len(rows) < total: | |
| cause = ("Loopy read only those rows" if page_read and len(records) <= ROW_LIMIT | |
| else f"a snapshot keeps the first {ROW_LIMIT}") | |
| notes.insert(0, _rows_sentence(len(rows), total, cause)) | |
| return {"title": title, "kind_of": kind_of, | |
| "columns": [{"key": c["key"], "label": c["label"]} for c in columns], | |
| "rows": rows, "totals": totals, "row_count": total, "shown": len(rows), | |
| "notes": notes, "sources": [database]} | |
| def _record_list(session, name, args, result): | |
| if (not isinstance(result, dict) or not isinstance(result.get("records"), list) | |
| or result.get("groups")): | |
| return None | |
| database = _text(args.get("database")) or _text(result.get("database")) | |
| if database is None: | |
| return None | |
| query = args.get("query") if name == "database_object" else args | |
| filters = (query or {}).get("filters") | |
| title = f"{_text(result.get('label')) or database} records" | |
| return _table_content(session, database, title, "records", result, filters, True) | |
| def _view_payload(config, visible): | |
| keys = config.get("visible") | |
| keys = list(keys) if isinstance(keys, list) else None | |
| order = [k for k in config.get("order") or () if isinstance(k, str)] | |
| if keys is not None and order: | |
| keys = [k for k in order if k in keys] + [k for k in keys if k not in order] | |
| payload = {"limit": ROW_LIMIT} | |
| if keys is not None: | |
| payload["fields"] = [k for k in dict.fromkeys(keys) if k in visible] | |
| sorts = [{"field": rule.get("colId"), "direction": rule.get("dir")} | |
| for rule in config.get("sorts") or () if isinstance(rule, dict)] | |
| if sorts: | |
| payload["sorts"] = sorts[:8] | |
| filters = config.get("filters") | |
| if isinstance(filters, list) and filters: | |
| payload["filters"] = {"conj": config.get("filterConj") or "and", "nodes": filters} | |
| elif isinstance(filters, dict) and filters.get("nodes"): | |
| payload["filters"] = filters | |
| return payload | |
| def _view(session, args, result): | |
| database = _text(args.get("database")) or _text((result or {}).get("database")) | |
| draft = args.get("view") if isinstance(args.get("view"), dict) else {} | |
| view_id = _text((result or {}).get("view_id")) or _text(draft.get("id")) | |
| if database is None or view_id is None: | |
| return None | |
| view = _load_view(session, database, view_id) or draft | |
| config = view.get("config") if isinstance(view.get("config"), dict) else {} | |
| payload = _view_payload(config, _visible_keys(session, database)) | |
| answer = _query(session, database, payload) | |
| title = _text(view.get("name")) or "Saved View" | |
| return _table_content(session, database, title, "view", answer, payload.get("filters"), False, | |
| payload.get("fields")) | |
| def _app(session, args, result): | |
| app = (result or {}).get("application") | |
| app = app if isinstance(app, dict) else {} | |
| title = _text(app.get("name")) or _text(args.get("name")) or "App" | |
| compatible = app.get("compatibleDatabases") or args.get("compatibleDatabases") or [] | |
| database = (_text(app.get("sourceDatabase")) or _text(args.get("sourceDatabase")) | |
| or (_text(compatible[0]) if isinstance(compatible, list) and compatible else None)) | |
| if database is None: | |
| return {"title": title, "kind_of": "app", "columns": [], "rows": [], "totals": {"count": 0, "sum": {}}, | |
| "row_count": 0, "shown": 0, "notes": ["This App reads no database."], "sources": None} | |
| payload = {"limit": ROW_LIMIT} | |
| field_map = app.get("fieldMap") or args.get("fieldMap") | |
| if isinstance(field_map, dict) and field_map: | |
| visible = _visible_keys(session, database) | |
| payload["fields"] = [k for k in dict.fromkeys(field_map.values()) if k in visible] | |
| answer = _query(session, database, payload) | |
| return _table_content(session, database, title, "app", answer, None, False, payload.get("fields")) | |
| def _col_index(letters): | |
| n = 0 | |
| for ch in letters: | |
| n = n * 26 + ord(ch) - 64 | |
| return n | |
| def _col_name(n): | |
| out = "" | |
| while n: | |
| n, rem = divmod(n - 1, 26) | |
| out = chr(65 + rem) + out | |
| return out | |
| def _cell_shown(cell): | |
| for key in ("display", "text"): | |
| if isinstance(cell.get(key), str) and cell[key] != "": | |
| return cell[key] | |
| value = cell.get("value") | |
| return "" if value is None else value | |
| def _sheet(session, name, args, result): | |
| result = result if isinstance(result, dict) else {} | |
| nested = result.get("workbook") if isinstance(result.get("workbook"), dict) else {} | |
| workbook_id = (_text(args.get("workbook_id")) or _text(result.get("workbook_id")) | |
| or _text(nested.get("id")) or _text(result.get("id"))) | |
| if workbook_id is None: | |
| return None | |
| book = _load_workbook(session, workbook_id) | |
| tabs = [tab for tab in book.get("sheets") or () if isinstance(tab, dict)] | |
| if not tabs: | |
| return None | |
| wanted = args.get("tab") or args.get("worksheet") or result.get("tab") | |
| wanted = wanted.get("id") if isinstance(wanted, dict) else wanted | |
| tab = next((t for t in tabs if wanted is not None and wanted in (t.get("id"), t.get("name"))), tabs[0]) | |
| cells = {} | |
| for addr, cell in (tab.get("cells") or {}).items(): | |
| match = _ADDR.fullmatch(str(addr).strip().upper()) | |
| if match and isinstance(cell, dict): | |
| cells[(int(match.group(2)), _col_index(match.group(1)))] = cell | |
| sources, unnamed, checked = set(), False, {} | |
| # A displayed formula can read a binding on another tab. | |
| for source_tab in tabs: | |
| for cell in (source_tab.get("cells") or {}).values(): | |
| if not isinstance(cell, dict) or cell.get("binding") is None: | |
| continue | |
| binding = cell["binding"] | |
| database = _text(binding.get("database")) if isinstance(binding, dict) else None | |
| if database is None: | |
| unnamed = True | |
| else: | |
| sources.add(database) | |
| shown = {} | |
| for at, cell in cells.items(): | |
| binding = cell.get("binding") | |
| if binding is not None: | |
| database = _text(binding.get("database")) if isinstance(binding, dict) else None | |
| if database is None: | |
| continue | |
| memo = json.dumps(binding, sort_keys=True, default=str) | |
| if memo not in checked: | |
| checked[memo] = _binding_readable(session, binding, book, tab) | |
| if not checked[memo]: | |
| continue | |
| shown[at] = copy.deepcopy(_cell_shown(cell)) | |
| last_row = max((r for r, _ in cells), default=0) | |
| last_col = max((c for _, c in cells), default=0) | |
| notes = [] | |
| if last_col > MAX_SHEET_COLUMNS: | |
| notes.append(f"Showing the first {MAX_SHEET_COLUMNS} columns because a snapshot keeps " | |
| f"{MAX_SHEET_COLUMNS} columns. Open in Loopable to see every column.") | |
| width = min(last_col, MAX_SHEET_COLUMNS) | |
| rows = [[shown.get((r, c), "") for c in range(1, width + 1)] for r in range(1, min(last_row, ROW_LIMIT) + 1)] | |
| if last_row > ROW_LIMIT: | |
| notes.insert(0, _rows_sentence(ROW_LIMIT, last_row, f"a snapshot keeps the first {ROW_LIMIT}")) | |
| book_name = _text(book.get("name")) or "Sheet" | |
| tab_name = _text(tab.get("name")) | |
| return {"title": f"{book_name}, {tab_name}" if tab_name and len(tabs) > 1 else book_name, | |
| "kind_of": "sheet", | |
| "columns": [{"key": _col_name(c), "label": _col_name(c)} for c in range(1, width + 1)], | |
| "rows": rows, "totals": {"count": last_row, "sum": {}}, "row_count": last_row, | |
| "shown": len(rows), "notes": notes, "sources": None if unnamed else sorted(sources)} | |
| def _size(content): | |
| return len(json.dumps(content, sort_keys=True, default=str).encode("utf-8")) | |
| def _finish(content, frozen_at): | |
| """Apply the byte bound, join the notes and stamp the time, keeping totals over all rows.""" | |
| content = json.loads(json.dumps(content, default=str)) | |
| notes = content.pop("notes") | |
| content.update(frozen_at=frozen_at, note=" ".join(notes) or None) | |
| if _size(content) > MAX_CONTENT_BYTES: | |
| lo, hi, rows = 0, len(content["rows"]), content["rows"] | |
| notes = [n for n in notes if not re.match(r"Showing [\d,]+ of [\d,]+ rows because ", n)] | |
| def bounded(count): | |
| content["rows"] = rows[:count] | |
| content["shown"] = count | |
| content["note"] = " ".join([ | |
| _rows_sentence(count, content["row_count"], | |
| f"a snapshot keeps at most {MAX_CONTENT_BYTES // 1024} KB"), *notes]) | |
| return _size(content) <= MAX_CONTENT_BYTES | |
| if not bounded(0): | |
| raise ValueError("Snapshot labels and totals exceed the content bound.") | |
| while lo < hi: | |
| mid = (lo + hi + 1) // 2 | |
| lo, hi = (mid, hi) if bounded(mid) else (lo, mid - 1) | |
| bounded(lo) | |
| return {key: content[key] for key in ("title", "kind_of", "columns", "rows", "totals", "row_count", | |
| "shown", "note", "sources", "frozen_at", "_authorization") | |
| if key in content} | |
| def _retire_earlier(rt, username, turn): | |
| """Keep one active record-list snapshot per turn.""" | |
| for item_id in list(turn.get("artifacts") or ()): | |
| row = assistant_artifacts.get(rt, username, item_id) | |
| if (row and row.get("kind") == "snapshot" and row.get("status") == "active" | |
| and row.get("toolCall", {}).get("name") in RECORD_TOOLS): | |
| assistant_artifacts.set_status(rt, username, item_id, "deleted") | |
| def capture(session, door, name, args, result, turn): | |
| """Store a frozen snapshot of this tool call when it qualifies; return its id or None. Never raises.""" | |
| try: | |
| if (door not in assistant_artifacts.DOORS or not isinstance(name, str) | |
| or not isinstance(turn, dict) or not isinstance(turn.get("artifacts"), list)): | |
| return None | |
| args = args if isinstance(args, dict) else {} | |
| kind_of = _kind_of(name, args) | |
| if kind_of is None: | |
| return None | |
| build = {"records": lambda: _record_list(session, name, args, result), | |
| "sheet": lambda: _sheet(session, name, args, result), | |
| "view": lambda: _view(session, args, result), | |
| "app": lambda: _app(session, args, result)}[kind_of] | |
| content = build() | |
| if content is None: | |
| return None | |
| if content["sources"] is not None: | |
| if kind_of == "sheet": | |
| try: | |
| content["_authorization"] = _authorization_proof(session, content["sources"]) | |
| except Exception: | |
| pass # A blanked unreadable binding retains protected provenance. | |
| else: | |
| content["_authorization"] = _authorization_proof(session, content["sources"]) | |
| rt, username = session.runtime, session.uname | |
| frozen_at = _now() | |
| content = _finish(content, frozen_at) | |
| ref = assistant_artifacts.store_snapshot(rt, username, content) | |
| if kind_of == "records": | |
| _retire_earlier(rt, username, turn) | |
| item_id = assistant_artifacts.record(rt, username, { | |
| "kind": "snapshot", "ref": ref, "title": content["title"], "door": door, | |
| "toolCall": {"name": name, "args": copy.deepcopy(args)}, "createdAt": time.time(), | |
| "sources": content["sources"], "frozen_at": frozen_at}) | |
| turn["artifacts"].append(item_id) | |
| return item_id | |
| except Exception as exc: | |
| LOG.warning("snapshot not captured for %s: %s: %s", name, type(exc).__name__, exc) | |
| return None | |
| def read(rt, artifact_id): | |
| """Return the stored artifact and its frozen content; read only those two documents.""" | |
| row = assistant_artifacts.get_by_id(rt, artifact_id) | |
| if row is None or row.get("kind") != "snapshot" or row.get("status") == "deleted": | |
| return None | |
| content = assistant_artifacts.read_snapshot(rt, row) | |
| if content is None: | |
| return None | |
| return {"artifact": assistant_artifacts.c7(row), | |
| "snapshot": {key: value for key, value in content.items() if key != "_authorization"}} | |