"""C14: a principal's artifact index, in store documents of at most 200 items.""" import copy import datetime import hashlib import heapq import json import math import re import uuid import assistant_log import loopy_links CHUNK_ITEMS = 200 KINDS = {"view", "sheet", "report", "run", "query", "schedule", "pdf", "xlsx", "image", "snapshot"} FROZEN_KINDS = {"snapshot", "pdf", "xlsx", "image"} DOORS = {"app", "whatsapp", "wake"} STATUSES = {"active", "paused", "deleted"} AMAZON_PREFIX = "ut_amzn_" _ITEM_ID = re.compile(r"([0-9a-f]{24})\.([0-9]+)") _SNAPSHOT_REF = re.compile(r"[0-9a-f]{32}") _ARTIFACT_CHUNK = re.compile(r'assistant_artifacts_([0-9a-f]{24})_([0-9]+)') _FILE_KEY = re.compile(r'assistant_file_([0-9a-f]{32})') _SNAPSHOT_KEY = re.compile(r'assistant_snapshot_([0-9a-f]{24})_([0-9a-f]{32})') _EXPORT_DELETIONS_KEY = 'amazon_retained_export_deletions' def timestamp(value): if isinstance(value, (int, float)) and not isinstance(value, bool) and math.isfinite(value): dt = datetime.datetime.fromtimestamp(value, datetime.timezone.utc) elif isinstance(value, str) and value.strip(): dt = datetime.datetime.fromisoformat(value.replace("Z", "+00:00")) if dt.tzinfo is None: dt = dt.replace(tzinfo=datetime.timezone.utc) else: raise ValueError("An artifact needs a valid creation time.") return dt.astimezone(datetime.timezone.utc).isoformat() def _index(raw): raw = raw or {} return {"version": 1, "nextSeq": int(raw.get("nextSeq", 0)), "chunks": copy.deepcopy(raw.get("chunks") or {})} def _chunk_key(h, n): return f"assistant_artifacts_{h}_{n}" def _source_keys(sources): """Return normalized database keys, or None for unknown or malformed provenance.""" if not isinstance(sources, list) or not all(isinstance(key, str) and key.strip() for key in sources): return None return [key.strip() for key in sources] def is_amazon(sources): """R86 applies to Amazon databases and unknown or malformed provenance.""" keys = _source_keys(sources) return keys is None or any(key.startswith(AMAZON_PREFIX) for key in keys) def _optional_text(item, field, sentence): value = item.get(field) if value is not None and (not isinstance(value, str) or not value.strip()): raise ValueError(sentence) return value.strip() if value is not None else None def _c7_inputs(item, kind, args, created_at): """Validate the optional C7 inputs and return sources, amazon, file_id, mime and frozen_at.""" sources = item.get("sources") if sources is not None and _source_keys(sources) is None: raise ValueError("Artifact sources must be a list of database names.") if sources is None and kind in ("view", "query") and isinstance(args.get("database"), str) \ and args["database"].strip(): sources = [args["database"].strip()] elif sources is not None: sources = _source_keys(sources) if "amazon" in item and not isinstance(item["amazon"], bool): raise ValueError("The artifact Amazon flag must be true or false.") amazon = is_amazon(sources) or item.get("amazon") is True file_id = _optional_text(item, "file_id", "An artifact file id must be text.") mime = _optional_text(item, "mime", "An artifact file type must be text.") frozen = item.get("frozen_at") try: frozen = timestamp(frozen) if frozen is not None else None except (TypeError, ValueError, OverflowError, OSError) as exc: raise ValueError("An artifact needs a valid frozen time.") from exc if frozen is None and kind in FROZEN_KINDS: frozen = created_at return sources, amazon, file_id, mime, frozen def _app_database(rt, ref, args): compatible = args.get("compatibleDatabases") if isinstance(compatible, list) and compatible and isinstance(compatible[0], str) and compatible[0].strip(): return compatible[0] if isinstance(args.get("sourceDatabase"), str) and args["sourceDatabase"].strip(): return args["sourceDatabase"] try: import application_registry stored = application_registry.load(rt)["applications"].get(ref) or {} first = (stored.get("compatibleDatabases") or [None])[0] return first if isinstance(first, str) and first.strip() else None except Exception: # A registry read failure falls back to the artifact link. return None def _link(rt, row): """The absolute link that opens this row's work, or None with no public base.""" kind, ref, call = row["kind"], row["ref"], row["toolCall"] args = call["args"] # A fixture links module can carry only the artifact builder, so a missing builder means no link. build = lambda name, *values: (getattr(loopy_links, name, None) or (lambda *_: None))(*values) link = None if ref is not None: if kind == "sheet": link = build("sheet_link", ref) elif kind == "view": link = build("view_link", args.get("database"), ref) elif kind == "query": link = build("query_link", ref) elif kind == "run" and call["name"] == "app_upsert": link = build("application_link", ref, _app_database(rt, ref, args)) return link or loopy_links.artifact_link(row["id"]) def record(rt, username, item): """Validate and append exactly one C14 item; return its principal-bound id.""" from routes_query import _no_dashes if not isinstance(item, dict): raise ValueError("An artifact must be an object.") if item.get("door") == "slack": raise ValueError("Slack artifacts are never stored.") for field, allowed in (("kind", KINDS), ("door", DOORS)): if not isinstance(item.get(field), str) or item[field] not in allowed: raise ValueError(f"That artifact {field} is not available.") # Some writes (for example, a batch) have no single object reference. ref = item.get("ref") if "ref" not in item or (ref is not None and (not isinstance(ref, str) or not ref.strip())): raise ValueError("An artifact reference must be text or null.") if not isinstance(item.get("title"), str) or not item["title"].strip(): raise ValueError("An artifact needs a title.") call = item.get("toolCall") if (not isinstance(call, dict) or not isinstance(call.get("name"), str) or not call["name"].strip() or not isinstance(call.get("args"), dict)): raise ValueError("An artifact needs a tool name and object arguments.") try: json.dumps(call, allow_nan=False) at = timestamp(item.get("createdAt")) except (TypeError, ValueError, OverflowError, OSError) as exc: raise ValueError("An artifact needs JSON arguments and a valid creation time.") from exc sources, amazon, file_id, mime, frozen = _c7_inputs(item, item["kind"], call["args"], at) value = {"kind": item["kind"], "ref": ref, "title": _no_dashes(item["title"]).strip()[:200], "door": item["door"], "toolCall": {"name": call["name"], "args": copy.deepcopy(call["args"])}, "createdAt": at, "status": "active", "sources": sources, "amazon": amazon} value.update({key: got for key, got in (("file_id", file_id), ("mime", mime), ("frozen_at", frozen)) if got is not None}) h = assistant_log._hash(rt, username) allocated = [] def allocate(raw): index = _index(raw) seq = index["nextSeq"] n = seq // CHUNK_ITEMS index["nextSeq"] = seq + 1 index["chunks"][str(n)] = index["chunks"].get(str(n), 0) + 1 allocated[:] = [seq, n] return index rt.update(f"assistant_artifacts_{h}_index", allocate, flush="sync") seq, n = allocated row = {**value, "id": f"{h}.{seq}", "seq": seq} row["link"] = _link(rt, row) def insert(raw): existing = {entry["seq"]: entry for entry in (raw or {}).get("items", [])} existing.setdefault(seq, row) if len(existing) > CHUNK_ITEMS: raise ValueError("An artifact chunk is full.") return {"version": 1, "items": [existing[key] for key in sorted(existing)]} rt.update(_chunk_key(h, n), insert, flush="sync") return row["id"] def _seq(h, item_id): if not isinstance(item_id, str) or not item_id.startswith(h + "."): return None suffix = item_id[len(h) + 1:] return int(suffix) if suffix.isdecimal() else None def _list_items(rt, username, *, cursor=None, limit=50): """Newest creation first, with an opaque boundary belonging only to this caller.""" h = assistant_log._hash(rt, username) index = _index(rt.get(f"assistant_artifacts_{h}_index")) boundary = None if cursor: boundary = get(rt, username, cursor) if boundary is None: raise ValueError("That artifact cursor does not exist.") limit = max(1, min(CHUNK_ITEMS, int(limit))) order = lambda row: (row["createdAt"], row["seq"]) candidates = (row for row in _rows(rt, h, index) if boundary is None or order(row) < order(boundary)) # Imported schedules can carry an earlier creation time than the preceding allocation. # Keep only one page plus lookahead in memory while visiting bounded documents. rows = heapq.nlargest(limit + 1, candidates, key=order) more = len(rows) > limit rows = rows[:limit] return {"items": rows, "nextCursor": rows[-1]["id"] if more else None} def _rows(rt, h, index): for n in range((index["nextSeq"] - 1) // CHUNK_ITEMS, -1, -1): if not index["chunks"].get(str(n)): continue chunk = rt.get(_chunk_key(h, n)) or {} yield from (copy.deepcopy(row) for row in reversed(chunk.get("items", []))) def get(rt, username, item_id): h = assistant_log._hash(rt, username) seq = _seq(h, item_id) if seq is None: return None chunk = rt.get(_chunk_key(h, seq // CHUNK_ITEMS)) or {} return next((copy.deepcopy(row) for row in chunk.get("items", []) if row["id"] == item_id), None) def get_by_id(rt, item_id): """Read one row by its `h.seq` id alone, for a page that has no username.""" match = _ITEM_ID.fullmatch(item_id) if isinstance(item_id, str) else None if match is None: return None h, seq = match.group(1), int(match.group(2)) chunk = rt.get(_chunk_key(h, seq // CHUNK_ITEMS)) or {} return next((copy.deepcopy(row) for row in chunk.get("items", []) if row.get("id") == item_id), None) def _snapshot_key(h, ref): # The content lives in its own document: a read is billed at the document's text size. return f"assistant_snapshot_{h}_{ref}" def store_snapshot(rt, username, content): """Write one frozen snapshot content document and return its `ref`.""" if not isinstance(content, dict): raise ValueError("A snapshot must be an object.") json.dumps(content, allow_nan=False) ref = uuid.uuid4().hex frozen = copy.deepcopy(content) rt.update(_snapshot_key(assistant_log._hash(rt, username), ref), lambda _raw: frozen, flush="sync") return ref def read_snapshot(rt, row): """Read the content document of one snapshot row, or None; it touches no other key.""" if not isinstance(row, dict) or row.get("kind") != "snapshot" or not isinstance(row.get("id"), str): return None match, ref = _ITEM_ID.fullmatch(row["id"]), row.get("ref") if match is None or not isinstance(ref, str) or not _SNAPSHOT_REF.fullmatch(ref): return None content = rt.get(_snapshot_key(match.group(1), ref)) return copy.deepcopy(content) if content else None def c7(row): """Project C7 with Amazon protection for unknown provenance, including malformed legacy sources.""" sources = _source_keys(row.get("sources")) out = {"id": row["id"], "kind": row["kind"], "title": row["title"], "created_at": row["createdAt"], "frozen_at": row.get("frozen_at"), "sources": sources, "amazon": is_amazon(sources) or row.get("amazon", False) is not False, "link": row["link"] if "link" in row else loopy_links.artifact_link(row["id"])} file_id = row.get("file_id") or (row.get("ref") if row["kind"] == "pdf" else None) if file_id: out["file_id"] = file_id if row.get("mime"): out["mime"] = row["mime"] return out def link_fields(row): """The link a tool result carries, or a plain note when no public base is set.""" link = (row or {}).get("link") return {"link": link} if link else {"link": None, "linkNote": loopy_links.NO_LINK_NOTE} def set_status(rt, username, item_id, status): if not isinstance(status, str) or status not in STATUSES: raise ValueError("That artifact status is not available.") h = assistant_log._hash(rt, username) seq = _seq(h, item_id) if seq is None or get(rt, username, item_id) is None: return None changed = [] def update(raw): doc = copy.deepcopy(raw) for row in doc.get("items", []): if row["id"] == item_id: row["status"] = status changed[:] = [copy.deepcopy(row)] return doc rt.update(_chunk_key(h, seq // CHUNK_ITEMS), update, flush="sync") return changed[0] if changed else None def _delete_query(session, args): from routes_query import _artifact_or_404, delete_query _artifact_or_404(session, args.get("qid")) return delete_query(args["qid"], session) def _amazon_origin(row): """Distinguish established Amazon origin from an access flag applied to unknown provenance.""" sources = _source_keys(row.get('sources')) if sources is not None and any(source.startswith(AMAZON_PREFIX) for source in sources): return True if sources is None or ('amazon' in row and type(row['amazon']) is not bool): return None return row.get('amazon') is True def _origin_union(first, second): if first is True or second is True: return True return False if first is False or second is False else None def _retained_export_objects(rt): """Inventory all durable principals and retry locators without returning artifact values.""" import assistant_files tenant = assistant_log._tenant_of(rt) objects, attributed_documents, attributed_paths = {}, set(), set() keys = set(rt.list_keys('assistant_')) paths = set(rt.list_paths('assistant_files/')) def add(object_id, descriptor, origin): if origin is not False: objects[object_id] = {**descriptor, 'unknown': origin is None} def file_objects(file_id, origin, expected_hash=None): key = assistant_files._key(file_id) metadata = rt.get(key, fresh=True) if key in keys else None if metadata is None: return origin attributed_documents.add(key) valid = (isinstance(metadata, dict) and metadata.get('id') == file_id and metadata.get('tenant') == tenant and isinstance(metadata.get('owner'), str) and metadata['owner'] and metadata.get('ownerHash') == assistant_log._hash(rt, metadata['owner']) and (expected_hash is None or metadata['ownerHash'] == expected_hash) and metadata.get('mime') in assistant_files._FORMATS) if not valid: add('document:' + key, {'kind': 'document', 'key': key}, None) return None origin = _origin_union(origin, _amazon_origin(metadata)) path = assistant_files._path(metadata) attributed_paths.add(path) add('document:' + key, {'kind': 'document', 'key': key}, origin) if path in paths: add('blob:' + path, {'kind': 'blob', 'path': path}, origin) return origin for key in sorted(keys): match = _ARTIFACT_CHUNK.fullmatch(key) if match is None: if (key.startswith('assistant_artifacts_') and not re.fullmatch(r'assistant_artifacts_[0-9a-f]{24}_index', key)): add('unattributed:' + hashlib.sha256(key.encode()).hexdigest(), {'kind': 'unknown'}, None) continue chunk = rt.get(key, fresh=True) if not isinstance(chunk, dict) or not isinstance(chunk.get('items'), list): add('unattributed:' + hashlib.sha256(key.encode()).hexdigest(), {'kind': 'unknown'}, None) continue for index, row in enumerate(chunk['items']): item_id = row.get('id') if isinstance(row, dict) else None item_match = _ITEM_ID.fullmatch(item_id) if isinstance(item_id, str) else None if item_match is None or item_match[1] != match[1] or int(item_match[2]) // CHUNK_ITEMS != int(match[2]): add('unattributed:' + hashlib.sha256(f'{key}:{index}'.encode()).hexdigest(), {'kind': 'unknown'}, None) continue origin = _amazon_origin(row) if row.get('kind') == 'snapshot': ref = row.get('ref') if isinstance(ref, str) and _SNAPSHOT_REF.fullmatch(ref): snapshot_key = _snapshot_key(match[1], ref) attributed_documents.add(snapshot_key) if snapshot_key in keys: origin = _origin_union(origin, _amazon_origin(rt.get(snapshot_key, fresh=True) or {})) add('document:' + snapshot_key, {'kind': 'document', 'key': snapshot_key}, origin) else: origin = None file_id = row.get('file_id') or (row.get('ref') if row.get('kind') in ('pdf', 'xlsx', 'image') else None) if isinstance(file_id, str) and assistant_files._FILE_ID.fullmatch(file_id): origin = file_objects(file_id, origin, match[1]) add('artifact:' + item_id, {'kind': 'artifact', 'chunk': key, 'id': item_id}, origin) for key in sorted(keys - attributed_documents): match = _FILE_KEY.fullmatch(key) if match: file_objects(match[1], _amazon_origin(rt.get(key, fresh=True) or {})) elif _SNAPSHOT_KEY.fullmatch(key): add('document:' + key, {'kind': 'document', 'key': key}, _amazon_origin(rt.get(key, fresh=True) or {})) elif (key.startswith('assistant_snapshot_') or key.startswith('assistant_file_') and not key.startswith('assistant_file_token_')): add('unattributed:' + hashlib.sha256(key.encode()).hexdigest(), {'kind': 'unknown'}, None) for path in paths - attributed_paths: add('blob:' + path, {'kind': 'blob', 'path': path}, None) plan = rt.get(_EXPORT_DELETIONS_KEY, fresh=True) or {} if plan and (plan.get('tenant') != tenant or not isinstance(plan.get('objects'), dict)): raise RuntimeError('Retained export deletion inventory is unavailable.') for object_id, descriptor in plan.get('objects', {}).items(): kind = descriptor.get('kind') if kind == 'document' and (_FILE_KEY.fullmatch(descriptor.get('key', '')) or _SNAPSHOT_KEY.fullmatch(descriptor.get('key', ''))): present = descriptor['key'] in keys elif kind == 'blob' and re.fullmatch(r'assistant_files/[0-9a-f]{24}/[0-9a-f]{32}\.(pdf|xlsx|png)', descriptor.get('path', '')): present = descriptor['path'] in paths elif kind == 'artifact' and _ARTIFACT_CHUNK.fullmatch(descriptor.get('chunk', '')): present = any(row.get('id') == descriptor.get('id') for row in (rt.get(descriptor['chunk'], fresh=True) or {}).get('items', [])) else: raise RuntimeError('Retained export deletion locator is invalid.') if present: objects.setdefault(object_id, descriptor) # A saved locator retains provenance after its artifact or metadata is removed. if not descriptor.get('unknown'): objects[object_id] = descriptor return objects def amazon_deletion_inventory(rt): """Return retained Amazon objects and unresolved provenance, including deleted-status payloads.""" return sorted(_retained_export_objects(rt)) def purge_amazon_object(rt, object_id, *, checked_at): """Retain retry locators before physical deletion and require fresh absence readback.""" objects = _retained_export_objects(rt) target = objects.get(object_id) if target is None: return if target.get('unknown'): raise RuntimeError('Retained export provenance needs resolution before deletion.') tenant = assistant_log._tenant_of(rt) def remember(raw): if raw and raw.get('tenant') != tenant: raise RuntimeError('Retained export deletion inventory belongs to another tenant.') entries = dict(raw.get('objects') or {}) entries.update({key: value for key, value in objects.items() if not value.get('unknown')}) return {'tenant': tenant, 'objects': entries} rt.update(_EXPORT_DELETIONS_KEY, remember, flush='sync') if target['kind'] == 'artifact': def remove(raw): return {**raw, 'items': [row for row in raw.get('items', []) if row.get('id') != target['id']]} rt.update(target['chunk'], remove, flush='sync') elif target['kind'] == 'blob': if rt.delete_path(target['path']) is False: raise RuntimeError('Retained export bytes could not be deleted.') else: if rt.delete_document(target['key']) is False: raise RuntimeError('Retained export document could not be deleted.') if object_id in _retained_export_objects(rt): raise RuntimeError('Retained export deletion readback found residue.') def register_amazon_deletion_owner(rt, register): """Mount retained export ownership through the runtime's injected Amazon registrar.""" register(rt, 'service_retained_exports', inventory=amazon_deletion_inventory, delete=purge_amazon_object, fixture=False) from harness.runtime import register_deletion_owner_initializer register_deletion_owner_initializer('assistant_artifacts', register_amazon_deletion_owner) try: import agent_tools except ImportError: pass else: agent_tools.register("query_view_delete", { "type": "object", "description": "Delete one of your saved Query answers after confirmation.", "properties": {"qid": {"type": "string"}}, "required": ["qid"], "additionalProperties": False}, _delete_query, effect="delete")