loopable / api /assistant_artifacts.py
fsanyoto's picture
Deploy AIOS web (React glide grid + FastAPI slice)
81a8217 verified
Raw History Blame Contribute Delete
22.6 kB
"""C14: a principal's artifact index, in store documents of at most 200 items."""
import copy
import datetime
import hashlib
import heapq
import json
import math
import re
import uuid
import assistant_log
import loopy_links
CHUNK_ITEMS = 200
KINDS = {"view", "sheet", "report", "run", "query", "schedule", "pdf", "xlsx", "image", "snapshot"}
FROZEN_KINDS = {"snapshot", "pdf", "xlsx", "image"}
DOORS = {"app", "whatsapp", "wake"}
STATUSES = {"active", "paused", "deleted"}
AMAZON_PREFIX = "ut_amzn_"
_ITEM_ID = re.compile(r"([0-9a-f]{24})\.([0-9]+)")
_SNAPSHOT_REF = re.compile(r"[0-9a-f]{32}")
_ARTIFACT_CHUNK = re.compile(r'assistant_artifacts_([0-9a-f]{24})_([0-9]+)')
_FILE_KEY = re.compile(r'assistant_file_([0-9a-f]{32})')
_SNAPSHOT_KEY = re.compile(r'assistant_snapshot_([0-9a-f]{24})_([0-9a-f]{32})')
_EXPORT_DELETIONS_KEY = 'amazon_retained_export_deletions'
def timestamp(value):
if isinstance(value, (int, float)) and not isinstance(value, bool) and math.isfinite(value):
dt = datetime.datetime.fromtimestamp(value, datetime.timezone.utc)
elif isinstance(value, str) and value.strip():
dt = datetime.datetime.fromisoformat(value.replace("Z", "+00:00"))
if dt.tzinfo is None:
dt = dt.replace(tzinfo=datetime.timezone.utc)
else:
raise ValueError("An artifact needs a valid creation time.")
return dt.astimezone(datetime.timezone.utc).isoformat()
def _index(raw):
raw = raw or {}
return {"version": 1, "nextSeq": int(raw.get("nextSeq", 0)),
"chunks": copy.deepcopy(raw.get("chunks") or {})}
def _chunk_key(h, n):
return f"assistant_artifacts_{h}_{n}"
def _source_keys(sources):
"""Return normalized database keys, or None for unknown or malformed provenance."""
if not isinstance(sources, list) or not all(isinstance(key, str) and key.strip() for key in sources):
return None
return [key.strip() for key in sources]
def is_amazon(sources):
"""R86 applies to Amazon databases and unknown or malformed provenance."""
keys = _source_keys(sources)
return keys is None or any(key.startswith(AMAZON_PREFIX) for key in keys)
def _optional_text(item, field, sentence):
value = item.get(field)
if value is not None and (not isinstance(value, str) or not value.strip()):
raise ValueError(sentence)
return value.strip() if value is not None else None
def _c7_inputs(item, kind, args, created_at):
"""Validate the optional C7 inputs and return sources, amazon, file_id, mime and frozen_at."""
sources = item.get("sources")
if sources is not None and _source_keys(sources) is None:
raise ValueError("Artifact sources must be a list of database names.")
if sources is None and kind in ("view", "query") and isinstance(args.get("database"), str) \
and args["database"].strip():
sources = [args["database"].strip()]
elif sources is not None:
sources = _source_keys(sources)
if "amazon" in item and not isinstance(item["amazon"], bool):
raise ValueError("The artifact Amazon flag must be true or false.")
amazon = is_amazon(sources) or item.get("amazon") is True
file_id = _optional_text(item, "file_id", "An artifact file id must be text.")
mime = _optional_text(item, "mime", "An artifact file type must be text.")
frozen = item.get("frozen_at")
try:
frozen = timestamp(frozen) if frozen is not None else None
except (TypeError, ValueError, OverflowError, OSError) as exc:
raise ValueError("An artifact needs a valid frozen time.") from exc
if frozen is None and kind in FROZEN_KINDS:
frozen = created_at
return sources, amazon, file_id, mime, frozen
def _app_database(rt, ref, args):
compatible = args.get("compatibleDatabases")
if isinstance(compatible, list) and compatible and isinstance(compatible[0], str) and compatible[0].strip():
return compatible[0]
if isinstance(args.get("sourceDatabase"), str) and args["sourceDatabase"].strip():
return args["sourceDatabase"]
try:
import application_registry
stored = application_registry.load(rt)["applications"].get(ref) or {}
first = (stored.get("compatibleDatabases") or [None])[0]
return first if isinstance(first, str) and first.strip() else None
except Exception:
# A registry read failure falls back to the artifact link.
return None
def _link(rt, row):
"""The absolute link that opens this row's work, or None with no public base."""
kind, ref, call = row["kind"], row["ref"], row["toolCall"]
args = call["args"]
# A fixture links module can carry only the artifact builder, so a missing builder means no link.
build = lambda name, *values: (getattr(loopy_links, name, None) or (lambda *_: None))(*values)
link = None
if ref is not None:
if kind == "sheet":
link = build("sheet_link", ref)
elif kind == "view":
link = build("view_link", args.get("database"), ref)
elif kind == "query":
link = build("query_link", ref)
elif kind == "run" and call["name"] == "app_upsert":
link = build("application_link", ref, _app_database(rt, ref, args))
return link or loopy_links.artifact_link(row["id"])
def record(rt, username, item):
"""Validate and append exactly one C14 item; return its principal-bound id."""
from routes_query import _no_dashes
if not isinstance(item, dict):
raise ValueError("An artifact must be an object.")
if item.get("door") == "slack":
raise ValueError("Slack artifacts are never stored.")
for field, allowed in (("kind", KINDS), ("door", DOORS)):
if not isinstance(item.get(field), str) or item[field] not in allowed:
raise ValueError(f"That artifact {field} is not available.")
# Some writes (for example, a batch) have no single object reference.
ref = item.get("ref")
if "ref" not in item or (ref is not None and (not isinstance(ref, str) or not ref.strip())):
raise ValueError("An artifact reference must be text or null.")
if not isinstance(item.get("title"), str) or not item["title"].strip():
raise ValueError("An artifact needs a title.")
call = item.get("toolCall")
if (not isinstance(call, dict) or not isinstance(call.get("name"), str)
or not call["name"].strip() or not isinstance(call.get("args"), dict)):
raise ValueError("An artifact needs a tool name and object arguments.")
try:
json.dumps(call, allow_nan=False)
at = timestamp(item.get("createdAt"))
except (TypeError, ValueError, OverflowError, OSError) as exc:
raise ValueError("An artifact needs JSON arguments and a valid creation time.") from exc
sources, amazon, file_id, mime, frozen = _c7_inputs(item, item["kind"], call["args"], at)
value = {"kind": item["kind"], "ref": ref, "title": _no_dashes(item["title"]).strip()[:200],
"door": item["door"], "toolCall": {"name": call["name"], "args": copy.deepcopy(call["args"])},
"createdAt": at, "status": "active", "sources": sources, "amazon": amazon}
value.update({key: got for key, got in (("file_id", file_id), ("mime", mime), ("frozen_at", frozen))
if got is not None})
h = assistant_log._hash(rt, username)
allocated = []
def allocate(raw):
index = _index(raw)
seq = index["nextSeq"]
n = seq // CHUNK_ITEMS
index["nextSeq"] = seq + 1
index["chunks"][str(n)] = index["chunks"].get(str(n), 0) + 1
allocated[:] = [seq, n]
return index
rt.update(f"assistant_artifacts_{h}_index", allocate, flush="sync")
seq, n = allocated
row = {**value, "id": f"{h}.{seq}", "seq": seq}
row["link"] = _link(rt, row)
def insert(raw):
existing = {entry["seq"]: entry for entry in (raw or {}).get("items", [])}
existing.setdefault(seq, row)
if len(existing) > CHUNK_ITEMS:
raise ValueError("An artifact chunk is full.")
return {"version": 1, "items": [existing[key] for key in sorted(existing)]}
rt.update(_chunk_key(h, n), insert, flush="sync")
return row["id"]
def _seq(h, item_id):
if not isinstance(item_id, str) or not item_id.startswith(h + "."):
return None
suffix = item_id[len(h) + 1:]
return int(suffix) if suffix.isdecimal() else None
def _list_items(rt, username, *, cursor=None, limit=50):
"""Newest creation first, with an opaque boundary belonging only to this caller."""
h = assistant_log._hash(rt, username)
index = _index(rt.get(f"assistant_artifacts_{h}_index"))
boundary = None
if cursor:
boundary = get(rt, username, cursor)
if boundary is None:
raise ValueError("That artifact cursor does not exist.")
limit = max(1, min(CHUNK_ITEMS, int(limit)))
order = lambda row: (row["createdAt"], row["seq"])
candidates = (row for row in _rows(rt, h, index) if boundary is None or order(row) < order(boundary))
# Imported schedules can carry an earlier creation time than the preceding allocation.
# Keep only one page plus lookahead in memory while visiting bounded documents.
rows = heapq.nlargest(limit + 1, candidates, key=order)
more = len(rows) > limit
rows = rows[:limit]
return {"items": rows, "nextCursor": rows[-1]["id"] if more else None}
def _rows(rt, h, index):
for n in range((index["nextSeq"] - 1) // CHUNK_ITEMS, -1, -1):
if not index["chunks"].get(str(n)):
continue
chunk = rt.get(_chunk_key(h, n)) or {}
yield from (copy.deepcopy(row) for row in reversed(chunk.get("items", [])))
def get(rt, username, item_id):
h = assistant_log._hash(rt, username)
seq = _seq(h, item_id)
if seq is None:
return None
chunk = rt.get(_chunk_key(h, seq // CHUNK_ITEMS)) or {}
return next((copy.deepcopy(row) for row in chunk.get("items", []) if row["id"] == item_id), None)
def get_by_id(rt, item_id):
"""Read one row by its `h.seq` id alone, for a page that has no username."""
match = _ITEM_ID.fullmatch(item_id) if isinstance(item_id, str) else None
if match is None:
return None
h, seq = match.group(1), int(match.group(2))
chunk = rt.get(_chunk_key(h, seq // CHUNK_ITEMS)) or {}
return next((copy.deepcopy(row) for row in chunk.get("items", []) if row.get("id") == item_id), None)
def _snapshot_key(h, ref):
# The content lives in its own document: a read is billed at the document's text size.
return f"assistant_snapshot_{h}_{ref}"
def store_snapshot(rt, username, content):
"""Write one frozen snapshot content document and return its `ref`."""
if not isinstance(content, dict):
raise ValueError("A snapshot must be an object.")
json.dumps(content, allow_nan=False)
ref = uuid.uuid4().hex
frozen = copy.deepcopy(content)
rt.update(_snapshot_key(assistant_log._hash(rt, username), ref), lambda _raw: frozen, flush="sync")
return ref
def read_snapshot(rt, row):
"""Read the content document of one snapshot row, or None; it touches no other key."""
if not isinstance(row, dict) or row.get("kind") != "snapshot" or not isinstance(row.get("id"), str):
return None
match, ref = _ITEM_ID.fullmatch(row["id"]), row.get("ref")
if match is None or not isinstance(ref, str) or not _SNAPSHOT_REF.fullmatch(ref):
return None
content = rt.get(_snapshot_key(match.group(1), ref))
return copy.deepcopy(content) if content else None
def c7(row):
"""Project C7 with Amazon protection for unknown provenance, including malformed legacy sources."""
sources = _source_keys(row.get("sources"))
out = {"id": row["id"], "kind": row["kind"], "title": row["title"], "created_at": row["createdAt"],
"frozen_at": row.get("frozen_at"), "sources": sources,
"amazon": is_amazon(sources) or row.get("amazon", False) is not False,
"link": row["link"] if "link" in row else loopy_links.artifact_link(row["id"])}
file_id = row.get("file_id") or (row.get("ref") if row["kind"] == "pdf" else None)
if file_id:
out["file_id"] = file_id
if row.get("mime"):
out["mime"] = row["mime"]
return out
def link_fields(row):
"""The link a tool result carries, or a plain note when no public base is set."""
link = (row or {}).get("link")
return {"link": link} if link else {"link": None, "linkNote": loopy_links.NO_LINK_NOTE}
def set_status(rt, username, item_id, status):
if not isinstance(status, str) or status not in STATUSES:
raise ValueError("That artifact status is not available.")
h = assistant_log._hash(rt, username)
seq = _seq(h, item_id)
if seq is None or get(rt, username, item_id) is None:
return None
changed = []
def update(raw):
doc = copy.deepcopy(raw)
for row in doc.get("items", []):
if row["id"] == item_id:
row["status"] = status
changed[:] = [copy.deepcopy(row)]
return doc
rt.update(_chunk_key(h, seq // CHUNK_ITEMS), update, flush="sync")
return changed[0] if changed else None
def _delete_query(session, args):
from routes_query import _artifact_or_404, delete_query
_artifact_or_404(session, args.get("qid"))
return delete_query(args["qid"], session)
def _amazon_origin(row):
"""Distinguish established Amazon origin from an access flag applied to unknown provenance."""
sources = _source_keys(row.get('sources'))
if sources is not None and any(source.startswith(AMAZON_PREFIX) for source in sources):
return True
if sources is None or ('amazon' in row and type(row['amazon']) is not bool):
return None
return row.get('amazon') is True
def _origin_union(first, second):
if first is True or second is True:
return True
return False if first is False or second is False else None
def _retained_export_objects(rt):
"""Inventory all durable principals and retry locators without returning artifact values."""
import assistant_files
tenant = assistant_log._tenant_of(rt)
objects, attributed_documents, attributed_paths = {}, set(), set()
keys = set(rt.list_keys('assistant_'))
paths = set(rt.list_paths('assistant_files/'))
def add(object_id, descriptor, origin):
if origin is not False:
objects[object_id] = {**descriptor, 'unknown': origin is None}
def file_objects(file_id, origin, expected_hash=None):
key = assistant_files._key(file_id)
metadata = rt.get(key, fresh=True) if key in keys else None
if metadata is None:
return origin
attributed_documents.add(key)
valid = (isinstance(metadata, dict) and metadata.get('id') == file_id
and metadata.get('tenant') == tenant
and isinstance(metadata.get('owner'), str) and metadata['owner']
and metadata.get('ownerHash') == assistant_log._hash(rt, metadata['owner'])
and (expected_hash is None or metadata['ownerHash'] == expected_hash)
and metadata.get('mime') in assistant_files._FORMATS)
if not valid:
add('document:' + key, {'kind': 'document', 'key': key}, None)
return None
origin = _origin_union(origin, _amazon_origin(metadata))
path = assistant_files._path(metadata)
attributed_paths.add(path)
add('document:' + key, {'kind': 'document', 'key': key}, origin)
if path in paths:
add('blob:' + path, {'kind': 'blob', 'path': path}, origin)
return origin
for key in sorted(keys):
match = _ARTIFACT_CHUNK.fullmatch(key)
if match is None:
if (key.startswith('assistant_artifacts_')
and not re.fullmatch(r'assistant_artifacts_[0-9a-f]{24}_index', key)):
add('unattributed:' + hashlib.sha256(key.encode()).hexdigest(),
{'kind': 'unknown'}, None)
continue
chunk = rt.get(key, fresh=True)
if not isinstance(chunk, dict) or not isinstance(chunk.get('items'), list):
add('unattributed:' + hashlib.sha256(key.encode()).hexdigest(),
{'kind': 'unknown'}, None)
continue
for index, row in enumerate(chunk['items']):
item_id = row.get('id') if isinstance(row, dict) else None
item_match = _ITEM_ID.fullmatch(item_id) if isinstance(item_id, str) else None
if item_match is None or item_match[1] != match[1] or int(item_match[2]) // CHUNK_ITEMS != int(match[2]):
add('unattributed:' + hashlib.sha256(f'{key}:{index}'.encode()).hexdigest(),
{'kind': 'unknown'}, None)
continue
origin = _amazon_origin(row)
if row.get('kind') == 'snapshot':
ref = row.get('ref')
if isinstance(ref, str) and _SNAPSHOT_REF.fullmatch(ref):
snapshot_key = _snapshot_key(match[1], ref)
attributed_documents.add(snapshot_key)
if snapshot_key in keys:
origin = _origin_union(origin, _amazon_origin(rt.get(snapshot_key, fresh=True) or {}))
add('document:' + snapshot_key, {'kind': 'document', 'key': snapshot_key}, origin)
else:
origin = None
file_id = row.get('file_id') or (row.get('ref') if row.get('kind') in ('pdf', 'xlsx', 'image') else None)
if isinstance(file_id, str) and assistant_files._FILE_ID.fullmatch(file_id):
origin = file_objects(file_id, origin, match[1])
add('artifact:' + item_id, {'kind': 'artifact', 'chunk': key, 'id': item_id}, origin)
for key in sorted(keys - attributed_documents):
match = _FILE_KEY.fullmatch(key)
if match:
file_objects(match[1], _amazon_origin(rt.get(key, fresh=True) or {}))
elif _SNAPSHOT_KEY.fullmatch(key):
add('document:' + key, {'kind': 'document', 'key': key},
_amazon_origin(rt.get(key, fresh=True) or {}))
elif (key.startswith('assistant_snapshot_')
or key.startswith('assistant_file_') and not key.startswith('assistant_file_token_')):
add('unattributed:' + hashlib.sha256(key.encode()).hexdigest(),
{'kind': 'unknown'}, None)
for path in paths - attributed_paths:
add('blob:' + path, {'kind': 'blob', 'path': path}, None)
plan = rt.get(_EXPORT_DELETIONS_KEY, fresh=True) or {}
if plan and (plan.get('tenant') != tenant or not isinstance(plan.get('objects'), dict)):
raise RuntimeError('Retained export deletion inventory is unavailable.')
for object_id, descriptor in plan.get('objects', {}).items():
kind = descriptor.get('kind')
if kind == 'document' and (_FILE_KEY.fullmatch(descriptor.get('key', ''))
or _SNAPSHOT_KEY.fullmatch(descriptor.get('key', ''))):
present = descriptor['key'] in keys
elif kind == 'blob' and re.fullmatch(r'assistant_files/[0-9a-f]{24}/[0-9a-f]{32}\.(pdf|xlsx|png)',
descriptor.get('path', '')):
present = descriptor['path'] in paths
elif kind == 'artifact' and _ARTIFACT_CHUNK.fullmatch(descriptor.get('chunk', '')):
present = any(row.get('id') == descriptor.get('id') for row in
(rt.get(descriptor['chunk'], fresh=True) or {}).get('items', []))
else:
raise RuntimeError('Retained export deletion locator is invalid.')
if present:
objects.setdefault(object_id, descriptor)
# A saved locator retains provenance after its artifact or metadata is removed.
if not descriptor.get('unknown'):
objects[object_id] = descriptor
return objects
def amazon_deletion_inventory(rt):
"""Return retained Amazon objects and unresolved provenance, including deleted-status payloads."""
return sorted(_retained_export_objects(rt))
def purge_amazon_object(rt, object_id, *, checked_at):
"""Retain retry locators before physical deletion and require fresh absence readback."""
objects = _retained_export_objects(rt)
target = objects.get(object_id)
if target is None:
return
if target.get('unknown'):
raise RuntimeError('Retained export provenance needs resolution before deletion.')
tenant = assistant_log._tenant_of(rt)
def remember(raw):
if raw and raw.get('tenant') != tenant:
raise RuntimeError('Retained export deletion inventory belongs to another tenant.')
entries = dict(raw.get('objects') or {})
entries.update({key: value for key, value in objects.items() if not value.get('unknown')})
return {'tenant': tenant, 'objects': entries}
rt.update(_EXPORT_DELETIONS_KEY, remember, flush='sync')
if target['kind'] == 'artifact':
def remove(raw):
return {**raw, 'items': [row for row in raw.get('items', []) if row.get('id') != target['id']]}
rt.update(target['chunk'], remove, flush='sync')
elif target['kind'] == 'blob':
if rt.delete_path(target['path']) is False:
raise RuntimeError('Retained export bytes could not be deleted.')
else:
if rt.delete_document(target['key']) is False:
raise RuntimeError('Retained export document could not be deleted.')
if object_id in _retained_export_objects(rt):
raise RuntimeError('Retained export deletion readback found residue.')
def register_amazon_deletion_owner(rt, register):
"""Mount retained export ownership through the runtime's injected Amazon registrar."""
register(rt, 'service_retained_exports', inventory=amazon_deletion_inventory,
delete=purge_amazon_object, fixture=False)
from harness.runtime import register_deletion_owner_initializer
register_deletion_owner_initializer('assistant_artifacts', register_amazon_deletion_owner)
try:
import agent_tools
except ImportError:
pass
else:
agent_tools.register("query_view_delete", {
"type": "object", "description": "Delete one of your saved Query answers after confirmation.",
"properties": {"qid": {"type": "string"}}, "required": ["qid"],
"additionalProperties": False}, _delete_query, effect="delete")