Download api/assistant_artifacts.py from fsanyoto/loopable: direct link, hf CLI and curl.
- Browser
- Download file 22.6 kB
-
https://huggingface.co/spaces/fsanyoto/loopable/resolve/9291c0100f4e7b8da2efcbbf1ebdea72ee1663e7/api/assistant_artifacts.py
- Command line
-
hf download hf://spaces/fsanyoto/loopable@9291c0100f4e7b8da2efcbbf1ebdea72ee1663e7/api/assistant_artifacts.py
-
curl -L -o assistant_artifacts.py https://huggingface.co/spaces/fsanyoto/loopable/resolve/9291c0100f4e7b8da2efcbbf1ebdea72ee1663e7/api/assistant_artifacts.py
22.6 kB
| """C14: a principal's artifact index, in store documents of at most 200 items.""" | |
| import copy | |
| import datetime | |
| import hashlib | |
| import heapq | |
| import json | |
| import math | |
| import re | |
| import uuid | |
| import assistant_log | |
| import loopy_links | |
| CHUNK_ITEMS = 200 | |
| KINDS = {"view", "sheet", "report", "run", "query", "schedule", "pdf", "xlsx", "image", "snapshot"} | |
| FROZEN_KINDS = {"snapshot", "pdf", "xlsx", "image"} | |
| DOORS = {"app", "whatsapp", "wake"} | |
| STATUSES = {"active", "paused", "deleted"} | |
| AMAZON_PREFIX = "ut_amzn_" | |
| _ITEM_ID = re.compile(r"([0-9a-f]{24})\.([0-9]+)") | |
| _SNAPSHOT_REF = re.compile(r"[0-9a-f]{32}") | |
| _ARTIFACT_CHUNK = re.compile(r'assistant_artifacts_([0-9a-f]{24})_([0-9]+)') | |
| _FILE_KEY = re.compile(r'assistant_file_([0-9a-f]{32})') | |
| _SNAPSHOT_KEY = re.compile(r'assistant_snapshot_([0-9a-f]{24})_([0-9a-f]{32})') | |
| _EXPORT_DELETIONS_KEY = 'amazon_retained_export_deletions' | |
| def timestamp(value): | |
| if isinstance(value, (int, float)) and not isinstance(value, bool) and math.isfinite(value): | |
| dt = datetime.datetime.fromtimestamp(value, datetime.timezone.utc) | |
| elif isinstance(value, str) and value.strip(): | |
| dt = datetime.datetime.fromisoformat(value.replace("Z", "+00:00")) | |
| if dt.tzinfo is None: | |
| dt = dt.replace(tzinfo=datetime.timezone.utc) | |
| else: | |
| raise ValueError("An artifact needs a valid creation time.") | |
| return dt.astimezone(datetime.timezone.utc).isoformat() | |
| def _index(raw): | |
| raw = raw or {} | |
| return {"version": 1, "nextSeq": int(raw.get("nextSeq", 0)), | |
| "chunks": copy.deepcopy(raw.get("chunks") or {})} | |
| def _chunk_key(h, n): | |
| return f"assistant_artifacts_{h}_{n}" | |
| def _source_keys(sources): | |
| """Return normalized database keys, or None for unknown or malformed provenance.""" | |
| if not isinstance(sources, list) or not all(isinstance(key, str) and key.strip() for key in sources): | |
| return None | |
| return [key.strip() for key in sources] | |
| def is_amazon(sources): | |
| """R86 applies to Amazon databases and unknown or malformed provenance.""" | |
| keys = _source_keys(sources) | |
| return keys is None or any(key.startswith(AMAZON_PREFIX) for key in keys) | |
| def _optional_text(item, field, sentence): | |
| value = item.get(field) | |
| if value is not None and (not isinstance(value, str) or not value.strip()): | |
| raise ValueError(sentence) | |
| return value.strip() if value is not None else None | |
| def _c7_inputs(item, kind, args, created_at): | |
| """Validate the optional C7 inputs and return sources, amazon, file_id, mime and frozen_at.""" | |
| sources = item.get("sources") | |
| if sources is not None and _source_keys(sources) is None: | |
| raise ValueError("Artifact sources must be a list of database names.") | |
| if sources is None and kind in ("view", "query") and isinstance(args.get("database"), str) \ | |
| and args["database"].strip(): | |
| sources = [args["database"].strip()] | |
| elif sources is not None: | |
| sources = _source_keys(sources) | |
| if "amazon" in item and not isinstance(item["amazon"], bool): | |
| raise ValueError("The artifact Amazon flag must be true or false.") | |
| amazon = is_amazon(sources) or item.get("amazon") is True | |
| file_id = _optional_text(item, "file_id", "An artifact file id must be text.") | |
| mime = _optional_text(item, "mime", "An artifact file type must be text.") | |
| frozen = item.get("frozen_at") | |
| try: | |
| frozen = timestamp(frozen) if frozen is not None else None | |
| except (TypeError, ValueError, OverflowError, OSError) as exc: | |
| raise ValueError("An artifact needs a valid frozen time.") from exc | |
| if frozen is None and kind in FROZEN_KINDS: | |
| frozen = created_at | |
| return sources, amazon, file_id, mime, frozen | |
| def _app_database(rt, ref, args): | |
| compatible = args.get("compatibleDatabases") | |
| if isinstance(compatible, list) and compatible and isinstance(compatible[0], str) and compatible[0].strip(): | |
| return compatible[0] | |
| if isinstance(args.get("sourceDatabase"), str) and args["sourceDatabase"].strip(): | |
| return args["sourceDatabase"] | |
| try: | |
| import application_registry | |
| stored = application_registry.load(rt)["applications"].get(ref) or {} | |
| first = (stored.get("compatibleDatabases") or [None])[0] | |
| return first if isinstance(first, str) and first.strip() else None | |
| except Exception: | |
| # A registry read failure falls back to the artifact link. | |
| return None | |
| def _link(rt, row): | |
| """The absolute link that opens this row's work, or None with no public base.""" | |
| kind, ref, call = row["kind"], row["ref"], row["toolCall"] | |
| args = call["args"] | |
| # A fixture links module can carry only the artifact builder, so a missing builder means no link. | |
| build = lambda name, *values: (getattr(loopy_links, name, None) or (lambda *_: None))(*values) | |
| link = None | |
| if ref is not None: | |
| if kind == "sheet": | |
| link = build("sheet_link", ref) | |
| elif kind == "view": | |
| link = build("view_link", args.get("database"), ref) | |
| elif kind == "query": | |
| link = build("query_link", ref) | |
| elif kind == "run" and call["name"] == "app_upsert": | |
| link = build("application_link", ref, _app_database(rt, ref, args)) | |
| return link or loopy_links.artifact_link(row["id"]) | |
| def record(rt, username, item): | |
| """Validate and append exactly one C14 item; return its principal-bound id.""" | |
| from routes_query import _no_dashes | |
| if not isinstance(item, dict): | |
| raise ValueError("An artifact must be an object.") | |
| if item.get("door") == "slack": | |
| raise ValueError("Slack artifacts are never stored.") | |
| for field, allowed in (("kind", KINDS), ("door", DOORS)): | |
| if not isinstance(item.get(field), str) or item[field] not in allowed: | |
| raise ValueError(f"That artifact {field} is not available.") | |
| # Some writes (for example, a batch) have no single object reference. | |
| ref = item.get("ref") | |
| if "ref" not in item or (ref is not None and (not isinstance(ref, str) or not ref.strip())): | |
| raise ValueError("An artifact reference must be text or null.") | |
| if not isinstance(item.get("title"), str) or not item["title"].strip(): | |
| raise ValueError("An artifact needs a title.") | |
| call = item.get("toolCall") | |
| if (not isinstance(call, dict) or not isinstance(call.get("name"), str) | |
| or not call["name"].strip() or not isinstance(call.get("args"), dict)): | |
| raise ValueError("An artifact needs a tool name and object arguments.") | |
| try: | |
| json.dumps(call, allow_nan=False) | |
| at = timestamp(item.get("createdAt")) | |
| except (TypeError, ValueError, OverflowError, OSError) as exc: | |
| raise ValueError("An artifact needs JSON arguments and a valid creation time.") from exc | |
| sources, amazon, file_id, mime, frozen = _c7_inputs(item, item["kind"], call["args"], at) | |
| value = {"kind": item["kind"], "ref": ref, "title": _no_dashes(item["title"]).strip()[:200], | |
| "door": item["door"], "toolCall": {"name": call["name"], "args": copy.deepcopy(call["args"])}, | |
| "createdAt": at, "status": "active", "sources": sources, "amazon": amazon} | |
| value.update({key: got for key, got in (("file_id", file_id), ("mime", mime), ("frozen_at", frozen)) | |
| if got is not None}) | |
| h = assistant_log._hash(rt, username) | |
| allocated = [] | |
| def allocate(raw): | |
| index = _index(raw) | |
| seq = index["nextSeq"] | |
| n = seq // CHUNK_ITEMS | |
| index["nextSeq"] = seq + 1 | |
| index["chunks"][str(n)] = index["chunks"].get(str(n), 0) + 1 | |
| allocated[:] = [seq, n] | |
| return index | |
| rt.update(f"assistant_artifacts_{h}_index", allocate, flush="sync") | |
| seq, n = allocated | |
| row = {**value, "id": f"{h}.{seq}", "seq": seq} | |
| row["link"] = _link(rt, row) | |
| def insert(raw): | |
| existing = {entry["seq"]: entry for entry in (raw or {}).get("items", [])} | |
| existing.setdefault(seq, row) | |
| if len(existing) > CHUNK_ITEMS: | |
| raise ValueError("An artifact chunk is full.") | |
| return {"version": 1, "items": [existing[key] for key in sorted(existing)]} | |
| rt.update(_chunk_key(h, n), insert, flush="sync") | |
| return row["id"] | |
| def _seq(h, item_id): | |
| if not isinstance(item_id, str) or not item_id.startswith(h + "."): | |
| return None | |
| suffix = item_id[len(h) + 1:] | |
| return int(suffix) if suffix.isdecimal() else None | |
| def _list_items(rt, username, *, cursor=None, limit=50): | |
| """Newest creation first, with an opaque boundary belonging only to this caller.""" | |
| h = assistant_log._hash(rt, username) | |
| index = _index(rt.get(f"assistant_artifacts_{h}_index")) | |
| boundary = None | |
| if cursor: | |
| boundary = get(rt, username, cursor) | |
| if boundary is None: | |
| raise ValueError("That artifact cursor does not exist.") | |
| limit = max(1, min(CHUNK_ITEMS, int(limit))) | |
| order = lambda row: (row["createdAt"], row["seq"]) | |
| candidates = (row for row in _rows(rt, h, index) if boundary is None or order(row) < order(boundary)) | |
| # Imported schedules can carry an earlier creation time than the preceding allocation. | |
| # Keep only one page plus lookahead in memory while visiting bounded documents. | |
| rows = heapq.nlargest(limit + 1, candidates, key=order) | |
| more = len(rows) > limit | |
| rows = rows[:limit] | |
| return {"items": rows, "nextCursor": rows[-1]["id"] if more else None} | |
| def _rows(rt, h, index): | |
| for n in range((index["nextSeq"] - 1) // CHUNK_ITEMS, -1, -1): | |
| if not index["chunks"].get(str(n)): | |
| continue | |
| chunk = rt.get(_chunk_key(h, n)) or {} | |
| yield from (copy.deepcopy(row) for row in reversed(chunk.get("items", []))) | |
| def get(rt, username, item_id): | |
| h = assistant_log._hash(rt, username) | |
| seq = _seq(h, item_id) | |
| if seq is None: | |
| return None | |
| chunk = rt.get(_chunk_key(h, seq // CHUNK_ITEMS)) or {} | |
| return next((copy.deepcopy(row) for row in chunk.get("items", []) if row["id"] == item_id), None) | |
| def get_by_id(rt, item_id): | |
| """Read one row by its `h.seq` id alone, for a page that has no username.""" | |
| match = _ITEM_ID.fullmatch(item_id) if isinstance(item_id, str) else None | |
| if match is None: | |
| return None | |
| h, seq = match.group(1), int(match.group(2)) | |
| chunk = rt.get(_chunk_key(h, seq // CHUNK_ITEMS)) or {} | |
| return next((copy.deepcopy(row) for row in chunk.get("items", []) if row.get("id") == item_id), None) | |
| def _snapshot_key(h, ref): | |
| # The content lives in its own document: a read is billed at the document's text size. | |
| return f"assistant_snapshot_{h}_{ref}" | |
| def store_snapshot(rt, username, content): | |
| """Write one frozen snapshot content document and return its `ref`.""" | |
| if not isinstance(content, dict): | |
| raise ValueError("A snapshot must be an object.") | |
| json.dumps(content, allow_nan=False) | |
| ref = uuid.uuid4().hex | |
| frozen = copy.deepcopy(content) | |
| rt.update(_snapshot_key(assistant_log._hash(rt, username), ref), lambda _raw: frozen, flush="sync") | |
| return ref | |
| def read_snapshot(rt, row): | |
| """Read the content document of one snapshot row, or None; it touches no other key.""" | |
| if not isinstance(row, dict) or row.get("kind") != "snapshot" or not isinstance(row.get("id"), str): | |
| return None | |
| match, ref = _ITEM_ID.fullmatch(row["id"]), row.get("ref") | |
| if match is None or not isinstance(ref, str) or not _SNAPSHOT_REF.fullmatch(ref): | |
| return None | |
| content = rt.get(_snapshot_key(match.group(1), ref)) | |
| return copy.deepcopy(content) if content else None | |
| def c7(row): | |
| """Project C7 with Amazon protection for unknown provenance, including malformed legacy sources.""" | |
| sources = _source_keys(row.get("sources")) | |
| out = {"id": row["id"], "kind": row["kind"], "title": row["title"], "created_at": row["createdAt"], | |
| "frozen_at": row.get("frozen_at"), "sources": sources, | |
| "amazon": is_amazon(sources) or row.get("amazon", False) is not False, | |
| "link": row["link"] if "link" in row else loopy_links.artifact_link(row["id"])} | |
| file_id = row.get("file_id") or (row.get("ref") if row["kind"] == "pdf" else None) | |
| if file_id: | |
| out["file_id"] = file_id | |
| if row.get("mime"): | |
| out["mime"] = row["mime"] | |
| return out | |
| def link_fields(row): | |
| """The link a tool result carries, or a plain note when no public base is set.""" | |
| link = (row or {}).get("link") | |
| return {"link": link} if link else {"link": None, "linkNote": loopy_links.NO_LINK_NOTE} | |
| def set_status(rt, username, item_id, status): | |
| if not isinstance(status, str) or status not in STATUSES: | |
| raise ValueError("That artifact status is not available.") | |
| h = assistant_log._hash(rt, username) | |
| seq = _seq(h, item_id) | |
| if seq is None or get(rt, username, item_id) is None: | |
| return None | |
| changed = [] | |
| def update(raw): | |
| doc = copy.deepcopy(raw) | |
| for row in doc.get("items", []): | |
| if row["id"] == item_id: | |
| row["status"] = status | |
| changed[:] = [copy.deepcopy(row)] | |
| return doc | |
| rt.update(_chunk_key(h, seq // CHUNK_ITEMS), update, flush="sync") | |
| return changed[0] if changed else None | |
| def _delete_query(session, args): | |
| from routes_query import _artifact_or_404, delete_query | |
| _artifact_or_404(session, args.get("qid")) | |
| return delete_query(args["qid"], session) | |
| def _amazon_origin(row): | |
| """Distinguish established Amazon origin from an access flag applied to unknown provenance.""" | |
| sources = _source_keys(row.get('sources')) | |
| if sources is not None and any(source.startswith(AMAZON_PREFIX) for source in sources): | |
| return True | |
| if sources is None or ('amazon' in row and type(row['amazon']) is not bool): | |
| return None | |
| return row.get('amazon') is True | |
| def _origin_union(first, second): | |
| if first is True or second is True: | |
| return True | |
| return False if first is False or second is False else None | |
| def _retained_export_objects(rt): | |
| """Inventory all durable principals and retry locators without returning artifact values.""" | |
| import assistant_files | |
| tenant = assistant_log._tenant_of(rt) | |
| objects, attributed_documents, attributed_paths = {}, set(), set() | |
| keys = set(rt.list_keys('assistant_')) | |
| paths = set(rt.list_paths('assistant_files/')) | |
| def add(object_id, descriptor, origin): | |
| if origin is not False: | |
| objects[object_id] = {**descriptor, 'unknown': origin is None} | |
| def file_objects(file_id, origin, expected_hash=None): | |
| key = assistant_files._key(file_id) | |
| metadata = rt.get(key, fresh=True) if key in keys else None | |
| if metadata is None: | |
| return origin | |
| attributed_documents.add(key) | |
| valid = (isinstance(metadata, dict) and metadata.get('id') == file_id | |
| and metadata.get('tenant') == tenant | |
| and isinstance(metadata.get('owner'), str) and metadata['owner'] | |
| and metadata.get('ownerHash') == assistant_log._hash(rt, metadata['owner']) | |
| and (expected_hash is None or metadata['ownerHash'] == expected_hash) | |
| and metadata.get('mime') in assistant_files._FORMATS) | |
| if not valid: | |
| add('document:' + key, {'kind': 'document', 'key': key}, None) | |
| return None | |
| origin = _origin_union(origin, _amazon_origin(metadata)) | |
| path = assistant_files._path(metadata) | |
| attributed_paths.add(path) | |
| add('document:' + key, {'kind': 'document', 'key': key}, origin) | |
| if path in paths: | |
| add('blob:' + path, {'kind': 'blob', 'path': path}, origin) | |
| return origin | |
| for key in sorted(keys): | |
| match = _ARTIFACT_CHUNK.fullmatch(key) | |
| if match is None: | |
| if (key.startswith('assistant_artifacts_') | |
| and not re.fullmatch(r'assistant_artifacts_[0-9a-f]{24}_index', key)): | |
| add('unattributed:' + hashlib.sha256(key.encode()).hexdigest(), | |
| {'kind': 'unknown'}, None) | |
| continue | |
| chunk = rt.get(key, fresh=True) | |
| if not isinstance(chunk, dict) or not isinstance(chunk.get('items'), list): | |
| add('unattributed:' + hashlib.sha256(key.encode()).hexdigest(), | |
| {'kind': 'unknown'}, None) | |
| continue | |
| for index, row in enumerate(chunk['items']): | |
| item_id = row.get('id') if isinstance(row, dict) else None | |
| item_match = _ITEM_ID.fullmatch(item_id) if isinstance(item_id, str) else None | |
| if item_match is None or item_match[1] != match[1] or int(item_match[2]) // CHUNK_ITEMS != int(match[2]): | |
| add('unattributed:' + hashlib.sha256(f'{key}:{index}'.encode()).hexdigest(), | |
| {'kind': 'unknown'}, None) | |
| continue | |
| origin = _amazon_origin(row) | |
| if row.get('kind') == 'snapshot': | |
| ref = row.get('ref') | |
| if isinstance(ref, str) and _SNAPSHOT_REF.fullmatch(ref): | |
| snapshot_key = _snapshot_key(match[1], ref) | |
| attributed_documents.add(snapshot_key) | |
| if snapshot_key in keys: | |
| origin = _origin_union(origin, _amazon_origin(rt.get(snapshot_key, fresh=True) or {})) | |
| add('document:' + snapshot_key, {'kind': 'document', 'key': snapshot_key}, origin) | |
| else: | |
| origin = None | |
| file_id = row.get('file_id') or (row.get('ref') if row.get('kind') in ('pdf', 'xlsx', 'image') else None) | |
| if isinstance(file_id, str) and assistant_files._FILE_ID.fullmatch(file_id): | |
| origin = file_objects(file_id, origin, match[1]) | |
| add('artifact:' + item_id, {'kind': 'artifact', 'chunk': key, 'id': item_id}, origin) | |
| for key in sorted(keys - attributed_documents): | |
| match = _FILE_KEY.fullmatch(key) | |
| if match: | |
| file_objects(match[1], _amazon_origin(rt.get(key, fresh=True) or {})) | |
| elif _SNAPSHOT_KEY.fullmatch(key): | |
| add('document:' + key, {'kind': 'document', 'key': key}, | |
| _amazon_origin(rt.get(key, fresh=True) or {})) | |
| elif (key.startswith('assistant_snapshot_') | |
| or key.startswith('assistant_file_') and not key.startswith('assistant_file_token_')): | |
| add('unattributed:' + hashlib.sha256(key.encode()).hexdigest(), | |
| {'kind': 'unknown'}, None) | |
| for path in paths - attributed_paths: | |
| add('blob:' + path, {'kind': 'blob', 'path': path}, None) | |
| plan = rt.get(_EXPORT_DELETIONS_KEY, fresh=True) or {} | |
| if plan and (plan.get('tenant') != tenant or not isinstance(plan.get('objects'), dict)): | |
| raise RuntimeError('Retained export deletion inventory is unavailable.') | |
| for object_id, descriptor in plan.get('objects', {}).items(): | |
| kind = descriptor.get('kind') | |
| if kind == 'document' and (_FILE_KEY.fullmatch(descriptor.get('key', '')) | |
| or _SNAPSHOT_KEY.fullmatch(descriptor.get('key', ''))): | |
| present = descriptor['key'] in keys | |
| elif kind == 'blob' and re.fullmatch(r'assistant_files/[0-9a-f]{24}/[0-9a-f]{32}\.(pdf|xlsx|png)', | |
| descriptor.get('path', '')): | |
| present = descriptor['path'] in paths | |
| elif kind == 'artifact' and _ARTIFACT_CHUNK.fullmatch(descriptor.get('chunk', '')): | |
| present = any(row.get('id') == descriptor.get('id') for row in | |
| (rt.get(descriptor['chunk'], fresh=True) or {}).get('items', [])) | |
| else: | |
| raise RuntimeError('Retained export deletion locator is invalid.') | |
| if present: | |
| objects.setdefault(object_id, descriptor) | |
| # A saved locator retains provenance after its artifact or metadata is removed. | |
| if not descriptor.get('unknown'): | |
| objects[object_id] = descriptor | |
| return objects | |
| def amazon_deletion_inventory(rt): | |
| """Return retained Amazon objects and unresolved provenance, including deleted-status payloads.""" | |
| return sorted(_retained_export_objects(rt)) | |
| def purge_amazon_object(rt, object_id, *, checked_at): | |
| """Retain retry locators before physical deletion and require fresh absence readback.""" | |
| objects = _retained_export_objects(rt) | |
| target = objects.get(object_id) | |
| if target is None: | |
| return | |
| if target.get('unknown'): | |
| raise RuntimeError('Retained export provenance needs resolution before deletion.') | |
| tenant = assistant_log._tenant_of(rt) | |
| def remember(raw): | |
| if raw and raw.get('tenant') != tenant: | |
| raise RuntimeError('Retained export deletion inventory belongs to another tenant.') | |
| entries = dict(raw.get('objects') or {}) | |
| entries.update({key: value for key, value in objects.items() if not value.get('unknown')}) | |
| return {'tenant': tenant, 'objects': entries} | |
| rt.update(_EXPORT_DELETIONS_KEY, remember, flush='sync') | |
| if target['kind'] == 'artifact': | |
| def remove(raw): | |
| return {**raw, 'items': [row for row in raw.get('items', []) if row.get('id') != target['id']]} | |
| rt.update(target['chunk'], remove, flush='sync') | |
| elif target['kind'] == 'blob': | |
| if rt.delete_path(target['path']) is False: | |
| raise RuntimeError('Retained export bytes could not be deleted.') | |
| else: | |
| if rt.delete_document(target['key']) is False: | |
| raise RuntimeError('Retained export document could not be deleted.') | |
| if object_id in _retained_export_objects(rt): | |
| raise RuntimeError('Retained export deletion readback found residue.') | |
| def register_amazon_deletion_owner(rt, register): | |
| """Mount retained export ownership through the runtime's injected Amazon registrar.""" | |
| register(rt, 'service_retained_exports', inventory=amazon_deletion_inventory, | |
| delete=purge_amazon_object, fixture=False) | |
| from harness.runtime import register_deletion_owner_initializer | |
| register_deletion_owner_initializer('assistant_artifacts', register_amazon_deletion_owner) | |
| try: | |
| import agent_tools | |
| except ImportError: | |
| pass | |
| else: | |
| agent_tools.register("query_view_delete", { | |
| "type": "object", "description": "Delete one of your saved Query answers after confirmation.", | |
| "properties": {"qid": {"type": "string"}}, "required": ["qid"], | |
| "additionalProperties": False}, _delete_query, effect="delete") | |