Spaces:
Running on CPU Upgrade
Running on CPU Upgrade
File size: 12,180 Bytes
816fa8a 89af3b2 816fa8a 89af3b2 816fa8a 89af3b2 816fa8a 89af3b2 816fa8a 89af3b2 816fa8a 89af3b2 816fa8a 89af3b2 816fa8a 89af3b2 816fa8a 89af3b2 816fa8a 89af3b2 816fa8a 89af3b2 816fa8a 89af3b2 816fa8a 89af3b2 816fa8a 89af3b2 816fa8a 89af3b2 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 | """Store assistant files and durable capabilities under their tenant and owner."""
import base64
import copy
import hashlib
import hmac
import importlib
import json
import math
import re
import secrets
import time
from urllib.parse import urlsplit, urlunsplit
import assistant_artifacts
import assistant_log
MAX_TTL = 3600
XLSX_MIME = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"
IMAGE_MIME = "image/png"
_FORMATS = {"application/pdf": ("pdf", "pdf"), XLSX_MIME: ("xlsx", "xlsx"), IMAGE_MIME: ("png", "image")}
_FILE_ID = re.compile(r"[0-9a-f]{32}")
_TENANT = re.compile(r"[a-z0-9][a-z0-9_-]{0,62}")
_TOKEN_PART = re.compile(r"[A-Za-z0-9_-]{48}")
def _links():
"""Load the lane I contract only when a link is requested."""
try:
return importlib.import_module("loopy_links")
except ModuleNotFoundError as exc:
if exc.name != "loopy_links":
raise
return None
def _artifact_link(artifact_id):
links = _links()
return links.artifact_link(artifact_id) if links is not None else None
def _download_link(token):
links = _links()
app = links.app_link("#/assistant") if links is not None else None
parsed = urlsplit(app or "")
if (parsed.scheme != "https" or not parsed.netloc or parsed.username
or parsed.password or parsed.query):
raise ValueError("A secure public link is not configured.")
path = parsed.path.rstrip("/") + "/api/v1/files/dl/" + token
return urlunsplit((parsed.scheme, parsed.netloc, path, "", ""))
def _bytes_runtime(rt):
# A module fallback could put another tenant's bytes in the default store.
if not all(callable(getattr(rt, name, None)) for name in ("upload_bytes", "download_bytes")):
raise RuntimeError("File storage is not available in this workspace.")
return rt
def _key(file_id):
return f"assistant_file_{file_id}"
def _path(item):
extension = _FORMATS[item["mime"]][0]
return f"assistant_files/{item['ownerHash']}/{item['id']}.{extension}"
def metadata(rt, file_id):
"""Read internal metadata in this tenant without granting its caller ownership."""
if not isinstance(file_id, str) or not _FILE_ID.fullmatch(file_id):
return None
item = rt.get(_key(file_id))
if (not isinstance(item, dict) or item.get("id") != file_id
or item.get("tenant") != assistant_log._tenant_of(rt)
or not isinstance(item.get("owner"), str) or not item["owner"]
or item.get("ownerHash") != assistant_log._hash(rt, item["owner"])
or not isinstance(item.get("mime"), str) or item["mime"] not in _FORMATS
or not isinstance(item.get("size"), int) or item["size"] < 0
or not isinstance(item.get("artifactId"), str)):
return None
return copy.deepcopy(item)
def owned_metadata(rt, username, file_id):
item = metadata(rt, file_id)
return item if item and item["ownerHash"] == assistant_log._hash(rt, username) else None
def read_bytes(rt, item):
"""Read bytes only after the caller has authorized the metadata row."""
data = _bytes_runtime(rt).download_bytes(_path(item))
if data is None:
return None
if not isinstance(data, (bytes, bytearray, memoryview)) or len(data) != item["size"]:
raise RuntimeError("This file could not be read.")
return bytes(data)
def _public_metadata(item):
fields = ("id", "filename", "size", "mime", "artifactId", "createdAt")
result = {**{key: item[key] for key in fields}, "link": _artifact_link(item["artifactId"])}
if item["mime"] in (XLSX_MIME, IMAGE_MIME):
result.update(file_id=item["id"], bytes=item["size"], created_at=item["createdAt"],
frozen_at=item["createdAt"], sources=copy.deepcopy(item.get("sources")),
amazon=item.get("amazon", True))
return result
def save(rt, username, *, title, data, mime, door, tool_call, sources=None, amazon=False):
"""Save one file and its replayable artifact with validated source provenance."""
h = assistant_log._hash(rt, username)
if not isinstance(title, str) or not title.strip():
raise ValueError("A file needs a title.")
if not isinstance(mime, str) or mime not in _FORMATS or not isinstance(data, (bytes, bytearray, memoryview)):
raise ValueError("A file needs bytes and a supported media type.")
extension, kind = _FORMATS[mime]
import agent_runtime
if agent_runtime._PRODUCING_TURN.get() is not None:
proof = agent_runtime.producing_turn_sources(rt, username)
if proof is None:
sources = None
else:
if sources is not None and assistant_artifacts._source_keys(sources) is None:
raise ValueError("A file needs valid source provenance.")
sources = sorted(set(proof) | set(sources or []))
amazon = amazon or assistant_artifacts.is_amazon(sources)
if mime == XLSX_MIME:
from loopy_xlsx import MAX_BYTES
if len(data) > MAX_BYTES:
raise ValueError("Excel export exceeds the 20 MiB file limit. Filter the rows or select fewer fields and ask again. No file was stored.")
if sources is None:
raise ValueError("An Excel file needs validated source provenance. Read its sources and ask again.")
if mime == IMAGE_MIME:
from loopy_image import MAX_BYTES
if len(data) > MAX_BYTES:
raise ValueError("The image exceeds the 20 MiB limit. Select fewer fields and ask again. No file was stored.")
if sources is None:
raise ValueError("An image needs validated source provenance. Read its sources and ask again.")
if (sources is not None and assistant_artifacts._source_keys(sources) is None
or type(amazon) is not bool):
raise ValueError("A file needs valid source provenance and its Amazon flag.")
sources = list(dict.fromkeys(assistant_artifacts._source_keys(sources))) if sources is not None else None
amazon = amazon or assistant_artifacts.is_amazon(sources)
if kind not in assistant_artifacts.KINDS:
raise ValueError(f"{kind.capitalize()} artifacts are not available on this build. Update the artifact registry and ask again. No file was stored.")
if not isinstance(door, str) or door not in assistant_artifacts.DOORS:
raise ValueError("That file door is not available.")
if (not isinstance(tool_call, dict) or not isinstance(tool_call.get("name"), str)
or not tool_call["name"].strip() or not isinstance(tool_call.get("args"), dict)):
raise ValueError("A file needs a tool name and object arguments.")
json.dumps(tool_call, allow_nan=False)
from routes_query import _no_dashes
title = _no_dashes(title).strip()[:200]
file_id = secrets.token_hex(16)
item = {"id": file_id, "owner": username, "ownerHash": h,
"tenant": assistant_log._tenant_of(rt), "title": title,
"filename": (re.sub(r"[^A-Za-z0-9._-]+", "_", title).strip("._")[:100] or "document") + "." + extension,
"size": len(data), "mime": mime, "door": door,
"createdAt": assistant_artifacts.timestamp(time.time()),
"toolCall": copy.deepcopy(tool_call), "sources": copy.deepcopy(sources), "amazon": amazon}
_bytes_runtime(rt).upload_bytes(_path(item), bytes(data), message=f"Save assistant {kind.upper()}")
item["artifactId"] = assistant_artifacts.record(rt, username, {
"kind": kind, "ref": file_id, "title": title, "door": door,
"toolCall": tool_call, "createdAt": item["createdAt"],
"created_at": item["createdAt"], "frozen_at": item["createdAt"],
"sources": sources, "amazon": amazon, "file_id": file_id, "mime": mime})
rt.update(_key(file_id), lambda _raw: copy.deepcopy(item), flush="sync")
return {"id": file_id, "artifactId": item["artifactId"], "link": _artifact_link(item["artifactId"]),
"file_id": file_id, "mime": mime, "bytes": len(data)}
def created_in_turn(rt, username, since):
"""Return all owned file metadata created at or after the turn's start."""
boundary = assistant_artifacts.timestamp(since)
h = assistant_log._hash(rt, username)
index = assistant_artifacts._index(rt.get(f"assistant_artifacts_{h}_index"))
items = []
for row in assistant_artifacts._rows(rt, h, index):
if row["kind"] not in ("pdf", "xlsx", "image") or row["status"] == "deleted" or row["createdAt"] < boundary:
continue
item = owned_metadata(rt, username, row["ref"])
if item and item["createdAt"] >= boundary:
items.append(_public_metadata(item))
return sorted(items, key=lambda item: (item["createdAt"], item["id"]), reverse=True)
def _token_parts(token):
if not isinstance(token, str) or token.count(".") != 1:
return None
tenant, random_part = token.split(".")
if not _TENANT.fullmatch(tenant) or not _TOKEN_PART.fullmatch(random_part):
return None
raw = base64.b64decode(random_part, altchars=b"-_", validate=True)
return tenant, raw[:12].hex(), hashlib.sha256(token.encode("ascii")).hexdigest()
def _token_key(h, digest):
return f"assistant_file_token_{h}_{digest}"
def capability_url(rt, file_id, ttl=MAX_TTL):
"""Mint a durable download capability with a maximum lifetime of one hour."""
if isinstance(ttl, bool) or not isinstance(ttl, (int, float)) or not math.isfinite(ttl) or ttl <= 0:
raise ValueError("A file link needs a positive lifetime.")
item = metadata(rt, file_id)
if item is None:
raise ValueError("That file does not exist.")
if assistant_artifacts.is_amazon(item.get("sources")) or item.get("amazon") is not False:
raise ValueError("Amazon files cannot use a public download link. Download the file in the app instead.")
tenant = assistant_log._tenant_of(rt)
if not _TENANT.fullmatch(tenant):
raise ValueError("A file link needs a valid tenant.")
# The opaque locator selects a document; only a stored token grants access.
random_part = base64.urlsafe_b64encode(bytes.fromhex(item["ownerHash"]) + secrets.token_bytes(24)).decode("ascii")
token = tenant + "." + random_part
link = _download_link(token)
_, h, digest = _token_parts(token)
now = time.time()
entry = {"version": 1, "tokenHash": digest, "tenant": tenant, "owner": item["owner"],
"ownerHash": h, "fileId": file_id, "createdAt": now,
"expiresAt": now + min(ttl, MAX_TTL)}
rt.update(_token_key(h, digest), lambda _raw: entry, flush="sync")
return link
def resolve_capability(token):
"""Resolve a recorded token to its owned file and expiry state without a session."""
parts = _token_parts(token)
if parts is None:
return None
tenant, h, digest = parts
from harness.runtime import get_runtime
try:
rt = get_runtime(tenant)
if assistant_log._tenant_of(rt) != tenant:
return None
entry = rt.get(_token_key(h, digest))
if (not isinstance(entry, dict) or entry.get("tenant") != tenant
or entry.get("ownerHash") != h or not isinstance(entry.get("tokenHash"), str)
or not hmac.compare_digest(entry["tokenHash"], digest)
or not isinstance(entry.get("owner"), str) or not entry["owner"]
or assistant_log._hash(rt, entry["owner"]) != h):
return None
born, expires = entry.get("createdAt"), entry.get("expiresAt")
if (any(isinstance(value, bool) or not isinstance(value, (int, float))
or not math.isfinite(value) for value in (born, expires))
or not 0 < expires - born <= MAX_TTL):
return None
item = owned_metadata(rt, entry["owner"], entry.get("fileId"))
if item is None:
return None
if assistant_artifacts.is_amazon(item.get("sources")) or item.get("amazon") is not False:
return None
now = time.time()
if now < born:
return None
return rt, item, now >= expires
except Exception:
return None
|