runloopable / api /catalog_store.py
fsanyoto's picture
Deploy AIOS web (React glide grid + FastAPI slice)
8873406 verified
Raw History Blame Contribute Delete
24.3 kB
"""Tenant catalog records with atomic App indexes and lossless legacy migration."""
from __future__ import annotations
import base64
import copy
import hashlib
import json
import app_rules
import app_templates
import application_registry as applications
from core import write_execution
from deps import err
STORE_KEY = "catalog_records_v1"
DEFAULT_LIMIT = 100
MAX_LIMIT = 200
_ALIASES = {"intro": "section", "gallery": "grid"}
_EDITABLE = {"name", "pages", "thumbnail", "paper", "orientation", "quality", "brand", "fields"}
_SNAPSHOT_KEYS = {"id", "tenant", "templateKind", "sourceDatabase", "compatibleDatabases",
"fieldMap", "navPlacement", "createdBy", "share"}
def _document(raw):
if raw is None or raw == {}:
return {"schema": 1, "catalogs": {}, "byApp": {}, "migrations": {},
"appSnapshots": {}, "nextOrder": 1}
if (not isinstance(raw, dict) or raw.get("schema") != 1
or any(not isinstance(raw.get(key), dict) for key in
("catalogs", "byApp", "migrations", "appSnapshots"))
or type(raw.get("nextOrder")) is not int or raw["nextOrder"] < 1):
raise err(503, "catalog_store_invalid", "Catalog storage is invalid. Ask an administrator to restore it.")
return copy.deepcopy(raw)
def _load(runtime):
return _document(applications._read_strict(runtime, STORE_KEY))
def _update(runtime, change):
result = {}
def apply(raw):
doc = _document(raw)
result["value"] = copy.deepcopy(change(doc))
return doc
# TenantRuntime.update is the atomic JSON document door, also captured by write previews.
saved = runtime.update(STORE_KEY, apply, flush="sync")
if saved is False or saved is None:
raise err(503, "catalog_save_failed", "The catalog save was not confirmed. Reload the library before trying again.")
return result["value"]
def _identity(*parts):
encoded = json.dumps(parts, ensure_ascii=True, separators=(",", ":"))
return "cat_legacy_" + hashlib.sha256(encoded.encode("utf-8")).hexdigest()[:32]
def _text(value, label):
if not isinstance(value, str) or not value.strip():
raise err(400, "invalid_catalog", f"Give {label} a nonempty text value.")
if "\u2013" in value or "\u2014" in value:
raise err(400, "invalid_catalog", f"Replace long dashes in {label} with plain punctuation.")
return value
def normalize_pages(pages, *, legacy=False, identity=""):
"""Keep page order and source payload while adding the shared page contract."""
if not isinstance(pages, list):
raise err(400, "invalid_catalog", "Catalog pages must be a list. Send an empty list for a new catalog.")
out, seen = [], set()
for index, raw in enumerate(pages):
if not isinstance(raw, dict):
raise err(400, "invalid_catalog", "Each catalog page must be an object. Replace the malformed page.")
page = copy.deepcopy(raw)
page_id = page.get("id")
if legacy and (not isinstance(page_id, str) or not page_id):
page_id = _identity(identity, "page", index)
_text(page_id, "the page id")
if page_id in seen:
if not legacy:
raise err(400, "invalid_catalog", "Page ids must be unique. Give the repeated page a new id.")
page_id = _identity(identity, "page", index, page_id)
seen.add(page_id)
kind = page.get("kind", "section" if legacy else None)
_text(kind, "the page kind")
kind = _ALIASES.get(kind, kind)
template = page.get("template", kind + "-stacked")
_text(template, "the page template")
bindings = page.get("bindings", {})
if not isinstance(bindings, dict):
raise err(400, "invalid_catalog", "Page bindings must be an object. Replace the malformed bindings.")
bindings = copy.deepcopy(bindings)
for key in ("title", "body", "products", "codes", "imageCode", "layout"):
if key in raw:
bindings.setdefault(key, copy.deepcopy(raw[key]))
if legacy:
# The legacy carrier is archival content, never an authoritative product value.
bindings.setdefault("legacy", copy.deepcopy(raw))
for key, value in raw.items():
if key not in {"id", "kind", "template", "bindings", "design", "price", "prices"}:
bindings.setdefault(key, copy.deepcopy(value))
for key in ("title", "body"):
if key in bindings and not isinstance(bindings[key], str):
raise err(400, "invalid_catalog", f"Page {key} must be text. Replace the malformed value.")
products = bindings.get("products", bindings.get("codes", []))
if (not isinstance(products, list)
or any(not isinstance(code, str) or not code.strip() for code in products)):
raise err(400, "invalid_catalog", "Bind products to a list of source key strings. Fetch prices from the source.")
if "products" in bindings or "codes" in bindings:
bindings["products"] = copy.deepcopy(products)
page.update(id=page_id, kind=kind, template=template, bindings=bindings)
out.append(page)
return out
def _snapshot(app):
# R33: a snapshot keeps a source or field map only when it differs from the C1 binding.
kept = _SNAPSHOT_KEYS - {"sourceDatabase", "fieldMap"}
row = {key: copy.deepcopy(value) for key, value in app.items() if key in kept}
row.update(copy.deepcopy(app_templates.own_binding(app)))
return row
def migrate_snapshots(runtime, app_ids):
"""R33: drop a C1 binding from the named Apps' catalog snapshots. Writes only on a change."""
wanted = set(app_ids)
def stale(doc):
return sorted(app_id for app_id, row in doc["appSnapshots"].items()
if app_id in wanted and isinstance(row, dict)
and applications._c1_rewrite(row) is not None)
if not stale(_load(runtime)):
return "unchanged"
def apply(doc):
changed = stale(doc)
for app_id in changed:
row = applications._c1_rewrite(doc["appSnapshots"][app_id])
doc["appSnapshots"][app_id] = {key: value for key, value in row.items() if key in _SNAPSHOT_KEYS}
return changed
return f"{len(_update(runtime, apply))} rewritten"
def _insert(doc, record):
catalog_id, app_id = record["id"], record["appId"]
if catalog_id == "templates" or catalog_id in doc["catalogs"]:
raise err(409, "catalog_id_conflict", "This catalog id is already used. Create a new catalog with a new id.")
doc["catalogs"][catalog_id] = copy.deepcopy(record)
doc["byApp"].setdefault(app_id, {})[catalog_id] = doc["nextOrder"]
doc["nextOrder"] += 1
def migrate_legacy(runtime, app):
"""Copy plural display.catalogs once without changing either legacy display key."""
if app.get("templateKind") != "catalog":
return {}
app_id, snapshot = app["id"], _snapshot(app)
existing = _load(runtime)
if app_id in existing["migrations"] and existing["appSnapshots"].get(app_id) == snapshot:
return copy.deepcopy(existing["migrations"][app_id])
display = (app.get("config") or {}).get("display") or {}
legacy = display.get("catalogs", [])
if not isinstance(legacy, list):
raise err(409, "catalog_migration_invalid", "Legacy catalogs must be a list. Restore the App content before deleting it.")
def apply(doc):
doc["appSnapshots"][app_id] = snapshot
if app_id in doc["migrations"]:
return doc["migrations"][app_id]
migrated = {}
for index, raw in enumerate(legacy):
if not isinstance(raw, dict):
raise err(409, "catalog_migration_invalid", "A legacy catalog is malformed. Restore its content before deleting the App.")
source_id = raw.get("id")
catalog_id = source_id if isinstance(source_id, str) and source_id else _identity(app_id, index)
attempt = 0
while catalog_id == "templates" or catalog_id in doc["catalogs"]:
catalog_id = _identity(app_id, index, source_id, attempt)
attempt += 1
record = copy.deepcopy(raw)
stamp = raw.get("updatedAt") or app.get("updatedAt") or app.get("createdAt")
author = raw.get("updatedBy", app.get("updatedBy", app.get("createdBy")))
record.update(id=catalog_id, appId=app_id,
name=raw.get("name") if isinstance(raw.get("name"), str) else "Untitled catalog",
pages=normalize_pages(raw.get("pages", []), legacy=True,
identity=_identity(app_id, index)),
thumbnail=raw.get("thumbnail") if isinstance(raw.get("thumbnail"), str) else None,
updatedAt=stamp if isinstance(stamp, str) else "1970-01-01T00:00:00+00:00",
updatedBy=author if isinstance(author, str) else None)
# A new colour or design rule must never erase existing authored content.
_insert(doc, record)
migrated[str(index)] = catalog_id
doc["migrations"][app_id] = migrated
return migrated
return _update(runtime, apply)
def require_app(session, app_id, *, write=False, allow_orphan=False):
"""Admit the App and its current source schema before returning any catalog content."""
from core import perm_scope
from routes_external import _assert_database_scope, _assert_write_scope
runtime = session.runtime
bound = getattr(runtime, "key", None) or getattr(getattr(runtime, "tenant", None), "key", None)
if bound is not None and bound != session.tenant:
raise err(403, "catalog_tenant_forbidden", "This catalog belongs to another workspace. Open the correct workspace.")
if not runtime.available():
raise err(503, "store_unavailable", "Catalog storage is unavailable. Try again when the store is ready.")
app = applications.load(runtime, strict=True)["applications"].get(app_id)
if app is None and allow_orphan and session.admin and not write:
app = _load(runtime)["appSnapshots"].get(app_id)
if (not isinstance(app, dict) or app.get("tenant") != session.tenant
or app.get("templateKind") != "catalog" or app.get("retired") is True):
raise err(404, "catalog_app_not_found", "Catalog App not found. Open an existing Catalog App.")
may = applications.may_edit if write else applications.may_read
if not may(app, session.uname, session.admin):
raise err(403, "catalog_forbidden", "Your account cannot access this Catalog App. Ask its owner for access.")
# R33: Catalog reads Odoo products through its C1 binding unless the record holds its own.
source = app_templates.bound(app).get("sourceDatabase")
_assert_database_scope(session, source)
session.require(source)
if not perm_scope.may_read(session.user, source, st=runtime):
raise err(403, "catalog_source_forbidden", "The catalog source is unavailable to your account. Ask an administrator for access.")
if write:
_assert_write_scope(session, "catalogs")
contracts = applications.chart_field_contracts(session, [source])
fields = applications._contract_fields(source, field_contracts=contracts)
# Projected C1 fields retain their optional roles; authored legacy maps stay strict.
if not app_templates.own_binding(app):
app = {key: value for key, value in app.items() if key not in {"sourceDatabase", "fieldMap"}}
try:
app = applications.bound_application(app, fields)
except applications.ApplicationError:
raise err(403, "catalog_fields_forbidden", "The Catalog App binds unavailable fields. Ask its owner to rebind readable fields.") from None
return app, fields
def _binding_wall(record, app, fields):
"""Keep source and field references within the App's permission-filtered schema."""
maps = [record.get("fields", {})]
for page in record.get("pages", []):
bindings = page.get("bindings", {})
maps.extend(bindings.get(key, {}) for key in ("fields", "fieldMap"))
for key, value in bindings.items():
if key in {"database", "sourceDatabase"} and value != app["sourceDatabase"]:
raise err(403, "catalog_source_forbidden", "Page bindings must use the App source. Rebind this page to its App.")
if key.endswith("Field") or key in {"price", "prices", "unitPrice", "unit_price"}:
maps.append({key: value})
for mapping in maps:
if not isinstance(mapping, dict):
raise err(400, "invalid_catalog", "Field bindings must be objects. Bind each role to a readable source field.")
for value in mapping.values():
if not isinstance(value, str) or value not in fields:
raise err(403, "catalog_fields_forbidden", "Bind catalog fields to readable source keys. Fetch product prices from the source.")
def _record(session, catalog_id, *, write=False):
record = _load(session.runtime)["catalogs"].get(catalog_id)
if record is None:
apps = applications.load(session.runtime, strict=True)["applications"]
for app in apps.values():
if not isinstance(app, dict):
continue
legacy = ((app.get("config") or {}).get("display") or {}).get("catalogs", [])
if isinstance(legacy, list) and any(isinstance(row, dict) and row.get("id") == catalog_id for row in legacy):
require_app(session, app["id"], write=write)
migrate_legacy(session.runtime, app)
record = _load(session.runtime)["catalogs"].get(catalog_id)
break
if record is None:
raise err(404, "catalog_not_found", "Catalog not found. Open a catalog from the library.")
app, fields = require_app(session, record["appId"], write=write, allow_orphan=True)
_binding_wall(record, app, fields)
return record, app, fields
def read_catalog(session, catalog_id):
return copy.deepcopy(_record(session, catalog_id)[0])
def list_catalogs(session, app_id, *, cursor=None, limit=DEFAULT_LIMIT):
"""Page insertion order with an App-bound exclusive sequence cursor, default 100 and maximum 200 per read."""
if type(limit) is not int or not 1 <= limit <= MAX_LIMIT:
raise err(400, "invalid_catalog_limit", "Use a catalog page limit from 1 to 200. Follow nextCursor to read more.")
after = 0
if cursor is not None:
try:
if not isinstance(cursor, str) or not cursor or len(cursor) > 2048:
raise ValueError()
decoded = json.loads(base64.b64decode(cursor + "=" * (-len(cursor) % 4), altchars=b"-_", validate=True))
version, bound_app, after = decoded
if version != 1 or bound_app != app_id or type(after) is not int or after < 1:
raise ValueError()
except (ValueError, TypeError, UnicodeError):
raise err(400, "invalid_catalog_cursor", "This catalog cursor is invalid for this App. Restart from the first page.") from None
app, fields = require_app(session, app_id, allow_orphan=True)
# An orphan snapshot contains no legacy content and cannot recreate deleted records.
if "config" in app:
migrate_legacy(session.runtime, app)
doc = _load(session.runtime)
ordered = sorted((order, key) for key, order in doc["byApp"].get(app_id, {}).items()
if order > after and key in doc["catalogs"])
page = ordered[:limit]
records = [copy.deepcopy(doc["catalogs"][key]) for _order, key in page]
for record in records:
_binding_wall(record, app, fields)
next_cursor = None
if len(ordered) > limit:
encoded = json.dumps([1, app_id, page[-1][0]], separators=(",", ":")).encode("utf-8")
next_cursor = base64.urlsafe_b64encode(encoded).decode("ascii").rstrip("=")
return {"catalogs": records, "nextCursor": next_cursor}
def _payload(payload):
if not isinstance(payload, dict) or set(payload) - _EDITABLE:
raise err(400, "invalid_catalog", "Send catalog name, pages, thumbnail or print settings. Catalog ids and audit fields are server-managed.")
return copy.deepcopy(payload)
def _validated(record, app, fields):
_text(record.get("name"), "the catalog name")
record["pages"] = normalize_pages(record.get("pages", []))
if record.get("thumbnail") is not None and not isinstance(record["thumbnail"], str):
raise err(400, "invalid_catalog", "Catalog thumbnail must be text or null. Replace the malformed value.")
_binding_wall(record, app, fields)
app_rules.validate_catalog_display({"catalogs": [record]})
return record
def create_catalog(session, app_id, payload):
raw = _payload(payload)
app, fields = require_app(session, app_id, write=True)
catalog_id = "cat_" + write_execution.new_id()
stamp = write_execution.utcnow().isoformat()
candidate = _validated({"name": "Untitled catalog", "pages": [], "thumbnail": None,
**raw, "id": catalog_id, "appId": app_id,
"updatedAt": stamp, "updatedBy": session.uname or None}, app, fields)
if catalog_id in _load(session.runtime)["catalogs"]:
raise err(409, "catalog_id_conflict", "This catalog id is already used. Create a new catalog with a new id.")
migrate_legacy(session.runtime, app)
def apply(doc):
current, fields = require_app(session, app_id, write=True)
record = _validated(copy.deepcopy(candidate), current, fields)
_insert(doc, record)
doc["appSnapshots"][app_id] = _snapshot(current)
return record
return _update(session.runtime, apply)
def update_catalog(session, catalog_id, payload):
raw = _payload(payload)
_record(session, catalog_id, write=True)
stamp = write_execution.utcnow().isoformat()
def apply(doc):
record = doc["catalogs"].get(catalog_id)
if record is None:
raise err(404, "catalog_not_found", "Catalog not found. Reopen the library before saving.")
app, fields = require_app(session, record["appId"], write=True)
record = _validated({**record, **raw, "updatedAt": stamp,
"updatedBy": session.uname or None}, app, fields)
doc["catalogs"][catalog_id] = record
doc["appSnapshots"][record["appId"]] = _snapshot(app)
return record
return _update(session.runtime, apply)
def duplicate_catalog(session, catalog_id, payload=None):
raw = {} if payload is None else _payload(payload)
if set(raw) - {"name"}:
raise err(400, "invalid_catalog", "Duplicate accepts only a name. Edit the new catalog after copying it.")
_record(session, catalog_id, write=True)
duplicate_id = "cat_" + write_execution.new_id()
stamp = write_execution.utcnow().isoformat()
def apply(doc):
source = doc["catalogs"].get(catalog_id)
if source is None:
raise err(404, "catalog_not_found", "Catalog not found. Reopen the library before copying it.")
app, fields = require_app(session, source["appId"], write=True)
copied = {key: copy.deepcopy(value) for key, value in source.items() if key in _EDITABLE}
copied.update(id=duplicate_id, appId=source["appId"],
name=raw.get("name", source["name"] + " copy"),
updatedAt=stamp, updatedBy=session.uname or None)
record = _validated(copied, app, fields)
_insert(doc, record)
doc["appSnapshots"][source["appId"]] = _snapshot(app)
return record
return _update(session.runtime, apply)
def delete_catalog(session, catalog_id):
_record(session, catalog_id, write=True)
def apply(doc):
record = doc["catalogs"].get(catalog_id)
if record is None:
raise err(404, "catalog_not_found", "Catalog not found. Reopen the library before deleting it.")
require_app(session, record["appId"], write=True)
del doc["catalogs"][catalog_id]
doc["byApp"].get(record["appId"], {}).pop(catalog_id, None)
return {"deleted": catalog_id}
return _update(session.runtime, apply)
# Long PDF exports: an MCP caller polls one of these through operation_status.
EXPORT_KEY = "catalog_exports_v1"
EXPORT_RETENTION_SECONDS = 24 * 3600
# A running export older than this lost its worker (a restart), so it reads as failed.
EXPORT_STALE_SECONDS = 30 * 60
def _export_bindings(session):
from routes_external import _operation_bindings
return _operation_bindings(session)
def _exports(raw, now):
doc = raw if isinstance(raw, dict) and isinstance(raw.get("exports"), dict) else {"exports": {}}
doc = copy.deepcopy(doc)
doc["exports"] = {key: value for key, value in doc["exports"].items()
if isinstance(value, dict) and float(value.get("retainUntil") or 0) >= now}
return doc
def start_export(session, catalog_id):
"""Record a running export bound to this caller's account and connection, and return its id."""
import secrets
import time
operation_id = secrets.token_hex(32)
now = time.time()
item = {"bindings": _export_bindings(session), "catalogId": catalog_id, "status": "running",
"startedAt": now, "retainUntil": now + EXPORT_RETENTION_SECONDS}
def apply(raw):
doc = _exports(raw, now)
doc["exports"][operation_id] = item
return doc
session.runtime.update(EXPORT_KEY, apply, flush="sync")
return operation_id
def _export_error(error):
from fastapi import HTTPException
from core import sheets
if isinstance(error, sheets.SheetRefusal):
return {"code": error.code, "message": error.message, "status": 409}
if isinstance(error, HTTPException) and isinstance(error.detail, dict):
info = error.detail.get("error") or {}
if isinstance(info.get("code"), str) and isinstance(info.get("message"), str):
return {"code": info["code"], "message": info["message"], "status": error.status_code}
return {"code": "catalog_export_failed", "status": 503,
"message": "The PDF export did not finish. Call apps.catalog_pdf again with the same catalog_id."}
def finish_export(session, operation_id, result, error=None):
"""Store the worker's outcome on its export record."""
import time
now = time.time()
def apply(raw):
doc = _exports(raw, now)
item = doc["exports"].get(operation_id)
if item is not None:
item.update(status="failed" if error is not None else "done", finishedAt=now,
retainUntil=now + EXPORT_RETENTION_SECONDS)
if error is not None:
item["error"] = _export_error(error)
else:
item["result"] = copy.deepcopy(result)
return doc
session.runtime.update(EXPORT_KEY, apply, flush="sync")
def export_status(session, operation_id):
"""Read this caller's export by id, or None when no export of theirs has that id."""
import time
now = time.time()
try:
raw = session.runtime.get(EXPORT_KEY, fresh=True)
except Exception:
# The confirmation read that follows reports an unavailable store with its own code.
return None
item = _exports(raw, now)["exports"].get(operation_id)
if not isinstance(item, dict) or item.get("bindings") != _export_bindings(session):
return None
base = {"operation_id": operation_id, "catalog_id": item.get("catalogId")}
if item.get("status") == "done":
return {**copy.deepcopy(item["result"]), **base, "state": "done", "retry": False}
if item.get("status") == "failed" or now - float(item.get("startedAt") or 0) > EXPORT_STALE_SECONDS:
error = item.get("error") or _export_error(None)
return {**base, "ok": False, "state": "failed", "retry": False, "error": copy.deepcopy(error)}
return {**base, "state": "running", "retry": True,
"message": "The PDF is still rendering. Call operation_status again shortly."}