Spaces:
Running on CPU Upgrade
Running on CPU Upgrade
Download api/catalog_store.py from fsanyoto/runloopable: direct link, hf CLI and curl.
- Browser
- Download file 24.3 kB
-
https://huggingface.co/spaces/fsanyoto/runloopable/resolve/main/api/catalog_store.py
- Command line
-
hf download hf://spaces/fsanyoto/runloopable/api/catalog_store.py
-
curl -L -o catalog_store.py https://huggingface.co/spaces/fsanyoto/runloopable/resolve/main/api/catalog_store.py
24.3 kB
| """Tenant catalog records with atomic App indexes and lossless legacy migration.""" | |
| from __future__ import annotations | |
| import base64 | |
| import copy | |
| import hashlib | |
| import json | |
| import app_rules | |
| import app_templates | |
| import application_registry as applications | |
| from core import write_execution | |
| from deps import err | |
| STORE_KEY = "catalog_records_v1" | |
| DEFAULT_LIMIT = 100 | |
| MAX_LIMIT = 200 | |
| _ALIASES = {"intro": "section", "gallery": "grid"} | |
| _EDITABLE = {"name", "pages", "thumbnail", "paper", "orientation", "quality", "brand", "fields"} | |
| _SNAPSHOT_KEYS = {"id", "tenant", "templateKind", "sourceDatabase", "compatibleDatabases", | |
| "fieldMap", "navPlacement", "createdBy", "share"} | |
| def _document(raw): | |
| if raw is None or raw == {}: | |
| return {"schema": 1, "catalogs": {}, "byApp": {}, "migrations": {}, | |
| "appSnapshots": {}, "nextOrder": 1} | |
| if (not isinstance(raw, dict) or raw.get("schema") != 1 | |
| or any(not isinstance(raw.get(key), dict) for key in | |
| ("catalogs", "byApp", "migrations", "appSnapshots")) | |
| or type(raw.get("nextOrder")) is not int or raw["nextOrder"] < 1): | |
| raise err(503, "catalog_store_invalid", "Catalog storage is invalid. Ask an administrator to restore it.") | |
| return copy.deepcopy(raw) | |
| def _load(runtime): | |
| return _document(applications._read_strict(runtime, STORE_KEY)) | |
| def _update(runtime, change): | |
| result = {} | |
| def apply(raw): | |
| doc = _document(raw) | |
| result["value"] = copy.deepcopy(change(doc)) | |
| return doc | |
| # TenantRuntime.update is the atomic JSON document door, also captured by write previews. | |
| saved = runtime.update(STORE_KEY, apply, flush="sync") | |
| if saved is False or saved is None: | |
| raise err(503, "catalog_save_failed", "The catalog save was not confirmed. Reload the library before trying again.") | |
| return result["value"] | |
| def _identity(*parts): | |
| encoded = json.dumps(parts, ensure_ascii=True, separators=(",", ":")) | |
| return "cat_legacy_" + hashlib.sha256(encoded.encode("utf-8")).hexdigest()[:32] | |
| def _text(value, label): | |
| if not isinstance(value, str) or not value.strip(): | |
| raise err(400, "invalid_catalog", f"Give {label} a nonempty text value.") | |
| if "\u2013" in value or "\u2014" in value: | |
| raise err(400, "invalid_catalog", f"Replace long dashes in {label} with plain punctuation.") | |
| return value | |
| def normalize_pages(pages, *, legacy=False, identity=""): | |
| """Keep page order and source payload while adding the shared page contract.""" | |
| if not isinstance(pages, list): | |
| raise err(400, "invalid_catalog", "Catalog pages must be a list. Send an empty list for a new catalog.") | |
| out, seen = [], set() | |
| for index, raw in enumerate(pages): | |
| if not isinstance(raw, dict): | |
| raise err(400, "invalid_catalog", "Each catalog page must be an object. Replace the malformed page.") | |
| page = copy.deepcopy(raw) | |
| page_id = page.get("id") | |
| if legacy and (not isinstance(page_id, str) or not page_id): | |
| page_id = _identity(identity, "page", index) | |
| _text(page_id, "the page id") | |
| if page_id in seen: | |
| if not legacy: | |
| raise err(400, "invalid_catalog", "Page ids must be unique. Give the repeated page a new id.") | |
| page_id = _identity(identity, "page", index, page_id) | |
| seen.add(page_id) | |
| kind = page.get("kind", "section" if legacy else None) | |
| _text(kind, "the page kind") | |
| kind = _ALIASES.get(kind, kind) | |
| template = page.get("template", kind + "-stacked") | |
| _text(template, "the page template") | |
| bindings = page.get("bindings", {}) | |
| if not isinstance(bindings, dict): | |
| raise err(400, "invalid_catalog", "Page bindings must be an object. Replace the malformed bindings.") | |
| bindings = copy.deepcopy(bindings) | |
| for key in ("title", "body", "products", "codes", "imageCode", "layout"): | |
| if key in raw: | |
| bindings.setdefault(key, copy.deepcopy(raw[key])) | |
| if legacy: | |
| # The legacy carrier is archival content, never an authoritative product value. | |
| bindings.setdefault("legacy", copy.deepcopy(raw)) | |
| for key, value in raw.items(): | |
| if key not in {"id", "kind", "template", "bindings", "design", "price", "prices"}: | |
| bindings.setdefault(key, copy.deepcopy(value)) | |
| for key in ("title", "body"): | |
| if key in bindings and not isinstance(bindings[key], str): | |
| raise err(400, "invalid_catalog", f"Page {key} must be text. Replace the malformed value.") | |
| products = bindings.get("products", bindings.get("codes", [])) | |
| if (not isinstance(products, list) | |
| or any(not isinstance(code, str) or not code.strip() for code in products)): | |
| raise err(400, "invalid_catalog", "Bind products to a list of source key strings. Fetch prices from the source.") | |
| if "products" in bindings or "codes" in bindings: | |
| bindings["products"] = copy.deepcopy(products) | |
| page.update(id=page_id, kind=kind, template=template, bindings=bindings) | |
| out.append(page) | |
| return out | |
| def _snapshot(app): | |
| # R33: a snapshot keeps a source or field map only when it differs from the C1 binding. | |
| kept = _SNAPSHOT_KEYS - {"sourceDatabase", "fieldMap"} | |
| row = {key: copy.deepcopy(value) for key, value in app.items() if key in kept} | |
| row.update(copy.deepcopy(app_templates.own_binding(app))) | |
| return row | |
| def migrate_snapshots(runtime, app_ids): | |
| """R33: drop a C1 binding from the named Apps' catalog snapshots. Writes only on a change.""" | |
| wanted = set(app_ids) | |
| def stale(doc): | |
| return sorted(app_id for app_id, row in doc["appSnapshots"].items() | |
| if app_id in wanted and isinstance(row, dict) | |
| and applications._c1_rewrite(row) is not None) | |
| if not stale(_load(runtime)): | |
| return "unchanged" | |
| def apply(doc): | |
| changed = stale(doc) | |
| for app_id in changed: | |
| row = applications._c1_rewrite(doc["appSnapshots"][app_id]) | |
| doc["appSnapshots"][app_id] = {key: value for key, value in row.items() if key in _SNAPSHOT_KEYS} | |
| return changed | |
| return f"{len(_update(runtime, apply))} rewritten" | |
| def _insert(doc, record): | |
| catalog_id, app_id = record["id"], record["appId"] | |
| if catalog_id == "templates" or catalog_id in doc["catalogs"]: | |
| raise err(409, "catalog_id_conflict", "This catalog id is already used. Create a new catalog with a new id.") | |
| doc["catalogs"][catalog_id] = copy.deepcopy(record) | |
| doc["byApp"].setdefault(app_id, {})[catalog_id] = doc["nextOrder"] | |
| doc["nextOrder"] += 1 | |
| def migrate_legacy(runtime, app): | |
| """Copy plural display.catalogs once without changing either legacy display key.""" | |
| if app.get("templateKind") != "catalog": | |
| return {} | |
| app_id, snapshot = app["id"], _snapshot(app) | |
| existing = _load(runtime) | |
| if app_id in existing["migrations"] and existing["appSnapshots"].get(app_id) == snapshot: | |
| return copy.deepcopy(existing["migrations"][app_id]) | |
| display = (app.get("config") or {}).get("display") or {} | |
| legacy = display.get("catalogs", []) | |
| if not isinstance(legacy, list): | |
| raise err(409, "catalog_migration_invalid", "Legacy catalogs must be a list. Restore the App content before deleting it.") | |
| def apply(doc): | |
| doc["appSnapshots"][app_id] = snapshot | |
| if app_id in doc["migrations"]: | |
| return doc["migrations"][app_id] | |
| migrated = {} | |
| for index, raw in enumerate(legacy): | |
| if not isinstance(raw, dict): | |
| raise err(409, "catalog_migration_invalid", "A legacy catalog is malformed. Restore its content before deleting the App.") | |
| source_id = raw.get("id") | |
| catalog_id = source_id if isinstance(source_id, str) and source_id else _identity(app_id, index) | |
| attempt = 0 | |
| while catalog_id == "templates" or catalog_id in doc["catalogs"]: | |
| catalog_id = _identity(app_id, index, source_id, attempt) | |
| attempt += 1 | |
| record = copy.deepcopy(raw) | |
| stamp = raw.get("updatedAt") or app.get("updatedAt") or app.get("createdAt") | |
| author = raw.get("updatedBy", app.get("updatedBy", app.get("createdBy"))) | |
| record.update(id=catalog_id, appId=app_id, | |
| name=raw.get("name") if isinstance(raw.get("name"), str) else "Untitled catalog", | |
| pages=normalize_pages(raw.get("pages", []), legacy=True, | |
| identity=_identity(app_id, index)), | |
| thumbnail=raw.get("thumbnail") if isinstance(raw.get("thumbnail"), str) else None, | |
| updatedAt=stamp if isinstance(stamp, str) else "1970-01-01T00:00:00+00:00", | |
| updatedBy=author if isinstance(author, str) else None) | |
| # A new colour or design rule must never erase existing authored content. | |
| _insert(doc, record) | |
| migrated[str(index)] = catalog_id | |
| doc["migrations"][app_id] = migrated | |
| return migrated | |
| return _update(runtime, apply) | |
| def require_app(session, app_id, *, write=False, allow_orphan=False): | |
| """Admit the App and its current source schema before returning any catalog content.""" | |
| from core import perm_scope | |
| from routes_external import _assert_database_scope, _assert_write_scope | |
| runtime = session.runtime | |
| bound = getattr(runtime, "key", None) or getattr(getattr(runtime, "tenant", None), "key", None) | |
| if bound is not None and bound != session.tenant: | |
| raise err(403, "catalog_tenant_forbidden", "This catalog belongs to another workspace. Open the correct workspace.") | |
| if not runtime.available(): | |
| raise err(503, "store_unavailable", "Catalog storage is unavailable. Try again when the store is ready.") | |
| app = applications.load(runtime, strict=True)["applications"].get(app_id) | |
| if app is None and allow_orphan and session.admin and not write: | |
| app = _load(runtime)["appSnapshots"].get(app_id) | |
| if (not isinstance(app, dict) or app.get("tenant") != session.tenant | |
| or app.get("templateKind") != "catalog" or app.get("retired") is True): | |
| raise err(404, "catalog_app_not_found", "Catalog App not found. Open an existing Catalog App.") | |
| may = applications.may_edit if write else applications.may_read | |
| if not may(app, session.uname, session.admin): | |
| raise err(403, "catalog_forbidden", "Your account cannot access this Catalog App. Ask its owner for access.") | |
| # R33: Catalog reads Odoo products through its C1 binding unless the record holds its own. | |
| source = app_templates.bound(app).get("sourceDatabase") | |
| _assert_database_scope(session, source) | |
| session.require(source) | |
| if not perm_scope.may_read(session.user, source, st=runtime): | |
| raise err(403, "catalog_source_forbidden", "The catalog source is unavailable to your account. Ask an administrator for access.") | |
| if write: | |
| _assert_write_scope(session, "catalogs") | |
| contracts = applications.chart_field_contracts(session, [source]) | |
| fields = applications._contract_fields(source, field_contracts=contracts) | |
| # Projected C1 fields retain their optional roles; authored legacy maps stay strict. | |
| if not app_templates.own_binding(app): | |
| app = {key: value for key, value in app.items() if key not in {"sourceDatabase", "fieldMap"}} | |
| try: | |
| app = applications.bound_application(app, fields) | |
| except applications.ApplicationError: | |
| raise err(403, "catalog_fields_forbidden", "The Catalog App binds unavailable fields. Ask its owner to rebind readable fields.") from None | |
| return app, fields | |
| def _binding_wall(record, app, fields): | |
| """Keep source and field references within the App's permission-filtered schema.""" | |
| maps = [record.get("fields", {})] | |
| for page in record.get("pages", []): | |
| bindings = page.get("bindings", {}) | |
| maps.extend(bindings.get(key, {}) for key in ("fields", "fieldMap")) | |
| for key, value in bindings.items(): | |
| if key in {"database", "sourceDatabase"} and value != app["sourceDatabase"]: | |
| raise err(403, "catalog_source_forbidden", "Page bindings must use the App source. Rebind this page to its App.") | |
| if key.endswith("Field") or key in {"price", "prices", "unitPrice", "unit_price"}: | |
| maps.append({key: value}) | |
| for mapping in maps: | |
| if not isinstance(mapping, dict): | |
| raise err(400, "invalid_catalog", "Field bindings must be objects. Bind each role to a readable source field.") | |
| for value in mapping.values(): | |
| if not isinstance(value, str) or value not in fields: | |
| raise err(403, "catalog_fields_forbidden", "Bind catalog fields to readable source keys. Fetch product prices from the source.") | |
| def _record(session, catalog_id, *, write=False): | |
| record = _load(session.runtime)["catalogs"].get(catalog_id) | |
| if record is None: | |
| apps = applications.load(session.runtime, strict=True)["applications"] | |
| for app in apps.values(): | |
| if not isinstance(app, dict): | |
| continue | |
| legacy = ((app.get("config") or {}).get("display") or {}).get("catalogs", []) | |
| if isinstance(legacy, list) and any(isinstance(row, dict) and row.get("id") == catalog_id for row in legacy): | |
| require_app(session, app["id"], write=write) | |
| migrate_legacy(session.runtime, app) | |
| record = _load(session.runtime)["catalogs"].get(catalog_id) | |
| break | |
| if record is None: | |
| raise err(404, "catalog_not_found", "Catalog not found. Open a catalog from the library.") | |
| app, fields = require_app(session, record["appId"], write=write, allow_orphan=True) | |
| _binding_wall(record, app, fields) | |
| return record, app, fields | |
| def read_catalog(session, catalog_id): | |
| return copy.deepcopy(_record(session, catalog_id)[0]) | |
| def list_catalogs(session, app_id, *, cursor=None, limit=DEFAULT_LIMIT): | |
| """Page insertion order with an App-bound exclusive sequence cursor, default 100 and maximum 200 per read.""" | |
| if type(limit) is not int or not 1 <= limit <= MAX_LIMIT: | |
| raise err(400, "invalid_catalog_limit", "Use a catalog page limit from 1 to 200. Follow nextCursor to read more.") | |
| after = 0 | |
| if cursor is not None: | |
| try: | |
| if not isinstance(cursor, str) or not cursor or len(cursor) > 2048: | |
| raise ValueError() | |
| decoded = json.loads(base64.b64decode(cursor + "=" * (-len(cursor) % 4), altchars=b"-_", validate=True)) | |
| version, bound_app, after = decoded | |
| if version != 1 or bound_app != app_id or type(after) is not int or after < 1: | |
| raise ValueError() | |
| except (ValueError, TypeError, UnicodeError): | |
| raise err(400, "invalid_catalog_cursor", "This catalog cursor is invalid for this App. Restart from the first page.") from None | |
| app, fields = require_app(session, app_id, allow_orphan=True) | |
| # An orphan snapshot contains no legacy content and cannot recreate deleted records. | |
| if "config" in app: | |
| migrate_legacy(session.runtime, app) | |
| doc = _load(session.runtime) | |
| ordered = sorted((order, key) for key, order in doc["byApp"].get(app_id, {}).items() | |
| if order > after and key in doc["catalogs"]) | |
| page = ordered[:limit] | |
| records = [copy.deepcopy(doc["catalogs"][key]) for _order, key in page] | |
| for record in records: | |
| _binding_wall(record, app, fields) | |
| next_cursor = None | |
| if len(ordered) > limit: | |
| encoded = json.dumps([1, app_id, page[-1][0]], separators=(",", ":")).encode("utf-8") | |
| next_cursor = base64.urlsafe_b64encode(encoded).decode("ascii").rstrip("=") | |
| return {"catalogs": records, "nextCursor": next_cursor} | |
| def _payload(payload): | |
| if not isinstance(payload, dict) or set(payload) - _EDITABLE: | |
| raise err(400, "invalid_catalog", "Send catalog name, pages, thumbnail or print settings. Catalog ids and audit fields are server-managed.") | |
| return copy.deepcopy(payload) | |
| def _validated(record, app, fields): | |
| _text(record.get("name"), "the catalog name") | |
| record["pages"] = normalize_pages(record.get("pages", [])) | |
| if record.get("thumbnail") is not None and not isinstance(record["thumbnail"], str): | |
| raise err(400, "invalid_catalog", "Catalog thumbnail must be text or null. Replace the malformed value.") | |
| _binding_wall(record, app, fields) | |
| app_rules.validate_catalog_display({"catalogs": [record]}) | |
| return record | |
| def create_catalog(session, app_id, payload): | |
| raw = _payload(payload) | |
| app, fields = require_app(session, app_id, write=True) | |
| catalog_id = "cat_" + write_execution.new_id() | |
| stamp = write_execution.utcnow().isoformat() | |
| candidate = _validated({"name": "Untitled catalog", "pages": [], "thumbnail": None, | |
| **raw, "id": catalog_id, "appId": app_id, | |
| "updatedAt": stamp, "updatedBy": session.uname or None}, app, fields) | |
| if catalog_id in _load(session.runtime)["catalogs"]: | |
| raise err(409, "catalog_id_conflict", "This catalog id is already used. Create a new catalog with a new id.") | |
| migrate_legacy(session.runtime, app) | |
| def apply(doc): | |
| current, fields = require_app(session, app_id, write=True) | |
| record = _validated(copy.deepcopy(candidate), current, fields) | |
| _insert(doc, record) | |
| doc["appSnapshots"][app_id] = _snapshot(current) | |
| return record | |
| return _update(session.runtime, apply) | |
| def update_catalog(session, catalog_id, payload): | |
| raw = _payload(payload) | |
| _record(session, catalog_id, write=True) | |
| stamp = write_execution.utcnow().isoformat() | |
| def apply(doc): | |
| record = doc["catalogs"].get(catalog_id) | |
| if record is None: | |
| raise err(404, "catalog_not_found", "Catalog not found. Reopen the library before saving.") | |
| app, fields = require_app(session, record["appId"], write=True) | |
| record = _validated({**record, **raw, "updatedAt": stamp, | |
| "updatedBy": session.uname or None}, app, fields) | |
| doc["catalogs"][catalog_id] = record | |
| doc["appSnapshots"][record["appId"]] = _snapshot(app) | |
| return record | |
| return _update(session.runtime, apply) | |
| def duplicate_catalog(session, catalog_id, payload=None): | |
| raw = {} if payload is None else _payload(payload) | |
| if set(raw) - {"name"}: | |
| raise err(400, "invalid_catalog", "Duplicate accepts only a name. Edit the new catalog after copying it.") | |
| _record(session, catalog_id, write=True) | |
| duplicate_id = "cat_" + write_execution.new_id() | |
| stamp = write_execution.utcnow().isoformat() | |
| def apply(doc): | |
| source = doc["catalogs"].get(catalog_id) | |
| if source is None: | |
| raise err(404, "catalog_not_found", "Catalog not found. Reopen the library before copying it.") | |
| app, fields = require_app(session, source["appId"], write=True) | |
| copied = {key: copy.deepcopy(value) for key, value in source.items() if key in _EDITABLE} | |
| copied.update(id=duplicate_id, appId=source["appId"], | |
| name=raw.get("name", source["name"] + " copy"), | |
| updatedAt=stamp, updatedBy=session.uname or None) | |
| record = _validated(copied, app, fields) | |
| _insert(doc, record) | |
| doc["appSnapshots"][source["appId"]] = _snapshot(app) | |
| return record | |
| return _update(session.runtime, apply) | |
| def delete_catalog(session, catalog_id): | |
| _record(session, catalog_id, write=True) | |
| def apply(doc): | |
| record = doc["catalogs"].get(catalog_id) | |
| if record is None: | |
| raise err(404, "catalog_not_found", "Catalog not found. Reopen the library before deleting it.") | |
| require_app(session, record["appId"], write=True) | |
| del doc["catalogs"][catalog_id] | |
| doc["byApp"].get(record["appId"], {}).pop(catalog_id, None) | |
| return {"deleted": catalog_id} | |
| return _update(session.runtime, apply) | |
| # Long PDF exports: an MCP caller polls one of these through operation_status. | |
| EXPORT_KEY = "catalog_exports_v1" | |
| EXPORT_RETENTION_SECONDS = 24 * 3600 | |
| # A running export older than this lost its worker (a restart), so it reads as failed. | |
| EXPORT_STALE_SECONDS = 30 * 60 | |
| def _export_bindings(session): | |
| from routes_external import _operation_bindings | |
| return _operation_bindings(session) | |
| def _exports(raw, now): | |
| doc = raw if isinstance(raw, dict) and isinstance(raw.get("exports"), dict) else {"exports": {}} | |
| doc = copy.deepcopy(doc) | |
| doc["exports"] = {key: value for key, value in doc["exports"].items() | |
| if isinstance(value, dict) and float(value.get("retainUntil") or 0) >= now} | |
| return doc | |
| def start_export(session, catalog_id): | |
| """Record a running export bound to this caller's account and connection, and return its id.""" | |
| import secrets | |
| import time | |
| operation_id = secrets.token_hex(32) | |
| now = time.time() | |
| item = {"bindings": _export_bindings(session), "catalogId": catalog_id, "status": "running", | |
| "startedAt": now, "retainUntil": now + EXPORT_RETENTION_SECONDS} | |
| def apply(raw): | |
| doc = _exports(raw, now) | |
| doc["exports"][operation_id] = item | |
| return doc | |
| session.runtime.update(EXPORT_KEY, apply, flush="sync") | |
| return operation_id | |
| def _export_error(error): | |
| from fastapi import HTTPException | |
| from core import sheets | |
| if isinstance(error, sheets.SheetRefusal): | |
| return {"code": error.code, "message": error.message, "status": 409} | |
| if isinstance(error, HTTPException) and isinstance(error.detail, dict): | |
| info = error.detail.get("error") or {} | |
| if isinstance(info.get("code"), str) and isinstance(info.get("message"), str): | |
| return {"code": info["code"], "message": info["message"], "status": error.status_code} | |
| return {"code": "catalog_export_failed", "status": 503, | |
| "message": "The PDF export did not finish. Call apps.catalog_pdf again with the same catalog_id."} | |
| def finish_export(session, operation_id, result, error=None): | |
| """Store the worker's outcome on its export record.""" | |
| import time | |
| now = time.time() | |
| def apply(raw): | |
| doc = _exports(raw, now) | |
| item = doc["exports"].get(operation_id) | |
| if item is not None: | |
| item.update(status="failed" if error is not None else "done", finishedAt=now, | |
| retainUntil=now + EXPORT_RETENTION_SECONDS) | |
| if error is not None: | |
| item["error"] = _export_error(error) | |
| else: | |
| item["result"] = copy.deepcopy(result) | |
| return doc | |
| session.runtime.update(EXPORT_KEY, apply, flush="sync") | |
| def export_status(session, operation_id): | |
| """Read this caller's export by id, or None when no export of theirs has that id.""" | |
| import time | |
| now = time.time() | |
| try: | |
| raw = session.runtime.get(EXPORT_KEY, fresh=True) | |
| except Exception: | |
| # The confirmation read that follows reports an unavailable store with its own code. | |
| return None | |
| item = _exports(raw, now)["exports"].get(operation_id) | |
| if not isinstance(item, dict) or item.get("bindings") != _export_bindings(session): | |
| return None | |
| base = {"operation_id": operation_id, "catalog_id": item.get("catalogId")} | |
| if item.get("status") == "done": | |
| return {**copy.deepcopy(item["result"]), **base, "state": "done", "retry": False} | |
| if item.get("status") == "failed" or now - float(item.get("startedAt") or 0) > EXPORT_STALE_SECONDS: | |
| error = item.get("error") or _export_error(None) | |
| return {**base, "ok": False, "state": "failed", "retry": False, "error": copy.deepcopy(error)} | |
| return {**base, "state": "running", "retry": True, | |
| "message": "The PDF is still rendering. Call operation_status again shortly."} | |