"""Retain edit responses in the tenant store and serialize replays before applying them.""" from contextlib import contextmanager from contextvars import ContextVar from copy import deepcopy import hashlib import json import threading import time TABLE_NAME = 'edit_receipts' _OFFLINE_LOCK = threading.RLock() _ACTIVE = ContextVar('edit_receipt_operation', default=None) # Reserve half of store_pg's eight connections for callbacks across all tenant handles. _PG_SLOTS = threading.BoundedSemaphore(4) class ReceiptUnavailable(RuntimeError): """The edit cannot receive a durable acknowledgement.""" class CheckedStore: """Require synchronous acknowledged writes from the event's existing store handle.""" def __init__(self, st): self.st = st def __getattr__(self, name): return getattr(self.st, name) def _store(self): return self.st._store() if callable(getattr(self.st, '_store', None)) else self.st def update(self, name, fn, flush='sync', **kwargs): result = self.st.update(name, fn, flush='sync', **kwargs) if result is None or result is False: raise ReceiptUnavailable('The edit write was not saved. Retry this opId.') return result def op_id(value): """Keep the complete client identity without truncation or normalization.""" if not isinstance(value, str) or not value.strip(): raise ValueError('opId must be a non-empty string.') return value def operation(scope, kind, pid=None): return json.dumps([str(scope), str(kind), None if pid is None else str(pid)], separators=(',', ':')) def current_op_id(): active = _ACTIVE.get() return active['key'][3] if active is not None else None def observe_event(result): active = _ACTIVE.get() if active is not None: active['event'] = deepcopy(result) def current_event(): active = _ACTIVE.get() return deepcopy(active.get('event')) if active is not None else None def _backend(st): from core import store, store_pg bound = st._store() if callable(getattr(st, '_store', None)) else st # The default runtime can hold the module facade instead of a PgStore. if bound is store and store.backend() == 'pg': bound = store._d() if isinstance(bound, store_pg.PgStore): if getattr(st, 'store_namespace', ''): raise ReceiptUnavailable('Edit receipts require a dedicated tenant schema.') if getattr(st, 'key', bound.tenant_slug) != bound.tenant_slug: raise ReceiptUnavailable('The edit receipt store belongs to another tenant.') return PgReceipts(bound) # Detached stores retain their normal get/update contract without opening Postgres. return DocumentReceipts(st) def run_once(st, principal, action, identity, apply): """Return the first committed response and skip every later application of its identity.""" identity = op_id(identity) key = (id(st), str(principal), str(action), identity) if _ACTIVE.get() is not None and _ACTIVE.get()['key'] == key: return apply() def invoke(): token = _ACTIVE.set({'key': key}) try: return apply() finally: _ACTIVE.reset(token) result = _backend(st).run(str(principal), str(action), identity, invoke) # Replays also retry committed intents, after the outer receipt and transaction reset. try: from core import received_changes if not received_changes._uncommitted(): received_changes.dispatch_pending(st) except Exception as error: print('[edit_receipts] received-change delivery deferred: ' + type(error).__name__) return result class DocumentReceipts: """Keep offline and legacy stores hermetic, with serialization within their process.""" def __init__(self, st): self.st = st def run(self, principal, action, identity, apply): key = json.dumps([principal, action, identity], separators=(',', ':')) with _OFFLINE_LOCK: prior = (self.st.get(TABLE_NAME) or {}).get(key) if prior is not None: return deepcopy(prior['result']) result = apply() def save(current): current = dict(current or {}) current[key] = {'result': deepcopy(result)} return current stored = self.st.update(TABLE_NAME, save, flush='sync') if stored is True: stored = self.st.get(TABLE_NAME) if not isinstance(stored, dict) or key not in stored: raise ReceiptUnavailable('The edit receipt was not saved. Retry this opId.') return deepcopy(stored[key]['result']) class PgReceipts: """Use the bound tenant schema and a transaction lock for each principal and operation.""" def __init__(self, bound): from core import store_pg self.slug = bound.tenant_slug self.schema = store_pg._schema_for(self.slug) self.table = f'"{self.schema}"."{TABLE_NAME}"' def _connection(self): from core import store_pg return store_pg._pool().connection() def ensure(self): self._write_guard() with self._connection() as con: con.execute('SELECT pg_advisory_xact_lock(hashtextextended(%s, 0))', (self.table,)) con.execute(f'''CREATE TABLE IF NOT EXISTS {self.table} ( receipt_key BYTEA PRIMARY KEY, principal TEXT NOT NULL, operation TEXT NOT NULL, op_id TEXT NOT NULL, result JSONB NOT NULL, created_at TIMESTAMPTZ NOT NULL DEFAULT now())''') def _write_guard(self): from core import store_pg store_pg.check_write(self.slug, 'store edit receipt') @contextmanager def _admission(self): from core import request_timing, store_pg timeout = request_timing.remaining_budget(store_pg._pg_limits()['acquire']) if not _PG_SLOTS.acquire(timeout=timeout): request_timing.remaining_budget() raise store_pg.PostgresOperationTimeout('Edit receipt admission timed out. Retry this opId.') try: request_timing.remaining_budget() yield finally: _PG_SLOTS.release() def run(self, principal, action, identity, apply): from core import request_timing, store_pg limits = store_pg._pg_limits() deadline = time.monotonic() + limits['acquire'] + limits['statement'] with request_timing.budget_scope(deadline, store_pg.PostgresOperationTimeout): with self._admission(): return self._run(principal, action, identity, apply) def _run(self, principal, action, identity, apply): from core import request_timing, store_pg self.ensure() store = store_pg.PgStore(self.slug) lock = json.dumps([self.table, principal, action, identity], separators=(',', ':')) receipt_key = hashlib.sha256(lock.encode('utf-8')).digest() # Poll without keeping a pool connection that the current writer may need. while True: request_timing.remaining_budget() with self._connection() as con: acquired = con.execute( 'SELECT pg_try_advisory_xact_lock(hashtextextended(%s, 0))', (lock,)).fetchone()[0] if acquired: # The callback may leave this transaction idle, within the existing database budget. idle_ms = max(1, int(request_timing.remaining_budget( store_pg._pg_limits()['idle']) * 1000)) con.execute("SELECT set_config('idle_in_transaction_session_timeout', %s, true)", (f'{idle_ms}ms',)) prior = con.execute(f'SELECT result FROM {self.table} ' 'WHERE receipt_key=%s', (receipt_key,)).fetchone() if prior is not None: result = deepcopy(prior[0]) break with store.transaction(con): result = apply() request_timing.remaining_budget() con.execute(f'INSERT INTO {self.table} ' '(receipt_key, principal, operation, op_id, result) ' 'VALUES (%s, %s, %s, %s, %s::jsonb)', (receipt_key, principal, action, identity, json.dumps(result))) # The connection context commits before the caller receives the answer. break time.sleep(request_timing.remaining_budget(0.01)) # Deliver only committed activity. The durable intent survives a delivery outage. from core import activity_log try: activity_log.drain(store) except Exception as error: print('[edit_receipts] activity delivery deferred: ' + type(error).__name__) return deepcopy(result)