File size: 9,086 Bytes
7f72224 82f0f29 7f72224 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 | """Retain edit responses in the tenant store and serialize replays before applying them."""
from contextlib import contextmanager
from contextvars import ContextVar
from copy import deepcopy
import hashlib
import json
import threading
import time
TABLE_NAME = 'edit_receipts'
_OFFLINE_LOCK = threading.RLock()
_ACTIVE = ContextVar('edit_receipt_operation', default=None)
# Reserve half of store_pg's eight connections for callbacks across all tenant handles.
_PG_SLOTS = threading.BoundedSemaphore(4)
class ReceiptUnavailable(RuntimeError):
"""The edit cannot receive a durable acknowledgement."""
class CheckedStore:
"""Require synchronous acknowledged writes from the event's existing store handle."""
def __init__(self, st):
self.st = st
def __getattr__(self, name):
return getattr(self.st, name)
def _store(self):
return self.st._store() if callable(getattr(self.st, '_store', None)) else self.st
def update(self, name, fn, flush='sync', **kwargs):
result = self.st.update(name, fn, flush='sync', **kwargs)
if result is None or result is False:
raise ReceiptUnavailable('The edit write was not saved. Retry this opId.')
return result
def op_id(value):
"""Keep the complete client identity without truncation or normalization."""
if not isinstance(value, str) or not value.strip():
raise ValueError('opId must be a non-empty string.')
return value
def operation(scope, kind, pid=None):
return json.dumps([str(scope), str(kind), None if pid is None else str(pid)],
separators=(',', ':'))
def current_op_id():
active = _ACTIVE.get()
return active['key'][3] if active is not None else None
def observe_event(result):
active = _ACTIVE.get()
if active is not None:
active['event'] = deepcopy(result)
def current_event():
active = _ACTIVE.get()
return deepcopy(active.get('event')) if active is not None else None
def _backend(st):
from core import store, store_pg
bound = st._store() if callable(getattr(st, '_store', None)) else st
# The default runtime can hold the module facade instead of a PgStore.
if bound is store and store.backend() == 'pg':
bound = store._d()
if isinstance(bound, store_pg.PgStore):
if getattr(st, 'store_namespace', ''):
raise ReceiptUnavailable('Edit receipts require a dedicated tenant schema.')
if getattr(st, 'key', bound.tenant_slug) != bound.tenant_slug:
raise ReceiptUnavailable('The edit receipt store belongs to another tenant.')
return PgReceipts(bound)
# Detached stores retain their normal get/update contract without opening Postgres.
return DocumentReceipts(st)
def run_once(st, principal, action, identity, apply):
"""Return the first committed response and skip every later application of its identity."""
identity = op_id(identity)
key = (id(st), str(principal), str(action), identity)
if _ACTIVE.get() is not None and _ACTIVE.get()['key'] == key:
return apply()
def invoke():
token = _ACTIVE.set({'key': key})
try:
return apply()
finally:
_ACTIVE.reset(token)
result = _backend(st).run(str(principal), str(action), identity, invoke)
# Replays also retry committed intents, after the outer receipt and transaction reset.
try:
from core import received_changes
if not received_changes._uncommitted():
received_changes.dispatch_pending(st)
except Exception as error:
print('[edit_receipts] received-change delivery deferred: ' + type(error).__name__)
return result
class DocumentReceipts:
"""Keep offline and legacy stores hermetic, with serialization within their process."""
def __init__(self, st):
self.st = st
def run(self, principal, action, identity, apply):
key = json.dumps([principal, action, identity], separators=(',', ':'))
with _OFFLINE_LOCK:
prior = (self.st.get(TABLE_NAME) or {}).get(key)
if prior is not None:
return deepcopy(prior['result'])
result = apply()
def save(current):
current = dict(current or {})
current[key] = {'result': deepcopy(result)}
return current
stored = self.st.update(TABLE_NAME, save, flush='sync')
if stored is True:
stored = self.st.get(TABLE_NAME)
if not isinstance(stored, dict) or key not in stored:
raise ReceiptUnavailable('The edit receipt was not saved. Retry this opId.')
return deepcopy(stored[key]['result'])
class PgReceipts:
"""Use the bound tenant schema and a transaction lock for each principal and operation."""
def __init__(self, bound):
from core import store_pg
self.slug = bound.tenant_slug
self.schema = store_pg._schema_for(self.slug)
self.table = f'"{self.schema}"."{TABLE_NAME}"'
def _connection(self):
from core import store_pg
return store_pg._pool().connection()
def ensure(self):
self._write_guard()
with self._connection() as con:
con.execute('SELECT pg_advisory_xact_lock(hashtextextended(%s, 0))', (self.table,))
con.execute(f'''CREATE TABLE IF NOT EXISTS {self.table} (
receipt_key BYTEA PRIMARY KEY,
principal TEXT NOT NULL, operation TEXT NOT NULL, op_id TEXT NOT NULL,
result JSONB NOT NULL, created_at TIMESTAMPTZ NOT NULL DEFAULT now())''')
def _write_guard(self):
from core import store_pg
store_pg.check_write(self.slug, 'store edit receipt')
@contextmanager
def _admission(self):
from core import request_timing, store_pg
timeout = request_timing.remaining_budget(store_pg._pg_limits()['acquire'])
if not _PG_SLOTS.acquire(timeout=timeout):
request_timing.remaining_budget()
raise store_pg.PostgresOperationTimeout('Edit receipt admission timed out. Retry this opId.')
try:
request_timing.remaining_budget()
yield
finally:
_PG_SLOTS.release()
def run(self, principal, action, identity, apply):
from core import request_timing, store_pg
limits = store_pg._pg_limits()
deadline = time.monotonic() + limits['acquire'] + limits['statement']
with request_timing.budget_scope(deadline, store_pg.PostgresOperationTimeout):
with self._admission():
return self._run(principal, action, identity, apply)
def _run(self, principal, action, identity, apply):
from core import request_timing, store_pg
self.ensure()
store = store_pg.PgStore(self.slug)
lock = json.dumps([self.table, principal, action, identity], separators=(',', ':'))
receipt_key = hashlib.sha256(lock.encode('utf-8')).digest()
# Poll without keeping a pool connection that the current writer may need.
while True:
request_timing.remaining_budget()
with self._connection() as con:
acquired = con.execute(
'SELECT pg_try_advisory_xact_lock(hashtextextended(%s, 0))',
(lock,)).fetchone()[0]
if acquired:
# The callback may leave this transaction idle, within the existing database budget.
idle_ms = max(1, int(request_timing.remaining_budget(
store_pg._pg_limits()['idle']) * 1000))
con.execute("SELECT set_config('idle_in_transaction_session_timeout', %s, true)",
(f'{idle_ms}ms',))
prior = con.execute(f'SELECT result FROM {self.table} '
'WHERE receipt_key=%s', (receipt_key,)).fetchone()
if prior is not None:
result = deepcopy(prior[0])
break
with store.transaction(con):
result = apply()
request_timing.remaining_budget()
con.execute(f'INSERT INTO {self.table} '
'(receipt_key, principal, operation, op_id, result) '
'VALUES (%s, %s, %s, %s, %s::jsonb)',
(receipt_key, principal, action, identity, json.dumps(result)))
# The connection context commits before the caller receives the answer.
break
time.sleep(request_timing.remaining_budget(0.01))
# Deliver only committed activity. The durable intent survives a delivery outage.
from core import activity_log
try:
activity_log.drain(store)
except Exception as error:
print('[edit_receipts] activity delivery deferred: ' + type(error).__name__)
return deepcopy(result)
|