File size: 9,086 Bytes
7f72224
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
82f0f29
 
 
 
 
 
 
 
 
7f72224
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
"""Retain edit responses in the tenant store and serialize replays before applying them."""
from contextlib import contextmanager
from contextvars import ContextVar
from copy import deepcopy
import hashlib
import json
import threading
import time


TABLE_NAME = 'edit_receipts'
_OFFLINE_LOCK = threading.RLock()
_ACTIVE = ContextVar('edit_receipt_operation', default=None)
# Reserve half of store_pg's eight connections for callbacks across all tenant handles.
_PG_SLOTS = threading.BoundedSemaphore(4)


class ReceiptUnavailable(RuntimeError):
    """The edit cannot receive a durable acknowledgement."""


class CheckedStore:
    """Require synchronous acknowledged writes from the event's existing store handle."""

    def __init__(self, st):
        self.st = st

    def __getattr__(self, name):
        return getattr(self.st, name)

    def _store(self):
        return self.st._store() if callable(getattr(self.st, '_store', None)) else self.st

    def update(self, name, fn, flush='sync', **kwargs):
        result = self.st.update(name, fn, flush='sync', **kwargs)
        if result is None or result is False:
            raise ReceiptUnavailable('The edit write was not saved. Retry this opId.')
        return result


def op_id(value):
    """Keep the complete client identity without truncation or normalization."""
    if not isinstance(value, str) or not value.strip():
        raise ValueError('opId must be a non-empty string.')
    return value


def operation(scope, kind, pid=None):
    return json.dumps([str(scope), str(kind), None if pid is None else str(pid)],
                      separators=(',', ':'))


def current_op_id():
    active = _ACTIVE.get()
    return active['key'][3] if active is not None else None


def observe_event(result):
    active = _ACTIVE.get()
    if active is not None:
        active['event'] = deepcopy(result)


def current_event():
    active = _ACTIVE.get()
    return deepcopy(active.get('event')) if active is not None else None


def _backend(st):
    from core import store, store_pg
    bound = st._store() if callable(getattr(st, '_store', None)) else st
    # The default runtime can hold the module facade instead of a PgStore.
    if bound is store and store.backend() == 'pg':
        bound = store._d()
    if isinstance(bound, store_pg.PgStore):
        if getattr(st, 'store_namespace', ''):
            raise ReceiptUnavailable('Edit receipts require a dedicated tenant schema.')
        if getattr(st, 'key', bound.tenant_slug) != bound.tenant_slug:
            raise ReceiptUnavailable('The edit receipt store belongs to another tenant.')
        return PgReceipts(bound)
    # Detached stores retain their normal get/update contract without opening Postgres.
    return DocumentReceipts(st)


def run_once(st, principal, action, identity, apply):
    """Return the first committed response and skip every later application of its identity."""
    identity = op_id(identity)
    key = (id(st), str(principal), str(action), identity)
    if _ACTIVE.get() is not None and _ACTIVE.get()['key'] == key:
        return apply()

    def invoke():
        token = _ACTIVE.set({'key': key})
        try:
            return apply()
        finally:
            _ACTIVE.reset(token)

    result = _backend(st).run(str(principal), str(action), identity, invoke)
    # Replays also retry committed intents, after the outer receipt and transaction reset.
    try:
        from core import received_changes
        if not received_changes._uncommitted():
            received_changes.dispatch_pending(st)
    except Exception as error:
        print('[edit_receipts] received-change delivery deferred: ' + type(error).__name__)
    return result


class DocumentReceipts:
    """Keep offline and legacy stores hermetic, with serialization within their process."""

    def __init__(self, st):
        self.st = st

    def run(self, principal, action, identity, apply):
        key = json.dumps([principal, action, identity], separators=(',', ':'))
        with _OFFLINE_LOCK:
            prior = (self.st.get(TABLE_NAME) or {}).get(key)
            if prior is not None:
                return deepcopy(prior['result'])
            result = apply()

            def save(current):
                current = dict(current or {})
                current[key] = {'result': deepcopy(result)}
                return current

            stored = self.st.update(TABLE_NAME, save, flush='sync')
            if stored is True:
                stored = self.st.get(TABLE_NAME)
            if not isinstance(stored, dict) or key not in stored:
                raise ReceiptUnavailable('The edit receipt was not saved. Retry this opId.')
            return deepcopy(stored[key]['result'])


class PgReceipts:
    """Use the bound tenant schema and a transaction lock for each principal and operation."""

    def __init__(self, bound):
        from core import store_pg
        self.slug = bound.tenant_slug
        self.schema = store_pg._schema_for(self.slug)
        self.table = f'"{self.schema}"."{TABLE_NAME}"'

    def _connection(self):
        from core import store_pg
        return store_pg._pool().connection()

    def ensure(self):
        self._write_guard()
        with self._connection() as con:
            con.execute('SELECT pg_advisory_xact_lock(hashtextextended(%s, 0))', (self.table,))
            con.execute(f'''CREATE TABLE IF NOT EXISTS {self.table} (
                receipt_key BYTEA PRIMARY KEY,
                principal TEXT NOT NULL, operation TEXT NOT NULL, op_id TEXT NOT NULL,
                result JSONB NOT NULL, created_at TIMESTAMPTZ NOT NULL DEFAULT now())''')

    def _write_guard(self):
        from core import store_pg
        store_pg.check_write(self.slug, 'store edit receipt')

    @contextmanager
    def _admission(self):
        from core import request_timing, store_pg
        timeout = request_timing.remaining_budget(store_pg._pg_limits()['acquire'])
        if not _PG_SLOTS.acquire(timeout=timeout):
            request_timing.remaining_budget()
            raise store_pg.PostgresOperationTimeout('Edit receipt admission timed out. Retry this opId.')
        try:
            request_timing.remaining_budget()
            yield
        finally:
            _PG_SLOTS.release()

    def run(self, principal, action, identity, apply):
        from core import request_timing, store_pg
        limits = store_pg._pg_limits()
        deadline = time.monotonic() + limits['acquire'] + limits['statement']
        with request_timing.budget_scope(deadline, store_pg.PostgresOperationTimeout):
            with self._admission():
                return self._run(principal, action, identity, apply)

    def _run(self, principal, action, identity, apply):
        from core import request_timing, store_pg
        self.ensure()
        store = store_pg.PgStore(self.slug)
        lock = json.dumps([self.table, principal, action, identity], separators=(',', ':'))
        receipt_key = hashlib.sha256(lock.encode('utf-8')).digest()
        # Poll without keeping a pool connection that the current writer may need.
        while True:
            request_timing.remaining_budget()
            with self._connection() as con:
                acquired = con.execute(
                    'SELECT pg_try_advisory_xact_lock(hashtextextended(%s, 0))',
                    (lock,)).fetchone()[0]
                if acquired:
                    # The callback may leave this transaction idle, within the existing database budget.
                    idle_ms = max(1, int(request_timing.remaining_budget(
                        store_pg._pg_limits()['idle']) * 1000))
                    con.execute("SELECT set_config('idle_in_transaction_session_timeout', %s, true)",
                                (f'{idle_ms}ms',))
                    prior = con.execute(f'SELECT result FROM {self.table} '
                                        'WHERE receipt_key=%s', (receipt_key,)).fetchone()
                    if prior is not None:
                        result = deepcopy(prior[0])
                        break
                    with store.transaction(con):
                        result = apply()
                    request_timing.remaining_budget()
                    con.execute(f'INSERT INTO {self.table} '
                                '(receipt_key, principal, operation, op_id, result) '
                                'VALUES (%s, %s, %s, %s, %s::jsonb)',
                                (receipt_key, principal, action, identity, json.dumps(result)))
                    # The connection context commits before the caller receives the answer.
                    break
            time.sleep(request_timing.remaining_budget(0.01))
        # Deliver only committed activity. The durable intent survives a delivery outage.
        from core import activity_log
        try:
            activity_log.drain(store)
        except Exception as error:
            print('[edit_receipts] activity delivery deferred: ' + type(error).__name__)
        return deepcopy(result)