File size: 14,243 Bytes
b2aa446
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
"""One store-backed limiter for AI and metric live source calls, per tenant, across processes.

Scope (R64): the Live tool, the in-app assistant and the metric cross-checks admit through here.
Screens and the sync pass never do: `core/odoo.py::search_read` and `_breaker_call` stay unwrapped.
Defaults (R65): 30 live calls per minute, one in flight, rows per call unlimited (rule 1).

The `tenant` argument is the tenant-bound store handle (`session.runtime`, or any object with
`get`, `revision` and `update`). State lives in that store, never in process memory, so every
worker process sees the same in-flight slot. A refusal raises `LiveLimitRefused` at once. Nothing
here queues or sleeps.
"""
import time as _time
import uuid
from contextvars import ContextVar
from datetime import datetime, timezone

#: The admin-set limits document.
LIMITS_KEY = 'live_limits'
#: The admission state document: recent call times and the in-flight slot.
STATE_KEY = 'live_limiter_state'

DEFAULT_CALLS_PER_MINUTE = 30
DEFAULT_ONE_IN_FLIGHT = True
#: None is unlimited (rule 1). A set value refuses a larger read with cause and fix.
DEFAULT_MAX_ROWS_PER_CALL = None

#: An in-flight slot expires after this many seconds, so a crashed holder blocks for 2 minutes at most.
HOLD_SECONDS = 120
WINDOW_SECONDS = 60
#: The bounds a PUT accepts for calls per minute.
CALLS_PER_MINUTE_MIN = 1
CALLS_PER_MINUTE_MAX = 10000
#: Compare-and-swap attempts before a contended admit refuses. Each attempt re-reads, none waits.
CAS_ATTEMPTS = 3

FIX_RAISE = 'An administrator can raise the live call limit under Connectors.'
FIX_IN_FLIGHT = ('Try again when the running call ends, or an administrator can allow more than '
                 'one call at a time under Connectors.')
FIX_ROWS = 'Ask for fewer rows, or an administrator can raise the row limit under Connectors.'
FIX_RETRY = 'Try again in a moment.'

_FIELDS = ('callsPerMinute', 'oneInFlight', 'maxRowsPerCall')
_ADMISSION_SCOPE = ContextVar('live_limiter_admission_scope', default=None)


def _now():
    return _time.time()


def iso_utc(epoch):
    """`epoch` as an ISO 8601 UTC string, or None."""
    if epoch is None:
        return None
    return datetime.fromtimestamp(float(epoch), timezone.utc).isoformat(
        timespec='seconds').replace('+00:00', 'Z')


class LiveLimitRefused(RuntimeError):
    """A live call the tenant's limits refuse. Carries `cause`, `next_allowed_at` and `fix`.

    `cause` is one of `rate`, `in_flight`, `rows`, `contended`, `unreadable`, `expired`. `next_allowed_at` is
    ISO UTC, or None when waiting does not help (`rows`).
    """

    def __init__(self, cause, message, next_allowed_at=None, fix=FIX_RAISE, limits=None):
        super().__init__(message)
        self.cause = cause
        self.message = message
        self.next_allowed_at = next_allowed_at
        self.fix = fix
        self.limits = dict(limits or {})

    def as_dict(self):
        """The refusal as a client payload."""
        return {'code': 'live_limit_refused', 'cause': self.cause, 'message': self.message,
                'nextAllowedAt': self.next_allowed_at, 'fix': self.fix}


def defaults():
    return {'callsPerMinute': DEFAULT_CALLS_PER_MINUTE, 'oneInFlight': DEFAULT_ONE_IN_FLIGHT,
            'maxRowsPerCall': DEFAULT_MAX_ROWS_PER_CALL}


def _clean_stored(doc):
    """The stored limits over the defaults. A malformed stored value falls back to its default."""
    out = defaults()
    doc = doc if isinstance(doc, dict) else {}
    clean, errors = validate_limits({k: doc[k] for k in _FIELDS if k in doc})
    for key in _FIELDS:
        if key in clean and key not in errors:
            out[key] = clean[key]
    return out


def read_limits(tenant, fresh=True):
    """The tenant's current limits: `{callsPerMinute, oneInFlight, maxRowsPerCall, holdSeconds}`."""
    doc = tenant.get(LIMITS_KEY, fresh=fresh) or {}
    out = _clean_stored(doc)
    out['holdSeconds'] = HOLD_SECONDS
    out['updatedAt'] = doc.get('updatedAt') if isinstance(doc, dict) else None
    out['updatedBy'] = doc.get('updatedBy') if isinstance(doc, dict) else None
    return out


def _is_int(value):
    return isinstance(value, int) and not isinstance(value, bool)


def validate_limits(body):
    """`(clean, errors)` for a limits change. Every field is optional. An unknown field is an error."""
    clean, errors = {}, {}
    if not isinstance(body, dict):
        return {}, {'body': 'Send the limits as an object.'}
    for key in body:
        if key not in _FIELDS:
            errors[key] = 'This is not a live limit setting.'
    if 'callsPerMinute' in body:
        value = body['callsPerMinute']
        if _is_int(value) and CALLS_PER_MINUTE_MIN <= value <= CALLS_PER_MINUTE_MAX:
            clean['callsPerMinute'] = value
        else:
            errors['callsPerMinute'] = (f'Use a whole number from {CALLS_PER_MINUTE_MIN} to '
                                        f'{CALLS_PER_MINUTE_MAX}.')
    if 'oneInFlight' in body:
        value = body['oneInFlight']
        if isinstance(value, bool):
            clean['oneInFlight'] = value
        else:
            errors['oneInFlight'] = 'Use true or false.'
    if 'maxRowsPerCall' in body:
        value = body['maxRowsPerCall']
        if value is None or (_is_int(value) and value >= 1):
            clean['maxRowsPerCall'] = value
        else:
            errors['maxRowsPerCall'] = 'Use a whole number of at least 1, or null for no limit.'
    return clean, errors


def write_limits(tenant, clean, by=''):
    """Store validated limits over the current ones. Returns `read_limits` after the write."""
    stamp = iso_utc(_now())

    def _up(cur):
        cur = cur if isinstance(cur, dict) else {}
        cur.update({k: clean[k] for k in _FIELDS if k in clean})
        cur['updatedAt'] = stamp
        cur['updatedBy'] = str(by or '')
        return cur

    tenant.update(LIMITS_KEY, _up)
    return read_limits(tenant)


def _recent(state, now):
    calls = state.get('calls') if isinstance(state, dict) else None
    return sorted(float(t) for t in (calls or []) if isinstance(t, (int, float))
                  and now - WINDOW_SECONDS < float(t) <= now + 1)


def _slot(state, now):
    """The live in-flight slot, or None when free or expired."""
    slot = state.get('inFlight') if isinstance(state, dict) else None
    if not isinstance(slot, dict):
        return None
    try:
        until = float(slot.get('until'))
    except (TypeError, ValueError):
        return None
    return slot if until > now else None


def _decide(limits, state, now):
    """A `LiveLimitRefused` for this state, or None to admit."""
    if limits['oneInFlight']:
        slot = _slot(state, now)
        if slot is not None:
            return LiveLimitRefused(
                'in_flight', 'Another live call for this workspace is still running.',
                iso_utc(float(slot['until'])), FIX_IN_FLIGHT, limits)
    recent = _recent(state, now)
    cap = int(limits['callsPerMinute'])
    if len(recent) >= cap:
        return LiveLimitRefused(
            'rate', f'This workspace reached its limit of {cap} live calls per minute.',
            iso_utc(recent[-cap] + WINDOW_SECONDS), FIX_RAISE, limits)
    return None


def _rows_refusal(limits, rows):
    cap = limits.get('maxRowsPerCall')
    if cap is None or rows is None:
        return None
    if int(rows) > int(cap):
        return LiveLimitRefused(
            'rows', f'This read asks for {int(rows)} rows and the limit is {int(cap)} per call.',
            None, FIX_ROWS, limits)
    return None


def _rev(tenant):
    return int((tenant.revision(STATE_KEY) or {}).get('rev') or 0)


def _stale_errors():
    from core import view_audit
    from core.store import StoreConflict
    return (view_audit.ViewRevisionStale, StoreConflict)


class _RefusedInside(Exception):
    def __init__(self, refusal):
        super().__init__(str(refusal))
        self.refusal = refusal


class Admission:
    """One admitted live call. Release it with `with`, or call `release()` in a `finally`."""

    def __init__(self, tenant, token, limits, admitted_at):
        self.tenant = tenant
        self.token = token
        self.limits = limits
        self.admitted_at = admitted_at
        self.expires_at = admitted_at + HOLD_SECONDS
        self.released = False

    def check_rows(self, rows):
        """Refuse a read whose row count is over the set max rows per call."""
        refusal = _rows_refusal(self.limits, rows)
        if refusal is not None:
            raise refusal

    def release(self):
        """Free the in-flight slot if it is still ours. Never raises."""
        if self.released:
            return
        self.released = True
        token = self.token

        def _free(cur):
            cur = cur if isinstance(cur, dict) else {}
            slot = cur.get('inFlight')
            if isinstance(slot, dict) and slot.get('token') == token:
                cur['inFlight'] = None
            return cur

        stale = _stale_errors()
        for _ in range(CAS_ATTEMPTS):
            try:
                self.tenant.update(STATE_KEY, _free, expect_rev=_rev(self.tenant))
                return
            except stale:
                continue
            except Exception:                                             # noqa: BLE001
                return
        # The free is token-matched, so an unguarded write cannot free another holder's slot.
        try:
            self.tenant.update(STATE_KEY, _free)
        except Exception:                                                 # noqa: BLE001
            pass                    # the slot expires after HOLD_SECONDS

    def __enter__(self):
        return self

    def __exit__(self, *_exc):
        self.release()
        return False


def _scope_reusable(admission, tenant):
    """Reuse only the same store handle's unreleased, unexpired admission."""
    if admission.tenant is not tenant or admission.released or _now() >= admission.expires_at:
        return False
    if admission.token is None:
        return True
    try:
        slot = _slot(tenant.get(STATE_KEY, fresh=True) or {}, _now())
    except Exception as exc:
        raise LiveLimitRefused('unreadable', 'The live admission could not be checked.',
                               iso_utc(_now()), FIX_RETRY) from exc
    return slot is not None and slot.get('token') == admission.token


class _AdmissionScope:
    """A trusted nested scope borrows an admission without releasing its owner's slot."""

    def __init__(self, admission, owns):
        self.admission = admission
        self.owns = owns
        self.context_token = None

    def __enter__(self):
        try:
            if not _scope_reusable(self.admission, self.admission.tenant):
                raise LiveLimitRefused('expired', 'The live admission has ended. Try again.',
                                       iso_utc(_now()), FIX_RETRY)
            self.context_token = _ADMISSION_SCOPE.set(self.admission)
        except Exception:
            if self.owns:
                self.admission.release()
            raise
        return self.admission

    def __exit__(self, *_exc):
        try:
            _ADMISSION_SCOPE.reset(self.context_token)
        finally:
            if self.owns:
                self.admission.release()
        return False


def scoped_admit(tenant, expected_rows=None, caller=''):
    """Eagerly admit a trusted operation, borrowing only its active same-tenant parent scope."""
    active = _ADMISSION_SCOPE.get()
    if active is not None and _scope_reusable(active, tenant):
        active.check_rows(expected_rows)
        return _AdmissionScope(active, owns=False)
    return _AdmissionScope(admit(tenant, expected_rows=expected_rows, caller=caller), owns=True)


def admit(tenant, expected_rows=None, caller=''):
    """Admit one live call for `tenant` or raise `LiveLimitRefused`. Returns an `Admission`."""
    try:
        limits = read_limits(tenant)
    except Exception as exc:                                              # noqa: BLE001
        raise LiveLimitRefused('unreadable', 'The live call limits could not be read.',
                               iso_utc(_now()), FIX_RETRY) from exc
    refusal = _rows_refusal(limits, expected_rows)
    if refusal is not None:
        raise refusal
    stale = _stale_errors()
    for _ in range(CAS_ATTEMPTS):
        now = _now()
        try:
            rev = _rev(tenant)
            state = tenant.get(STATE_KEY, fresh=True) or {}
        except Exception as exc:                                          # noqa: BLE001
            raise LiveLimitRefused('unreadable', 'The live call state could not be read.',
                                   iso_utc(now), FIX_RETRY, limits) from exc
        refusal = _decide(limits, state, now)
        if refusal is not None:
            raise refusal
        token = uuid.uuid4().hex

        def _claim(cur, now=now, token=token):
            cur = cur if isinstance(cur, dict) else {}
            inside = _decide(limits, cur, now)
            if inside is not None:
                raise _RefusedInside(inside)
            keep = max(int(limits['callsPerMinute']), 1)
            cur['calls'] = (_recent(cur, now) + [now])[-keep:]
            if limits['oneInFlight']:
                cur['inFlight'] = {'token': token, 'since': now, 'until': now + HOLD_SECONDS,
                                   'caller': str(caller or '')[:80]}
            return cur

        try:
            tenant.update(STATE_KEY, _claim, expect_rev=rev)
        except _RefusedInside as inside:
            raise inside.refusal from None
        except stale:
            continue
        except Exception as exc:                                          # noqa: BLE001
            raise LiveLimitRefused('unreadable', 'The live call state could not be saved.',
                                   iso_utc(_now()), FIX_RETRY, limits) from exc
        return Admission(tenant, token if limits['oneInFlight'] else None, limits, now)
    raise LiveLimitRefused('contended', 'Other live calls changed the limit state at the same time.',
                           iso_utc(_now()), FIX_RETRY, limits)