"""One store-backed limiter for AI and metric live source calls, per tenant, across processes. Scope (R64): the Live tool, the in-app assistant and the metric cross-checks admit through here. Screens and the sync pass never do: `core/odoo.py::search_read` and `_breaker_call` stay unwrapped. Defaults (R65): 30 live calls per minute, one in flight, rows per call unlimited (rule 1). The `tenant` argument is the tenant-bound store handle (`session.runtime`, or any object with `get`, `revision` and `update`). State lives in that store, never in process memory, so every worker process sees the same in-flight slot. A refusal raises `LiveLimitRefused` at once. Nothing here queues or sleeps. """ import time as _time import uuid from contextvars import ContextVar from datetime import datetime, timezone #: The admin-set limits document. LIMITS_KEY = 'live_limits' #: The admission state document: recent call times and the in-flight slot. STATE_KEY = 'live_limiter_state' DEFAULT_CALLS_PER_MINUTE = 30 DEFAULT_ONE_IN_FLIGHT = True #: None is unlimited (rule 1). A set value refuses a larger read with cause and fix. DEFAULT_MAX_ROWS_PER_CALL = None #: An in-flight slot expires after this many seconds, so a crashed holder blocks for 2 minutes at most. HOLD_SECONDS = 120 WINDOW_SECONDS = 60 #: The bounds a PUT accepts for calls per minute. CALLS_PER_MINUTE_MIN = 1 CALLS_PER_MINUTE_MAX = 10000 #: Compare-and-swap attempts before a contended admit refuses. Each attempt re-reads, none waits. CAS_ATTEMPTS = 3 FIX_RAISE = 'An administrator can raise the live call limit under Connectors.' FIX_IN_FLIGHT = ('Try again when the running call ends, or an administrator can allow more than ' 'one call at a time under Connectors.') FIX_ROWS = 'Ask for fewer rows, or an administrator can raise the row limit under Connectors.' FIX_RETRY = 'Try again in a moment.' _FIELDS = ('callsPerMinute', 'oneInFlight', 'maxRowsPerCall') _ADMISSION_SCOPE = ContextVar('live_limiter_admission_scope', default=None) def _now(): return _time.time() def iso_utc(epoch): """`epoch` as an ISO 8601 UTC string, or None.""" if epoch is None: return None return datetime.fromtimestamp(float(epoch), timezone.utc).isoformat( timespec='seconds').replace('+00:00', 'Z') class LiveLimitRefused(RuntimeError): """A live call the tenant's limits refuse. Carries `cause`, `next_allowed_at` and `fix`. `cause` is one of `rate`, `in_flight`, `rows`, `contended`, `unreadable`, `expired`. `next_allowed_at` is ISO UTC, or None when waiting does not help (`rows`). """ def __init__(self, cause, message, next_allowed_at=None, fix=FIX_RAISE, limits=None): super().__init__(message) self.cause = cause self.message = message self.next_allowed_at = next_allowed_at self.fix = fix self.limits = dict(limits or {}) def as_dict(self): """The refusal as a client payload.""" return {'code': 'live_limit_refused', 'cause': self.cause, 'message': self.message, 'nextAllowedAt': self.next_allowed_at, 'fix': self.fix} def defaults(): return {'callsPerMinute': DEFAULT_CALLS_PER_MINUTE, 'oneInFlight': DEFAULT_ONE_IN_FLIGHT, 'maxRowsPerCall': DEFAULT_MAX_ROWS_PER_CALL} def _clean_stored(doc): """The stored limits over the defaults. A malformed stored value falls back to its default.""" out = defaults() doc = doc if isinstance(doc, dict) else {} clean, errors = validate_limits({k: doc[k] for k in _FIELDS if k in doc}) for key in _FIELDS: if key in clean and key not in errors: out[key] = clean[key] return out def read_limits(tenant, fresh=True): """The tenant's current limits: `{callsPerMinute, oneInFlight, maxRowsPerCall, holdSeconds}`.""" doc = tenant.get(LIMITS_KEY, fresh=fresh) or {} out = _clean_stored(doc) out['holdSeconds'] = HOLD_SECONDS out['updatedAt'] = doc.get('updatedAt') if isinstance(doc, dict) else None out['updatedBy'] = doc.get('updatedBy') if isinstance(doc, dict) else None return out def _is_int(value): return isinstance(value, int) and not isinstance(value, bool) def validate_limits(body): """`(clean, errors)` for a limits change. Every field is optional. An unknown field is an error.""" clean, errors = {}, {} if not isinstance(body, dict): return {}, {'body': 'Send the limits as an object.'} for key in body: if key not in _FIELDS: errors[key] = 'This is not a live limit setting.' if 'callsPerMinute' in body: value = body['callsPerMinute'] if _is_int(value) and CALLS_PER_MINUTE_MIN <= value <= CALLS_PER_MINUTE_MAX: clean['callsPerMinute'] = value else: errors['callsPerMinute'] = (f'Use a whole number from {CALLS_PER_MINUTE_MIN} to ' f'{CALLS_PER_MINUTE_MAX}.') if 'oneInFlight' in body: value = body['oneInFlight'] if isinstance(value, bool): clean['oneInFlight'] = value else: errors['oneInFlight'] = 'Use true or false.' if 'maxRowsPerCall' in body: value = body['maxRowsPerCall'] if value is None or (_is_int(value) and value >= 1): clean['maxRowsPerCall'] = value else: errors['maxRowsPerCall'] = 'Use a whole number of at least 1, or null for no limit.' return clean, errors def write_limits(tenant, clean, by=''): """Store validated limits over the current ones. Returns `read_limits` after the write.""" stamp = iso_utc(_now()) def _up(cur): cur = cur if isinstance(cur, dict) else {} cur.update({k: clean[k] for k in _FIELDS if k in clean}) cur['updatedAt'] = stamp cur['updatedBy'] = str(by or '') return cur tenant.update(LIMITS_KEY, _up) return read_limits(tenant) def _recent(state, now): calls = state.get('calls') if isinstance(state, dict) else None return sorted(float(t) for t in (calls or []) if isinstance(t, (int, float)) and now - WINDOW_SECONDS < float(t) <= now + 1) def _slot(state, now): """The live in-flight slot, or None when free or expired.""" slot = state.get('inFlight') if isinstance(state, dict) else None if not isinstance(slot, dict): return None try: until = float(slot.get('until')) except (TypeError, ValueError): return None return slot if until > now else None def _decide(limits, state, now): """A `LiveLimitRefused` for this state, or None to admit.""" if limits['oneInFlight']: slot = _slot(state, now) if slot is not None: return LiveLimitRefused( 'in_flight', 'Another live call for this workspace is still running.', iso_utc(float(slot['until'])), FIX_IN_FLIGHT, limits) recent = _recent(state, now) cap = int(limits['callsPerMinute']) if len(recent) >= cap: return LiveLimitRefused( 'rate', f'This workspace reached its limit of {cap} live calls per minute.', iso_utc(recent[-cap] + WINDOW_SECONDS), FIX_RAISE, limits) return None def _rows_refusal(limits, rows): cap = limits.get('maxRowsPerCall') if cap is None or rows is None: return None if int(rows) > int(cap): return LiveLimitRefused( 'rows', f'This read asks for {int(rows)} rows and the limit is {int(cap)} per call.', None, FIX_ROWS, limits) return None def _rev(tenant): return int((tenant.revision(STATE_KEY) or {}).get('rev') or 0) def _stale_errors(): from core import view_audit from core.store import StoreConflict return (view_audit.ViewRevisionStale, StoreConflict) class _RefusedInside(Exception): def __init__(self, refusal): super().__init__(str(refusal)) self.refusal = refusal class Admission: """One admitted live call. Release it with `with`, or call `release()` in a `finally`.""" def __init__(self, tenant, token, limits, admitted_at): self.tenant = tenant self.token = token self.limits = limits self.admitted_at = admitted_at self.expires_at = admitted_at + HOLD_SECONDS self.released = False def check_rows(self, rows): """Refuse a read whose row count is over the set max rows per call.""" refusal = _rows_refusal(self.limits, rows) if refusal is not None: raise refusal def release(self): """Free the in-flight slot if it is still ours. Never raises.""" if self.released: return self.released = True token = self.token def _free(cur): cur = cur if isinstance(cur, dict) else {} slot = cur.get('inFlight') if isinstance(slot, dict) and slot.get('token') == token: cur['inFlight'] = None return cur stale = _stale_errors() for _ in range(CAS_ATTEMPTS): try: self.tenant.update(STATE_KEY, _free, expect_rev=_rev(self.tenant)) return except stale: continue except Exception: # noqa: BLE001 return # The free is token-matched, so an unguarded write cannot free another holder's slot. try: self.tenant.update(STATE_KEY, _free) except Exception: # noqa: BLE001 pass # the slot expires after HOLD_SECONDS def __enter__(self): return self def __exit__(self, *_exc): self.release() return False def _scope_reusable(admission, tenant): """Reuse only the same store handle's unreleased, unexpired admission.""" if admission.tenant is not tenant or admission.released or _now() >= admission.expires_at: return False if admission.token is None: return True try: slot = _slot(tenant.get(STATE_KEY, fresh=True) or {}, _now()) except Exception as exc: raise LiveLimitRefused('unreadable', 'The live admission could not be checked.', iso_utc(_now()), FIX_RETRY) from exc return slot is not None and slot.get('token') == admission.token class _AdmissionScope: """A trusted nested scope borrows an admission without releasing its owner's slot.""" def __init__(self, admission, owns): self.admission = admission self.owns = owns self.context_token = None def __enter__(self): try: if not _scope_reusable(self.admission, self.admission.tenant): raise LiveLimitRefused('expired', 'The live admission has ended. Try again.', iso_utc(_now()), FIX_RETRY) self.context_token = _ADMISSION_SCOPE.set(self.admission) except Exception: if self.owns: self.admission.release() raise return self.admission def __exit__(self, *_exc): try: _ADMISSION_SCOPE.reset(self.context_token) finally: if self.owns: self.admission.release() return False def scoped_admit(tenant, expected_rows=None, caller=''): """Eagerly admit a trusted operation, borrowing only its active same-tenant parent scope.""" active = _ADMISSION_SCOPE.get() if active is not None and _scope_reusable(active, tenant): active.check_rows(expected_rows) return _AdmissionScope(active, owns=False) return _AdmissionScope(admit(tenant, expected_rows=expected_rows, caller=caller), owns=True) def admit(tenant, expected_rows=None, caller=''): """Admit one live call for `tenant` or raise `LiveLimitRefused`. Returns an `Admission`.""" try: limits = read_limits(tenant) except Exception as exc: # noqa: BLE001 raise LiveLimitRefused('unreadable', 'The live call limits could not be read.', iso_utc(_now()), FIX_RETRY) from exc refusal = _rows_refusal(limits, expected_rows) if refusal is not None: raise refusal stale = _stale_errors() for _ in range(CAS_ATTEMPTS): now = _now() try: rev = _rev(tenant) state = tenant.get(STATE_KEY, fresh=True) or {} except Exception as exc: # noqa: BLE001 raise LiveLimitRefused('unreadable', 'The live call state could not be read.', iso_utc(now), FIX_RETRY, limits) from exc refusal = _decide(limits, state, now) if refusal is not None: raise refusal token = uuid.uuid4().hex def _claim(cur, now=now, token=token): cur = cur if isinstance(cur, dict) else {} inside = _decide(limits, cur, now) if inside is not None: raise _RefusedInside(inside) keep = max(int(limits['callsPerMinute']), 1) cur['calls'] = (_recent(cur, now) + [now])[-keep:] if limits['oneInFlight']: cur['inFlight'] = {'token': token, 'since': now, 'until': now + HOLD_SECONDS, 'caller': str(caller or '')[:80]} return cur try: tenant.update(STATE_KEY, _claim, expect_rev=rev) except _RefusedInside as inside: raise inside.refusal from None except stale: continue except Exception as exc: # noqa: BLE001 raise LiveLimitRefused('unreadable', 'The live call state could not be saved.', iso_utc(_now()), FIX_RETRY, limits) from exc return Admission(tenant, token if limits['oneInFlight'] else None, limits, now) raise LiveLimitRefused('contended', 'Other live calls changed the limit state at the same time.', iso_utc(_now()), FIX_RETRY, limits)