runloopable / platform /core /live_limiter.py
fsanyoto's picture
Deploy AIOS web (React glide grid + FastAPI slice)
8133df9 verified
Raw History Blame Contribute Delete
14.2 kB
"""One store-backed limiter for AI and metric live source calls, per tenant, across processes.
Scope (R64): the Live tool, the in-app assistant and the metric cross-checks admit through here.
Screens and the sync pass never do: `core/odoo.py::search_read` and `_breaker_call` stay unwrapped.
Defaults (R65): 30 live calls per minute, one in flight, rows per call unlimited (rule 1).
The `tenant` argument is the tenant-bound store handle (`session.runtime`, or any object with
`get`, `revision` and `update`). State lives in that store, never in process memory, so every
worker process sees the same in-flight slot. A refusal raises `LiveLimitRefused` at once. Nothing
here queues or sleeps.
"""
import time as _time
import uuid
from contextvars import ContextVar
from datetime import datetime, timezone
#: The admin-set limits document.
LIMITS_KEY = 'live_limits'
#: The admission state document: recent call times and the in-flight slot.
STATE_KEY = 'live_limiter_state'
DEFAULT_CALLS_PER_MINUTE = 30
DEFAULT_ONE_IN_FLIGHT = True
#: None is unlimited (rule 1). A set value refuses a larger read with cause and fix.
DEFAULT_MAX_ROWS_PER_CALL = None
#: An in-flight slot expires after this many seconds, so a crashed holder blocks for 2 minutes at most.
HOLD_SECONDS = 120
WINDOW_SECONDS = 60
#: The bounds a PUT accepts for calls per minute.
CALLS_PER_MINUTE_MIN = 1
CALLS_PER_MINUTE_MAX = 10000
#: Compare-and-swap attempts before a contended admit refuses. Each attempt re-reads, none waits.
CAS_ATTEMPTS = 3
FIX_RAISE = 'An administrator can raise the live call limit under Connectors.'
FIX_IN_FLIGHT = ('Try again when the running call ends, or an administrator can allow more than '
'one call at a time under Connectors.')
FIX_ROWS = 'Ask for fewer rows, or an administrator can raise the row limit under Connectors.'
FIX_RETRY = 'Try again in a moment.'
_FIELDS = ('callsPerMinute', 'oneInFlight', 'maxRowsPerCall')
_ADMISSION_SCOPE = ContextVar('live_limiter_admission_scope', default=None)
def _now():
return _time.time()
def iso_utc(epoch):
"""`epoch` as an ISO 8601 UTC string, or None."""
if epoch is None:
return None
return datetime.fromtimestamp(float(epoch), timezone.utc).isoformat(
timespec='seconds').replace('+00:00', 'Z')
class LiveLimitRefused(RuntimeError):
"""A live call the tenant's limits refuse. Carries `cause`, `next_allowed_at` and `fix`.
`cause` is one of `rate`, `in_flight`, `rows`, `contended`, `unreadable`, `expired`. `next_allowed_at` is
ISO UTC, or None when waiting does not help (`rows`).
"""
def __init__(self, cause, message, next_allowed_at=None, fix=FIX_RAISE, limits=None):
super().__init__(message)
self.cause = cause
self.message = message
self.next_allowed_at = next_allowed_at
self.fix = fix
self.limits = dict(limits or {})
def as_dict(self):
"""The refusal as a client payload."""
return {'code': 'live_limit_refused', 'cause': self.cause, 'message': self.message,
'nextAllowedAt': self.next_allowed_at, 'fix': self.fix}
def defaults():
return {'callsPerMinute': DEFAULT_CALLS_PER_MINUTE, 'oneInFlight': DEFAULT_ONE_IN_FLIGHT,
'maxRowsPerCall': DEFAULT_MAX_ROWS_PER_CALL}
def _clean_stored(doc):
"""The stored limits over the defaults. A malformed stored value falls back to its default."""
out = defaults()
doc = doc if isinstance(doc, dict) else {}
clean, errors = validate_limits({k: doc[k] for k in _FIELDS if k in doc})
for key in _FIELDS:
if key in clean and key not in errors:
out[key] = clean[key]
return out
def read_limits(tenant, fresh=True):
"""The tenant's current limits: `{callsPerMinute, oneInFlight, maxRowsPerCall, holdSeconds}`."""
doc = tenant.get(LIMITS_KEY, fresh=fresh) or {}
out = _clean_stored(doc)
out['holdSeconds'] = HOLD_SECONDS
out['updatedAt'] = doc.get('updatedAt') if isinstance(doc, dict) else None
out['updatedBy'] = doc.get('updatedBy') if isinstance(doc, dict) else None
return out
def _is_int(value):
return isinstance(value, int) and not isinstance(value, bool)
def validate_limits(body):
"""`(clean, errors)` for a limits change. Every field is optional. An unknown field is an error."""
clean, errors = {}, {}
if not isinstance(body, dict):
return {}, {'body': 'Send the limits as an object.'}
for key in body:
if key not in _FIELDS:
errors[key] = 'This is not a live limit setting.'
if 'callsPerMinute' in body:
value = body['callsPerMinute']
if _is_int(value) and CALLS_PER_MINUTE_MIN <= value <= CALLS_PER_MINUTE_MAX:
clean['callsPerMinute'] = value
else:
errors['callsPerMinute'] = (f'Use a whole number from {CALLS_PER_MINUTE_MIN} to '
f'{CALLS_PER_MINUTE_MAX}.')
if 'oneInFlight' in body:
value = body['oneInFlight']
if isinstance(value, bool):
clean['oneInFlight'] = value
else:
errors['oneInFlight'] = 'Use true or false.'
if 'maxRowsPerCall' in body:
value = body['maxRowsPerCall']
if value is None or (_is_int(value) and value >= 1):
clean['maxRowsPerCall'] = value
else:
errors['maxRowsPerCall'] = 'Use a whole number of at least 1, or null for no limit.'
return clean, errors
def write_limits(tenant, clean, by=''):
"""Store validated limits over the current ones. Returns `read_limits` after the write."""
stamp = iso_utc(_now())
def _up(cur):
cur = cur if isinstance(cur, dict) else {}
cur.update({k: clean[k] for k in _FIELDS if k in clean})
cur['updatedAt'] = stamp
cur['updatedBy'] = str(by or '')
return cur
tenant.update(LIMITS_KEY, _up)
return read_limits(tenant)
def _recent(state, now):
calls = state.get('calls') if isinstance(state, dict) else None
return sorted(float(t) for t in (calls or []) if isinstance(t, (int, float))
and now - WINDOW_SECONDS < float(t) <= now + 1)
def _slot(state, now):
"""The live in-flight slot, or None when free or expired."""
slot = state.get('inFlight') if isinstance(state, dict) else None
if not isinstance(slot, dict):
return None
try:
until = float(slot.get('until'))
except (TypeError, ValueError):
return None
return slot if until > now else None
def _decide(limits, state, now):
"""A `LiveLimitRefused` for this state, or None to admit."""
if limits['oneInFlight']:
slot = _slot(state, now)
if slot is not None:
return LiveLimitRefused(
'in_flight', 'Another live call for this workspace is still running.',
iso_utc(float(slot['until'])), FIX_IN_FLIGHT, limits)
recent = _recent(state, now)
cap = int(limits['callsPerMinute'])
if len(recent) >= cap:
return LiveLimitRefused(
'rate', f'This workspace reached its limit of {cap} live calls per minute.',
iso_utc(recent[-cap] + WINDOW_SECONDS), FIX_RAISE, limits)
return None
def _rows_refusal(limits, rows):
cap = limits.get('maxRowsPerCall')
if cap is None or rows is None:
return None
if int(rows) > int(cap):
return LiveLimitRefused(
'rows', f'This read asks for {int(rows)} rows and the limit is {int(cap)} per call.',
None, FIX_ROWS, limits)
return None
def _rev(tenant):
return int((tenant.revision(STATE_KEY) or {}).get('rev') or 0)
def _stale_errors():
from core import view_audit
from core.store import StoreConflict
return (view_audit.ViewRevisionStale, StoreConflict)
class _RefusedInside(Exception):
def __init__(self, refusal):
super().__init__(str(refusal))
self.refusal = refusal
class Admission:
"""One admitted live call. Release it with `with`, or call `release()` in a `finally`."""
def __init__(self, tenant, token, limits, admitted_at):
self.tenant = tenant
self.token = token
self.limits = limits
self.admitted_at = admitted_at
self.expires_at = admitted_at + HOLD_SECONDS
self.released = False
def check_rows(self, rows):
"""Refuse a read whose row count is over the set max rows per call."""
refusal = _rows_refusal(self.limits, rows)
if refusal is not None:
raise refusal
def release(self):
"""Free the in-flight slot if it is still ours. Never raises."""
if self.released:
return
self.released = True
token = self.token
def _free(cur):
cur = cur if isinstance(cur, dict) else {}
slot = cur.get('inFlight')
if isinstance(slot, dict) and slot.get('token') == token:
cur['inFlight'] = None
return cur
stale = _stale_errors()
for _ in range(CAS_ATTEMPTS):
try:
self.tenant.update(STATE_KEY, _free, expect_rev=_rev(self.tenant))
return
except stale:
continue
except Exception: # noqa: BLE001
return
# The free is token-matched, so an unguarded write cannot free another holder's slot.
try:
self.tenant.update(STATE_KEY, _free)
except Exception: # noqa: BLE001
pass # the slot expires after HOLD_SECONDS
def __enter__(self):
return self
def __exit__(self, *_exc):
self.release()
return False
def _scope_reusable(admission, tenant):
"""Reuse only the same store handle's unreleased, unexpired admission."""
if admission.tenant is not tenant or admission.released or _now() >= admission.expires_at:
return False
if admission.token is None:
return True
try:
slot = _slot(tenant.get(STATE_KEY, fresh=True) or {}, _now())
except Exception as exc:
raise LiveLimitRefused('unreadable', 'The live admission could not be checked.',
iso_utc(_now()), FIX_RETRY) from exc
return slot is not None and slot.get('token') == admission.token
class _AdmissionScope:
"""A trusted nested scope borrows an admission without releasing its owner's slot."""
def __init__(self, admission, owns):
self.admission = admission
self.owns = owns
self.context_token = None
def __enter__(self):
try:
if not _scope_reusable(self.admission, self.admission.tenant):
raise LiveLimitRefused('expired', 'The live admission has ended. Try again.',
iso_utc(_now()), FIX_RETRY)
self.context_token = _ADMISSION_SCOPE.set(self.admission)
except Exception:
if self.owns:
self.admission.release()
raise
return self.admission
def __exit__(self, *_exc):
try:
_ADMISSION_SCOPE.reset(self.context_token)
finally:
if self.owns:
self.admission.release()
return False
def scoped_admit(tenant, expected_rows=None, caller=''):
"""Eagerly admit a trusted operation, borrowing only its active same-tenant parent scope."""
active = _ADMISSION_SCOPE.get()
if active is not None and _scope_reusable(active, tenant):
active.check_rows(expected_rows)
return _AdmissionScope(active, owns=False)
return _AdmissionScope(admit(tenant, expected_rows=expected_rows, caller=caller), owns=True)
def admit(tenant, expected_rows=None, caller=''):
"""Admit one live call for `tenant` or raise `LiveLimitRefused`. Returns an `Admission`."""
try:
limits = read_limits(tenant)
except Exception as exc: # noqa: BLE001
raise LiveLimitRefused('unreadable', 'The live call limits could not be read.',
iso_utc(_now()), FIX_RETRY) from exc
refusal = _rows_refusal(limits, expected_rows)
if refusal is not None:
raise refusal
stale = _stale_errors()
for _ in range(CAS_ATTEMPTS):
now = _now()
try:
rev = _rev(tenant)
state = tenant.get(STATE_KEY, fresh=True) or {}
except Exception as exc: # noqa: BLE001
raise LiveLimitRefused('unreadable', 'The live call state could not be read.',
iso_utc(now), FIX_RETRY, limits) from exc
refusal = _decide(limits, state, now)
if refusal is not None:
raise refusal
token = uuid.uuid4().hex
def _claim(cur, now=now, token=token):
cur = cur if isinstance(cur, dict) else {}
inside = _decide(limits, cur, now)
if inside is not None:
raise _RefusedInside(inside)
keep = max(int(limits['callsPerMinute']), 1)
cur['calls'] = (_recent(cur, now) + [now])[-keep:]
if limits['oneInFlight']:
cur['inFlight'] = {'token': token, 'since': now, 'until': now + HOLD_SECONDS,
'caller': str(caller or '')[:80]}
return cur
try:
tenant.update(STATE_KEY, _claim, expect_rev=rev)
except _RefusedInside as inside:
raise inside.refusal from None
except stale:
continue
except Exception as exc: # noqa: BLE001
raise LiveLimitRefused('unreadable', 'The live call state could not be saved.',
iso_utc(_now()), FIX_RETRY, limits) from exc
return Admission(tenant, token if limits['oneInFlight'] else None, limits, now)
raise LiveLimitRefused('contended', 'Other live calls changed the limit state at the same time.',
iso_utc(_now()), FIX_RETRY, limits)