Spaces:
Running on CPU Upgrade
Running on CPU Upgrade
File size: 14,243 Bytes
8133df9 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 | """One store-backed limiter for AI and metric live source calls, per tenant, across processes.
Scope (R64): the Live tool, the in-app assistant and the metric cross-checks admit through here.
Screens and the sync pass never do: `core/odoo.py::search_read` and `_breaker_call` stay unwrapped.
Defaults (R65): 30 live calls per minute, one in flight, rows per call unlimited (rule 1).
The `tenant` argument is the tenant-bound store handle (`session.runtime`, or any object with
`get`, `revision` and `update`). State lives in that store, never in process memory, so every
worker process sees the same in-flight slot. A refusal raises `LiveLimitRefused` at once. Nothing
here queues or sleeps.
"""
import time as _time
import uuid
from contextvars import ContextVar
from datetime import datetime, timezone
#: The admin-set limits document.
LIMITS_KEY = 'live_limits'
#: The admission state document: recent call times and the in-flight slot.
STATE_KEY = 'live_limiter_state'
DEFAULT_CALLS_PER_MINUTE = 30
DEFAULT_ONE_IN_FLIGHT = True
#: None is unlimited (rule 1). A set value refuses a larger read with cause and fix.
DEFAULT_MAX_ROWS_PER_CALL = None
#: An in-flight slot expires after this many seconds, so a crashed holder blocks for 2 minutes at most.
HOLD_SECONDS = 120
WINDOW_SECONDS = 60
#: The bounds a PUT accepts for calls per minute.
CALLS_PER_MINUTE_MIN = 1
CALLS_PER_MINUTE_MAX = 10000
#: Compare-and-swap attempts before a contended admit refuses. Each attempt re-reads, none waits.
CAS_ATTEMPTS = 3
FIX_RAISE = 'An administrator can raise the live call limit under Connectors.'
FIX_IN_FLIGHT = ('Try again when the running call ends, or an administrator can allow more than '
'one call at a time under Connectors.')
FIX_ROWS = 'Ask for fewer rows, or an administrator can raise the row limit under Connectors.'
FIX_RETRY = 'Try again in a moment.'
_FIELDS = ('callsPerMinute', 'oneInFlight', 'maxRowsPerCall')
_ADMISSION_SCOPE = ContextVar('live_limiter_admission_scope', default=None)
def _now():
return _time.time()
def iso_utc(epoch):
"""`epoch` as an ISO 8601 UTC string, or None."""
if epoch is None:
return None
return datetime.fromtimestamp(float(epoch), timezone.utc).isoformat(
timespec='seconds').replace('+00:00', 'Z')
class LiveLimitRefused(RuntimeError):
"""A live call the tenant's limits refuse. Carries `cause`, `next_allowed_at` and `fix`.
`cause` is one of `rate`, `in_flight`, `rows`, `contended`, `unreadable`, `expired`. `next_allowed_at` is
ISO UTC, or None when waiting does not help (`rows`).
"""
def __init__(self, cause, message, next_allowed_at=None, fix=FIX_RAISE, limits=None):
super().__init__(message)
self.cause = cause
self.message = message
self.next_allowed_at = next_allowed_at
self.fix = fix
self.limits = dict(limits or {})
def as_dict(self):
"""The refusal as a client payload."""
return {'code': 'live_limit_refused', 'cause': self.cause, 'message': self.message,
'nextAllowedAt': self.next_allowed_at, 'fix': self.fix}
def defaults():
return {'callsPerMinute': DEFAULT_CALLS_PER_MINUTE, 'oneInFlight': DEFAULT_ONE_IN_FLIGHT,
'maxRowsPerCall': DEFAULT_MAX_ROWS_PER_CALL}
def _clean_stored(doc):
"""The stored limits over the defaults. A malformed stored value falls back to its default."""
out = defaults()
doc = doc if isinstance(doc, dict) else {}
clean, errors = validate_limits({k: doc[k] for k in _FIELDS if k in doc})
for key in _FIELDS:
if key in clean and key not in errors:
out[key] = clean[key]
return out
def read_limits(tenant, fresh=True):
"""The tenant's current limits: `{callsPerMinute, oneInFlight, maxRowsPerCall, holdSeconds}`."""
doc = tenant.get(LIMITS_KEY, fresh=fresh) or {}
out = _clean_stored(doc)
out['holdSeconds'] = HOLD_SECONDS
out['updatedAt'] = doc.get('updatedAt') if isinstance(doc, dict) else None
out['updatedBy'] = doc.get('updatedBy') if isinstance(doc, dict) else None
return out
def _is_int(value):
return isinstance(value, int) and not isinstance(value, bool)
def validate_limits(body):
"""`(clean, errors)` for a limits change. Every field is optional. An unknown field is an error."""
clean, errors = {}, {}
if not isinstance(body, dict):
return {}, {'body': 'Send the limits as an object.'}
for key in body:
if key not in _FIELDS:
errors[key] = 'This is not a live limit setting.'
if 'callsPerMinute' in body:
value = body['callsPerMinute']
if _is_int(value) and CALLS_PER_MINUTE_MIN <= value <= CALLS_PER_MINUTE_MAX:
clean['callsPerMinute'] = value
else:
errors['callsPerMinute'] = (f'Use a whole number from {CALLS_PER_MINUTE_MIN} to '
f'{CALLS_PER_MINUTE_MAX}.')
if 'oneInFlight' in body:
value = body['oneInFlight']
if isinstance(value, bool):
clean['oneInFlight'] = value
else:
errors['oneInFlight'] = 'Use true or false.'
if 'maxRowsPerCall' in body:
value = body['maxRowsPerCall']
if value is None or (_is_int(value) and value >= 1):
clean['maxRowsPerCall'] = value
else:
errors['maxRowsPerCall'] = 'Use a whole number of at least 1, or null for no limit.'
return clean, errors
def write_limits(tenant, clean, by=''):
"""Store validated limits over the current ones. Returns `read_limits` after the write."""
stamp = iso_utc(_now())
def _up(cur):
cur = cur if isinstance(cur, dict) else {}
cur.update({k: clean[k] for k in _FIELDS if k in clean})
cur['updatedAt'] = stamp
cur['updatedBy'] = str(by or '')
return cur
tenant.update(LIMITS_KEY, _up)
return read_limits(tenant)
def _recent(state, now):
calls = state.get('calls') if isinstance(state, dict) else None
return sorted(float(t) for t in (calls or []) if isinstance(t, (int, float))
and now - WINDOW_SECONDS < float(t) <= now + 1)
def _slot(state, now):
"""The live in-flight slot, or None when free or expired."""
slot = state.get('inFlight') if isinstance(state, dict) else None
if not isinstance(slot, dict):
return None
try:
until = float(slot.get('until'))
except (TypeError, ValueError):
return None
return slot if until > now else None
def _decide(limits, state, now):
"""A `LiveLimitRefused` for this state, or None to admit."""
if limits['oneInFlight']:
slot = _slot(state, now)
if slot is not None:
return LiveLimitRefused(
'in_flight', 'Another live call for this workspace is still running.',
iso_utc(float(slot['until'])), FIX_IN_FLIGHT, limits)
recent = _recent(state, now)
cap = int(limits['callsPerMinute'])
if len(recent) >= cap:
return LiveLimitRefused(
'rate', f'This workspace reached its limit of {cap} live calls per minute.',
iso_utc(recent[-cap] + WINDOW_SECONDS), FIX_RAISE, limits)
return None
def _rows_refusal(limits, rows):
cap = limits.get('maxRowsPerCall')
if cap is None or rows is None:
return None
if int(rows) > int(cap):
return LiveLimitRefused(
'rows', f'This read asks for {int(rows)} rows and the limit is {int(cap)} per call.',
None, FIX_ROWS, limits)
return None
def _rev(tenant):
return int((tenant.revision(STATE_KEY) or {}).get('rev') or 0)
def _stale_errors():
from core import view_audit
from core.store import StoreConflict
return (view_audit.ViewRevisionStale, StoreConflict)
class _RefusedInside(Exception):
def __init__(self, refusal):
super().__init__(str(refusal))
self.refusal = refusal
class Admission:
"""One admitted live call. Release it with `with`, or call `release()` in a `finally`."""
def __init__(self, tenant, token, limits, admitted_at):
self.tenant = tenant
self.token = token
self.limits = limits
self.admitted_at = admitted_at
self.expires_at = admitted_at + HOLD_SECONDS
self.released = False
def check_rows(self, rows):
"""Refuse a read whose row count is over the set max rows per call."""
refusal = _rows_refusal(self.limits, rows)
if refusal is not None:
raise refusal
def release(self):
"""Free the in-flight slot if it is still ours. Never raises."""
if self.released:
return
self.released = True
token = self.token
def _free(cur):
cur = cur if isinstance(cur, dict) else {}
slot = cur.get('inFlight')
if isinstance(slot, dict) and slot.get('token') == token:
cur['inFlight'] = None
return cur
stale = _stale_errors()
for _ in range(CAS_ATTEMPTS):
try:
self.tenant.update(STATE_KEY, _free, expect_rev=_rev(self.tenant))
return
except stale:
continue
except Exception: # noqa: BLE001
return
# The free is token-matched, so an unguarded write cannot free another holder's slot.
try:
self.tenant.update(STATE_KEY, _free)
except Exception: # noqa: BLE001
pass # the slot expires after HOLD_SECONDS
def __enter__(self):
return self
def __exit__(self, *_exc):
self.release()
return False
def _scope_reusable(admission, tenant):
"""Reuse only the same store handle's unreleased, unexpired admission."""
if admission.tenant is not tenant or admission.released or _now() >= admission.expires_at:
return False
if admission.token is None:
return True
try:
slot = _slot(tenant.get(STATE_KEY, fresh=True) or {}, _now())
except Exception as exc:
raise LiveLimitRefused('unreadable', 'The live admission could not be checked.',
iso_utc(_now()), FIX_RETRY) from exc
return slot is not None and slot.get('token') == admission.token
class _AdmissionScope:
"""A trusted nested scope borrows an admission without releasing its owner's slot."""
def __init__(self, admission, owns):
self.admission = admission
self.owns = owns
self.context_token = None
def __enter__(self):
try:
if not _scope_reusable(self.admission, self.admission.tenant):
raise LiveLimitRefused('expired', 'The live admission has ended. Try again.',
iso_utc(_now()), FIX_RETRY)
self.context_token = _ADMISSION_SCOPE.set(self.admission)
except Exception:
if self.owns:
self.admission.release()
raise
return self.admission
def __exit__(self, *_exc):
try:
_ADMISSION_SCOPE.reset(self.context_token)
finally:
if self.owns:
self.admission.release()
return False
def scoped_admit(tenant, expected_rows=None, caller=''):
"""Eagerly admit a trusted operation, borrowing only its active same-tenant parent scope."""
active = _ADMISSION_SCOPE.get()
if active is not None and _scope_reusable(active, tenant):
active.check_rows(expected_rows)
return _AdmissionScope(active, owns=False)
return _AdmissionScope(admit(tenant, expected_rows=expected_rows, caller=caller), owns=True)
def admit(tenant, expected_rows=None, caller=''):
"""Admit one live call for `tenant` or raise `LiveLimitRefused`. Returns an `Admission`."""
try:
limits = read_limits(tenant)
except Exception as exc: # noqa: BLE001
raise LiveLimitRefused('unreadable', 'The live call limits could not be read.',
iso_utc(_now()), FIX_RETRY) from exc
refusal = _rows_refusal(limits, expected_rows)
if refusal is not None:
raise refusal
stale = _stale_errors()
for _ in range(CAS_ATTEMPTS):
now = _now()
try:
rev = _rev(tenant)
state = tenant.get(STATE_KEY, fresh=True) or {}
except Exception as exc: # noqa: BLE001
raise LiveLimitRefused('unreadable', 'The live call state could not be read.',
iso_utc(now), FIX_RETRY, limits) from exc
refusal = _decide(limits, state, now)
if refusal is not None:
raise refusal
token = uuid.uuid4().hex
def _claim(cur, now=now, token=token):
cur = cur if isinstance(cur, dict) else {}
inside = _decide(limits, cur, now)
if inside is not None:
raise _RefusedInside(inside)
keep = max(int(limits['callsPerMinute']), 1)
cur['calls'] = (_recent(cur, now) + [now])[-keep:]
if limits['oneInFlight']:
cur['inFlight'] = {'token': token, 'since': now, 'until': now + HOLD_SECONDS,
'caller': str(caller or '')[:80]}
return cur
try:
tenant.update(STATE_KEY, _claim, expect_rev=rev)
except _RefusedInside as inside:
raise inside.refusal from None
except stale:
continue
except Exception as exc: # noqa: BLE001
raise LiveLimitRefused('unreadable', 'The live call state could not be saved.',
iso_utc(_now()), FIX_RETRY, limits) from exc
return Admission(tenant, token if limits['oneInFlight'] else None, limits, now)
raise LiveLimitRefused('contended', 'Other live calls changed the limit state at the same time.',
iso_utc(_now()), FIX_RETRY, limits)
|