"""Bounded, tenant-scoped security events. Refuse secrets before attempting any store I/O.""" from collections.abc import Mapping from collections import Counter from datetime import datetime, timezone import atexit import math import re import threading import time from harness import runtime from core import account_rows, freeze_notice, platform_admin AUDIT_KINDS = frozenset({ "auth.login", "auth.login_failed", "auth.lockout", "auth.mfa_enrolled", "auth.mfa_failed", "auth.logout", "admin.grant", "credential.added", "credential.rotated", "credential.revoked", "amazon.access", "amazon.sync", "amazon.throttled", "amazon.authorization_lost", "amazon.deletion_scheduled", "amazon.personal_read", "data.purge", "data.restore", "config.changed", "customer.personal_read", "external.query", "external.write", }) BUCKET = "security_events" MAX_EVENTS = 2000 EXTERNAL_KINDS = frozenset({"external.query", "external.write"}) RETENTION_SECONDS = 400 * 24 * 60 * 60 #: Kinds a write-through store queues and writes in batches, so one query never pays a document write. DEFERRED_KINDS = frozenset({"external.query"}) #: A tenant queue this long is written by the caller that fills it: the queue never grows past it. QUEUE_MAX = 200 FLUSH_INTERVAL_S = 2.0 _ATTRIBUTION_MAX = 256 _KEY_MAX = 128 def _key_name(key): return "".join(c for c in key.casefold() if c.isalnum()) # Restated from core.keychain._PREVIEW_FIELDS to avoid coupling a log to the keychain. # Keep the first six names and that tuple in step; client_secret is also an Amazon secret. SECRET_KEYS = frozenset(_key_name(key) for key in ( "api_key", "access_token", "value", "password", "token", "refresh_token", "client_secret", "secret", "private_key", "authorization", "cookie", "session", "mfa_secret", "recovery_code", "otpauth", "credential", )) _BEARER = re.compile(r"\bbearer\s+\S+", re.IGNORECASE) _JWT = re.compile(r"\beyJ[A-Za-z0-9_-]*\.") _LOOPABLE_CREDENTIAL = re.compile(r"\blp(?:b|o|r|c)1\.[A-Za-z0-9._~-]+") _INTENT_CREDENTIAL = re.compile( r"Atzr\||\bcfm1\.|\blp[borc]1\.|\b[A-Za-z0-9_-]{24,}\.[A-Za-z0-9_-]{24,}\b" r"|\b(?:password|api[_ -]?key|client[_ -]?secret|access[_ -]?token)\s*[:=]" r"|\b(?:sk-[A-Za-z0-9_-]{12,}|hf_[A-Za-z0-9]{12,}|gsk_[A-Za-z0-9_-]{12,})", re.IGNORECASE) class AuditRefused(ValueError): """A call-site error carrying only the field name and refusal reason, never its value.""" def __init__(self, key, reason): self.key = key self.reason = reason # Explaining a refusal with the value would put the secret into its traceback. super().__init__(f"audit refused key {key!r}: {reason}") def _string(key, value): if len(value) > 512: raise AuditRefused(key, "string exceeds 512 characters") if value.startswith("Atzr|"): raise AuditRefused(key, "Atzr refresh-token prefix") if _BEARER.search(value): raise AuditRefused(key, "bearer token shape") if _JWT.search(value): raise AuditRefused(key, "JWT token shape") if _LOOPABLE_CREDENTIAL.search(value): raise AuditRefused(key, "Loopable credential shape") def _attribution(key, value): if not isinstance(value, str) or not value.strip(): raise AuditRefused(key, "attribution requires a non-empty string") _string(key, value) value = value.strip() _string(key, value) if len(value) > _ATTRIBUTION_MAX: raise AuditRefused(key, "attribution exceeds 256 characters") return value.lower() if key == "tenant" else value def redact_intent(value): """Keep bounded intent prose while replacing credential patterns and invalid text.""" if not isinstance(value, str) or not 1 <= len(value) <= 500 or not value.strip(): return "[invalid]" if _INTENT_CREDENTIAL.search(value) or any(ord(char) < 32 for char in value): return "[redacted]" try: _string("intent", value) except AuditRefused: return "[redacted]" return value.strip() def event(kind, tenant, principal, detail=None, *, now=None) -> dict: """Validate without I/O; an explicit epoch makes this pure and deterministic for gates.""" if not isinstance(kind, str) or kind not in AUDIT_KINDS: raise AuditRefused("kind", "unknown audit kind") tenant = _attribution("tenant", tenant) principal = _attribution("principal", principal) if detail is None: detail = {} if not isinstance(detail, Mapping): raise AuditRefused("detail", "detail must be a flat mapping") clean = {} for key, value in detail.items(): if not isinstance(key, str) or not key.strip(): raise AuditRefused("detail", "detail keys must be non-empty strings") # A malformed key can itself contain a token. Never echo that key in diagnostics. _string("detail key", key) if len(key) > _KEY_MAX or any(ord(c) < 32 for c in key): raise AuditRefused("detail key", "key exceeds 128 characters or contains controls") if _key_name(key) in SECRET_KEYS: raise AuditRefused(key, "secret key name") if isinstance(value, (Mapping, list, tuple, set, frozenset)): raise AuditRefused(key, "nested value violates flat detail") if type(value) not in (str, int, bool, float): raise AuditRefused(key, "unsupported detail value type") if isinstance(value, str): _string(key, value) if isinstance(value, float) and not math.isfinite(value): raise AuditRefused(key, "detail number must be finite") clean[key] = value try: ts = float(time.time() if now is None else now) except (TypeError, ValueError, OverflowError): raise AuditRefused("ts", "timestamp must be finite epoch seconds") from None if not math.isfinite(ts): raise AuditRefused("ts", "timestamp must be finite epoch seconds") return {"ts": ts, "kind": kind, "tenant": tenant, "principal": principal, "detail": clean} def ordered(doc, tenant, limit=200, *, kinds=None) -> list[dict]: """Return ascending timestamps; a limit retains the most recent rows, still ascending. The stored tenant must match exactly even inside an already scoped runtime. This second wall protects readers if a future migration ever puts multiple tenants in one document. """ tenant = _attribution("tenant", tenant) limit = max(0, int(limit)) if not limit or not isinstance(doc, Mapping): return [] rows = doc.get("events") if not isinstance(rows, list): return [] retained = [] for row in rows: if not isinstance(row, dict) or row.get("tenant") != tenant: continue if kinds is not None and row.get("kind") not in kinds: continue if type(row.get("ts")) not in (int, float): # In particular, None must not ask event() for today's clock on a pure read. continue try: # Revalidate persisted rows too, so legacy/corrupt documents cannot leak a secret. retained.append(event(row.get("kind"), tenant, row.get("principal"), row.get("detail"), now=row["ts"])) except (AuditRefused, KeyError): continue return sorted(retained, key=lambda row: row["ts"])[-limit:] def _well_formed(row, tenant): """A stored row this writer may keep without revalidation. Readers still revalidate every row.""" return (isinstance(row, dict) and row.get("tenant") == tenant and type(row.get("ts")) in (int, float) and row.get("kind") in AUDIT_KINDS and isinstance(row.get("principal"), str) and isinstance(row.get("detail"), dict)) def _append_rows(doc, tenant, items): """`{"events": rows}` with `items` added, pruned by age and capped per partition. The common case is linear with no per-row validation. A document holding any row this tenant's writer did not produce takes the full `ordered()` rebuild, which drops it. """ stored = doc.get("events") if isinstance(doc, Mapping) else None stored = stored if isinstance(stored, list) else [] if all(_well_formed(row, tenant) for row in stored): rows = list(stored) else: rows = ordered(doc, tenant, MAX_EVENTS * 2) rows.extend(items) # A queued batch can land after a later sign-in, so restore ascending order when it did. if any(rows[i]["ts"] > rows[i + 1]["ts"] for i in range(len(rows) - 1)): rows.sort(key=lambda row: row["ts"]) cutoff = max(item["ts"] for item in items) - RETENTION_SECONDS if rows and rows[0]["ts"] < cutoff: rows = [row for row in rows if row["ts"] >= cutoff] # Sign-ins and external calls each own their 2000-event retention partition, oldest out first. external = sum(1 for row in rows if row["kind"] in EXTERNAL_KINDS) drop = {True: max(0, external - MAX_EVENTS), False: max(0, len(rows) - external - MAX_EVENTS)} if drop[True] or drop[False]: kept = [] for row in rows: side = row["kind"] in EXTERNAL_KINDS if drop[side]: drop[side] -= 1 continue kept.append(row) rows = kept return {"events": rows} def _write(tenant, items): st = runtime.get_runtime(str(tenant)) if account_rows.records_backend(st): account_rows.append_events(st, items, retention_seconds=RETENTION_SECONDS) return # Bare core.store.update is bound to tenant #0. Always use the tenant-scoped door. # ⭐⭐ flush="async" IS DELIBERATE AND IT IS NOT A PERFORMANCE TWEAK. The default # flush="sync" is a BLOCKING upload (core/store.py::update says so), and this writer sits # on the sign-in path: every login would then wait on a network round trip, and on the hf # backend every login would spend one of that repo's 256 commits per hour. The async # branch applies the mutation to the in-process cache and a coalescing worker commits it, # which is the same posture the table-workspace hot path has used since wave 7. # ⚠ THE COST, stated rather than hidden: a container that dies before the worker flushes # loses the events still in its cache. That is the same window `record` already accepts by # swallowing a store failure, and it is the right side of the trade: an audit log must # never be the reason a person cannot sign in. st.update( BUCKET, lambda doc: _append_rows(doc, tenant, items), flush="async") def _writes_through(tenant): """True when the tenant store ignores flush="async" and commits the whole document per call (Postgres).""" try: backend = runtime.get_runtime(str(tenant))._document_store() except Exception: return False return type(backend).__module__ == "core.store_pg" #: Store refusals per (tenant, bucket), read by the api layer's owner alert. #: Core never imports that alert path: the api layer installs a sink with set_loss_sink. _LOSSES = {} _LOSS_LOCK = threading.Lock() _LOSS_SINK = [] def set_loss_sink(fn): """Install the one callable told of every lost or recovered write; None removes it.""" _LOSS_SINK[:] = [fn] if fn is not None else [] def _tell_sink(entry): for fn in list(_LOSS_SINK): try: fn(entry) except Exception: pass def note_loss(tenant, bucket, exc, count=1): """Count a write the store refused. The cause keeps raw text: every reader must scrub it.""" try: key = (str(tenant), str(bucket)) with _LOSS_LOCK: entry = _LOSSES.setdefault(key, {"tenant": key[0], "bucket": key[1], "count": 0, "first": time.time()}) entry.update(count=entry["count"] + max(1, int(count)), lost=True, last=time.time(), cause=f"{type(exc).__name__}: {exc}"[:500]) snapshot = dict(entry) _tell_sink(snapshot) except Exception: pass def note_ok(tenant, bucket): """Mark a bucket with a past loss as writing again. Costs one dict read when nothing was lost.""" if not _LOSSES: return with _LOSS_LOCK: entry = _LOSSES.get((str(tenant), str(bucket))) if entry is None or not entry.get("lost"): return entry.update(lost=False, count=0, last=time.time()) snapshot = dict(entry) _tell_sink(snapshot) def lost_writes(tenant=None) -> list[dict]: """Every bucket this process saw refused or recovered, oldest first; `lost` says which.""" with _LOSS_LOCK: rows = [dict(entry) for entry in _LOSSES.values() if tenant is None or entry["tenant"] == str(tenant)] return sorted(rows, key=lambda row: (row["first"], row["bucket"])) _QUEUE = {} _QUEUE_LOCK = threading.Lock() _WAKE = threading.Event() _FLUSHER = [] def _flush_loop(): while True: _WAKE.wait(FLUSH_INTERVAL_S) _WAKE.clear() try: flush_pending() except Exception: pass def _enqueue(tenant, item): with _QUEUE_LOCK: pending = _QUEUE.setdefault(tenant, []) pending.append(item) full = len(pending) >= QUEUE_MAX if not _FLUSHER: _FLUSHER.append(threading.Thread(target=_flush_loop, name="audit-flush", daemon=True)) _FLUSHER[0].start() atexit.register(flush_pending) if full: flush_pending(tenant) def flush_pending(tenant=None) -> int: """Write queued events in one update per tenant. Returns how many rows were handed to the store.""" with _QUEUE_LOCK: names = list(_QUEUE) if tenant is None else [_attribution("tenant", tenant)] batches = {name: _QUEUE.pop(name) for name in names if _QUEUE.get(name)} written = 0 for name, items in batches.items(): try: _write(name, items) written += len(items) except Exception as exc: # Exception text can contain credentials from a provider or connection string. print(f"[aios-api] audit lost {len(items)} queued event(s) for tenant {name!r}: " f"store unavailable ({type(exc).__name__}).") note_loss(name, BUCKET, exc, len(items)) continue note_ok(name, BUCKET) return written def record(kind, tenant, principal, detail=None) -> dict | None: """Refusals raise before I/O. Store failures lose the event, never the user's sign-in.""" item = event(kind, tenant, principal, detail) if freeze_notice.enabled(): return None tenant = item["tenant"] # Row storage commits one event immediately, including audited queries. try: _write(tenant, [item]) except Exception as exc: # Exception text can contain credentials from a provider or connection string. print(f"[aios-api] audit lost {item['kind']} for tenant {tenant!r}: " f"store unavailable ({type(exc).__name__}).") note_loss(tenant, BUCKET, exc) return None note_ok(tenant, BUCKET) return item def usage_counts(st, tenant, *, since, until, now, kinds, actor=None): """Count retained metadata in SQL and include the same bounded local queue as audit reads.""" tenant = _attribution("tenant", tenant) since = max(float(since), float(now) - RETENTION_SECONDS) rows = account_rows.event_counts(st, tenant, since=since, until=until, now=now, kinds=kinds, actor=actor) if rows is None: return None counts = Counter({(day, who, kind): count for day, who, kind, count in rows}) with _QUEUE_LOCK: queued = list(_QUEUE.get(tenant, ())) # Queue entries remain subject to the canonical validator and never become durable on a GET. for row in ordered({'events': queued}, tenant, len(queued), kinds=EXTERNAL_KINDS | {'auth.login'}): if not since <= row['ts'] < until or row['ts'] > now or actor is not None and row['principal'] != actor: continue kind = ('sign_in' if row['kind'] == 'auth.login' else 'mcp' if row['detail'].get('via') == 'mcp' else 'api') if kind in kinds: day = datetime.fromtimestamp(row['ts'], timezone.utc).date().isoformat() counts[(day, row['principal'], kind)] += 1 return [(*key, count) for key, count in sorted(counts.items())] def read(tenant, limit=200, now=None, *, kinds=None, viewer=None) -> list[dict]: """Read one tenant's retained events. Read failures propagate instead of claiming no history.""" tenant = _attribution("tenant", tenant) now = time.time() if now is None else float(now) st = runtime.get_runtime(str(tenant)) if account_rows.records_backend(st): doc = {'events': account_rows.read_events(st, tenant, since=now - RETENTION_SECONDS, limit=limit, kinds=kinds)} else: doc = st.get(BUCKET) # A read can include an older process-local queue without making it durable. with _QUEUE_LOCK: queued = list(_QUEUE.get(tenant, ())) if queued: doc = {'events': list((doc or {}).get('events') or []) + queued} rows = ordered(doc, tenant, limit, kinds=kinds) rows = [row for row in rows if row["ts"] >= now - RETENTION_SECONDS] if not platform_admin.is_platform_admin(viewer): rows = [{**row, "detail": {key: value for key, value in row["detail"].items() if key != "intent"}} for row in rows] return rows def external_write_receipt(tenant, principal, *, tool, auth_type, affected_count, diff_digest, revision_before, revision_after, confirmation_id, activity_detail=None) -> dict: """Emit and return a value-free C7 write receipt. Target ids, field names, record values and arguments deliberately do not belong here. The receipt proves who confirmed which normalized diff and how many objects moved; the concrete before/after stays in the short-lived preview response instead of the retained security log. """ detail = { "surface": "ai_api", "tool": str(tool or "")[:128], "authentication": str(auth_type or "")[:32], "affected_count": max(0, int(affected_count or 0)), "diff_digest": str(diff_digest or "")[:128], "revision_before": str(revision_before or "")[:128], "revision_after": str(revision_after or "")[:128], "confirmation_id": str(confirmation_id or "")[:128], } if activity_detail is not None: detail["write_tool"] = detail["tool"] detail.update(activity_detail) item = event("external.write", tenant, principal, detail) stored = record("external.write", tenant, principal, detail) return {"event": item, "persisted": stored is not None}