loopable / platform /core /audit.py
fsanyoto's picture
Deploy AIOS web (React glide grid + FastAPI slice)
c4a1388 verified
Raw History Blame
18.9 kB
"""Bounded, tenant-scoped security events. Refuse secrets before attempting any store I/O."""
from collections.abc import Mapping
from collections import Counter
from datetime import datetime, timezone
import atexit
import math
import re
import threading
import time
from harness import runtime
from core import account_rows, freeze_notice, platform_admin
AUDIT_KINDS = frozenset({
"auth.login", "auth.login_failed", "auth.lockout", "auth.mfa_enrolled",
"auth.mfa_failed", "auth.logout", "admin.grant", "credential.added",
"credential.rotated", "credential.revoked", "amazon.access", "amazon.sync", "amazon.throttled",
"amazon.authorization_lost", "amazon.deletion_scheduled", "amazon.personal_read", "data.purge", "data.restore",
"config.changed", "customer.personal_read",
"external.query",
"external.write",
})
BUCKET = "security_events"
MAX_EVENTS = 2000
EXTERNAL_KINDS = frozenset({"external.query", "external.write"})
RETENTION_SECONDS = 400 * 24 * 60 * 60
#: Kinds a write-through store queues and writes in batches, so one query never pays a document write.
DEFERRED_KINDS = frozenset({"external.query"})
#: A tenant queue this long is written by the caller that fills it: the queue never grows past it.
QUEUE_MAX = 200
FLUSH_INTERVAL_S = 2.0
_ATTRIBUTION_MAX = 256
_KEY_MAX = 128
def _key_name(key):
return "".join(c for c in key.casefold() if c.isalnum())
# Restated from core.keychain._PREVIEW_FIELDS to avoid coupling a log to the keychain.
# Keep the first six names and that tuple in step; client_secret is also an Amazon secret.
SECRET_KEYS = frozenset(_key_name(key) for key in (
"api_key", "access_token", "value", "password", "token", "refresh_token",
"client_secret", "secret", "private_key", "authorization", "cookie", "session",
"mfa_secret", "recovery_code", "otpauth", "credential",
))
_BEARER = re.compile(r"\bbearer\s+\S+", re.IGNORECASE)
_JWT = re.compile(r"\beyJ[A-Za-z0-9_-]*\.")
_LOOPABLE_CREDENTIAL = re.compile(r"\blp(?:b|o|r|c)1\.[A-Za-z0-9._~-]+")
_INTENT_CREDENTIAL = re.compile(
r"Atzr\||\bcfm1\.|\blp[borc]1\.|\b[A-Za-z0-9_-]{24,}\.[A-Za-z0-9_-]{24,}\b"
r"|\b(?:password|api[_ -]?key|client[_ -]?secret|access[_ -]?token)\s*[:=]"
r"|\b(?:sk-[A-Za-z0-9_-]{12,}|hf_[A-Za-z0-9]{12,}|gsk_[A-Za-z0-9_-]{12,})", re.IGNORECASE)
class AuditRefused(ValueError):
"""A call-site error carrying only the field name and refusal reason, never its value."""
def __init__(self, key, reason):
self.key = key
self.reason = reason
# Explaining a refusal with the value would put the secret into its traceback.
super().__init__(f"audit refused key {key!r}: {reason}")
def _string(key, value):
if len(value) > 512:
raise AuditRefused(key, "string exceeds 512 characters")
if value.startswith("Atzr|"):
raise AuditRefused(key, "Atzr refresh-token prefix")
if _BEARER.search(value):
raise AuditRefused(key, "bearer token shape")
if _JWT.search(value):
raise AuditRefused(key, "JWT token shape")
if _LOOPABLE_CREDENTIAL.search(value):
raise AuditRefused(key, "Loopable credential shape")
def _attribution(key, value):
if not isinstance(value, str) or not value.strip():
raise AuditRefused(key, "attribution requires a non-empty string")
_string(key, value)
value = value.strip()
_string(key, value)
if len(value) > _ATTRIBUTION_MAX:
raise AuditRefused(key, "attribution exceeds 256 characters")
return value.lower() if key == "tenant" else value
def redact_intent(value):
"""Keep bounded intent prose while replacing credential patterns and invalid text."""
if not isinstance(value, str) or not 1 <= len(value) <= 500 or not value.strip():
return "[invalid]"
if _INTENT_CREDENTIAL.search(value) or any(ord(char) < 32 for char in value):
return "[redacted]"
try:
_string("intent", value)
except AuditRefused:
return "[redacted]"
return value.strip()
def event(kind, tenant, principal, detail=None, *, now=None) -> dict:
"""Validate without I/O; an explicit epoch makes this pure and deterministic for gates."""
if not isinstance(kind, str) or kind not in AUDIT_KINDS:
raise AuditRefused("kind", "unknown audit kind")
tenant = _attribution("tenant", tenant)
principal = _attribution("principal", principal)
if detail is None:
detail = {}
if not isinstance(detail, Mapping):
raise AuditRefused("detail", "detail must be a flat mapping")
clean = {}
for key, value in detail.items():
if not isinstance(key, str) or not key.strip():
raise AuditRefused("detail", "detail keys must be non-empty strings")
# A malformed key can itself contain a token. Never echo that key in diagnostics.
_string("detail key", key)
if len(key) > _KEY_MAX or any(ord(c) < 32 for c in key):
raise AuditRefused("detail key", "key exceeds 128 characters or contains controls")
if _key_name(key) in SECRET_KEYS:
raise AuditRefused(key, "secret key name")
if isinstance(value, (Mapping, list, tuple, set, frozenset)):
raise AuditRefused(key, "nested value violates flat detail")
if type(value) not in (str, int, bool, float):
raise AuditRefused(key, "unsupported detail value type")
if isinstance(value, str):
_string(key, value)
if isinstance(value, float) and not math.isfinite(value):
raise AuditRefused(key, "detail number must be finite")
clean[key] = value
try:
ts = float(time.time() if now is None else now)
except (TypeError, ValueError, OverflowError):
raise AuditRefused("ts", "timestamp must be finite epoch seconds") from None
if not math.isfinite(ts):
raise AuditRefused("ts", "timestamp must be finite epoch seconds")
return {"ts": ts, "kind": kind, "tenant": tenant, "principal": principal, "detail": clean}
def ordered(doc, tenant, limit=200, *, kinds=None) -> list[dict]:
"""Return ascending timestamps; a limit retains the most recent rows, still ascending.
The stored tenant must match exactly even inside an already scoped runtime. This second
wall protects readers if a future migration ever puts multiple tenants in one document.
"""
tenant = _attribution("tenant", tenant)
limit = max(0, int(limit))
if not limit or not isinstance(doc, Mapping):
return []
rows = doc.get("events")
if not isinstance(rows, list):
return []
retained = []
for row in rows:
if not isinstance(row, dict) or row.get("tenant") != tenant:
continue
if kinds is not None and row.get("kind") not in kinds:
continue
if type(row.get("ts")) not in (int, float):
# In particular, None must not ask event() for today's clock on a pure read.
continue
try:
# Revalidate persisted rows too, so legacy/corrupt documents cannot leak a secret.
retained.append(event(row.get("kind"), tenant, row.get("principal"),
row.get("detail"), now=row["ts"]))
except (AuditRefused, KeyError):
continue
return sorted(retained, key=lambda row: row["ts"])[-limit:]
def _well_formed(row, tenant):
"""A stored row this writer may keep without revalidation. Readers still revalidate every row."""
return (isinstance(row, dict) and row.get("tenant") == tenant
and type(row.get("ts")) in (int, float) and row.get("kind") in AUDIT_KINDS
and isinstance(row.get("principal"), str) and isinstance(row.get("detail"), dict))
def _append_rows(doc, tenant, items):
"""`{"events": rows}` with `items` added, pruned by age and capped per partition.
The common case is linear with no per-row validation. A document holding any row this tenant's
writer did not produce takes the full `ordered()` rebuild, which drops it.
"""
stored = doc.get("events") if isinstance(doc, Mapping) else None
stored = stored if isinstance(stored, list) else []
if all(_well_formed(row, tenant) for row in stored):
rows = list(stored)
else:
rows = ordered(doc, tenant, MAX_EVENTS * 2)
rows.extend(items)
# A queued batch can land after a later sign-in, so restore ascending order when it did.
if any(rows[i]["ts"] > rows[i + 1]["ts"] for i in range(len(rows) - 1)):
rows.sort(key=lambda row: row["ts"])
cutoff = max(item["ts"] for item in items) - RETENTION_SECONDS
if rows and rows[0]["ts"] < cutoff:
rows = [row for row in rows if row["ts"] >= cutoff]
# Sign-ins and external calls each own their 2000-event retention partition, oldest out first.
external = sum(1 for row in rows if row["kind"] in EXTERNAL_KINDS)
drop = {True: max(0, external - MAX_EVENTS), False: max(0, len(rows) - external - MAX_EVENTS)}
if drop[True] or drop[False]:
kept = []
for row in rows:
side = row["kind"] in EXTERNAL_KINDS
if drop[side]:
drop[side] -= 1
continue
kept.append(row)
rows = kept
return {"events": rows}
def _write(tenant, items):
st = runtime.get_runtime(str(tenant))
if account_rows.records_backend(st):
account_rows.append_events(st, items, retention_seconds=RETENTION_SECONDS)
return
# Bare core.store.update is bound to tenant #0. Always use the tenant-scoped door.
# ⭐⭐ flush="async" IS DELIBERATE AND IT IS NOT A PERFORMANCE TWEAK. The default
# flush="sync" is a BLOCKING upload (core/store.py::update says so), and this writer sits
# on the sign-in path: every login would then wait on a network round trip, and on the hf
# backend every login would spend one of that repo's 256 commits per hour. The async
# branch applies the mutation to the in-process cache and a coalescing worker commits it,
# which is the same posture the table-workspace hot path has used since wave 7.
# ⚠ THE COST, stated rather than hidden: a container that dies before the worker flushes
# loses the events still in its cache. That is the same window `record` already accepts by
# swallowing a store failure, and it is the right side of the trade: an audit log must
# never be the reason a person cannot sign in.
st.update(
BUCKET, lambda doc: _append_rows(doc, tenant, items), flush="async")
def _writes_through(tenant):
"""True when the tenant store ignores flush="async" and commits the whole document per call (Postgres)."""
try:
backend = runtime.get_runtime(str(tenant))._document_store()
except Exception:
return False
return type(backend).__module__ == "core.store_pg"
#: Store refusals per (tenant, bucket), read by the api layer's owner alert.
#: Core never imports that alert path: the api layer installs a sink with set_loss_sink.
_LOSSES = {}
_LOSS_LOCK = threading.Lock()
_LOSS_SINK = []
def set_loss_sink(fn):
"""Install the one callable told of every lost or recovered write; None removes it."""
_LOSS_SINK[:] = [fn] if fn is not None else []
def _tell_sink(entry):
for fn in list(_LOSS_SINK):
try:
fn(entry)
except Exception:
pass
def note_loss(tenant, bucket, exc, count=1):
"""Count a write the store refused. The cause keeps raw text: every reader must scrub it."""
try:
key = (str(tenant), str(bucket))
with _LOSS_LOCK:
entry = _LOSSES.setdefault(key, {"tenant": key[0], "bucket": key[1], "count": 0,
"first": time.time()})
entry.update(count=entry["count"] + max(1, int(count)), lost=True, last=time.time(),
cause=f"{type(exc).__name__}: {exc}"[:500])
snapshot = dict(entry)
_tell_sink(snapshot)
except Exception:
pass
def note_ok(tenant, bucket):
"""Mark a bucket with a past loss as writing again. Costs one dict read when nothing was lost."""
if not _LOSSES:
return
with _LOSS_LOCK:
entry = _LOSSES.get((str(tenant), str(bucket)))
if entry is None or not entry.get("lost"):
return
entry.update(lost=False, count=0, last=time.time())
snapshot = dict(entry)
_tell_sink(snapshot)
def lost_writes(tenant=None) -> list[dict]:
"""Every bucket this process saw refused or recovered, oldest first; `lost` says which."""
with _LOSS_LOCK:
rows = [dict(entry) for entry in _LOSSES.values()
if tenant is None or entry["tenant"] == str(tenant)]
return sorted(rows, key=lambda row: (row["first"], row["bucket"]))
_QUEUE = {}
_QUEUE_LOCK = threading.Lock()
_WAKE = threading.Event()
_FLUSHER = []
def _flush_loop():
while True:
_WAKE.wait(FLUSH_INTERVAL_S)
_WAKE.clear()
try:
flush_pending()
except Exception:
pass
def _enqueue(tenant, item):
with _QUEUE_LOCK:
pending = _QUEUE.setdefault(tenant, [])
pending.append(item)
full = len(pending) >= QUEUE_MAX
if not _FLUSHER:
_FLUSHER.append(threading.Thread(target=_flush_loop, name="audit-flush", daemon=True))
_FLUSHER[0].start()
atexit.register(flush_pending)
if full:
flush_pending(tenant)
def flush_pending(tenant=None) -> int:
"""Write queued events in one update per tenant. Returns how many rows were handed to the store."""
with _QUEUE_LOCK:
names = list(_QUEUE) if tenant is None else [_attribution("tenant", tenant)]
batches = {name: _QUEUE.pop(name) for name in names if _QUEUE.get(name)}
written = 0
for name, items in batches.items():
try:
_write(name, items)
written += len(items)
except Exception as exc:
# Exception text can contain credentials from a provider or connection string.
print(f"[aios-api] audit lost {len(items)} queued event(s) for tenant {name!r}: "
f"store unavailable ({type(exc).__name__}).")
note_loss(name, BUCKET, exc, len(items))
continue
note_ok(name, BUCKET)
return written
def record(kind, tenant, principal, detail=None) -> dict | None:
"""Refusals raise before I/O. Store failures lose the event, never the user's sign-in."""
item = event(kind, tenant, principal, detail)
if freeze_notice.enabled():
return None
tenant = item["tenant"]
# Row storage commits one event immediately, including audited queries.
try:
_write(tenant, [item])
except Exception as exc:
# Exception text can contain credentials from a provider or connection string.
print(f"[aios-api] audit lost {item['kind']} for tenant {tenant!r}: "
f"store unavailable ({type(exc).__name__}).")
note_loss(tenant, BUCKET, exc)
return None
note_ok(tenant, BUCKET)
return item
def usage_counts(st, tenant, *, since, until, now, kinds, actor=None):
"""Count retained metadata in SQL and include the same bounded local queue as audit reads."""
tenant = _attribution("tenant", tenant)
since = max(float(since), float(now) - RETENTION_SECONDS)
rows = account_rows.event_counts(st, tenant, since=since, until=until, now=now,
kinds=kinds, actor=actor)
if rows is None:
return None
counts = Counter({(day, who, kind): count for day, who, kind, count in rows})
with _QUEUE_LOCK:
queued = list(_QUEUE.get(tenant, ()))
# Queue entries remain subject to the canonical validator and never become durable on a GET.
for row in ordered({'events': queued}, tenant, len(queued), kinds=EXTERNAL_KINDS | {'auth.login'}):
if not since <= row['ts'] < until or row['ts'] > now or actor is not None and row['principal'] != actor:
continue
kind = ('sign_in' if row['kind'] == 'auth.login' else
'mcp' if row['detail'].get('via') == 'mcp' else 'api')
if kind in kinds:
day = datetime.fromtimestamp(row['ts'], timezone.utc).date().isoformat()
counts[(day, row['principal'], kind)] += 1
return [(*key, count) for key, count in sorted(counts.items())]
def read(tenant, limit=200, now=None, *, kinds=None, viewer=None) -> list[dict]:
"""Read one tenant's retained events. Read failures propagate instead of claiming no history."""
tenant = _attribution("tenant", tenant)
now = time.time() if now is None else float(now)
st = runtime.get_runtime(str(tenant))
if account_rows.records_backend(st):
doc = {'events': account_rows.read_events(st, tenant, since=now - RETENTION_SECONDS,
limit=limit, kinds=kinds)}
else:
doc = st.get(BUCKET)
# A read can include an older process-local queue without making it durable.
with _QUEUE_LOCK:
queued = list(_QUEUE.get(tenant, ()))
if queued:
doc = {'events': list((doc or {}).get('events') or []) + queued}
rows = ordered(doc, tenant, limit, kinds=kinds)
rows = [row for row in rows if row["ts"] >= now - RETENTION_SECONDS]
if not platform_admin.is_platform_admin(viewer):
rows = [{**row, "detail": {key: value for key, value in row["detail"].items() if key != "intent"}}
for row in rows]
return rows
def external_write_receipt(tenant, principal, *, tool, auth_type, affected_count,
diff_digest, revision_before, revision_after,
confirmation_id, activity_detail=None) -> dict:
"""Emit and return a value-free C7 write receipt.
Target ids, field names, record values and arguments deliberately do not belong here. The
receipt proves who confirmed which normalized diff and how many objects moved; the concrete
before/after stays in the short-lived preview response instead of the retained security log.
"""
detail = {
"surface": "ai_api",
"tool": str(tool or "")[:128],
"authentication": str(auth_type or "")[:32],
"affected_count": max(0, int(affected_count or 0)),
"diff_digest": str(diff_digest or "")[:128],
"revision_before": str(revision_before or "")[:128],
"revision_after": str(revision_after or "")[:128],
"confirmation_id": str(confirmation_id or "")[:128],
}
if activity_detail is not None:
detail["write_tool"] = detail["tool"]
detail.update(activity_detail)
item = event("external.write", tenant, principal, detail)
stored = record("external.write", tenant, principal, detail)
return {"event": item, "persisted": stored is not None}