File size: 18,946 Bytes
992a41f c4a1388 a60104f 992a41f a60104f 992a41f 9e15897 992a41f 85f794d 81a8217 b2aa446 cfa8541 992a41f c72f0a1 992a41f a60104f 992a41f cfa8541 affe37f 992a41f cfa8541 992a41f affe37f 992a41f c72f0a1 992a41f c72f0a1 992a41f a60104f 9e15897 a60104f 9e15897 a60104f b2aa446 a60104f b2aa446 a60104f 992a41f 5e98fbd 992a41f 9e15897 992a41f a60104f 992a41f b2aa446 992a41f b2aa446 992a41f c4a1388 affe37f 992a41f 9e15897 c72f0a1 affe37f cfa8541 c72f0a1 cfa8541 c72f0a1 cfa8541 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 | """Bounded, tenant-scoped security events. Refuse secrets before attempting any store I/O."""
from collections.abc import Mapping
from collections import Counter
from datetime import datetime, timezone
import atexit
import math
import re
import threading
import time
from harness import runtime
from core import account_rows, freeze_notice, platform_admin
AUDIT_KINDS = frozenset({
"auth.login", "auth.login_failed", "auth.lockout", "auth.mfa_enrolled",
"auth.mfa_failed", "auth.logout", "admin.grant", "credential.added",
"credential.rotated", "credential.revoked", "amazon.access", "amazon.sync", "amazon.throttled",
"amazon.authorization_lost", "amazon.deletion_scheduled", "amazon.personal_read", "data.purge", "data.restore",
"config.changed", "customer.personal_read",
"external.query",
"external.write",
})
BUCKET = "security_events"
MAX_EVENTS = 2000
EXTERNAL_KINDS = frozenset({"external.query", "external.write"})
RETENTION_SECONDS = 400 * 24 * 60 * 60
#: Kinds a write-through store queues and writes in batches, so one query never pays a document write.
DEFERRED_KINDS = frozenset({"external.query"})
#: A tenant queue this long is written by the caller that fills it: the queue never grows past it.
QUEUE_MAX = 200
FLUSH_INTERVAL_S = 2.0
_ATTRIBUTION_MAX = 256
_KEY_MAX = 128
def _key_name(key):
return "".join(c for c in key.casefold() if c.isalnum())
# Restated from core.keychain._PREVIEW_FIELDS to avoid coupling a log to the keychain.
# Keep the first six names and that tuple in step; client_secret is also an Amazon secret.
SECRET_KEYS = frozenset(_key_name(key) for key in (
"api_key", "access_token", "value", "password", "token", "refresh_token",
"client_secret", "secret", "private_key", "authorization", "cookie", "session",
"mfa_secret", "recovery_code", "otpauth", "credential",
))
_BEARER = re.compile(r"\bbearer\s+\S+", re.IGNORECASE)
_JWT = re.compile(r"\beyJ[A-Za-z0-9_-]*\.")
_LOOPABLE_CREDENTIAL = re.compile(r"\blp(?:b|o|r|c)1\.[A-Za-z0-9._~-]+")
_INTENT_CREDENTIAL = re.compile(
r"Atzr\||\bcfm1\.|\blp[borc]1\.|\b[A-Za-z0-9_-]{24,}\.[A-Za-z0-9_-]{24,}\b"
r"|\b(?:password|api[_ -]?key|client[_ -]?secret|access[_ -]?token)\s*[:=]"
r"|\b(?:sk-[A-Za-z0-9_-]{12,}|hf_[A-Za-z0-9]{12,}|gsk_[A-Za-z0-9_-]{12,})", re.IGNORECASE)
class AuditRefused(ValueError):
"""A call-site error carrying only the field name and refusal reason, never its value."""
def __init__(self, key, reason):
self.key = key
self.reason = reason
# Explaining a refusal with the value would put the secret into its traceback.
super().__init__(f"audit refused key {key!r}: {reason}")
def _string(key, value):
if len(value) > 512:
raise AuditRefused(key, "string exceeds 512 characters")
if value.startswith("Atzr|"):
raise AuditRefused(key, "Atzr refresh-token prefix")
if _BEARER.search(value):
raise AuditRefused(key, "bearer token shape")
if _JWT.search(value):
raise AuditRefused(key, "JWT token shape")
if _LOOPABLE_CREDENTIAL.search(value):
raise AuditRefused(key, "Loopable credential shape")
def _attribution(key, value):
if not isinstance(value, str) or not value.strip():
raise AuditRefused(key, "attribution requires a non-empty string")
_string(key, value)
value = value.strip()
_string(key, value)
if len(value) > _ATTRIBUTION_MAX:
raise AuditRefused(key, "attribution exceeds 256 characters")
return value.lower() if key == "tenant" else value
def redact_intent(value):
"""Keep bounded intent prose while replacing credential patterns and invalid text."""
if not isinstance(value, str) or not 1 <= len(value) <= 500 or not value.strip():
return "[invalid]"
if _INTENT_CREDENTIAL.search(value) or any(ord(char) < 32 for char in value):
return "[redacted]"
try:
_string("intent", value)
except AuditRefused:
return "[redacted]"
return value.strip()
def event(kind, tenant, principal, detail=None, *, now=None) -> dict:
"""Validate without I/O; an explicit epoch makes this pure and deterministic for gates."""
if not isinstance(kind, str) or kind not in AUDIT_KINDS:
raise AuditRefused("kind", "unknown audit kind")
tenant = _attribution("tenant", tenant)
principal = _attribution("principal", principal)
if detail is None:
detail = {}
if not isinstance(detail, Mapping):
raise AuditRefused("detail", "detail must be a flat mapping")
clean = {}
for key, value in detail.items():
if not isinstance(key, str) or not key.strip():
raise AuditRefused("detail", "detail keys must be non-empty strings")
# A malformed key can itself contain a token. Never echo that key in diagnostics.
_string("detail key", key)
if len(key) > _KEY_MAX or any(ord(c) < 32 for c in key):
raise AuditRefused("detail key", "key exceeds 128 characters or contains controls")
if _key_name(key) in SECRET_KEYS:
raise AuditRefused(key, "secret key name")
if isinstance(value, (Mapping, list, tuple, set, frozenset)):
raise AuditRefused(key, "nested value violates flat detail")
if type(value) not in (str, int, bool, float):
raise AuditRefused(key, "unsupported detail value type")
if isinstance(value, str):
_string(key, value)
if isinstance(value, float) and not math.isfinite(value):
raise AuditRefused(key, "detail number must be finite")
clean[key] = value
try:
ts = float(time.time() if now is None else now)
except (TypeError, ValueError, OverflowError):
raise AuditRefused("ts", "timestamp must be finite epoch seconds") from None
if not math.isfinite(ts):
raise AuditRefused("ts", "timestamp must be finite epoch seconds")
return {"ts": ts, "kind": kind, "tenant": tenant, "principal": principal, "detail": clean}
def ordered(doc, tenant, limit=200, *, kinds=None) -> list[dict]:
"""Return ascending timestamps; a limit retains the most recent rows, still ascending.
The stored tenant must match exactly even inside an already scoped runtime. This second
wall protects readers if a future migration ever puts multiple tenants in one document.
"""
tenant = _attribution("tenant", tenant)
limit = max(0, int(limit))
if not limit or not isinstance(doc, Mapping):
return []
rows = doc.get("events")
if not isinstance(rows, list):
return []
retained = []
for row in rows:
if not isinstance(row, dict) or row.get("tenant") != tenant:
continue
if kinds is not None and row.get("kind") not in kinds:
continue
if type(row.get("ts")) not in (int, float):
# In particular, None must not ask event() for today's clock on a pure read.
continue
try:
# Revalidate persisted rows too, so legacy/corrupt documents cannot leak a secret.
retained.append(event(row.get("kind"), tenant, row.get("principal"),
row.get("detail"), now=row["ts"]))
except (AuditRefused, KeyError):
continue
return sorted(retained, key=lambda row: row["ts"])[-limit:]
def _well_formed(row, tenant):
"""A stored row this writer may keep without revalidation. Readers still revalidate every row."""
return (isinstance(row, dict) and row.get("tenant") == tenant
and type(row.get("ts")) in (int, float) and row.get("kind") in AUDIT_KINDS
and isinstance(row.get("principal"), str) and isinstance(row.get("detail"), dict))
def _append_rows(doc, tenant, items):
"""`{"events": rows}` with `items` added, pruned by age and capped per partition.
The common case is linear with no per-row validation. A document holding any row this tenant's
writer did not produce takes the full `ordered()` rebuild, which drops it.
"""
stored = doc.get("events") if isinstance(doc, Mapping) else None
stored = stored if isinstance(stored, list) else []
if all(_well_formed(row, tenant) for row in stored):
rows = list(stored)
else:
rows = ordered(doc, tenant, MAX_EVENTS * 2)
rows.extend(items)
# A queued batch can land after a later sign-in, so restore ascending order when it did.
if any(rows[i]["ts"] > rows[i + 1]["ts"] for i in range(len(rows) - 1)):
rows.sort(key=lambda row: row["ts"])
cutoff = max(item["ts"] for item in items) - RETENTION_SECONDS
if rows and rows[0]["ts"] < cutoff:
rows = [row for row in rows if row["ts"] >= cutoff]
# Sign-ins and external calls each own their 2000-event retention partition, oldest out first.
external = sum(1 for row in rows if row["kind"] in EXTERNAL_KINDS)
drop = {True: max(0, external - MAX_EVENTS), False: max(0, len(rows) - external - MAX_EVENTS)}
if drop[True] or drop[False]:
kept = []
for row in rows:
side = row["kind"] in EXTERNAL_KINDS
if drop[side]:
drop[side] -= 1
continue
kept.append(row)
rows = kept
return {"events": rows}
def _write(tenant, items):
st = runtime.get_runtime(str(tenant))
if account_rows.records_backend(st):
account_rows.append_events(st, items, retention_seconds=RETENTION_SECONDS)
return
# Bare core.store.update is bound to tenant #0. Always use the tenant-scoped door.
# ⭐⭐ flush="async" IS DELIBERATE AND IT IS NOT A PERFORMANCE TWEAK. The default
# flush="sync" is a BLOCKING upload (core/store.py::update says so), and this writer sits
# on the sign-in path: every login would then wait on a network round trip, and on the hf
# backend every login would spend one of that repo's 256 commits per hour. The async
# branch applies the mutation to the in-process cache and a coalescing worker commits it,
# which is the same posture the table-workspace hot path has used since wave 7.
# ⚠ THE COST, stated rather than hidden: a container that dies before the worker flushes
# loses the events still in its cache. That is the same window `record` already accepts by
# swallowing a store failure, and it is the right side of the trade: an audit log must
# never be the reason a person cannot sign in.
st.update(
BUCKET, lambda doc: _append_rows(doc, tenant, items), flush="async")
def _writes_through(tenant):
"""True when the tenant store ignores flush="async" and commits the whole document per call (Postgres)."""
try:
backend = runtime.get_runtime(str(tenant))._document_store()
except Exception:
return False
return type(backend).__module__ == "core.store_pg"
#: Store refusals per (tenant, bucket), read by the api layer's owner alert.
#: Core never imports that alert path: the api layer installs a sink with set_loss_sink.
_LOSSES = {}
_LOSS_LOCK = threading.Lock()
_LOSS_SINK = []
def set_loss_sink(fn):
"""Install the one callable told of every lost or recovered write; None removes it."""
_LOSS_SINK[:] = [fn] if fn is not None else []
def _tell_sink(entry):
for fn in list(_LOSS_SINK):
try:
fn(entry)
except Exception:
pass
def note_loss(tenant, bucket, exc, count=1):
"""Count a write the store refused. The cause keeps raw text: every reader must scrub it."""
try:
key = (str(tenant), str(bucket))
with _LOSS_LOCK:
entry = _LOSSES.setdefault(key, {"tenant": key[0], "bucket": key[1], "count": 0,
"first": time.time()})
entry.update(count=entry["count"] + max(1, int(count)), lost=True, last=time.time(),
cause=f"{type(exc).__name__}: {exc}"[:500])
snapshot = dict(entry)
_tell_sink(snapshot)
except Exception:
pass
def note_ok(tenant, bucket):
"""Mark a bucket with a past loss as writing again. Costs one dict read when nothing was lost."""
if not _LOSSES:
return
with _LOSS_LOCK:
entry = _LOSSES.get((str(tenant), str(bucket)))
if entry is None or not entry.get("lost"):
return
entry.update(lost=False, count=0, last=time.time())
snapshot = dict(entry)
_tell_sink(snapshot)
def lost_writes(tenant=None) -> list[dict]:
"""Every bucket this process saw refused or recovered, oldest first; `lost` says which."""
with _LOSS_LOCK:
rows = [dict(entry) for entry in _LOSSES.values()
if tenant is None or entry["tenant"] == str(tenant)]
return sorted(rows, key=lambda row: (row["first"], row["bucket"]))
_QUEUE = {}
_QUEUE_LOCK = threading.Lock()
_WAKE = threading.Event()
_FLUSHER = []
def _flush_loop():
while True:
_WAKE.wait(FLUSH_INTERVAL_S)
_WAKE.clear()
try:
flush_pending()
except Exception:
pass
def _enqueue(tenant, item):
with _QUEUE_LOCK:
pending = _QUEUE.setdefault(tenant, [])
pending.append(item)
full = len(pending) >= QUEUE_MAX
if not _FLUSHER:
_FLUSHER.append(threading.Thread(target=_flush_loop, name="audit-flush", daemon=True))
_FLUSHER[0].start()
atexit.register(flush_pending)
if full:
flush_pending(tenant)
def flush_pending(tenant=None) -> int:
"""Write queued events in one update per tenant. Returns how many rows were handed to the store."""
with _QUEUE_LOCK:
names = list(_QUEUE) if tenant is None else [_attribution("tenant", tenant)]
batches = {name: _QUEUE.pop(name) for name in names if _QUEUE.get(name)}
written = 0
for name, items in batches.items():
try:
_write(name, items)
written += len(items)
except Exception as exc:
# Exception text can contain credentials from a provider or connection string.
print(f"[aios-api] audit lost {len(items)} queued event(s) for tenant {name!r}: "
f"store unavailable ({type(exc).__name__}).")
note_loss(name, BUCKET, exc, len(items))
continue
note_ok(name, BUCKET)
return written
def record(kind, tenant, principal, detail=None) -> dict | None:
"""Refusals raise before I/O. Store failures lose the event, never the user's sign-in."""
item = event(kind, tenant, principal, detail)
if freeze_notice.enabled():
return None
tenant = item["tenant"]
# Row storage commits one event immediately, including audited queries.
try:
_write(tenant, [item])
except Exception as exc:
# Exception text can contain credentials from a provider or connection string.
print(f"[aios-api] audit lost {item['kind']} for tenant {tenant!r}: "
f"store unavailable ({type(exc).__name__}).")
note_loss(tenant, BUCKET, exc)
return None
note_ok(tenant, BUCKET)
return item
def usage_counts(st, tenant, *, since, until, now, kinds, actor=None):
"""Count retained metadata in SQL and include the same bounded local queue as audit reads."""
tenant = _attribution("tenant", tenant)
since = max(float(since), float(now) - RETENTION_SECONDS)
rows = account_rows.event_counts(st, tenant, since=since, until=until, now=now,
kinds=kinds, actor=actor)
if rows is None:
return None
counts = Counter({(day, who, kind): count for day, who, kind, count in rows})
with _QUEUE_LOCK:
queued = list(_QUEUE.get(tenant, ()))
# Queue entries remain subject to the canonical validator and never become durable on a GET.
for row in ordered({'events': queued}, tenant, len(queued), kinds=EXTERNAL_KINDS | {'auth.login'}):
if not since <= row['ts'] < until or row['ts'] > now or actor is not None and row['principal'] != actor:
continue
kind = ('sign_in' if row['kind'] == 'auth.login' else
'mcp' if row['detail'].get('via') == 'mcp' else 'api')
if kind in kinds:
day = datetime.fromtimestamp(row['ts'], timezone.utc).date().isoformat()
counts[(day, row['principal'], kind)] += 1
return [(*key, count) for key, count in sorted(counts.items())]
def read(tenant, limit=200, now=None, *, kinds=None, viewer=None) -> list[dict]:
"""Read one tenant's retained events. Read failures propagate instead of claiming no history."""
tenant = _attribution("tenant", tenant)
now = time.time() if now is None else float(now)
st = runtime.get_runtime(str(tenant))
if account_rows.records_backend(st):
doc = {'events': account_rows.read_events(st, tenant, since=now - RETENTION_SECONDS,
limit=limit, kinds=kinds)}
else:
doc = st.get(BUCKET)
# A read can include an older process-local queue without making it durable.
with _QUEUE_LOCK:
queued = list(_QUEUE.get(tenant, ()))
if queued:
doc = {'events': list((doc or {}).get('events') or []) + queued}
rows = ordered(doc, tenant, limit, kinds=kinds)
rows = [row for row in rows if row["ts"] >= now - RETENTION_SECONDS]
if not platform_admin.is_platform_admin(viewer):
rows = [{**row, "detail": {key: value for key, value in row["detail"].items() if key != "intent"}}
for row in rows]
return rows
def external_write_receipt(tenant, principal, *, tool, auth_type, affected_count,
diff_digest, revision_before, revision_after,
confirmation_id, activity_detail=None) -> dict:
"""Emit and return a value-free C7 write receipt.
Target ids, field names, record values and arguments deliberately do not belong here. The
receipt proves who confirmed which normalized diff and how many objects moved; the concrete
before/after stays in the short-lived preview response instead of the retained security log.
"""
detail = {
"surface": "ai_api",
"tool": str(tool or "")[:128],
"authentication": str(auth_type or "")[:32],
"affected_count": max(0, int(affected_count or 0)),
"diff_digest": str(diff_digest or "")[:128],
"revision_before": str(revision_before or "")[:128],
"revision_after": str(revision_after or "")[:128],
"confirmation_id": str(confirmation_id or "")[:128],
}
if activity_detail is not None:
detail["write_tool"] = detail["tool"]
detail.update(activity_detail)
item = event("external.write", tenant, principal, detail)
stored = record("external.write", tenant, principal, detail)
return {"event": item, "persisted": stored is not None}
|