File size: 18,946 Bytes
367a695
 
023301a
 
07a68d0
367a695
 
07a68d0
367a695
 
 
f462120
367a695
 
 
 
 
8729c1d
dec006b
8133df9
2e7e617
 
367a695
 
 
dec006b
367a695
07a68d0
 
 
 
 
367a695
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2e7e617
103a492
 
 
 
367a695
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2e7e617
 
367a695
 
 
 
 
 
 
 
 
 
 
 
 
103a492
 
 
 
 
 
 
 
 
 
 
 
 
367a695
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
dec006b
367a695
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
dec006b
 
367a695
 
 
 
 
 
 
 
 
 
 
 
07a68d0
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
f462120
 
 
 
07a68d0
 
 
 
 
 
 
 
 
 
 
f462120
07a68d0
 
 
 
 
 
 
 
 
 
 
 
8133df9
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
07a68d0
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
8133df9
 
 
07a68d0
 
 
367a695
 
 
971c29d
 
367a695
f462120
367a695
07a68d0
367a695
 
 
 
8133df9
367a695
8133df9
367a695
 
 
023301a
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
103a492
367a695
 
 
f462120
 
 
 
 
 
 
 
 
 
 
dec006b
103a492
 
 
 
 
2e7e617
 
 
 
dec006b
2e7e617
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
dec006b
 
 
2e7e617
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
"""Bounded, tenant-scoped security events. Refuse secrets before attempting any store I/O."""
from collections.abc import Mapping
from collections import Counter
from datetime import datetime, timezone
import atexit
import math
import re
import threading
import time

from harness import runtime
from core import account_rows, freeze_notice, platform_admin


AUDIT_KINDS = frozenset({
    "auth.login", "auth.login_failed", "auth.lockout", "auth.mfa_enrolled",
    "auth.mfa_failed", "auth.logout", "admin.grant", "credential.added",
    "credential.rotated", "credential.revoked", "amazon.access", "amazon.sync", "amazon.throttled",
    "amazon.authorization_lost", "amazon.deletion_scheduled", "amazon.personal_read", "data.purge", "data.restore",
    "config.changed", "customer.personal_read",
    "external.query",
    "external.write",
})
BUCKET = "security_events"
MAX_EVENTS = 2000
EXTERNAL_KINDS = frozenset({"external.query", "external.write"})
RETENTION_SECONDS = 400 * 24 * 60 * 60
#: Kinds a write-through store queues and writes in batches, so one query never pays a document write.
DEFERRED_KINDS = frozenset({"external.query"})
#: A tenant queue this long is written by the caller that fills it: the queue never grows past it.
QUEUE_MAX = 200
FLUSH_INTERVAL_S = 2.0
_ATTRIBUTION_MAX = 256
_KEY_MAX = 128


def _key_name(key):
    return "".join(c for c in key.casefold() if c.isalnum())


# Restated from core.keychain._PREVIEW_FIELDS to avoid coupling a log to the keychain.
# Keep the first six names and that tuple in step; client_secret is also an Amazon secret.
SECRET_KEYS = frozenset(_key_name(key) for key in (
    "api_key", "access_token", "value", "password", "token", "refresh_token",
    "client_secret", "secret", "private_key", "authorization", "cookie", "session",
    "mfa_secret", "recovery_code", "otpauth", "credential",
))
_BEARER = re.compile(r"\bbearer\s+\S+", re.IGNORECASE)
_JWT = re.compile(r"\beyJ[A-Za-z0-9_-]*\.")
_LOOPABLE_CREDENTIAL = re.compile(r"\blp(?:b|o|r|c)1\.[A-Za-z0-9._~-]+")
_INTENT_CREDENTIAL = re.compile(
    r"Atzr\||\bcfm1\.|\blp[borc]1\.|\b[A-Za-z0-9_-]{24,}\.[A-Za-z0-9_-]{24,}\b"
    r"|\b(?:password|api[_ -]?key|client[_ -]?secret|access[_ -]?token)\s*[:=]"
    r"|\b(?:sk-[A-Za-z0-9_-]{12,}|hf_[A-Za-z0-9]{12,}|gsk_[A-Za-z0-9_-]{12,})", re.IGNORECASE)


class AuditRefused(ValueError):
    """A call-site error carrying only the field name and refusal reason, never its value."""

    def __init__(self, key, reason):
        self.key = key
        self.reason = reason
        # Explaining a refusal with the value would put the secret into its traceback.
        super().__init__(f"audit refused key {key!r}: {reason}")


def _string(key, value):
    if len(value) > 512:
        raise AuditRefused(key, "string exceeds 512 characters")
    if value.startswith("Atzr|"):
        raise AuditRefused(key, "Atzr refresh-token prefix")
    if _BEARER.search(value):
        raise AuditRefused(key, "bearer token shape")
    if _JWT.search(value):
        raise AuditRefused(key, "JWT token shape")
    if _LOOPABLE_CREDENTIAL.search(value):
        raise AuditRefused(key, "Loopable credential shape")


def _attribution(key, value):
    if not isinstance(value, str) or not value.strip():
        raise AuditRefused(key, "attribution requires a non-empty string")
    _string(key, value)
    value = value.strip()
    _string(key, value)
    if len(value) > _ATTRIBUTION_MAX:
        raise AuditRefused(key, "attribution exceeds 256 characters")
    return value.lower() if key == "tenant" else value


def redact_intent(value):
    """Keep bounded intent prose while replacing credential patterns and invalid text."""
    if not isinstance(value, str) or not 1 <= len(value) <= 500 or not value.strip():
        return "[invalid]"
    if _INTENT_CREDENTIAL.search(value) or any(ord(char) < 32 for char in value):
        return "[redacted]"
    try:
        _string("intent", value)
    except AuditRefused:
        return "[redacted]"
    return value.strip()


def event(kind, tenant, principal, detail=None, *, now=None) -> dict:
    """Validate without I/O; an explicit epoch makes this pure and deterministic for gates."""
    if not isinstance(kind, str) or kind not in AUDIT_KINDS:
        raise AuditRefused("kind", "unknown audit kind")
    tenant = _attribution("tenant", tenant)
    principal = _attribution("principal", principal)
    if detail is None:
        detail = {}
    if not isinstance(detail, Mapping):
        raise AuditRefused("detail", "detail must be a flat mapping")
    clean = {}
    for key, value in detail.items():
        if not isinstance(key, str) or not key.strip():
            raise AuditRefused("detail", "detail keys must be non-empty strings")
        # A malformed key can itself contain a token. Never echo that key in diagnostics.
        _string("detail key", key)
        if len(key) > _KEY_MAX or any(ord(c) < 32 for c in key):
            raise AuditRefused("detail key", "key exceeds 128 characters or contains controls")
        if _key_name(key) in SECRET_KEYS:
            raise AuditRefused(key, "secret key name")
        if isinstance(value, (Mapping, list, tuple, set, frozenset)):
            raise AuditRefused(key, "nested value violates flat detail")
        if type(value) not in (str, int, bool, float):
            raise AuditRefused(key, "unsupported detail value type")
        if isinstance(value, str):
            _string(key, value)
        if isinstance(value, float) and not math.isfinite(value):
            raise AuditRefused(key, "detail number must be finite")
        clean[key] = value
    try:
        ts = float(time.time() if now is None else now)
    except (TypeError, ValueError, OverflowError):
        raise AuditRefused("ts", "timestamp must be finite epoch seconds") from None
    if not math.isfinite(ts):
        raise AuditRefused("ts", "timestamp must be finite epoch seconds")
    return {"ts": ts, "kind": kind, "tenant": tenant, "principal": principal, "detail": clean}


def ordered(doc, tenant, limit=200, *, kinds=None) -> list[dict]:
    """Return ascending timestamps; a limit retains the most recent rows, still ascending.

    The stored tenant must match exactly even inside an already scoped runtime. This second
    wall protects readers if a future migration ever puts multiple tenants in one document.
    """
    tenant = _attribution("tenant", tenant)
    limit = max(0, int(limit))
    if not limit or not isinstance(doc, Mapping):
        return []
    rows = doc.get("events")
    if not isinstance(rows, list):
        return []
    retained = []
    for row in rows:
        if not isinstance(row, dict) or row.get("tenant") != tenant:
            continue
        if kinds is not None and row.get("kind") not in kinds:
            continue
        if type(row.get("ts")) not in (int, float):
            # In particular, None must not ask event() for today's clock on a pure read.
            continue
        try:
            # Revalidate persisted rows too, so legacy/corrupt documents cannot leak a secret.
            retained.append(event(row.get("kind"), tenant, row.get("principal"),
                                  row.get("detail"), now=row["ts"]))
        except (AuditRefused, KeyError):
            continue
    return sorted(retained, key=lambda row: row["ts"])[-limit:]


def _well_formed(row, tenant):
    """A stored row this writer may keep without revalidation. Readers still revalidate every row."""
    return (isinstance(row, dict) and row.get("tenant") == tenant
            and type(row.get("ts")) in (int, float) and row.get("kind") in AUDIT_KINDS
            and isinstance(row.get("principal"), str) and isinstance(row.get("detail"), dict))


def _append_rows(doc, tenant, items):
    """`{"events": rows}` with `items` added, pruned by age and capped per partition.

    The common case is linear with no per-row validation. A document holding any row this tenant's
    writer did not produce takes the full `ordered()` rebuild, which drops it.
    """
    stored = doc.get("events") if isinstance(doc, Mapping) else None
    stored = stored if isinstance(stored, list) else []
    if all(_well_formed(row, tenant) for row in stored):
        rows = list(stored)
    else:
        rows = ordered(doc, tenant, MAX_EVENTS * 2)
    rows.extend(items)
    # A queued batch can land after a later sign-in, so restore ascending order when it did.
    if any(rows[i]["ts"] > rows[i + 1]["ts"] for i in range(len(rows) - 1)):
        rows.sort(key=lambda row: row["ts"])
    cutoff = max(item["ts"] for item in items) - RETENTION_SECONDS
    if rows and rows[0]["ts"] < cutoff:
        rows = [row for row in rows if row["ts"] >= cutoff]
    # Sign-ins and external calls each own their 2000-event retention partition, oldest out first.
    external = sum(1 for row in rows if row["kind"] in EXTERNAL_KINDS)
    drop = {True: max(0, external - MAX_EVENTS), False: max(0, len(rows) - external - MAX_EVENTS)}
    if drop[True] or drop[False]:
        kept = []
        for row in rows:
            side = row["kind"] in EXTERNAL_KINDS
            if drop[side]:
                drop[side] -= 1
                continue
            kept.append(row)
        rows = kept
    return {"events": rows}


def _write(tenant, items):
    st = runtime.get_runtime(str(tenant))
    if account_rows.records_backend(st):
        account_rows.append_events(st, items, retention_seconds=RETENTION_SECONDS)
        return
    # Bare core.store.update is bound to tenant #0. Always use the tenant-scoped door.
    # ⭐⭐ flush="async" IS DELIBERATE AND IT IS NOT A PERFORMANCE TWEAK. The default
    # flush="sync" is a BLOCKING upload (core/store.py::update says so), and this writer sits
    # on the sign-in path: every login would then wait on a network round trip, and on the hf
    # backend every login would spend one of that repo's 256 commits per hour. The async
    # branch applies the mutation to the in-process cache and a coalescing worker commits it,
    # which is the same posture the table-workspace hot path has used since wave 7.
    # ⚠ THE COST, stated rather than hidden: a container that dies before the worker flushes
    # loses the events still in its cache. That is the same window `record` already accepts by
    # swallowing a store failure, and it is the right side of the trade: an audit log must
    # never be the reason a person cannot sign in.
    st.update(
        BUCKET, lambda doc: _append_rows(doc, tenant, items), flush="async")


def _writes_through(tenant):
    """True when the tenant store ignores flush="async" and commits the whole document per call (Postgres)."""
    try:
        backend = runtime.get_runtime(str(tenant))._document_store()
    except Exception:
        return False
    return type(backend).__module__ == "core.store_pg"


#: Store refusals per (tenant, bucket), read by the api layer's owner alert.
#: Core never imports that alert path: the api layer installs a sink with set_loss_sink.
_LOSSES = {}
_LOSS_LOCK = threading.Lock()
_LOSS_SINK = []


def set_loss_sink(fn):
    """Install the one callable told of every lost or recovered write; None removes it."""
    _LOSS_SINK[:] = [fn] if fn is not None else []


def _tell_sink(entry):
    for fn in list(_LOSS_SINK):
        try:
            fn(entry)
        except Exception:
            pass


def note_loss(tenant, bucket, exc, count=1):
    """Count a write the store refused. The cause keeps raw text: every reader must scrub it."""
    try:
        key = (str(tenant), str(bucket))
        with _LOSS_LOCK:
            entry = _LOSSES.setdefault(key, {"tenant": key[0], "bucket": key[1], "count": 0,
                                             "first": time.time()})
            entry.update(count=entry["count"] + max(1, int(count)), lost=True, last=time.time(),
                         cause=f"{type(exc).__name__}: {exc}"[:500])
            snapshot = dict(entry)
        _tell_sink(snapshot)
    except Exception:
        pass


def note_ok(tenant, bucket):
    """Mark a bucket with a past loss as writing again. Costs one dict read when nothing was lost."""
    if not _LOSSES:
        return
    with _LOSS_LOCK:
        entry = _LOSSES.get((str(tenant), str(bucket)))
        if entry is None or not entry.get("lost"):
            return
        entry.update(lost=False, count=0, last=time.time())
        snapshot = dict(entry)
    _tell_sink(snapshot)


def lost_writes(tenant=None) -> list[dict]:
    """Every bucket this process saw refused or recovered, oldest first; `lost` says which."""
    with _LOSS_LOCK:
        rows = [dict(entry) for entry in _LOSSES.values()
                if tenant is None or entry["tenant"] == str(tenant)]
    return sorted(rows, key=lambda row: (row["first"], row["bucket"]))


_QUEUE = {}
_QUEUE_LOCK = threading.Lock()
_WAKE = threading.Event()
_FLUSHER = []


def _flush_loop():
    while True:
        _WAKE.wait(FLUSH_INTERVAL_S)
        _WAKE.clear()
        try:
            flush_pending()
        except Exception:
            pass


def _enqueue(tenant, item):
    with _QUEUE_LOCK:
        pending = _QUEUE.setdefault(tenant, [])
        pending.append(item)
        full = len(pending) >= QUEUE_MAX
        if not _FLUSHER:
            _FLUSHER.append(threading.Thread(target=_flush_loop, name="audit-flush", daemon=True))
            _FLUSHER[0].start()
            atexit.register(flush_pending)
    if full:
        flush_pending(tenant)


def flush_pending(tenant=None) -> int:
    """Write queued events in one update per tenant. Returns how many rows were handed to the store."""
    with _QUEUE_LOCK:
        names = list(_QUEUE) if tenant is None else [_attribution("tenant", tenant)]
        batches = {name: _QUEUE.pop(name) for name in names if _QUEUE.get(name)}
    written = 0
    for name, items in batches.items():
        try:
            _write(name, items)
            written += len(items)
        except Exception as exc:
            # Exception text can contain credentials from a provider or connection string.
            print(f"[aios-api] audit lost {len(items)} queued event(s) for tenant {name!r}: "
                  f"store unavailable ({type(exc).__name__}).")
            note_loss(name, BUCKET, exc, len(items))
            continue
        note_ok(name, BUCKET)
    return written


def record(kind, tenant, principal, detail=None) -> dict | None:
    """Refusals raise before I/O. Store failures lose the event, never the user's sign-in."""
    item = event(kind, tenant, principal, detail)
    if freeze_notice.enabled():
        return None
    tenant = item["tenant"]
    # Row storage commits one event immediately, including audited queries.
    try:
        _write(tenant, [item])
    except Exception as exc:
        # Exception text can contain credentials from a provider or connection string.
        print(f"[aios-api] audit lost {item['kind']} for tenant {tenant!r}: "
              f"store unavailable ({type(exc).__name__}).")
        note_loss(tenant, BUCKET, exc)
        return None
    note_ok(tenant, BUCKET)
    return item


def usage_counts(st, tenant, *, since, until, now, kinds, actor=None):
    """Count retained metadata in SQL and include the same bounded local queue as audit reads."""
    tenant = _attribution("tenant", tenant)
    since = max(float(since), float(now) - RETENTION_SECONDS)
    rows = account_rows.event_counts(st, tenant, since=since, until=until, now=now,
                                     kinds=kinds, actor=actor)
    if rows is None:
        return None
    counts = Counter({(day, who, kind): count for day, who, kind, count in rows})
    with _QUEUE_LOCK:
        queued = list(_QUEUE.get(tenant, ()))
    # Queue entries remain subject to the canonical validator and never become durable on a GET.
    for row in ordered({'events': queued}, tenant, len(queued), kinds=EXTERNAL_KINDS | {'auth.login'}):
        if not since <= row['ts'] < until or row['ts'] > now or actor is not None and row['principal'] != actor:
            continue
        kind = ('sign_in' if row['kind'] == 'auth.login' else
                'mcp' if row['detail'].get('via') == 'mcp' else 'api')
        if kind in kinds:
            day = datetime.fromtimestamp(row['ts'], timezone.utc).date().isoformat()
            counts[(day, row['principal'], kind)] += 1
    return [(*key, count) for key, count in sorted(counts.items())]


def read(tenant, limit=200, now=None, *, kinds=None, viewer=None) -> list[dict]:
    """Read one tenant's retained events. Read failures propagate instead of claiming no history."""
    tenant = _attribution("tenant", tenant)
    now = time.time() if now is None else float(now)
    st = runtime.get_runtime(str(tenant))
    if account_rows.records_backend(st):
        doc = {'events': account_rows.read_events(st, tenant, since=now - RETENTION_SECONDS,
                                                 limit=limit, kinds=kinds)}
    else:
        doc = st.get(BUCKET)
    # A read can include an older process-local queue without making it durable.
    with _QUEUE_LOCK:
        queued = list(_QUEUE.get(tenant, ()))
    if queued:
        doc = {'events': list((doc or {}).get('events') or []) + queued}
    rows = ordered(doc, tenant, limit, kinds=kinds)
    rows = [row for row in rows if row["ts"] >= now - RETENTION_SECONDS]
    if not platform_admin.is_platform_admin(viewer):
        rows = [{**row, "detail": {key: value for key, value in row["detail"].items() if key != "intent"}}
                for row in rows]
    return rows


def external_write_receipt(tenant, principal, *, tool, auth_type, affected_count,
                           diff_digest, revision_before, revision_after,
                           confirmation_id, activity_detail=None) -> dict:
    """Emit and return a value-free C7 write receipt.

    Target ids, field names, record values and arguments deliberately do not belong here. The
    receipt proves who confirmed which normalized diff and how many objects moved; the concrete
    before/after stays in the short-lived preview response instead of the retained security log.
    """
    detail = {
        "surface": "ai_api",
        "tool": str(tool or "")[:128],
        "authentication": str(auth_type or "")[:32],
        "affected_count": max(0, int(affected_count or 0)),
        "diff_digest": str(diff_digest or "")[:128],
        "revision_before": str(revision_before or "")[:128],
        "revision_after": str(revision_after or "")[:128],
        "confirmation_id": str(confirmation_id or "")[:128],
    }
    if activity_detail is not None:
        detail["write_tool"] = detail["tool"]
        detail.update(activity_detail)
    item = event("external.write", tenant, principal, detail)
    stored = record("external.write", tenant, principal, detail)
    return {"event": item, "persisted": stored is not None}