Spaces:
Running on CPU Upgrade
Running on CPU Upgrade
Download platform/core/account_rows.py from fsanyoto/runloopable: direct link, hf CLI and curl.
- Browser
- Download file 38.1 kB
-
https://huggingface.co/spaces/fsanyoto/runloopable/resolve/main/platform/core/account_rows.py
- Command line
-
hf download hf://spaces/fsanyoto/runloopable/platform/core/account_rows.py
-
curl -L -o account_rows.py https://huggingface.co/spaces/fsanyoto/runloopable/resolve/main/platform/core/account_rows.py
38.1 kB
| """Store accounts, security events and notifications in rows with explicit document migration.""" | |
| import argparse | |
| import copy | |
| import hashlib | |
| import json | |
| import math | |
| import time | |
| import uuid | |
| from contextlib import contextmanager | |
| from datetime import datetime, timezone | |
| from core import request_timing | |
| DOCUMENT_KEY = 'users' | |
| PREFIX = 'account-row:' | |
| LAYOUT_KEY = '__layout__' | |
| VERSION = 1 | |
| DELETED = {'__accountDeleted__': True} | |
| EVENTS_KEY = 'security_events' | |
| NOTIFICATIONS_KEY = 'alert_notifications' | |
| EVENT_PREFIX = 'security-event-row:' | |
| NOTICE_PREFIX = 'notification-row:' | |
| NOTIFICATION_TARGET = 200 | |
| RECORD_BUCKETS = (EVENTS_KEY, NOTIFICATIONS_KEY) | |
| def row_key(username): | |
| """Keep canonical account names separate from document and lockout keys.""" | |
| return PREFIX + str(username or '').strip().lower().encode('utf-8').hex() | |
| def is_marker(value): | |
| return (isinstance(value, dict) and value.get(LAYOUT_KEY) == 'rows' | |
| and value.get('accountRows') == VERSION) | |
| def _source(st): | |
| if st is None: | |
| from core import store | |
| return store | |
| return st | |
| def _pg_source(st): | |
| """Resolve only actual PostgreSQL handles, keeping supplied stores authoritative.""" | |
| from core import store, store_pg | |
| if st is store and store.backend() == 'pg': | |
| handle = store._d() | |
| return (store_pg, handle.tenant_slug) if isinstance(handle, store_pg.PgStore) else None | |
| if isinstance(st, store_pg.PgStore): | |
| return store_pg, st.tenant_slug | |
| return None | |
| def _row_text(pg, slug, key, fresh): | |
| """Validate only this row, without the store cache's tenant-wide metadata check.""" | |
| pool = pg._pool() | |
| doc = (pg._effective_slug(slug), key) | |
| cache_key = (pool,) + doc | |
| def fetch(): | |
| with pg._READ_FLIGHT_LOCK: | |
| generation = pg._WRITE_GENERATION.get(doc, 0) | |
| with pg._WHOLE_TEXT_LOCK: | |
| cached = pg._WHOLE_TEXT.get(cache_key) if not fresh else None | |
| if cached is not None: | |
| pg._WHOLE_TEXT.move_to_end(cache_key) | |
| row = pg._read(pool, pg._q( | |
| 'SELECT kv.rev, kv.updated_at, CASE WHEN kv.rev = %s AND kv.updated_at = %s ' | |
| 'THEN NULL ELSE kv.value::text END FROM {tbl} AS kv WHERE kv.key = %s', | |
| 'store_kv', slug), (cached[0] if cached else None, | |
| cached[1] if cached else None, key), one=True) | |
| text = row[2] if row else None | |
| request_timing.record_store('get', key, bytes_read=request_timing.text_bytes(text), count=0) | |
| if row and text is None: | |
| if cached is None or row[:2] != cached[:2]: | |
| raise RuntimeError('Account cache has no content for its validated revision') | |
| text = cached[2] | |
| pg._remember_whole(cache_key, generation, row[0] if row else None, | |
| row[1] if row else None, text) | |
| return text | |
| return fetch() if fresh else pg._shared_read(key, 'account-row', slug, fetch, pool=pool) | |
| def _row(pg, slug, username, fresh): | |
| key = row_key(username) | |
| text = pg._request_text(key, 'account-row', slug, | |
| lambda: _row_text(pg, slug, key, fresh), fresh=fresh) | |
| return pg._parse_read(key, 'get', text) | |
| def get(username, *, st=None, fresh=False): | |
| """Read one account using the store's revision-and-timestamp immutable text cache.""" | |
| st = _source(st) | |
| username = str(username or '').strip().lower() | |
| if not username: | |
| return None | |
| pg_source = _pg_source(st) | |
| if pg_source: | |
| pg, slug = pg_source | |
| record = _row(pg, slug, username, fresh) | |
| if record == DELETED: | |
| return None | |
| if record: | |
| return record | |
| # Before migration, project only this account from the legacy document. | |
| legacy = pg.get_entry(DOCUMENT_KEY, username, tenant_slug=slug) | |
| if legacy is not None: | |
| return legacy | |
| # A migration can commit between the row miss and the legacy projection. | |
| record = _row(pg, slug, username, True) | |
| return None if not record or record == DELETED else record | |
| read_entry = getattr(st, 'get_entry', None) | |
| if read_entry is not None and not fresh: | |
| return read_entry(DOCUMENT_KEY, username) | |
| return (st.get(DOCUMENT_KEY, fresh=fresh) or {}).get(username) | |
| def _read_rows(con, pg, slug, usernames=None, lock=False): | |
| if usernames is None: | |
| where, params = 'key LIKE %s', (PREFIX + '%',) | |
| else: | |
| keys = sorted({row_key(name) for name in usernames}) | |
| if not keys: | |
| return {} | |
| where = 'key IN (' + ', '.join('%s' for _ in keys) + ')' | |
| params = tuple(keys) | |
| rows = con.execute(pg._q('SELECT key, value::text FROM {tbl} WHERE ' + where | |
| + ' ORDER BY key' + (' FOR UPDATE' if lock else ''), | |
| 'store_kv', slug), params).fetchall() | |
| result = {} | |
| for key, text in rows: | |
| name = bytes.fromhex(key[len(PREFIX):]).decode('utf-8') | |
| if row_key(name) != key: | |
| raise ValueError('Account row has a noncanonical key') | |
| value = pg._parsed(text) | |
| if value != DELETED: | |
| result[name] = value | |
| request_timing.record_store('account_rows', key, | |
| bytes_read=request_timing.text_bytes(text)) | |
| return result | |
| def _head(con, pg, slug, lock=False): | |
| row = con.execute(pg._q('SELECT value::text FROM {tbl} WHERE key = %s' | |
| + (' FOR UPDATE' if lock else ''), 'store_kv', slug), | |
| (DOCUMENT_KEY,)).fetchone() | |
| return pg._parsed(row[0]) if row else {} | |
| def _settings(con, pg): | |
| limits = pg._pg_limits() | |
| for setting, seconds in (('statement_timeout', limits['statement']), | |
| ('lock_timeout', pg._lock_timeout_s()), | |
| ('idle_in_transaction_session_timeout', limits['idle'])): | |
| con.execute('SELECT set_config(%s, %s, true)', | |
| (setting, f'{int(seconds * 1000)}ms')) | |
| def _read_settings(con, pg, statement_seconds): | |
| limits = pg._pg_limits() | |
| for name, seconds in (('statement_timeout', statement_seconds), | |
| ('lock_timeout', pg._lock_timeout_s()), | |
| ('idle_in_transaction_session_timeout', limits['idle'])): | |
| milliseconds = max(1, int(request_timing.remaining_budget(seconds) * 1000)) | |
| con.execute("SELECT set_config(%s, LEAST(COALESCE(NULLIF(" | |
| "EXTRACT(epoch FROM current_setting(%s)::interval) * 1000, 0), %s), %s)" | |
| "::bigint::text || 'ms', true)", | |
| (name, name, milliseconds, milliseconds)) | |
| def read_snapshot(tenant_slug, *, require_stable=False, statement_timeout_s=None): | |
| """Own a bounded repeatable-read/read-only snapshot, or borrow the tenant's enlisted transaction. | |
| Borrowed reads retain isolation, writes and transaction ownership, and only tighten local timeouts. | |
| Stable verification refuses a borrowed transaction below repeatable read. Exceptions propagate. | |
| """ | |
| from core import store_pg as pg | |
| slug = pg._effective_slug(tenant_slug) | |
| limits = pg._pg_limits() | |
| seconds = limits['statement'] if statement_timeout_s is None else float(statement_timeout_s) | |
| if not math.isfinite(seconds) or seconds <= 0: | |
| raise ValueError('Statement timeout must be finite and positive') | |
| seconds = min(seconds, limits['statement']) | |
| with request_timing.budget_scope(time.monotonic() + limits['acquire'] + seconds, | |
| pg.PostgresOperationTimeout): | |
| enlisted = pg._CALLER_TRANSACTION.get() | |
| if enlisted is not None: | |
| con = enlisted.con | |
| if require_stable: | |
| isolation = con.execute('SHOW transaction_isolation').fetchone()[0] | |
| if isolation not in ('repeatable read', 'serializable'): | |
| raise ValueError('Verification requires a repeatable-read caller transaction') | |
| _read_settings(con, pg, seconds) | |
| yield con | |
| else: | |
| # HolderPool.connection queries timeout settings before yielding. Its autocommit door does not. | |
| with pg._pool().autocommit(timeout=request_timing.remaining_budget(limits['acquire'])) as con: | |
| with con.transaction(): | |
| con.execute('SET TRANSACTION ISOLATION LEVEL REPEATABLE READ, READ ONLY') | |
| _read_settings(con, pg, seconds) | |
| yield con | |
| request_timing.remaining_budget() | |
| def _write(con, pg, slug, key, value): | |
| payload = pg._payload(key, value, con, slug) | |
| con.execute(pg._q('INSERT INTO {tbl} (key, value) VALUES (%s, %s::jsonb) ' | |
| 'ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value, ' | |
| 'updated_at = now(), rev = {bare}.rev + 1', 'store_kv', slug), | |
| (key, payload)) | |
| request_timing.record_store('update', key, | |
| bytes_written=request_timing.text_bytes(payload)) | |
| def _all(st): | |
| pg_source = _pg_source(st) | |
| if not pg_source: | |
| return st.get(DOCUMENT_KEY) or {} | |
| pg, slug = pg_source | |
| with read_snapshot(slug) as con: | |
| head = _head(con, pg, slug) | |
| return _read_rows(con, pg, slug) if is_marker(head) else head | |
| class Registry(dict): | |
| """Load individual accounts on lookup and all rows only for enumeration.""" | |
| def __init__(self, st): | |
| self.st, self.loaded = st, False | |
| def _load(self): | |
| if not self.loaded: | |
| super().update(_all(self.st)) | |
| self.loaded = True | |
| return self | |
| def __bool__(self): | |
| # A lazy view must survive callers' `registry() or {}` without enumerating accounts. | |
| return True if not self.loaded else super().__len__() != 0 | |
| def get(self, key, default=None): | |
| if self.loaded: | |
| return super().get(key, default) | |
| record = get(key, st=self.st) | |
| return default if record is None else record | |
| def __getitem__(self, key): | |
| absent = object() | |
| record = self.get(key, absent) | |
| if record is absent: | |
| raise KeyError(key) | |
| return record | |
| def __contains__(self, key): | |
| return self.get(key) is not None | |
| def __iter__(self): | |
| return super(Registry, self._load()).__iter__() | |
| def __len__(self): | |
| return super(Registry, self._load()).__len__() | |
| def keys(self): | |
| return super(Registry, self._load()).keys() | |
| def items(self): | |
| return super(Registry, self._load()).items() | |
| def values(self): | |
| return super(Registry, self._load()).values() | |
| def copy(self): | |
| return dict(self.items()) | |
| def __deepcopy__(self, memo): | |
| return copy.deepcopy(self.copy(), memo) | |
| def registry(*, st=None): | |
| """Return a lazy PostgreSQL roster or the compatibility store's document.""" | |
| st = _source(st) | |
| return Registry(st) if _pg_source(st) else st.get(DOCUMENT_KEY) | |
| def resolve(identifier, *, st=None): | |
| """Prefer the canonical key, then find one email alias without loading the roster.""" | |
| st = _source(st) | |
| identifier = str(identifier or '').strip().lower() | |
| record = get(identifier, st=st, fresh=True) | |
| if record is not None or '@' not in identifier: | |
| return identifier, record | |
| pg_source = _pg_source(st) | |
| if pg_source: | |
| pg, slug = pg_source | |
| with pg._pool().connection() as con: | |
| # The legacy branch projects one matching record and never transfers the document. | |
| row = con.execute(pg._q( | |
| 'SELECT key, value::text FROM {tbl} WHERE key LIKE %s ' | |
| "AND lower(btrim(value ->> 'email')) = %s ORDER BY key LIMIT 1", | |
| 'store_kv', slug), (PREFIX + '%', identifier)).fetchone() | |
| if row: | |
| return bytes.fromhex(row[0][len(PREFIX):]).decode('utf-8'), pg._parsed(row[1]) | |
| row = con.execute(pg._q( | |
| 'SELECT e.key, e.value::text FROM {tbl} AS kv ' | |
| 'CROSS JOIN LATERAL jsonb_each(kv.value) AS e WHERE kv.key = %s ' | |
| "AND lower(btrim(e.value ->> 'email')) = %s ORDER BY e.key LIMIT 1", | |
| 'store_kv', slug), (DOCUMENT_KEY, identifier)).fetchone() | |
| return (row[0], pg._parsed(row[1])) if row else (identifier, None) | |
| for username, candidate in (st.get(DOCUMENT_KEY, fresh=True) or {}).items(): | |
| if (isinstance(candidate, dict) | |
| and str(candidate.get('email') or '').strip().lower() == identifier): | |
| return str(username).strip().lower(), candidate | |
| return identifier, None | |
| def update(username, fn, *, st=None, read_users=(), all_users=False, flush='sync'): | |
| """Apply a registry-shaped callback but persist changes only to the named account.""" | |
| st = _source(st) | |
| pg_source = _pg_source(st) | |
| if not pg_source: | |
| return st.update(DOCUMENT_KEY, fn, flush=flush) | |
| pg, slug = pg_source | |
| username = str(username or '').strip().lower() | |
| key = row_key(username) | |
| pg.check_write(slug, 'update account row') | |
| changed = False | |
| with pg._pool().connection() as con: | |
| with con.transaction(): | |
| _settings(con, pg) | |
| # Shared marker locks let different accounts write concurrently and fence migrations. | |
| mode_query = ( | |
| "SELECT value ->> '__layout__', value ->> 'accountRows' FROM {tbl} WHERE key = %s " | |
| ) | |
| mode = con.execute(pg._q(mode_query, 'store_kv', slug), (DOCUMENT_KEY,)).fetchone() | |
| row_mode = bool(mode and tuple(map(str, mode)) == ('rows', str(VERSION))) | |
| mode = con.execute(pg._q(mode_query + ( | |
| 'FOR UPDATE' if all_users or not row_mode else 'FOR SHARE'), 'store_kv', slug), | |
| (DOCUMENT_KEY,)).fetchone() | |
| if not mode or tuple(map(str, mode)) != ('rows', str(VERSION)): | |
| # The old writer must take its strict read after acquiring the migration lock. | |
| with pg._caller_transaction(con, slug): | |
| return st.update(DOCUMENT_KEY, fn, flush=flush) | |
| con.execute('SELECT pg_advisory_xact_lock(hashtextextended(%s, 0))', | |
| (f'account-row:{slug}:{username}',)) | |
| before = _read_rows(con, pg, slug, None if all_users else (username, *read_users), lock=True) | |
| candidate = copy.deepcopy(before) | |
| after = fn(candidate) | |
| after = candidate if after is None else after | |
| if not isinstance(after, dict): | |
| raise ValueError('An account writer must return a registry') | |
| if any(before.get(name) != after.get(name) for name in (before.keys() | after.keys()) | |
| if name != username): | |
| raise ValueError('An account writer cannot change another account') | |
| if before.get(username) != after.get(username): | |
| changed = True | |
| if username not in after: | |
| _write(con, pg, slug, key, DELETED) | |
| else: | |
| if not isinstance(after[username], dict): | |
| raise ValueError('An account must be a record') | |
| _write(con, pg, slug, key, after[username]) | |
| if changed: | |
| pg._bump_write_generation(key, slug) | |
| return after | |
| def _digest(document): | |
| text = json.dumps(document, sort_keys=True, ensure_ascii=True, separators=(',', ':')) | |
| return hashlib.sha256(text.encode('utf-8')).hexdigest() | |
| def _validate(document): | |
| if not isinstance(document, dict) or any( | |
| not name or name != name.strip().lower() or not isinstance(record, dict) | |
| for name, record in document.items()): | |
| raise ValueError('Account migration requires canonical keys and record values') | |
| def migrate(slug): | |
| """Copy and compare every account in one guarded transaction before writing the marker.""" | |
| from core import store_pg as pg | |
| pg.check_write(slug, 'migrate users to account rows') | |
| keys = [] | |
| with pg._pool().connection() as con: | |
| with con.transaction(): | |
| _settings(con, pg) | |
| source = _head(con, pg, slug, lock=True) | |
| if is_marker(source): | |
| return {'changed': False, 'ok': True, 'mode': 'rows'} | |
| _validate(source) | |
| if _read_rows(con, pg, slug): | |
| raise ValueError('Account row keys already exist without a migration marker') | |
| for username, record in source.items(): | |
| key = row_key(username) | |
| _write(con, pg, slug, key, record) | |
| keys.append(key) | |
| copied = _read_rows(con, pg, slug) | |
| if source != copied: | |
| raise ValueError('Account migration lost or changed a record') | |
| marker = {LAYOUT_KEY: 'rows', 'accountRows': VERSION, 'version': VERSION, | |
| 'migratedAt': datetime.now(timezone.utc).isoformat(), | |
| 'source': {'records': len(source), 'sha256': _digest(source)}} | |
| _write(con, pg, slug, DOCUMENT_KEY, marker) | |
| for key in [DOCUMENT_KEY, *keys]: | |
| pg._bump_write_generation(key, slug) | |
| return {'changed': True, 'ok': True, 'records': len(source), 'sha256': _digest(source)} | |
| def verify(slug): | |
| """Report the current row digest and whether the untouched migration still matches its source.""" | |
| from core import store_pg as pg | |
| with read_snapshot(slug, require_stable=True) as con: | |
| head = _head(con, pg, slug) | |
| document = _read_rows(con, pg, slug) if is_marker(head) else head | |
| _validate(document) | |
| digest = _digest(document) | |
| modified = False | |
| if is_marker(head): | |
| modified = bool(con.execute(pg._q( | |
| 'SELECT 1 FROM {tbl} WHERE key LIKE %s AND (rev > 1 OR updated_at > ' | |
| '(SELECT updated_at FROM {tbl} WHERE key = %s)) LIMIT 1', 'store_kv', slug), | |
| (PREFIX + '%', DOCUMENT_KEY)).fetchone()) | |
| source = head.get('source', {}) if is_marker(head) else {} | |
| matches = source.get('sha256') == digest if source else None | |
| return {'ok': matches is not False or modified, 'mode': 'rows' if is_marker(head) else 'document', | |
| 'records': len(document), 'sha256': digest, 'matches_source': matches, | |
| 'modified': modified} | |
| def rollback(slug): | |
| """Restore the current accounts, including post-migration edits, and verify before committing.""" | |
| from core import store_pg as pg | |
| pg.check_write(slug, 'roll account rows back to users document') | |
| keys = [] | |
| with pg._pool().connection() as con: | |
| with con.transaction(): | |
| _settings(con, pg) | |
| if not is_marker(_head(con, pg, slug, lock=True)): | |
| return {'changed': False, 'ok': True, 'mode': 'document'} | |
| document = _read_rows(con, pg, slug) | |
| _validate(document) | |
| _write(con, pg, slug, DOCUMENT_KEY, document) | |
| if _head(con, pg, slug) != document: | |
| raise ValueError('Account rollback lost or changed a record') | |
| keys = [row_key(name) for name in document] | |
| con.execute(pg._q('DELETE FROM {tbl} WHERE key LIKE %s', 'store_kv', slug), (PREFIX + '%',)) | |
| for key in [DOCUMENT_KEY, *keys]: | |
| pg._bump_write_generation(key, slug) | |
| return {'changed': True, 'ok': True, 'records': len(document), 'sha256': _digest(document)} | |
| def records_backend(st): | |
| """Resolve a bound PostgreSQL runtime without bypassing private Assistant previews.""" | |
| from harness.runtime import TenantRuntime | |
| direct = _pg_source(st) | |
| if direct: | |
| return direct | |
| if isinstance(st, TenantRuntime): | |
| backend = _pg_source(st._document_store()) | |
| if backend and st.store_namespace: | |
| raise ValueError('Record rows require an unprefixed tenant store') | |
| return backend | |
| if getattr(st, 'activity_preview', False): | |
| return records_backend(st._real) | |
| return None | |
| def notice_prefix(user): | |
| return NOTICE_PREFIX + str(user).encode('utf-8').hex() + ':' | |
| def notice_key(user, identity): | |
| return notice_prefix(user) + hashlib.sha256(str(identity).encode('utf-8')).hexdigest() | |
| def _record_mode(con, pg, slug, bucket): | |
| mode = con.execute(pg._q( | |
| "SELECT value ->> '__layout__', value ->> 'recordRows', " | |
| 'value = %s::jsonb OR value = %s::jsonb ' | |
| 'FROM {tbl} WHERE key = %s', 'store_kv', slug), | |
| ('{}', '{"events": []}', bucket)).fetchone() | |
| return not mode or bool(mode[2]) or tuple(map(str, mode[:2])) == ('rows', str(VERSION)) | |
| def record_rows_ready(slug, bucket): | |
| """True when the bucket's rows writer accepts writes: a rows marker or no document at all.""" | |
| from core import store_pg as pg | |
| if bucket not in RECORD_BUCKETS: | |
| raise ValueError('Unknown record bucket') | |
| with pg._pool().connection() as con, con.transaction(): | |
| _settings(con, pg) | |
| return _record_mode(con, pg, slug, bucket) | |
| def _record_lock(con, slug, bucket, *, exclusive=False): | |
| fn = 'pg_advisory_xact_lock' if exclusive else 'pg_advisory_xact_lock_shared' | |
| con.execute(f'SELECT {fn}(hashtextextended(%s, 0))', (f'record-rows:{slug}:{bucket}',)) | |
| def _require_records(con, pg, slug, bucket): | |
| _record_lock(con, slug, bucket) | |
| if not _record_mode(con, pg, slug, bucket): | |
| raise ValueError(f'Migrate {bucket} to rows before writing') | |
| def _record_values(con, pg, slug, prefix, *, owner=None, lock=False): | |
| params = [prefix + '%'] | |
| where = 'key LIKE %s' | |
| if owner is not None: | |
| where += " AND value ->> 'owner' = %s" | |
| params.append(str(owner)) | |
| result = con.execute(pg._q('SELECT key, value::text FROM {tbl} WHERE ' + where | |
| + ' ORDER BY key' + (' FOR UPDATE' if lock else ''), | |
| 'store_kv', slug), tuple(params)).fetchall() | |
| for key, text in result: | |
| request_timing.record_store('account_rows', key, bytes_read=request_timing.text_bytes(text)) | |
| values = {} | |
| for key, text in result: | |
| value = pg._parsed(text) | |
| if value: | |
| values[key] = value | |
| return values | |
| def append_events(st, items, *, retention_seconds): | |
| """Insert one immutable row per event and expire rows by time without a count cap.""" | |
| pg, slug = records_backend(st) | |
| if any(item.get('tenant') != slug for item in items): | |
| raise ValueError('Security event belongs to another tenant') | |
| pg.check_write(slug, 'append security event rows') | |
| if not items: | |
| return | |
| keys = [] | |
| with pg._pool().connection() as con: | |
| with con.transaction(): | |
| _settings(con, pg) | |
| _require_records(con, pg, slug, EVENTS_KEY) | |
| for item in items: | |
| key = EVENT_PREFIX + uuid.uuid4().hex | |
| _write(con, pg, slug, key, item) | |
| keys.append(key) | |
| cutoff = max(item['ts'] for item in items) - retention_seconds | |
| con.execute(pg._q("DELETE FROM {tbl} WHERE key LIKE %s AND value ->> 'tenant' = %s " | |
| "AND CAST(value ->> 'ts' AS double precision) < %s", | |
| 'store_kv', slug), (EVENT_PREFIX + '%', slug, cutoff)) | |
| for key in keys: | |
| pg._bump_write_generation(key, slug) | |
| def event_counts(st, tenant, *, since, until, now, kinds, actor=None): | |
| """Aggregate canonical security rows, or decline an unmigrated document without reading its payload.""" | |
| backend = records_backend(st) | |
| if backend is None: | |
| return None | |
| pg, slug = backend | |
| if slug != tenant: | |
| raise ValueError('Security reader belongs to another tenant') | |
| label = ("CASE WHEN value ->> 'kind' = 'auth.login' THEN 'sign_in' " | |
| "WHEN value -> 'detail' ->> 'via' = 'mcp' THEN 'mcp' ELSE 'api' END") | |
| stamp = "CAST(value ->> 'ts' AS double precision)" | |
| selected = sorted(set(kinds) & {'sign_in', 'mcp', 'api'}) | |
| with read_snapshot(slug) as con: | |
| if not _record_mode(con, pg, slug, EVENTS_KEY): | |
| return None | |
| if not selected: | |
| return [] | |
| where = (f"key LIKE %s AND value ->> 'tenant' = %s AND {stamp} >= %s " | |
| f"AND {stamp} < %s AND {stamp} <= %s " | |
| "AND value ->> 'kind' IN ('auth.login', 'external.query', 'external.write') " | |
| f"AND ({label}) IN ({', '.join('%s' for _ in selected)})") | |
| params = [EVENT_PREFIX + '%', tenant, since, until, now, *selected] | |
| if actor is not None: | |
| where += " AND value ->> 'principal' = %s" | |
| params.append(actor) | |
| rows = con.execute(pg._q( | |
| f"SELECT CAST(floor({stamp} / 86400) AS bigint), value ->> 'principal', " | |
| f"{label}, count(*) FROM {{tbl}} WHERE {where} GROUP BY 1, 2, 3", | |
| 'store_kv', slug), tuple(params)).fetchall() | |
| return [(datetime.fromtimestamp(day * 86400, timezone.utc).date().isoformat(), who, kind, count) | |
| for day, who, kind, count in rows] | |
| def read_events(st, tenant, *, since, limit, kinds=None): | |
| """Select retained events by tenant and time without draining the audit queue.""" | |
| pg, slug = records_backend(st) | |
| if slug != tenant: | |
| raise ValueError('Security reader belongs to another tenant') | |
| limit = max(0, int(limit)) | |
| if not limit or kinds is not None and not kinds: | |
| return [] | |
| with read_snapshot(slug) as con: | |
| if _record_mode(con, pg, slug, EVENTS_KEY): | |
| source, where, params = '{tbl}', 'key LIKE %s', [EVENT_PREFIX + '%'] | |
| value = 'value' | |
| else: | |
| source = "{tbl} AS kv CROSS JOIN LATERAL jsonb_array_elements(kv.value -> 'events') AS e(value)" | |
| where, params, value = 'kv.key = %s', [EVENTS_KEY], 'e.value' | |
| where += f" AND {value} ->> 'tenant' = %s AND CAST({value} ->> 'ts' AS double precision) >= %s" | |
| params.extend((tenant, since)) | |
| if kinds is not None: | |
| selected = sorted(kinds) | |
| where += f" AND {value} ->> 'kind' IN (" + ', '.join('%s' for _ in selected) + ')' | |
| params.extend(selected) | |
| params.append(limit) | |
| result = con.execute(pg._q(f'SELECT {value}::text FROM ' + source + ' WHERE ' + where | |
| + f" ORDER BY CAST({value} ->> 'ts' AS double precision) DESC, " | |
| + f'{value}::text DESC LIMIT %s', 'store_kv', slug), | |
| tuple(params)).fetchall() | |
| for text, in result: | |
| request_timing.record_store('account_rows', EVENTS_KEY, | |
| bytes_read=request_timing.text_bytes(text)) | |
| return [pg._parsed(text) for text, in reversed(result)] | |
| def notification_items(user, *, st): | |
| """Read only this recipient's rows, or their legacy projection before migration.""" | |
| if getattr(st, 'activity_preview', False): | |
| pg, slug = records_backend(st) | |
| with read_snapshot(slug) as con: | |
| if not _record_mode(con, pg, slug, NOTIFICATIONS_KEY): | |
| raise ValueError(f'Migrate {NOTIFICATIONS_KEY} to rows before previewing writes') | |
| values = [st.get(key) for key in st.list_keys(notice_prefix(user))] | |
| else: | |
| pg, slug = records_backend(st) | |
| with read_snapshot(slug) as con: | |
| if not _record_mode(con, pg, slug, NOTIFICATIONS_KEY): | |
| projected = con.execute(pg._q('SELECT value -> %s FROM {tbl} WHERE key = %s', | |
| 'store_kv', slug), (str(user), NOTIFICATIONS_KEY)).fetchone() | |
| return copy.deepcopy(projected[0]) if projected and projected[0] else [] | |
| values = _record_values(con, pg, slug, notice_prefix(user), owner=user).values() | |
| return [copy.deepcopy(value['item']) for value in values | |
| if value.get('owner') == str(user) and isinstance(value.get('item'), dict)] | |
| def update_notifications(user, fn, *, st): | |
| """Apply an inbox mutation while writing only changed rows owned by its recipient.""" | |
| if getattr(st, 'activity_preview', False): | |
| before = {notice_key(user, item['id']): {'owner': str(user), 'item': item} | |
| for item in notification_items(user, st=st)} | |
| after = _notification_candidate(user, before, fn) | |
| for key in before.keys() | after.keys(): | |
| if before.get(key) != after.get(key): | |
| st.update(key, lambda _old, value=after.get(key, {}): copy.deepcopy(value), flush='sync') | |
| return | |
| pg, slug = records_backend(st) | |
| pg.check_write(slug, 'update recipient notification rows') | |
| changed = [] | |
| with pg._pool().connection() as con: | |
| with con.transaction(): | |
| _settings(con, pg) | |
| _require_records(con, pg, slug, NOTIFICATIONS_KEY) | |
| con.execute('SELECT pg_advisory_xact_lock(hashtextextended(%s, 0))', | |
| (f'notification-owner:{slug}:{user}',)) | |
| before = _record_values(con, pg, slug, notice_prefix(user), owner=user, lock=True) | |
| after = _notification_candidate(user, before, fn) | |
| for key in sorted(before.keys() | after.keys()): | |
| if before.get(key) == after.get(key): | |
| continue | |
| if key in after: | |
| _write(con, pg, slug, key, after[key]) | |
| else: | |
| con.execute(pg._q("DELETE FROM {tbl} WHERE key = %s AND value ->> 'owner' = %s", | |
| 'store_kv', slug), (key, str(user))) | |
| changed.append(key) | |
| for key in changed: | |
| pg._bump_write_generation(key, slug) | |
| def retain_notifications(items): | |
| """Drop the oldest read rows above the working-inbox target, keeping every unread row.""" | |
| items = sorted(items, key=lambda item: (str(item.get('at') or ''), str(item.get('id') or ''))) | |
| drop = max(0, len(items) - NOTIFICATION_TARGET) | |
| kept = [] | |
| for item in items: | |
| if drop and item.get('read'): | |
| drop -= 1 | |
| else: | |
| kept.append(item) | |
| return kept | |
| def _notification_candidate(user, before, fn, *, retain=True): | |
| items = sorted((copy.deepcopy(value['item']) for value in before.values()), | |
| key=lambda item: (str(item.get('at') or ''), str(item.get('id') or ''))) | |
| after = fn(items) | |
| after = items if after is None else after | |
| if not isinstance(after, list) or any(not isinstance(item, dict) or not item.get('id') for item in after): | |
| raise ValueError('A notification must have an id') | |
| result = {notice_key(user, item['id']): {'owner': str(user), 'item': item} for item in after} | |
| if len(result) != len(after): | |
| raise ValueError('Notification ids must be unique within an inbox') | |
| if retain: | |
| return {notice_key(user, item['id']): {'owner': str(user), 'item': item} | |
| for item in retain_notifications(after)} | |
| return result | |
| def _record_document(bucket, values, *, empty_owners=()): | |
| if bucket == EVENTS_KEY: | |
| return {'events': sorted(values.values(), key=lambda item: (item['ts'], _digest(item)))} | |
| document = {owner: [] for owner in empty_owners} | |
| for value in values.values(): | |
| document.setdefault(value['owner'], []).append(value['item']) | |
| for items in document.values(): | |
| items.sort(key=lambda item: (str(item.get('at') or ''), str(item['id']))) | |
| return document | |
| def _record_copy(bucket, document): | |
| if bucket == EVENTS_KEY: | |
| from core import audit | |
| if not isinstance(document, dict) or set(document) - {'events'}: | |
| raise ValueError('Security events migration requires an events document') | |
| values = {} | |
| for index, item in enumerate(document.get('events', [])): | |
| if audit.event(item['kind'], item['tenant'], item['principal'], item['detail'], now=item['ts']) != item: | |
| raise ValueError('Security event migration changed an event') | |
| values[EVENT_PREFIX + f'{index:016x}:' + _digest(item)] = item | |
| return values | |
| if not isinstance(document, dict): | |
| raise ValueError('Notification migration requires an inbox mapping') | |
| result = {} | |
| for owner, items in document.items(): | |
| if not owner or not isinstance(items, list): | |
| raise ValueError('Notification migration requires named recipient lists') | |
| result.update(_notification_candidate(owner, {}, lambda _old, items=items: items, retain=False)) | |
| return result | |
| def _records_transaction(pg, slug, action): | |
| """Select the stable read snapshot only for verification, preserving migration writer locks.""" | |
| if action == 'verify': | |
| with read_snapshot(slug, require_stable=True) as con: | |
| yield con | |
| else: | |
| with pg._pool().connection() as con, con.transaction(): | |
| _settings(con, pg) | |
| yield con | |
| def records_action(action, slug, bucket): | |
| """Migrate, verify or roll back event and notification rows in one guarded transaction.""" | |
| from core import store_pg as pg | |
| if action not in ('migrate', 'verify', 'rollback') or bucket not in RECORD_BUCKETS: | |
| raise ValueError('Unknown record migration action or bucket') | |
| prefix = EVENT_PREFIX if bucket == EVENTS_KEY else NOTICE_PREFIX | |
| if action != 'verify': | |
| pg.check_write(slug, f'{action} {bucket} rows') | |
| changed = [] | |
| modified = False | |
| with _records_transaction(pg, slug, action) as con: | |
| if action != 'verify': | |
| _record_lock(con, slug, bucket, exclusive=True) | |
| head_row = con.execute(pg._q('SELECT value::text FROM {tbl} WHERE key = %s' | |
| + (' FOR UPDATE' if action != 'verify' else ''), | |
| 'store_kv', slug), (bucket,)).fetchone() | |
| head = pg._parsed(head_row[0]) if head_row else {} | |
| if not isinstance(head, dict): | |
| raise ValueError('Record migration requires a document mapping') | |
| marker = isinstance(head, dict) and head.get('recordRows') == VERSION and head.get(LAYOUT_KEY) == 'rows' | |
| values = _record_values(con, pg, slug, prefix) | |
| empty_head = not head or head == {'events': []} | |
| empty = head.get('emptyOwners', []) if marker else [name for name, box in head.items() if box == []] | |
| document = _record_document(bucket, values, empty_owners=empty) if marker else head | |
| if action == 'migrate' and not marker: | |
| if values and not empty_head: | |
| raise ValueError('Record rows already exist without a migration marker') | |
| source = values if empty_head else _record_copy(bucket, head) | |
| if bucket == EVENTS_KEY and any(item['tenant'] != slug for item in source.values()): | |
| raise ValueError('Security migration contains another tenant') | |
| for key, value in source.items(): | |
| if key not in values: | |
| _write(con, pg, slug, key, value) | |
| changed.append(key) | |
| copied = _record_values(con, pg, slug, prefix) | |
| if source != copied: | |
| raise ValueError('Record migration lost or changed a row') | |
| document = _record_document(bucket, copied, empty_owners=empty) | |
| marker_value = {LAYOUT_KEY: 'rows', 'recordRows': VERSION, 'bucket': bucket, | |
| 'migratedAt': datetime.now(timezone.utc).isoformat(), | |
| 'emptyOwners': empty, 'source': {'sha256': _digest(document), 'records': len(copied)}} | |
| _write(con, pg, slug, bucket, marker_value) | |
| changed.append(bucket) | |
| values = copied | |
| elif action == 'rollback' and (marker or empty_head and values): | |
| document = _record_document(bucket, values, empty_owners=empty) | |
| _record_copy(bucket, document) | |
| _write(con, pg, slug, bucket, document) | |
| check = con.execute(pg._q('SELECT value::text FROM {tbl} WHERE key = %s', | |
| 'store_kv', slug), (bucket,)).fetchone() | |
| if not check or pg._parsed(check[0]) != document: | |
| raise ValueError('Record rollback lost or changed a row') | |
| con.execute(pg._q('DELETE FROM {tbl} WHERE key LIKE %s', 'store_kv', slug), (prefix + '%',)) | |
| changed.extend([bucket, *values]) | |
| elif action == 'verify': | |
| document = _record_document(bucket, values, empty_owners=empty) if marker or empty_head else head | |
| values = _record_copy(bucket, document) | |
| if marker: | |
| modified = bool(con.execute(pg._q( | |
| 'SELECT 1 FROM {tbl} WHERE key LIKE %s AND (rev > 1 OR updated_at > ' | |
| '(SELECT updated_at FROM {tbl} WHERE key = %s)) LIMIT 1', 'store_kv', slug), | |
| (prefix + '%', bucket)).fetchone()) | |
| elif action == 'rollback': | |
| values = _record_copy(bucket, head) | |
| for key in changed: | |
| pg._bump_write_generation(key, slug) | |
| digest = _digest(document) | |
| matches = head.get('source', {}).get('sha256') == digest if marker else None | |
| return {'ok': action != 'verify' or matches is not False or modified, | |
| 'changed': bool(changed), 'records': len(values), 'sha256': digest, | |
| 'matches_source': matches, 'modified': modified} | |
| def main(argv=None): | |
| parser = argparse.ArgumentParser(description=__doc__) | |
| parser.add_argument('action', choices=('migrate', 'verify', 'rollback')) | |
| parser.add_argument('slug') | |
| parser.add_argument('--bucket', choices=(DOCUMENT_KEY, *RECORD_BUCKETS), default=DOCUMENT_KEY) | |
| args = parser.parse_args(argv) | |
| result = (globals()[args.action](args.slug) if args.bucket == DOCUMENT_KEY else | |
| records_action(args.action, args.slug, args.bucket)) | |
| print(json.dumps(result, sort_keys=True)) | |
| return 0 if result.get('ok') else 1 | |
| if __name__ == '__main__': | |
| raise SystemExit(main()) | |