Download platform/core/history_rows.py from fsanyoto/loopable: direct link, hf CLI and curl.
- Browser
- Download file 33 kB
-
https://huggingface.co/spaces/fsanyoto/loopable/resolve/main/platform/core/history_rows.py
- Command line
-
hf download hf://spaces/fsanyoto/loopable/platform/core/history_rows.py
-
curl -L -o history_rows.py https://huggingface.co/spaces/fsanyoto/loopable/resolve/main/platform/core/history_rows.py
33 kB
| """Append complete cell history to tenant-local PostgreSQL month partitions.""" | |
| from datetime import datetime, timedelta, timezone | |
| import hashlib | |
| import json | |
| import math | |
| import re | |
| TABLE_NAME = 'value_history_rows' | |
| RETENTION_DAYS = 90 | |
| _COLUMNS = 'id, "table", row_id, kind, actor, at, changes' | |
| DOCUMENT_KEY = 'value_history' | |
| LAYOUT_KEY = '__layout__' | |
| STATEMENT_TIMEOUT_S = 120 | |
| def _time(value): | |
| if isinstance(value, str): | |
| value = datetime.fromisoformat(value.replace('Z', '+00:00')) | |
| if value.tzinfo is None: | |
| value = value.replace(tzinfo=timezone.utc) | |
| return value.astimezone(timezone.utc) | |
| def _month(value): | |
| return _time(value).replace(day=1, hour=0, minute=0, second=0, microsecond=0) | |
| def _next_month(value): | |
| if value.month == 12: | |
| return value.replace(year=value.year + 1, month=1) | |
| return value.replace(month=value.month + 1) | |
| def _table(tenant_slug): | |
| from core import store_pg | |
| schema = store_pg._schema_for(tenant_slug) | |
| if not re.fullmatch(r'[a-zA-Z0-9_]+', schema): | |
| raise ValueError('Invalid history schema.') | |
| return f'"{schema}"."{TABLE_NAME}"' | |
| def ensure_schema(con, tenant_slug): | |
| """Create the partitioned parent and its record-time index inside the caller's transaction.""" | |
| table = _table(tenant_slug) | |
| con.execute(f'''CREATE TABLE IF NOT EXISTS {table} ( | |
| id TEXT NOT NULL, "table" TEXT NOT NULL, row_id TEXT NOT NULL, | |
| kind TEXT NOT NULL, actor TEXT NOT NULL, at TIMESTAMPTZ NOT NULL, | |
| changes JSONB NOT NULL, PRIMARY KEY (at, id)) PARTITION BY RANGE (at)''') | |
| con.execute(f'CREATE INDEX IF NOT EXISTS "{TABLE_NAME}_record_time" ' | |
| f'ON {table} ("table", row_id, at DESC, id DESC)') | |
| class FixtureAdapter: | |
| """Mark an explicitly supplied offline history backend.""" | |
| def backend(st): | |
| """Resolve only the bound tenant's PostgreSQL store or an explicit history fixture.""" | |
| from core import user_tables, store_pg | |
| while isinstance(st, user_tables._Lent): | |
| st = st._runtime | |
| if st is user_tables.store: | |
| st = st.handle() | |
| bound = st._store() if callable(getattr(st, '_store', None)) else st | |
| if bound is user_tables.store: | |
| bound = bound.handle(slug=getattr(st, 'key', None)) | |
| if isinstance(bound, FixtureAdapter): | |
| return bound | |
| fixture = getattr(bound, 'history_rows_fixture', None) | |
| if isinstance(fixture, FixtureAdapter): | |
| return fixture | |
| if not isinstance(bound, store_pg.PgStore): | |
| raise TypeError('History requires a bound PgStore or an explicit history fixture.') | |
| if getattr(st, 'store_namespace', '') or getattr(st, 'key', bound.tenant_slug) != bound.tenant_slug: | |
| raise ValueError('History runtime and store tenant do not match.') | |
| adapter = getattr(bound, '_history_rows_backend', None) | |
| if adapter is None: | |
| adapter = PgHistoryStore(bound) | |
| bound._history_rows_backend = adapter | |
| if adapter.tenant_slug != bound.tenant_slug: | |
| raise ValueError('History store binding changed.') | |
| return adapter | |
| def cipher_changes(tenant, table, changes, *, opening=False, definition=None): | |
| """Bind personal history values to their originating tenant, table and column.""" | |
| from core import row_cipher, tenant_keys, user_tables, store | |
| if not opening and tenant_keys.subkey(tenant, 'row') is None: | |
| return changes | |
| if opening: | |
| return {column: {side: row_cipher.open_value(tenant, table, column, value) | |
| for side, value in pair.items()} | |
| for column, pair in (changes or {}).items()} | |
| if definition is None: | |
| definition = user_tables.get_definition(table, st=store.handle(slug=tenant)) | |
| personal = row_cipher.personal_columns(table, definition) | |
| return {column: {side: row_cipher.seal(tenant, table, column, value) | |
| if column in personal else value for side, value in pair.items()} | |
| for column, pair in (changes or {}).items()} | |
| def cipher_event(tenant, event, *, opening=False, definition=None): | |
| return {**event, 'changes': cipher_changes(tenant, event['table'], event['changes'], | |
| opening=opening, definition=definition)} | |
| def seal_copy_batch(tenant, events, *, backup_path, definitions=None): | |
| """Prepare a verified migration batch and encrypted backup without publishing either.""" | |
| from core import blob_envelope | |
| events = list(events) | |
| backup = blob_envelope.seal(tenant, backup_path, _json(events).encode(), required=True) | |
| sealed = [cipher_event(tenant, event, definition=(definitions or {}).get(event['table'])) | |
| for event in events] | |
| if ([cipher_event(tenant, event, opening=True) for event in sealed] | |
| != [cipher_event(tenant, event, opening=True) for event in events]): | |
| raise MigrationMismatch('Personal history batch differs after encryption.') | |
| return sealed, backup | |
| def _event(row, tenant=None): | |
| identity, table_key, row_id, kind, actor, at, changes = row | |
| value = {'id': identity, 'table': table_key, 'rowId': row_id, 'kind': kind, | |
| 'actor': actor, 'at': _time(at).isoformat(), | |
| 'changes': json.loads(changes) if isinstance(changes, str) else changes} | |
| return cipher_event(tenant, value, opening=True) if tenant else value | |
| class PgHistoryStore: | |
| """Store immutable history rows in the bound tenant schema.""" | |
| def __init__(self, st): | |
| from core import store_pg | |
| if not isinstance(st, store_pg.PgStore): | |
| raise TypeError('A PgStore is required.') | |
| self.tenant_slug = st.tenant_slug | |
| self.table = _table(st.tenant_slug) | |
| self._months = set() | |
| def _write_guard(self): | |
| from core import store_pg | |
| store_pg.check_write(self.tenant_slug, 'append value history') | |
| def _connection(self): | |
| from core import store_pg | |
| store_pg._effective_slug(self.tenant_slug) | |
| return store_pg._pool().connection() | |
| def _prepare(self, con, at): | |
| start = _month(at) | |
| end = _next_month(start) | |
| partition = self.table[:-1] + '_' + start.strftime('%Y%m') + '"' | |
| if con.execute('SELECT to_regclass(%s)', (partition,)).fetchone()[0] is not None: | |
| return False | |
| con.execute('SELECT pg_advisory_xact_lock(hashtextextended(%s, 0))', (self.table,)) | |
| ensure_schema(con, self.tenant_slug) | |
| # Bounds are generated from UTC datetimes, never caller-supplied SQL. | |
| con.execute(f"CREATE TABLE IF NOT EXISTS {partition} PARTITION OF {self.table} " | |
| f"FOR VALUES FROM ('{start.isoformat()}') TO ('{end.isoformat()}')") | |
| return True | |
| def insert(self, event): | |
| """Insert one complete event without reading or rewriting a history document.""" | |
| from core import store_pg | |
| self._write_guard() | |
| at = _time(event['at']) | |
| month = _month(at) | |
| prepared = month not in self._months | |
| with self._connection() as con: | |
| if prepared and self._prepare(con, at): | |
| self._prune(con, datetime.now(timezone.utc)) | |
| con.execute(f'INSERT INTO {self.table} ({_COLUMNS}) ' | |
| 'VALUES (%s, %s, %s, %s, %s, %s, %s::jsonb)', | |
| (event['id'], event['table'], event['rowId'], event['kind'], | |
| event['actor'], at, json.dumps(cipher_changes( | |
| self.tenant_slug, event['table'], event['changes']), ensure_ascii=False, | |
| allow_nan=False))) | |
| # A caller-owned transaction can still roll back its DDL. | |
| if prepared and store_pg._CALLER_TRANSACTION.get() is None: | |
| self._months.add(month) | |
| return event | |
| def query(self, table_key, row_id, *, since, row_ids=None, event_id=None): | |
| """Select the record and retention window in SQL without a result cap.""" | |
| clauses = ['"table" = %s', 'at >= %s'] | |
| args = [str(table_key), _time(since)] | |
| if row_id is not None: | |
| clauses.append('row_id = %s') | |
| args.append(str(row_id)) | |
| if row_ids is not None: | |
| ids = list(dict.fromkeys(str(rid) for rid in row_ids)) | |
| if not ids: | |
| return [] | |
| clauses.append('row_id = ANY(%s)') | |
| args.append(ids) | |
| if event_id is not None: | |
| clauses.append('id = %s') | |
| args.append(str(event_id)) | |
| try: | |
| with self._connection() as con: | |
| cursor = con.execute(f'SELECT {_COLUMNS} FROM {self.table} WHERE ' | |
| + ' AND '.join(clauses) + ' ORDER BY at DESC, id DESC', args) | |
| events = [] | |
| while rows := cursor.fetchmany(256): | |
| events.extend(_event(row, self.tenant_slug) for row in rows) | |
| return events | |
| except Exception as exc: | |
| if getattr(exc, 'sqlstate', None) != '42P01': | |
| raise | |
| return [] | |
| def latest(self, table_keys): | |
| """Return one SQL-aggregated edit timestamp per requested table for freshness metadata.""" | |
| keys = list(dict.fromkeys(str(key) for key in table_keys)) | |
| if not keys: | |
| return {} | |
| try: | |
| with self._connection() as con: | |
| rows = con.execute(f'SELECT "table", max(at) FROM {self.table} ' | |
| 'WHERE "table" = ANY(%s) GROUP BY "table"', (keys,)).fetchall() | |
| return {key: _time(at).isoformat() for key, at in rows} | |
| except Exception as exc: | |
| if getattr(exc, 'sqlstate', None) != '42P01': | |
| raise | |
| return {} | |
| def change_counts(self, *, since, until=None, kinds=None, tables=None): | |
| """Return (table, actor, at, changed cell count) for every table's events in one statement. | |
| Values stay sealed: only the changes keys are counted, so no row is decrypted. | |
| """ | |
| clauses, args = ['at >= %s'], [_time(since)] | |
| if until is not None: | |
| # The upper bound lets month partitions prune on both sides. | |
| clauses.append('at < %s') | |
| args.append(_time(until)) | |
| for column, values in (('kind', kinds), ('"table"', tables)): | |
| if values is not None: | |
| values = list(dict.fromkeys(str(value) for value in values)) | |
| if not values: | |
| return [] | |
| clauses.append(f'{column} = ANY(%s)') | |
| args.append(values) | |
| try: | |
| with self._connection() as con: | |
| # A non-object payload is returned whole and counted the way _event parses it. | |
| cursor = con.execute( | |
| 'SELECT "table", actor, at, CASE WHEN jsonb_typeof(changes) = \'object\' ' | |
| 'THEN (SELECT count(*) FROM jsonb_object_keys(changes)) END, ' | |
| 'CASE WHEN jsonb_typeof(changes) = \'object\' THEN NULL ELSE changes END ' | |
| f'FROM {self.table} WHERE ' + ' AND '.join(clauses), args) | |
| counts = [] | |
| while rows := cursor.fetchmany(1024): | |
| for table_key, actor, at, changed, raw in rows: | |
| if changed is None: | |
| raw = (json.loads(raw) if isinstance(raw, str) else raw) or {} | |
| if not isinstance(raw, dict): | |
| raise ValueError('History changes must be a JSON object.') | |
| changed = len(raw) | |
| counts.append((table_key, actor, _time(at), int(changed))) | |
| return counts | |
| except Exception as exc: | |
| if getattr(exc, 'sqlstate', None) != '42P01': | |
| raise | |
| return [] | |
| def _prune(self, con, now): | |
| cutoff = _time(now) - timedelta(days=RETENTION_DAYS) | |
| dropped = [] | |
| con.execute('SELECT pg_advisory_xact_lock(hashtextextended(%s, 0))', (self.table,)) | |
| partitions = con.execute( | |
| 'SELECT child.relname FROM pg_inherits i ' | |
| 'JOIN pg_class child ON child.oid = i.inhrelid ' | |
| 'WHERE i.inhparent = to_regclass(%s)', (self.table,)).fetchall() | |
| for (name,) in partitions: | |
| match = re.fullmatch(TABLE_NAME + r'_(\d{4})(\d{2})', name) | |
| if match is None: | |
| continue | |
| start = datetime(int(match[1]), int(match[2]), 1, tzinfo=timezone.utc) | |
| if _next_month(start) <= cutoff: | |
| partition = self.table.rsplit('.', 1)[0] + f'."{name}"' | |
| con.execute(f'DROP TABLE {partition}') | |
| dropped.append(start) | |
| return dropped | |
| def prune(self, *, now=None): | |
| """Drop only whole month partitions whose upper bound precedes the retention cutoff.""" | |
| self._write_guard() | |
| now = _time(now) if now is not None else datetime.now(timezone.utc) | |
| with self._connection() as con: | |
| dropped = self._prune(con, now) | |
| self._months.difference_update(dropped) | |
| return len(dropped) | |
| class MigrationMismatch(RuntimeError): | |
| """Refuse a cutover whose event identities or complete payloads differ.""" | |
| def is_marker(value): | |
| return (isinstance(value, dict) and value.get(LAYOUT_KEY) == 'rows' | |
| and value.get('table') == TABLE_NAME and value.get('version') == 1) | |
| def _json(value): | |
| return json.dumps(value, ensure_ascii=False, sort_keys=True, separators=(',', ':'), | |
| allow_nan=False) | |
| def _events(value): | |
| if not isinstance(value, list): | |
| raise ValueError('Legacy value_history must be a JSON array.') | |
| result, identities = [], set() | |
| fields = {'id', 'table', 'rowId', 'kind', 'actor', 'at', 'changes'} | |
| for event in value: | |
| if (not isinstance(event, dict) or set(event) != fields | |
| or any(not isinstance(event[key], str) for key in fields - {'changes'}) | |
| or not event['id'] or not isinstance(event['changes'], dict)): | |
| raise ValueError('Legacy history event has an unsupported shape.') | |
| if event['id'] in identities: | |
| raise ValueError('Legacy history event identities must be unique.') | |
| identities.add(event['id']) | |
| try: | |
| result.append(dict(event, at=_time(event['at']).isoformat())) | |
| except (ValueError, OverflowError): | |
| raise ValueError('Legacy history event timestamp is invalid.') from None | |
| return result | |
| def _digest(events): | |
| ordered = sorted(_events(events), key=lambda event: event['id']) | |
| return {'events': len(ordered), | |
| 'ids_sha256': hashlib.sha256(_json([event['id'] for event in ordered]).encode()).hexdigest(), | |
| 'payload_sha256': hashlib.sha256(_json(ordered).encode()).hexdigest()} | |
| def _settings(con, seconds, *, read_only=False): | |
| if not math.isfinite(seconds) or seconds <= 0: | |
| raise ValueError('Statement timeout must be finite and positive.') | |
| if read_only: | |
| con.execute('SET TRANSACTION ISOLATION LEVEL REPEATABLE READ, READ ONLY') | |
| con.execute("SELECT set_config('statement_timeout', %s, true)", (f'{int(seconds * 1000)}ms',)) | |
| con.execute("SELECT set_config('lock_timeout', %s, true)", (f'{int(seconds * 1000)}ms',)) | |
| def _migration_guard(tenant_slug): | |
| from core import store_pg | |
| store_pg.check_write(tenant_slug, 'operate history row migration') | |
| if (store_pg.data_binding.is_production_deployment() | |
| or (tenant_slug in store_pg.PRODUCTION_TENANTS and not store_pg.isolated_staging_database())): | |
| raise PermissionError('History migration and measurement require isolated staging (R58).') | |
| def _document(con, tenant_slug, *, lock=False): | |
| from core import store_pg | |
| row = con.execute(store_pg._q('SELECT value::text FROM {tbl} WHERE key = %s' | |
| + (' FOR UPDATE' if lock else ''), 'store_kv', tenant_slug), | |
| (DOCUMENT_KEY,)).fetchone() | |
| return (json.loads(row[0]) if isinstance(row[0], str) else row[0]) if row else None | |
| def _write_document(con, tenant_slug, value): | |
| from core import store_pg | |
| con.execute(store_pg._q('INSERT INTO {tbl} (key, value) VALUES (%s, %s::jsonb) ' | |
| 'ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value, ' | |
| 'rev = {bare}.rev + 1, updated_at = now()', 'store_kv', tenant_slug), | |
| (DOCUMENT_KEY, _json(value))) | |
| def _all_rows(con, adapter, *, legacy=False, identities=None): | |
| sql = f'SELECT {_COLUMNS} FROM {adapter.table}' | |
| args = () | |
| if legacy: | |
| sql += ' WHERE legacy_backfill = true' | |
| elif identities is not None: | |
| if not identities: | |
| return [] | |
| sql += ' WHERE id = ANY(%s)' | |
| args = (list(identities),) | |
| cursor = con.execute(sql + ' ORDER BY at, id', args) | |
| result = [] | |
| while rows := cursor.fetchmany(256): | |
| result.extend(_event(row, adapter.tenant_slug) for row in rows) | |
| return result | |
| def _backfill(adapter, events): | |
| """Copy every legacy event on the enlisted connection without invoking retention.""" | |
| with adapter._connection() as con: | |
| for month in sorted({_month(event['at']) for event in events}): | |
| adapter._prepare(con, month) | |
| if events: | |
| with con.cursor() as cursor: | |
| # psycopg consumes parameters under its non-reentrant connection lock. | |
| # Cipher key/definition reads enlist this same connection, so prepare each | |
| # bounded batch before executemany acquires that lock. | |
| for offset in range(0, len(events), 256): | |
| params = [(event['id'], event['table'], event['rowId'], event['kind'], event['actor'], | |
| _time(event['at']), _json(cipher_changes( | |
| adapter.tenant_slug, event['table'], event['changes']))) | |
| for event in events[offset:offset + 256]] | |
| cursor.executemany(f'INSERT INTO {adapter.table} ({_COLUMNS}) ' | |
| 'VALUES (%s, %s, %s, %s, %s, %s, %s::jsonb) ' | |
| 'ON CONFLICT (at, id) DO NOTHING', params) | |
| def _verify_rows(con, adapter, stamp): | |
| source = stamp.get('source') | |
| copied = _digest(_all_rows(con, adapter, legacy=True)) | |
| current = _digest(_all_rows(con, adapter)) | |
| return {'ok': source == copied, 'mode': 'rows', 'source': source, 'copied': copied, | |
| 'current': current, 'appended_events': current['events'] - copied['events']} | |
| def rows_ready(tenant_slug): | |
| """True when the history document is a rows marker, an empty array, or absent. Read-only.""" | |
| from core import account_rows, store_pg | |
| with account_rows.read_snapshot(tenant_slug) as con: | |
| row = con.execute(store_pg._q( | |
| "SELECT (CASE WHEN value ->> '__layout__' IS NOT NULL THEN value END)::text, " | |
| "value::text = '[]' FROM {tbl} WHERE key = %s", 'store_kv', tenant_slug), | |
| (DOCUMENT_KEY,)).fetchone() | |
| return not row or bool(row[1]) or is_marker(json.loads(row[0]) if row[0] else None) | |
| def migrate(tenant_slug, *, statement_timeout_s=STATEMENT_TIMEOUT_S, dry_run=False): | |
| """Lock the legacy document, backfill and verify all events, then commit its small marker atomically.""" | |
| from core import store_pg | |
| st = store_pg.PgStore(tenant_slug) | |
| if dry_run: | |
| return dict(verify(st.tenant_slug, statement_timeout_s=statement_timeout_s), | |
| changed=False, dry_run=True) | |
| _migration_guard(st.tenant_slug) | |
| adapter = PgHistoryStore(st) | |
| with store_pg._pool().connection() as con: | |
| _settings(con, statement_timeout_s) | |
| with st.transaction(con): | |
| con.execute('SELECT pg_advisory_xact_lock(hashtextextended(%s, 0))', (adapter.table,)) | |
| # Insert only the absent lock target, so concurrent migrations share one document lock. | |
| con.execute(store_pg._q('INSERT INTO {tbl} (key, value) VALUES (%s, %s::jsonb) ' | |
| 'ON CONFLICT (key) DO NOTHING', 'store_kv', st.tenant_slug), | |
| (DOCUMENT_KEY, '[]')) | |
| original = _document(con, st.tenant_slug, lock=True) | |
| if is_marker(original): | |
| report = _verify_rows(con, adapter, original) | |
| if not report['ok']: | |
| raise MigrationMismatch('Existing history marker does not match copied events.') | |
| return dict(report, changed=False, reason='already rows') | |
| events = _events(original) | |
| ensure_schema(con, st.tenant_slug) | |
| con.execute(f'LOCK TABLE {adapter.table} IN SHARE ROW EXCLUSIVE MODE') | |
| con.execute(f'ALTER TABLE {adapter.table} ADD COLUMN IF NOT EXISTS ' | |
| 'legacy_backfill BOOLEAN NOT NULL DEFAULT false') | |
| # The first normal save must find its month without pruning the newly copied legacy months. | |
| adapter._prepare(con, datetime.now(timezone.utc)) | |
| _backfill(adapter, events) | |
| copied = _all_rows(con, adapter, identities=[event['id'] for event in events]) | |
| source = _digest(events) | |
| if _digest(copied) != source: | |
| raise MigrationMismatch('History backfill event count, identities or payloads differ.') | |
| con.execute(f'UPDATE {adapter.table} SET legacy_backfill = false WHERE legacy_backfill = true') | |
| if events: | |
| con.execute(f'UPDATE {adapter.table} SET legacy_backfill = true WHERE id = ANY(%s)', | |
| ([event['id'] for event in events],)) | |
| stamp = {LAYOUT_KEY: 'rows', 'version': 1, 'table': TABLE_NAME, | |
| 'migratedAt': datetime.now(timezone.utc).isoformat(), 'source': source} | |
| _write_document(con, st.tenant_slug, stamp) | |
| report = _verify_rows(con, adapter, stamp) | |
| if not report['ok']: | |
| raise MigrationMismatch('History marker and copied events differ.') | |
| store_pg._bump_write_generation(DOCUMENT_KEY, st.tenant_slug) | |
| return dict(report, changed=True, marker_bytes=len(_json(stamp).encode())) | |
| def verify(tenant_slug, *, statement_timeout_s=STATEMENT_TIMEOUT_S): | |
| """Check the entire legacy copy, including event identities and payloads, independently of later appends.""" | |
| from core import account_rows, store_pg | |
| st = store_pg.PgStore(tenant_slug) | |
| adapter = PgHistoryStore(st) | |
| with account_rows.read_snapshot(st.tenant_slug, require_stable=True, | |
| statement_timeout_s=statement_timeout_s) as con: | |
| value = _document(con, st.tenant_slug) | |
| if is_marker(value): | |
| return _verify_rows(con, adapter, value) | |
| return {'ok': True, 'mode': 'absent' if value is None else 'document', | |
| 'source': _digest([] if value is None else _events(value))} | |
| def rollback(tenant_slug, *, statement_timeout_s=STATEMENT_TIMEOUT_S, dry_run=False): | |
| """Restore a verified legacy document from every current row, retaining rows for active row-only readers.""" | |
| from core import store_pg | |
| st = store_pg.PgStore(tenant_slug) | |
| if dry_run: | |
| return dict(verify(st.tenant_slug, statement_timeout_s=statement_timeout_s), | |
| changed=False, dry_run=True) | |
| _migration_guard(st.tenant_slug) | |
| adapter = PgHistoryStore(st) | |
| with store_pg._pool().connection() as con: | |
| _settings(con, statement_timeout_s) | |
| with st.transaction(con): | |
| con.execute('SELECT pg_advisory_xact_lock(hashtextextended(%s, 0))', (adapter.table,)) | |
| stamp = _document(con, st.tenant_slug, lock=True) | |
| if not is_marker(stamp): | |
| return {'ok': True, 'changed': False, 'reason': 'not in row mode'} | |
| con.execute(f'LOCK TABLE {adapter.table} IN SHARE ROW EXCLUSIVE MODE') | |
| report = _verify_rows(con, adapter, stamp) | |
| if not report['ok']: | |
| raise MigrationMismatch('History rollback refused an incomplete legacy copy.') | |
| events = _all_rows(con, adapter) | |
| before = _digest(events) | |
| _write_document(con, st.tenant_slug, events) | |
| if _digest(_events(_document(con, st.tenant_slug))) != before: | |
| raise MigrationMismatch('History rollback document differs from current rows.') | |
| store_pg._bump_write_generation(DOCUMENT_KEY, st.tenant_slug) | |
| return {'ok': True, 'changed': True, 'mode': 'document', 'current': before, | |
| 'rows_retained': True} | |
| class WriteMeter: | |
| """Count UTF-8 SQL text and logical parameter bytes, excluding protocol, WAL and index writes.""" | |
| def __init__(self, connection): | |
| self.connection = connection | |
| self.statement_bytes = 0 | |
| self.parameter_bytes = 0 | |
| self.statements = 0 | |
| def __getattr__(self, name): | |
| return getattr(self.connection, name) | |
| def _record(self, sql, params): | |
| text = sql.as_string(self.connection) if hasattr(sql, 'as_string') else str(sql) | |
| mutates = re.match(r'^\s*(INSERT|UPDATE|DELETE|MERGE|CREATE|ALTER|DROP|TRUNCATE)\b', text, re.I) | |
| if re.match(r'^\s*COPY\b', text, re.I): | |
| raise ValueError('COPY byte measurement is unsupported.') | |
| if mutates or (re.match(r'^\s*WITH\b', text, re.I) | |
| and re.search(r'\b(INSERT|UPDATE|DELETE|MERGE)\b', text, re.I)): | |
| self.statement_bytes += len(text.encode()) | |
| values = params.values() if isinstance(params, dict) else (params or ()) | |
| for value in values: | |
| if isinstance(value, bytes): | |
| self.parameter_bytes += len(value) | |
| elif value is not None: | |
| encoded = (_json(value) if isinstance(value, (dict, list, tuple)) else | |
| value.isoformat() if isinstance(value, datetime) else str(value)) | |
| self.parameter_bytes += len(encoded.encode()) | |
| self.statements += 1 | |
| def execute(self, sql, params=None, **kwargs): | |
| self._record(sql, params) | |
| return self.connection.execute(sql, params, **kwargs) | |
| def cursor(self, *args, **kwargs): | |
| meter = self | |
| class Cursor: | |
| def __init__(self, cursor): | |
| self.cursor = cursor | |
| def __getattr__(self, name): | |
| return getattr(self.cursor, name) | |
| def __enter__(self): | |
| self.cursor.__enter__() | |
| return self | |
| def __exit__(self, *exc): | |
| return self.cursor.__exit__(*exc) | |
| def __iter__(self): | |
| return iter(self.cursor) | |
| def execute(self, sql, params=None, **options): | |
| meter._record(sql, params) | |
| self.cursor.execute(sql, params, **options) | |
| return self | |
| def executemany(self, sql, params_seq, **options): | |
| params_seq = list(params_seq) | |
| for params in params_seq: | |
| meter._record(sql, params) | |
| self.cursor.executemany(sql, params_seq, **options) | |
| return self | |
| def copy(self, *args, **options): | |
| raise ValueError('COPY byte measurement is unsupported.') | |
| return Cursor(self.connection.cursor(*args, **kwargs)) | |
| def report(self): | |
| return {'metric': 'sql_write_input_utf8_bytes', 'bytes': self.statement_bytes + self.parameter_bytes, | |
| 'statement_bytes': self.statement_bytes, 'parameter_bytes': self.parameter_bytes, | |
| 'write_statements': self.statements, | |
| 'excludes': ['PostgreSQL protocol and TLS', 'WAL', 'indexes', 'storage page writes'], | |
| 'parameter_encoding': 'UTF-8 strings, ISO datetimes, canonical JSON collections, str scalars, raw bytes, zero for NULL'} | |
| def measure_save(tenant_slug, table_key, row_id, values, *, actor, | |
| statement_timeout_s=STATEMENT_TIMEOUT_S): | |
| """Measure a staging native-table grid event, then roll back its cell, history and Activity writes.""" | |
| import uuid | |
| from core import grid_events, store_pg, table_store, user_tables | |
| st = store_pg.PgStore(tenant_slug) | |
| _migration_guard(st.tenant_slug) | |
| if not isinstance(values, dict) or len(values) != 1 or not actor: | |
| raise ValueError('Measure exactly one cell value.') | |
| if not str(table_key).startswith('ut_'): | |
| raise ValueError('Measure a native editable user-table record.') | |
| pid = int(row_id) | |
| with store_pg._pool().connection() as con: | |
| with con.transaction(force_rollback=True): | |
| _settings(con, statement_timeout_s) | |
| measured = WriteMeter(con) | |
| with st.transaction(measured): | |
| stamp = _document(con, st.tenant_slug) | |
| if not is_marker(stamp): | |
| raise ValueError('Migrate history before measuring a cell save.') | |
| if not user_tables.records_mutable(table_key, st=st): | |
| raise ValueError('Measure a native editable user-table record.') | |
| record = user_tables.get_record(table_key, pid, st=st) or {} | |
| if str(pid) not in (record.get('rows') or {}): | |
| raise ValueError('The measurement record does not exist.') | |
| table = _table(st.tenant_slug) | |
| count_sql = f'SELECT count(*) FROM {table} WHERE "table" = %s AND row_id = %s' | |
| count_args = (str(table_key), str(pid)) | |
| before = con.execute(count_sql, count_args).fetchone()[0] | |
| event_id = 'history-measure:' + uuid.uuid4().hex | |
| ctx = grid_events.EventCtx(uname=actor, admin=True, allowed_pids=frozenset({pid}), | |
| fields=record.get('fields') or [], st=st, scope_key=str(table_key), by=actor, | |
| table=table_store.make(f'{table_key}_table_workspace', st=st)) | |
| grid_events.handle_one({'id': event_id, 'type': 'overlay_patch', | |
| 'pid': pid, 'updates': values}, ctx) | |
| after = con.execute(count_sql, count_args).fetchone()[0] | |
| if (not ctx.out.writes.get(event_id, {}).get('ok') or after != before + 1 | |
| or _document(con, st.tenant_slug) != stamp): | |
| raise ValueError('The measurement did not save exactly one changed cell and history row.') | |
| report = measured.report() | |
| return dict(report, scope='staging native-table grid overlay_patch SQL write inputs', fixture=False, | |
| request_overhead='HTTP handlers, receipt middleware and asynchronous work excluded', | |
| rolled_back=True, ok=report['bytes'] < 65536, limit_bytes=65536) | |
| def main(argv=None): | |
| import argparse | |
| parser = argparse.ArgumentParser(prog='python -m core.history_rows') | |
| parser.add_argument('action', choices=('migrate', 'verify', 'rollback', 'measure-save')) | |
| parser.add_argument('slug') | |
| parser.add_argument('--dry-run', action='store_true') | |
| parser.add_argument('--statement-timeout', type=float, default=STATEMENT_TIMEOUT_S) | |
| parser.add_argument('--json', action='store_true', help='print value-free JSON (the default)') | |
| parser.add_argument('--table') | |
| parser.add_argument('--row') | |
| parser.add_argument('--values-file', help='one changed cell as a JSON object, never printed') | |
| parser.add_argument('--actor', default='history-byte-measurement') | |
| args = parser.parse_args(argv) | |
| try: | |
| if args.action == 'measure-save': | |
| if args.dry_run or not all((args.table, args.row, args.values_file)): | |
| parser.error('measure-save requires --table, --row and --values-file, without --dry-run') | |
| with open(args.values_file, encoding='utf-8') as source: | |
| values = json.load(source) | |
| result = measure_save(args.slug, args.table, args.row, values, actor=args.actor, | |
| statement_timeout_s=args.statement_timeout) | |
| else: | |
| options = {'statement_timeout_s': args.statement_timeout} | |
| if args.action != 'verify': | |
| options['dry_run'] = args.dry_run | |
| result = {'migrate': migrate, 'verify': verify, 'rollback': rollback}[args.action](args.slug, **options) | |
| except (MigrationMismatch, ValueError, PermissionError) as exc: | |
| result = {'ok': False, 'error': str(exc)} | |
| except Exception as exc: | |
| # Database exception text can contain rejected customer values. | |
| result = {'ok': False, 'error': type(exc).__name__, 'detail': 'History operation failed without committing.'} | |
| print(json.dumps(result, sort_keys=True)) | |
| return 0 if result['ok'] else 1 | |
| if __name__ == '__main__': | |
| from core import history_rows as _module | |
| raise SystemExit(_module.main()) | |