"""Append complete cell history to tenant-local PostgreSQL month partitions.""" from datetime import datetime, timedelta, timezone import hashlib import json import math import re TABLE_NAME = 'value_history_rows' RETENTION_DAYS = 90 _COLUMNS = 'id, "table", row_id, kind, actor, at, changes' DOCUMENT_KEY = 'value_history' LAYOUT_KEY = '__layout__' STATEMENT_TIMEOUT_S = 120 def _time(value): if isinstance(value, str): value = datetime.fromisoformat(value.replace('Z', '+00:00')) if value.tzinfo is None: value = value.replace(tzinfo=timezone.utc) return value.astimezone(timezone.utc) def _month(value): return _time(value).replace(day=1, hour=0, minute=0, second=0, microsecond=0) def _next_month(value): if value.month == 12: return value.replace(year=value.year + 1, month=1) return value.replace(month=value.month + 1) def _table(tenant_slug): from core import store_pg schema = store_pg._schema_for(tenant_slug) if not re.fullmatch(r'[a-zA-Z0-9_]+', schema): raise ValueError('Invalid history schema.') return f'"{schema}"."{TABLE_NAME}"' def ensure_schema(con, tenant_slug): """Create the partitioned parent and its record-time index inside the caller's transaction.""" table = _table(tenant_slug) con.execute(f'''CREATE TABLE IF NOT EXISTS {table} ( id TEXT NOT NULL, "table" TEXT NOT NULL, row_id TEXT NOT NULL, kind TEXT NOT NULL, actor TEXT NOT NULL, at TIMESTAMPTZ NOT NULL, changes JSONB NOT NULL, PRIMARY KEY (at, id)) PARTITION BY RANGE (at)''') con.execute(f'CREATE INDEX IF NOT EXISTS "{TABLE_NAME}_record_time" ' f'ON {table} ("table", row_id, at DESC, id DESC)') class FixtureAdapter: """Mark an explicitly supplied offline history backend.""" def backend(st): """Resolve only the bound tenant's PostgreSQL store or an explicit history fixture.""" from core import user_tables, store_pg while isinstance(st, user_tables._Lent): st = st._runtime if st is user_tables.store: st = st.handle() bound = st._store() if callable(getattr(st, '_store', None)) else st if bound is user_tables.store: bound = bound.handle(slug=getattr(st, 'key', None)) if isinstance(bound, FixtureAdapter): return bound fixture = getattr(bound, 'history_rows_fixture', None) if isinstance(fixture, FixtureAdapter): return fixture if not isinstance(bound, store_pg.PgStore): raise TypeError('History requires a bound PgStore or an explicit history fixture.') if getattr(st, 'store_namespace', '') or getattr(st, 'key', bound.tenant_slug) != bound.tenant_slug: raise ValueError('History runtime and store tenant do not match.') adapter = getattr(bound, '_history_rows_backend', None) if adapter is None: adapter = PgHistoryStore(bound) bound._history_rows_backend = adapter if adapter.tenant_slug != bound.tenant_slug: raise ValueError('History store binding changed.') return adapter def cipher_changes(tenant, table, changes, *, opening=False, definition=None): """Bind personal history values to their originating tenant, table and column.""" from core import row_cipher, tenant_keys, user_tables, store if not opening and tenant_keys.subkey(tenant, 'row') is None: return changes if opening: return {column: {side: row_cipher.open_value(tenant, table, column, value) for side, value in pair.items()} for column, pair in (changes or {}).items()} if definition is None: definition = user_tables.get_definition(table, st=store.handle(slug=tenant)) personal = row_cipher.personal_columns(table, definition) return {column: {side: row_cipher.seal(tenant, table, column, value) if column in personal else value for side, value in pair.items()} for column, pair in (changes or {}).items()} def cipher_event(tenant, event, *, opening=False, definition=None): return {**event, 'changes': cipher_changes(tenant, event['table'], event['changes'], opening=opening, definition=definition)} def seal_copy_batch(tenant, events, *, backup_path, definitions=None): """Prepare a verified migration batch and encrypted backup without publishing either.""" from core import blob_envelope events = list(events) backup = blob_envelope.seal(tenant, backup_path, _json(events).encode(), required=True) sealed = [cipher_event(tenant, event, definition=(definitions or {}).get(event['table'])) for event in events] if ([cipher_event(tenant, event, opening=True) for event in sealed] != [cipher_event(tenant, event, opening=True) for event in events]): raise MigrationMismatch('Personal history batch differs after encryption.') return sealed, backup def _event(row, tenant=None): identity, table_key, row_id, kind, actor, at, changes = row value = {'id': identity, 'table': table_key, 'rowId': row_id, 'kind': kind, 'actor': actor, 'at': _time(at).isoformat(), 'changes': json.loads(changes) if isinstance(changes, str) else changes} return cipher_event(tenant, value, opening=True) if tenant else value class PgHistoryStore: """Store immutable history rows in the bound tenant schema.""" def __init__(self, st): from core import store_pg if not isinstance(st, store_pg.PgStore): raise TypeError('A PgStore is required.') self.tenant_slug = st.tenant_slug self.table = _table(st.tenant_slug) self._months = set() def _write_guard(self): from core import store_pg store_pg.check_write(self.tenant_slug, 'append value history') def _connection(self): from core import store_pg store_pg._effective_slug(self.tenant_slug) return store_pg._pool().connection() def _prepare(self, con, at): start = _month(at) end = _next_month(start) partition = self.table[:-1] + '_' + start.strftime('%Y%m') + '"' if con.execute('SELECT to_regclass(%s)', (partition,)).fetchone()[0] is not None: return False con.execute('SELECT pg_advisory_xact_lock(hashtextextended(%s, 0))', (self.table,)) ensure_schema(con, self.tenant_slug) # Bounds are generated from UTC datetimes, never caller-supplied SQL. con.execute(f"CREATE TABLE IF NOT EXISTS {partition} PARTITION OF {self.table} " f"FOR VALUES FROM ('{start.isoformat()}') TO ('{end.isoformat()}')") return True def insert(self, event): """Insert one complete event without reading or rewriting a history document.""" from core import store_pg self._write_guard() at = _time(event['at']) month = _month(at) prepared = month not in self._months with self._connection() as con: if prepared and self._prepare(con, at): self._prune(con, datetime.now(timezone.utc)) con.execute(f'INSERT INTO {self.table} ({_COLUMNS}) ' 'VALUES (%s, %s, %s, %s, %s, %s, %s::jsonb)', (event['id'], event['table'], event['rowId'], event['kind'], event['actor'], at, json.dumps(cipher_changes( self.tenant_slug, event['table'], event['changes']), ensure_ascii=False, allow_nan=False))) # A caller-owned transaction can still roll back its DDL. if prepared and store_pg._CALLER_TRANSACTION.get() is None: self._months.add(month) return event def query(self, table_key, row_id, *, since, row_ids=None, event_id=None): """Select the record and retention window in SQL without a result cap.""" clauses = ['"table" = %s', 'at >= %s'] args = [str(table_key), _time(since)] if row_id is not None: clauses.append('row_id = %s') args.append(str(row_id)) if row_ids is not None: ids = list(dict.fromkeys(str(rid) for rid in row_ids)) if not ids: return [] clauses.append('row_id = ANY(%s)') args.append(ids) if event_id is not None: clauses.append('id = %s') args.append(str(event_id)) try: with self._connection() as con: cursor = con.execute(f'SELECT {_COLUMNS} FROM {self.table} WHERE ' + ' AND '.join(clauses) + ' ORDER BY at DESC, id DESC', args) events = [] while rows := cursor.fetchmany(256): events.extend(_event(row, self.tenant_slug) for row in rows) return events except Exception as exc: if getattr(exc, 'sqlstate', None) != '42P01': raise return [] def latest(self, table_keys): """Return one SQL-aggregated edit timestamp per requested table for freshness metadata.""" keys = list(dict.fromkeys(str(key) for key in table_keys)) if not keys: return {} try: with self._connection() as con: rows = con.execute(f'SELECT "table", max(at) FROM {self.table} ' 'WHERE "table" = ANY(%s) GROUP BY "table"', (keys,)).fetchall() return {key: _time(at).isoformat() for key, at in rows} except Exception as exc: if getattr(exc, 'sqlstate', None) != '42P01': raise return {} def change_counts(self, *, since, until=None, kinds=None, tables=None): """Return (table, actor, at, changed cell count) for every table's events in one statement. Values stay sealed: only the changes keys are counted, so no row is decrypted. """ clauses, args = ['at >= %s'], [_time(since)] if until is not None: # The upper bound lets month partitions prune on both sides. clauses.append('at < %s') args.append(_time(until)) for column, values in (('kind', kinds), ('"table"', tables)): if values is not None: values = list(dict.fromkeys(str(value) for value in values)) if not values: return [] clauses.append(f'{column} = ANY(%s)') args.append(values) try: with self._connection() as con: # A non-object payload is returned whole and counted the way _event parses it. cursor = con.execute( 'SELECT "table", actor, at, CASE WHEN jsonb_typeof(changes) = \'object\' ' 'THEN (SELECT count(*) FROM jsonb_object_keys(changes)) END, ' 'CASE WHEN jsonb_typeof(changes) = \'object\' THEN NULL ELSE changes END ' f'FROM {self.table} WHERE ' + ' AND '.join(clauses), args) counts = [] while rows := cursor.fetchmany(1024): for table_key, actor, at, changed, raw in rows: if changed is None: raw = (json.loads(raw) if isinstance(raw, str) else raw) or {} if not isinstance(raw, dict): raise ValueError('History changes must be a JSON object.') changed = len(raw) counts.append((table_key, actor, _time(at), int(changed))) return counts except Exception as exc: if getattr(exc, 'sqlstate', None) != '42P01': raise return [] def _prune(self, con, now): cutoff = _time(now) - timedelta(days=RETENTION_DAYS) dropped = [] con.execute('SELECT pg_advisory_xact_lock(hashtextextended(%s, 0))', (self.table,)) partitions = con.execute( 'SELECT child.relname FROM pg_inherits i ' 'JOIN pg_class child ON child.oid = i.inhrelid ' 'WHERE i.inhparent = to_regclass(%s)', (self.table,)).fetchall() for (name,) in partitions: match = re.fullmatch(TABLE_NAME + r'_(\d{4})(\d{2})', name) if match is None: continue start = datetime(int(match[1]), int(match[2]), 1, tzinfo=timezone.utc) if _next_month(start) <= cutoff: partition = self.table.rsplit('.', 1)[0] + f'."{name}"' con.execute(f'DROP TABLE {partition}') dropped.append(start) return dropped def prune(self, *, now=None): """Drop only whole month partitions whose upper bound precedes the retention cutoff.""" self._write_guard() now = _time(now) if now is not None else datetime.now(timezone.utc) with self._connection() as con: dropped = self._prune(con, now) self._months.difference_update(dropped) return len(dropped) class MigrationMismatch(RuntimeError): """Refuse a cutover whose event identities or complete payloads differ.""" def is_marker(value): return (isinstance(value, dict) and value.get(LAYOUT_KEY) == 'rows' and value.get('table') == TABLE_NAME and value.get('version') == 1) def _json(value): return json.dumps(value, ensure_ascii=False, sort_keys=True, separators=(',', ':'), allow_nan=False) def _events(value): if not isinstance(value, list): raise ValueError('Legacy value_history must be a JSON array.') result, identities = [], set() fields = {'id', 'table', 'rowId', 'kind', 'actor', 'at', 'changes'} for event in value: if (not isinstance(event, dict) or set(event) != fields or any(not isinstance(event[key], str) for key in fields - {'changes'}) or not event['id'] or not isinstance(event['changes'], dict)): raise ValueError('Legacy history event has an unsupported shape.') if event['id'] in identities: raise ValueError('Legacy history event identities must be unique.') identities.add(event['id']) try: result.append(dict(event, at=_time(event['at']).isoformat())) except (ValueError, OverflowError): raise ValueError('Legacy history event timestamp is invalid.') from None return result def _digest(events): ordered = sorted(_events(events), key=lambda event: event['id']) return {'events': len(ordered), 'ids_sha256': hashlib.sha256(_json([event['id'] for event in ordered]).encode()).hexdigest(), 'payload_sha256': hashlib.sha256(_json(ordered).encode()).hexdigest()} def _settings(con, seconds, *, read_only=False): if not math.isfinite(seconds) or seconds <= 0: raise ValueError('Statement timeout must be finite and positive.') if read_only: con.execute('SET TRANSACTION ISOLATION LEVEL REPEATABLE READ, READ ONLY') con.execute("SELECT set_config('statement_timeout', %s, true)", (f'{int(seconds * 1000)}ms',)) con.execute("SELECT set_config('lock_timeout', %s, true)", (f'{int(seconds * 1000)}ms',)) def _migration_guard(tenant_slug): from core import store_pg store_pg.check_write(tenant_slug, 'operate history row migration') if (store_pg.data_binding.is_production_deployment() or (tenant_slug in store_pg.PRODUCTION_TENANTS and not store_pg.isolated_staging_database())): raise PermissionError('History migration and measurement require isolated staging (R58).') def _document(con, tenant_slug, *, lock=False): from core import store_pg row = con.execute(store_pg._q('SELECT value::text FROM {tbl} WHERE key = %s' + (' FOR UPDATE' if lock else ''), 'store_kv', tenant_slug), (DOCUMENT_KEY,)).fetchone() return (json.loads(row[0]) if isinstance(row[0], str) else row[0]) if row else None def _write_document(con, tenant_slug, value): from core import store_pg con.execute(store_pg._q('INSERT INTO {tbl} (key, value) VALUES (%s, %s::jsonb) ' 'ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value, ' 'rev = {bare}.rev + 1, updated_at = now()', 'store_kv', tenant_slug), (DOCUMENT_KEY, _json(value))) def _all_rows(con, adapter, *, legacy=False, identities=None): sql = f'SELECT {_COLUMNS} FROM {adapter.table}' args = () if legacy: sql += ' WHERE legacy_backfill = true' elif identities is not None: if not identities: return [] sql += ' WHERE id = ANY(%s)' args = (list(identities),) cursor = con.execute(sql + ' ORDER BY at, id', args) result = [] while rows := cursor.fetchmany(256): result.extend(_event(row, adapter.tenant_slug) for row in rows) return result def _backfill(adapter, events): """Copy every legacy event on the enlisted connection without invoking retention.""" with adapter._connection() as con: for month in sorted({_month(event['at']) for event in events}): adapter._prepare(con, month) if events: with con.cursor() as cursor: # psycopg consumes parameters under its non-reentrant connection lock. # Cipher key/definition reads enlist this same connection, so prepare each # bounded batch before executemany acquires that lock. for offset in range(0, len(events), 256): params = [(event['id'], event['table'], event['rowId'], event['kind'], event['actor'], _time(event['at']), _json(cipher_changes( adapter.tenant_slug, event['table'], event['changes']))) for event in events[offset:offset + 256]] cursor.executemany(f'INSERT INTO {adapter.table} ({_COLUMNS}) ' 'VALUES (%s, %s, %s, %s, %s, %s, %s::jsonb) ' 'ON CONFLICT (at, id) DO NOTHING', params) def _verify_rows(con, adapter, stamp): source = stamp.get('source') copied = _digest(_all_rows(con, adapter, legacy=True)) current = _digest(_all_rows(con, adapter)) return {'ok': source == copied, 'mode': 'rows', 'source': source, 'copied': copied, 'current': current, 'appended_events': current['events'] - copied['events']} def rows_ready(tenant_slug): """True when the history document is a rows marker, an empty array, or absent. Read-only.""" from core import account_rows, store_pg with account_rows.read_snapshot(tenant_slug) as con: row = con.execute(store_pg._q( "SELECT (CASE WHEN value ->> '__layout__' IS NOT NULL THEN value END)::text, " "value::text = '[]' FROM {tbl} WHERE key = %s", 'store_kv', tenant_slug), (DOCUMENT_KEY,)).fetchone() return not row or bool(row[1]) or is_marker(json.loads(row[0]) if row[0] else None) def migrate(tenant_slug, *, statement_timeout_s=STATEMENT_TIMEOUT_S, dry_run=False): """Lock the legacy document, backfill and verify all events, then commit its small marker atomically.""" from core import store_pg st = store_pg.PgStore(tenant_slug) if dry_run: return dict(verify(st.tenant_slug, statement_timeout_s=statement_timeout_s), changed=False, dry_run=True) _migration_guard(st.tenant_slug) adapter = PgHistoryStore(st) with store_pg._pool().connection() as con: _settings(con, statement_timeout_s) with st.transaction(con): con.execute('SELECT pg_advisory_xact_lock(hashtextextended(%s, 0))', (adapter.table,)) # Insert only the absent lock target, so concurrent migrations share one document lock. con.execute(store_pg._q('INSERT INTO {tbl} (key, value) VALUES (%s, %s::jsonb) ' 'ON CONFLICT (key) DO NOTHING', 'store_kv', st.tenant_slug), (DOCUMENT_KEY, '[]')) original = _document(con, st.tenant_slug, lock=True) if is_marker(original): report = _verify_rows(con, adapter, original) if not report['ok']: raise MigrationMismatch('Existing history marker does not match copied events.') return dict(report, changed=False, reason='already rows') events = _events(original) ensure_schema(con, st.tenant_slug) con.execute(f'LOCK TABLE {adapter.table} IN SHARE ROW EXCLUSIVE MODE') con.execute(f'ALTER TABLE {adapter.table} ADD COLUMN IF NOT EXISTS ' 'legacy_backfill BOOLEAN NOT NULL DEFAULT false') # The first normal save must find its month without pruning the newly copied legacy months. adapter._prepare(con, datetime.now(timezone.utc)) _backfill(adapter, events) copied = _all_rows(con, adapter, identities=[event['id'] for event in events]) source = _digest(events) if _digest(copied) != source: raise MigrationMismatch('History backfill event count, identities or payloads differ.') con.execute(f'UPDATE {adapter.table} SET legacy_backfill = false WHERE legacy_backfill = true') if events: con.execute(f'UPDATE {adapter.table} SET legacy_backfill = true WHERE id = ANY(%s)', ([event['id'] for event in events],)) stamp = {LAYOUT_KEY: 'rows', 'version': 1, 'table': TABLE_NAME, 'migratedAt': datetime.now(timezone.utc).isoformat(), 'source': source} _write_document(con, st.tenant_slug, stamp) report = _verify_rows(con, adapter, stamp) if not report['ok']: raise MigrationMismatch('History marker and copied events differ.') store_pg._bump_write_generation(DOCUMENT_KEY, st.tenant_slug) return dict(report, changed=True, marker_bytes=len(_json(stamp).encode())) def verify(tenant_slug, *, statement_timeout_s=STATEMENT_TIMEOUT_S): """Check the entire legacy copy, including event identities and payloads, independently of later appends.""" from core import account_rows, store_pg st = store_pg.PgStore(tenant_slug) adapter = PgHistoryStore(st) with account_rows.read_snapshot(st.tenant_slug, require_stable=True, statement_timeout_s=statement_timeout_s) as con: value = _document(con, st.tenant_slug) if is_marker(value): return _verify_rows(con, adapter, value) return {'ok': True, 'mode': 'absent' if value is None else 'document', 'source': _digest([] if value is None else _events(value))} def rollback(tenant_slug, *, statement_timeout_s=STATEMENT_TIMEOUT_S, dry_run=False): """Restore a verified legacy document from every current row, retaining rows for active row-only readers.""" from core import store_pg st = store_pg.PgStore(tenant_slug) if dry_run: return dict(verify(st.tenant_slug, statement_timeout_s=statement_timeout_s), changed=False, dry_run=True) _migration_guard(st.tenant_slug) adapter = PgHistoryStore(st) with store_pg._pool().connection() as con: _settings(con, statement_timeout_s) with st.transaction(con): con.execute('SELECT pg_advisory_xact_lock(hashtextextended(%s, 0))', (adapter.table,)) stamp = _document(con, st.tenant_slug, lock=True) if not is_marker(stamp): return {'ok': True, 'changed': False, 'reason': 'not in row mode'} con.execute(f'LOCK TABLE {adapter.table} IN SHARE ROW EXCLUSIVE MODE') report = _verify_rows(con, adapter, stamp) if not report['ok']: raise MigrationMismatch('History rollback refused an incomplete legacy copy.') events = _all_rows(con, adapter) before = _digest(events) _write_document(con, st.tenant_slug, events) if _digest(_events(_document(con, st.tenant_slug))) != before: raise MigrationMismatch('History rollback document differs from current rows.') store_pg._bump_write_generation(DOCUMENT_KEY, st.tenant_slug) return {'ok': True, 'changed': True, 'mode': 'document', 'current': before, 'rows_retained': True} class WriteMeter: """Count UTF-8 SQL text and logical parameter bytes, excluding protocol, WAL and index writes.""" def __init__(self, connection): self.connection = connection self.statement_bytes = 0 self.parameter_bytes = 0 self.statements = 0 def __getattr__(self, name): return getattr(self.connection, name) def _record(self, sql, params): text = sql.as_string(self.connection) if hasattr(sql, 'as_string') else str(sql) mutates = re.match(r'^\s*(INSERT|UPDATE|DELETE|MERGE|CREATE|ALTER|DROP|TRUNCATE)\b', text, re.I) if re.match(r'^\s*COPY\b', text, re.I): raise ValueError('COPY byte measurement is unsupported.') if mutates or (re.match(r'^\s*WITH\b', text, re.I) and re.search(r'\b(INSERT|UPDATE|DELETE|MERGE)\b', text, re.I)): self.statement_bytes += len(text.encode()) values = params.values() if isinstance(params, dict) else (params or ()) for value in values: if isinstance(value, bytes): self.parameter_bytes += len(value) elif value is not None: encoded = (_json(value) if isinstance(value, (dict, list, tuple)) else value.isoformat() if isinstance(value, datetime) else str(value)) self.parameter_bytes += len(encoded.encode()) self.statements += 1 def execute(self, sql, params=None, **kwargs): self._record(sql, params) return self.connection.execute(sql, params, **kwargs) def cursor(self, *args, **kwargs): meter = self class Cursor: def __init__(self, cursor): self.cursor = cursor def __getattr__(self, name): return getattr(self.cursor, name) def __enter__(self): self.cursor.__enter__() return self def __exit__(self, *exc): return self.cursor.__exit__(*exc) def __iter__(self): return iter(self.cursor) def execute(self, sql, params=None, **options): meter._record(sql, params) self.cursor.execute(sql, params, **options) return self def executemany(self, sql, params_seq, **options): params_seq = list(params_seq) for params in params_seq: meter._record(sql, params) self.cursor.executemany(sql, params_seq, **options) return self def copy(self, *args, **options): raise ValueError('COPY byte measurement is unsupported.') return Cursor(self.connection.cursor(*args, **kwargs)) def report(self): return {'metric': 'sql_write_input_utf8_bytes', 'bytes': self.statement_bytes + self.parameter_bytes, 'statement_bytes': self.statement_bytes, 'parameter_bytes': self.parameter_bytes, 'write_statements': self.statements, 'excludes': ['PostgreSQL protocol and TLS', 'WAL', 'indexes', 'storage page writes'], 'parameter_encoding': 'UTF-8 strings, ISO datetimes, canonical JSON collections, str scalars, raw bytes, zero for NULL'} def measure_save(tenant_slug, table_key, row_id, values, *, actor, statement_timeout_s=STATEMENT_TIMEOUT_S): """Measure a staging native-table grid event, then roll back its cell, history and Activity writes.""" import uuid from core import grid_events, store_pg, table_store, user_tables st = store_pg.PgStore(tenant_slug) _migration_guard(st.tenant_slug) if not isinstance(values, dict) or len(values) != 1 or not actor: raise ValueError('Measure exactly one cell value.') if not str(table_key).startswith('ut_'): raise ValueError('Measure a native editable user-table record.') pid = int(row_id) with store_pg._pool().connection() as con: with con.transaction(force_rollback=True): _settings(con, statement_timeout_s) measured = WriteMeter(con) with st.transaction(measured): stamp = _document(con, st.tenant_slug) if not is_marker(stamp): raise ValueError('Migrate history before measuring a cell save.') if not user_tables.records_mutable(table_key, st=st): raise ValueError('Measure a native editable user-table record.') record = user_tables.get_record(table_key, pid, st=st) or {} if str(pid) not in (record.get('rows') or {}): raise ValueError('The measurement record does not exist.') table = _table(st.tenant_slug) count_sql = f'SELECT count(*) FROM {table} WHERE "table" = %s AND row_id = %s' count_args = (str(table_key), str(pid)) before = con.execute(count_sql, count_args).fetchone()[0] event_id = 'history-measure:' + uuid.uuid4().hex ctx = grid_events.EventCtx(uname=actor, admin=True, allowed_pids=frozenset({pid}), fields=record.get('fields') or [], st=st, scope_key=str(table_key), by=actor, table=table_store.make(f'{table_key}_table_workspace', st=st)) grid_events.handle_one({'id': event_id, 'type': 'overlay_patch', 'pid': pid, 'updates': values}, ctx) after = con.execute(count_sql, count_args).fetchone()[0] if (not ctx.out.writes.get(event_id, {}).get('ok') or after != before + 1 or _document(con, st.tenant_slug) != stamp): raise ValueError('The measurement did not save exactly one changed cell and history row.') report = measured.report() return dict(report, scope='staging native-table grid overlay_patch SQL write inputs', fixture=False, request_overhead='HTTP handlers, receipt middleware and asynchronous work excluded', rolled_back=True, ok=report['bytes'] < 65536, limit_bytes=65536) def main(argv=None): import argparse parser = argparse.ArgumentParser(prog='python -m core.history_rows') parser.add_argument('action', choices=('migrate', 'verify', 'rollback', 'measure-save')) parser.add_argument('slug') parser.add_argument('--dry-run', action='store_true') parser.add_argument('--statement-timeout', type=float, default=STATEMENT_TIMEOUT_S) parser.add_argument('--json', action='store_true', help='print value-free JSON (the default)') parser.add_argument('--table') parser.add_argument('--row') parser.add_argument('--values-file', help='one changed cell as a JSON object, never printed') parser.add_argument('--actor', default='history-byte-measurement') args = parser.parse_args(argv) try: if args.action == 'measure-save': if args.dry_run or not all((args.table, args.row, args.values_file)): parser.error('measure-save requires --table, --row and --values-file, without --dry-run') with open(args.values_file, encoding='utf-8') as source: values = json.load(source) result = measure_save(args.slug, args.table, args.row, values, actor=args.actor, statement_timeout_s=args.statement_timeout) else: options = {'statement_timeout_s': args.statement_timeout} if args.action != 'verify': options['dry_run'] = args.dry_run result = {'migrate': migrate, 'verify': verify, 'rollback': rollback}[args.action](args.slug, **options) except (MigrationMismatch, ValueError, PermissionError) as exc: result = {'ok': False, 'error': str(exc)} except Exception as exc: # Database exception text can contain rejected customer values. result = {'ok': False, 'error': type(exc).__name__, 'detail': 'History operation failed without committing.'} print(json.dumps(result, sort_keys=True)) return 0 if result['ok'] else 1 if __name__ == '__main__': from core import history_rows as _module raise SystemExit(_module.main())