runloopable / platform /core /history_rows.py
fsanyoto's picture
Deploy AIOS web (React glide grid + FastAPI slice)
023301a verified
Raw History Blame Contribute Delete
33 kB
"""Append complete cell history to tenant-local PostgreSQL month partitions."""
from datetime import datetime, timedelta, timezone
import hashlib
import json
import math
import re
TABLE_NAME = 'value_history_rows'
RETENTION_DAYS = 90
_COLUMNS = 'id, "table", row_id, kind, actor, at, changes'
DOCUMENT_KEY = 'value_history'
LAYOUT_KEY = '__layout__'
STATEMENT_TIMEOUT_S = 120
def _time(value):
if isinstance(value, str):
value = datetime.fromisoformat(value.replace('Z', '+00:00'))
if value.tzinfo is None:
value = value.replace(tzinfo=timezone.utc)
return value.astimezone(timezone.utc)
def _month(value):
return _time(value).replace(day=1, hour=0, minute=0, second=0, microsecond=0)
def _next_month(value):
if value.month == 12:
return value.replace(year=value.year + 1, month=1)
return value.replace(month=value.month + 1)
def _table(tenant_slug):
from core import store_pg
schema = store_pg._schema_for(tenant_slug)
if not re.fullmatch(r'[a-zA-Z0-9_]+', schema):
raise ValueError('Invalid history schema.')
return f'"{schema}"."{TABLE_NAME}"'
def ensure_schema(con, tenant_slug):
"""Create the partitioned parent and its record-time index inside the caller's transaction."""
table = _table(tenant_slug)
con.execute(f'''CREATE TABLE IF NOT EXISTS {table} (
id TEXT NOT NULL, "table" TEXT NOT NULL, row_id TEXT NOT NULL,
kind TEXT NOT NULL, actor TEXT NOT NULL, at TIMESTAMPTZ NOT NULL,
changes JSONB NOT NULL, PRIMARY KEY (at, id)) PARTITION BY RANGE (at)''')
con.execute(f'CREATE INDEX IF NOT EXISTS "{TABLE_NAME}_record_time" '
f'ON {table} ("table", row_id, at DESC, id DESC)')
class FixtureAdapter:
"""Mark an explicitly supplied offline history backend."""
def backend(st):
"""Resolve only the bound tenant's PostgreSQL store or an explicit history fixture."""
from core import user_tables, store_pg
while isinstance(st, user_tables._Lent):
st = st._runtime
if st is user_tables.store:
st = st.handle()
bound = st._store() if callable(getattr(st, '_store', None)) else st
if bound is user_tables.store:
bound = bound.handle(slug=getattr(st, 'key', None))
if isinstance(bound, FixtureAdapter):
return bound
fixture = getattr(bound, 'history_rows_fixture', None)
if isinstance(fixture, FixtureAdapter):
return fixture
if not isinstance(bound, store_pg.PgStore):
raise TypeError('History requires a bound PgStore or an explicit history fixture.')
if getattr(st, 'store_namespace', '') or getattr(st, 'key', bound.tenant_slug) != bound.tenant_slug:
raise ValueError('History runtime and store tenant do not match.')
adapter = getattr(bound, '_history_rows_backend', None)
if adapter is None:
adapter = PgHistoryStore(bound)
bound._history_rows_backend = adapter
if adapter.tenant_slug != bound.tenant_slug:
raise ValueError('History store binding changed.')
return adapter
def cipher_changes(tenant, table, changes, *, opening=False, definition=None):
"""Bind personal history values to their originating tenant, table and column."""
from core import row_cipher, tenant_keys, user_tables, store
if not opening and tenant_keys.subkey(tenant, 'row') is None:
return changes
if opening:
return {column: {side: row_cipher.open_value(tenant, table, column, value)
for side, value in pair.items()}
for column, pair in (changes or {}).items()}
if definition is None:
definition = user_tables.get_definition(table, st=store.handle(slug=tenant))
personal = row_cipher.personal_columns(table, definition)
return {column: {side: row_cipher.seal(tenant, table, column, value)
if column in personal else value for side, value in pair.items()}
for column, pair in (changes or {}).items()}
def cipher_event(tenant, event, *, opening=False, definition=None):
return {**event, 'changes': cipher_changes(tenant, event['table'], event['changes'],
opening=opening, definition=definition)}
def seal_copy_batch(tenant, events, *, backup_path, definitions=None):
"""Prepare a verified migration batch and encrypted backup without publishing either."""
from core import blob_envelope
events = list(events)
backup = blob_envelope.seal(tenant, backup_path, _json(events).encode(), required=True)
sealed = [cipher_event(tenant, event, definition=(definitions or {}).get(event['table']))
for event in events]
if ([cipher_event(tenant, event, opening=True) for event in sealed]
!= [cipher_event(tenant, event, opening=True) for event in events]):
raise MigrationMismatch('Personal history batch differs after encryption.')
return sealed, backup
def _event(row, tenant=None):
identity, table_key, row_id, kind, actor, at, changes = row
value = {'id': identity, 'table': table_key, 'rowId': row_id, 'kind': kind,
'actor': actor, 'at': _time(at).isoformat(),
'changes': json.loads(changes) if isinstance(changes, str) else changes}
return cipher_event(tenant, value, opening=True) if tenant else value
class PgHistoryStore:
"""Store immutable history rows in the bound tenant schema."""
def __init__(self, st):
from core import store_pg
if not isinstance(st, store_pg.PgStore):
raise TypeError('A PgStore is required.')
self.tenant_slug = st.tenant_slug
self.table = _table(st.tenant_slug)
self._months = set()
def _write_guard(self):
from core import store_pg
store_pg.check_write(self.tenant_slug, 'append value history')
def _connection(self):
from core import store_pg
store_pg._effective_slug(self.tenant_slug)
return store_pg._pool().connection()
def _prepare(self, con, at):
start = _month(at)
end = _next_month(start)
partition = self.table[:-1] + '_' + start.strftime('%Y%m') + '"'
if con.execute('SELECT to_regclass(%s)', (partition,)).fetchone()[0] is not None:
return False
con.execute('SELECT pg_advisory_xact_lock(hashtextextended(%s, 0))', (self.table,))
ensure_schema(con, self.tenant_slug)
# Bounds are generated from UTC datetimes, never caller-supplied SQL.
con.execute(f"CREATE TABLE IF NOT EXISTS {partition} PARTITION OF {self.table} "
f"FOR VALUES FROM ('{start.isoformat()}') TO ('{end.isoformat()}')")
return True
def insert(self, event):
"""Insert one complete event without reading or rewriting a history document."""
from core import store_pg
self._write_guard()
at = _time(event['at'])
month = _month(at)
prepared = month not in self._months
with self._connection() as con:
if prepared and self._prepare(con, at):
self._prune(con, datetime.now(timezone.utc))
con.execute(f'INSERT INTO {self.table} ({_COLUMNS}) '
'VALUES (%s, %s, %s, %s, %s, %s, %s::jsonb)',
(event['id'], event['table'], event['rowId'], event['kind'],
event['actor'], at, json.dumps(cipher_changes(
self.tenant_slug, event['table'], event['changes']), ensure_ascii=False,
allow_nan=False)))
# A caller-owned transaction can still roll back its DDL.
if prepared and store_pg._CALLER_TRANSACTION.get() is None:
self._months.add(month)
return event
def query(self, table_key, row_id, *, since, row_ids=None, event_id=None):
"""Select the record and retention window in SQL without a result cap."""
clauses = ['"table" = %s', 'at >= %s']
args = [str(table_key), _time(since)]
if row_id is not None:
clauses.append('row_id = %s')
args.append(str(row_id))
if row_ids is not None:
ids = list(dict.fromkeys(str(rid) for rid in row_ids))
if not ids:
return []
clauses.append('row_id = ANY(%s)')
args.append(ids)
if event_id is not None:
clauses.append('id = %s')
args.append(str(event_id))
try:
with self._connection() as con:
cursor = con.execute(f'SELECT {_COLUMNS} FROM {self.table} WHERE '
+ ' AND '.join(clauses) + ' ORDER BY at DESC, id DESC', args)
events = []
while rows := cursor.fetchmany(256):
events.extend(_event(row, self.tenant_slug) for row in rows)
return events
except Exception as exc:
if getattr(exc, 'sqlstate', None) != '42P01':
raise
return []
def latest(self, table_keys):
"""Return one SQL-aggregated edit timestamp per requested table for freshness metadata."""
keys = list(dict.fromkeys(str(key) for key in table_keys))
if not keys:
return {}
try:
with self._connection() as con:
rows = con.execute(f'SELECT "table", max(at) FROM {self.table} '
'WHERE "table" = ANY(%s) GROUP BY "table"', (keys,)).fetchall()
return {key: _time(at).isoformat() for key, at in rows}
except Exception as exc:
if getattr(exc, 'sqlstate', None) != '42P01':
raise
return {}
def change_counts(self, *, since, until=None, kinds=None, tables=None):
"""Return (table, actor, at, changed cell count) for every table's events in one statement.
Values stay sealed: only the changes keys are counted, so no row is decrypted.
"""
clauses, args = ['at >= %s'], [_time(since)]
if until is not None:
# The upper bound lets month partitions prune on both sides.
clauses.append('at < %s')
args.append(_time(until))
for column, values in (('kind', kinds), ('"table"', tables)):
if values is not None:
values = list(dict.fromkeys(str(value) for value in values))
if not values:
return []
clauses.append(f'{column} = ANY(%s)')
args.append(values)
try:
with self._connection() as con:
# A non-object payload is returned whole and counted the way _event parses it.
cursor = con.execute(
'SELECT "table", actor, at, CASE WHEN jsonb_typeof(changes) = \'object\' '
'THEN (SELECT count(*) FROM jsonb_object_keys(changes)) END, '
'CASE WHEN jsonb_typeof(changes) = \'object\' THEN NULL ELSE changes END '
f'FROM {self.table} WHERE ' + ' AND '.join(clauses), args)
counts = []
while rows := cursor.fetchmany(1024):
for table_key, actor, at, changed, raw in rows:
if changed is None:
raw = (json.loads(raw) if isinstance(raw, str) else raw) or {}
if not isinstance(raw, dict):
raise ValueError('History changes must be a JSON object.')
changed = len(raw)
counts.append((table_key, actor, _time(at), int(changed)))
return counts
except Exception as exc:
if getattr(exc, 'sqlstate', None) != '42P01':
raise
return []
def _prune(self, con, now):
cutoff = _time(now) - timedelta(days=RETENTION_DAYS)
dropped = []
con.execute('SELECT pg_advisory_xact_lock(hashtextextended(%s, 0))', (self.table,))
partitions = con.execute(
'SELECT child.relname FROM pg_inherits i '
'JOIN pg_class child ON child.oid = i.inhrelid '
'WHERE i.inhparent = to_regclass(%s)', (self.table,)).fetchall()
for (name,) in partitions:
match = re.fullmatch(TABLE_NAME + r'_(\d{4})(\d{2})', name)
if match is None:
continue
start = datetime(int(match[1]), int(match[2]), 1, tzinfo=timezone.utc)
if _next_month(start) <= cutoff:
partition = self.table.rsplit('.', 1)[0] + f'."{name}"'
con.execute(f'DROP TABLE {partition}')
dropped.append(start)
return dropped
def prune(self, *, now=None):
"""Drop only whole month partitions whose upper bound precedes the retention cutoff."""
self._write_guard()
now = _time(now) if now is not None else datetime.now(timezone.utc)
with self._connection() as con:
dropped = self._prune(con, now)
self._months.difference_update(dropped)
return len(dropped)
class MigrationMismatch(RuntimeError):
"""Refuse a cutover whose event identities or complete payloads differ."""
def is_marker(value):
return (isinstance(value, dict) and value.get(LAYOUT_KEY) == 'rows'
and value.get('table') == TABLE_NAME and value.get('version') == 1)
def _json(value):
return json.dumps(value, ensure_ascii=False, sort_keys=True, separators=(',', ':'),
allow_nan=False)
def _events(value):
if not isinstance(value, list):
raise ValueError('Legacy value_history must be a JSON array.')
result, identities = [], set()
fields = {'id', 'table', 'rowId', 'kind', 'actor', 'at', 'changes'}
for event in value:
if (not isinstance(event, dict) or set(event) != fields
or any(not isinstance(event[key], str) for key in fields - {'changes'})
or not event['id'] or not isinstance(event['changes'], dict)):
raise ValueError('Legacy history event has an unsupported shape.')
if event['id'] in identities:
raise ValueError('Legacy history event identities must be unique.')
identities.add(event['id'])
try:
result.append(dict(event, at=_time(event['at']).isoformat()))
except (ValueError, OverflowError):
raise ValueError('Legacy history event timestamp is invalid.') from None
return result
def _digest(events):
ordered = sorted(_events(events), key=lambda event: event['id'])
return {'events': len(ordered),
'ids_sha256': hashlib.sha256(_json([event['id'] for event in ordered]).encode()).hexdigest(),
'payload_sha256': hashlib.sha256(_json(ordered).encode()).hexdigest()}
def _settings(con, seconds, *, read_only=False):
if not math.isfinite(seconds) or seconds <= 0:
raise ValueError('Statement timeout must be finite and positive.')
if read_only:
con.execute('SET TRANSACTION ISOLATION LEVEL REPEATABLE READ, READ ONLY')
con.execute("SELECT set_config('statement_timeout', %s, true)", (f'{int(seconds * 1000)}ms',))
con.execute("SELECT set_config('lock_timeout', %s, true)", (f'{int(seconds * 1000)}ms',))
def _migration_guard(tenant_slug):
from core import store_pg
store_pg.check_write(tenant_slug, 'operate history row migration')
if (store_pg.data_binding.is_production_deployment()
or (tenant_slug in store_pg.PRODUCTION_TENANTS and not store_pg.isolated_staging_database())):
raise PermissionError('History migration and measurement require isolated staging (R58).')
def _document(con, tenant_slug, *, lock=False):
from core import store_pg
row = con.execute(store_pg._q('SELECT value::text FROM {tbl} WHERE key = %s'
+ (' FOR UPDATE' if lock else ''), 'store_kv', tenant_slug),
(DOCUMENT_KEY,)).fetchone()
return (json.loads(row[0]) if isinstance(row[0], str) else row[0]) if row else None
def _write_document(con, tenant_slug, value):
from core import store_pg
con.execute(store_pg._q('INSERT INTO {tbl} (key, value) VALUES (%s, %s::jsonb) '
'ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value, '
'rev = {bare}.rev + 1, updated_at = now()', 'store_kv', tenant_slug),
(DOCUMENT_KEY, _json(value)))
def _all_rows(con, adapter, *, legacy=False, identities=None):
sql = f'SELECT {_COLUMNS} FROM {adapter.table}'
args = ()
if legacy:
sql += ' WHERE legacy_backfill = true'
elif identities is not None:
if not identities:
return []
sql += ' WHERE id = ANY(%s)'
args = (list(identities),)
cursor = con.execute(sql + ' ORDER BY at, id', args)
result = []
while rows := cursor.fetchmany(256):
result.extend(_event(row, adapter.tenant_slug) for row in rows)
return result
def _backfill(adapter, events):
"""Copy every legacy event on the enlisted connection without invoking retention."""
with adapter._connection() as con:
for month in sorted({_month(event['at']) for event in events}):
adapter._prepare(con, month)
if events:
with con.cursor() as cursor:
# psycopg consumes parameters under its non-reentrant connection lock.
# Cipher key/definition reads enlist this same connection, so prepare each
# bounded batch before executemany acquires that lock.
for offset in range(0, len(events), 256):
params = [(event['id'], event['table'], event['rowId'], event['kind'], event['actor'],
_time(event['at']), _json(cipher_changes(
adapter.tenant_slug, event['table'], event['changes'])))
for event in events[offset:offset + 256]]
cursor.executemany(f'INSERT INTO {adapter.table} ({_COLUMNS}) '
'VALUES (%s, %s, %s, %s, %s, %s, %s::jsonb) '
'ON CONFLICT (at, id) DO NOTHING', params)
def _verify_rows(con, adapter, stamp):
source = stamp.get('source')
copied = _digest(_all_rows(con, adapter, legacy=True))
current = _digest(_all_rows(con, adapter))
return {'ok': source == copied, 'mode': 'rows', 'source': source, 'copied': copied,
'current': current, 'appended_events': current['events'] - copied['events']}
def rows_ready(tenant_slug):
"""True when the history document is a rows marker, an empty array, or absent. Read-only."""
from core import account_rows, store_pg
with account_rows.read_snapshot(tenant_slug) as con:
row = con.execute(store_pg._q(
"SELECT (CASE WHEN value ->> '__layout__' IS NOT NULL THEN value END)::text, "
"value::text = '[]' FROM {tbl} WHERE key = %s", 'store_kv', tenant_slug),
(DOCUMENT_KEY,)).fetchone()
return not row or bool(row[1]) or is_marker(json.loads(row[0]) if row[0] else None)
def migrate(tenant_slug, *, statement_timeout_s=STATEMENT_TIMEOUT_S, dry_run=False):
"""Lock the legacy document, backfill and verify all events, then commit its small marker atomically."""
from core import store_pg
st = store_pg.PgStore(tenant_slug)
if dry_run:
return dict(verify(st.tenant_slug, statement_timeout_s=statement_timeout_s),
changed=False, dry_run=True)
_migration_guard(st.tenant_slug)
adapter = PgHistoryStore(st)
with store_pg._pool().connection() as con:
_settings(con, statement_timeout_s)
with st.transaction(con):
con.execute('SELECT pg_advisory_xact_lock(hashtextextended(%s, 0))', (adapter.table,))
# Insert only the absent lock target, so concurrent migrations share one document lock.
con.execute(store_pg._q('INSERT INTO {tbl} (key, value) VALUES (%s, %s::jsonb) '
'ON CONFLICT (key) DO NOTHING', 'store_kv', st.tenant_slug),
(DOCUMENT_KEY, '[]'))
original = _document(con, st.tenant_slug, lock=True)
if is_marker(original):
report = _verify_rows(con, adapter, original)
if not report['ok']:
raise MigrationMismatch('Existing history marker does not match copied events.')
return dict(report, changed=False, reason='already rows')
events = _events(original)
ensure_schema(con, st.tenant_slug)
con.execute(f'LOCK TABLE {adapter.table} IN SHARE ROW EXCLUSIVE MODE')
con.execute(f'ALTER TABLE {adapter.table} ADD COLUMN IF NOT EXISTS '
'legacy_backfill BOOLEAN NOT NULL DEFAULT false')
# The first normal save must find its month without pruning the newly copied legacy months.
adapter._prepare(con, datetime.now(timezone.utc))
_backfill(adapter, events)
copied = _all_rows(con, adapter, identities=[event['id'] for event in events])
source = _digest(events)
if _digest(copied) != source:
raise MigrationMismatch('History backfill event count, identities or payloads differ.')
con.execute(f'UPDATE {adapter.table} SET legacy_backfill = false WHERE legacy_backfill = true')
if events:
con.execute(f'UPDATE {adapter.table} SET legacy_backfill = true WHERE id = ANY(%s)',
([event['id'] for event in events],))
stamp = {LAYOUT_KEY: 'rows', 'version': 1, 'table': TABLE_NAME,
'migratedAt': datetime.now(timezone.utc).isoformat(), 'source': source}
_write_document(con, st.tenant_slug, stamp)
report = _verify_rows(con, adapter, stamp)
if not report['ok']:
raise MigrationMismatch('History marker and copied events differ.')
store_pg._bump_write_generation(DOCUMENT_KEY, st.tenant_slug)
return dict(report, changed=True, marker_bytes=len(_json(stamp).encode()))
def verify(tenant_slug, *, statement_timeout_s=STATEMENT_TIMEOUT_S):
"""Check the entire legacy copy, including event identities and payloads, independently of later appends."""
from core import account_rows, store_pg
st = store_pg.PgStore(tenant_slug)
adapter = PgHistoryStore(st)
with account_rows.read_snapshot(st.tenant_slug, require_stable=True,
statement_timeout_s=statement_timeout_s) as con:
value = _document(con, st.tenant_slug)
if is_marker(value):
return _verify_rows(con, adapter, value)
return {'ok': True, 'mode': 'absent' if value is None else 'document',
'source': _digest([] if value is None else _events(value))}
def rollback(tenant_slug, *, statement_timeout_s=STATEMENT_TIMEOUT_S, dry_run=False):
"""Restore a verified legacy document from every current row, retaining rows for active row-only readers."""
from core import store_pg
st = store_pg.PgStore(tenant_slug)
if dry_run:
return dict(verify(st.tenant_slug, statement_timeout_s=statement_timeout_s),
changed=False, dry_run=True)
_migration_guard(st.tenant_slug)
adapter = PgHistoryStore(st)
with store_pg._pool().connection() as con:
_settings(con, statement_timeout_s)
with st.transaction(con):
con.execute('SELECT pg_advisory_xact_lock(hashtextextended(%s, 0))', (adapter.table,))
stamp = _document(con, st.tenant_slug, lock=True)
if not is_marker(stamp):
return {'ok': True, 'changed': False, 'reason': 'not in row mode'}
con.execute(f'LOCK TABLE {adapter.table} IN SHARE ROW EXCLUSIVE MODE')
report = _verify_rows(con, adapter, stamp)
if not report['ok']:
raise MigrationMismatch('History rollback refused an incomplete legacy copy.')
events = _all_rows(con, adapter)
before = _digest(events)
_write_document(con, st.tenant_slug, events)
if _digest(_events(_document(con, st.tenant_slug))) != before:
raise MigrationMismatch('History rollback document differs from current rows.')
store_pg._bump_write_generation(DOCUMENT_KEY, st.tenant_slug)
return {'ok': True, 'changed': True, 'mode': 'document', 'current': before,
'rows_retained': True}
class WriteMeter:
"""Count UTF-8 SQL text and logical parameter bytes, excluding protocol, WAL and index writes."""
def __init__(self, connection):
self.connection = connection
self.statement_bytes = 0
self.parameter_bytes = 0
self.statements = 0
def __getattr__(self, name):
return getattr(self.connection, name)
def _record(self, sql, params):
text = sql.as_string(self.connection) if hasattr(sql, 'as_string') else str(sql)
mutates = re.match(r'^\s*(INSERT|UPDATE|DELETE|MERGE|CREATE|ALTER|DROP|TRUNCATE)\b', text, re.I)
if re.match(r'^\s*COPY\b', text, re.I):
raise ValueError('COPY byte measurement is unsupported.')
if mutates or (re.match(r'^\s*WITH\b', text, re.I)
and re.search(r'\b(INSERT|UPDATE|DELETE|MERGE)\b', text, re.I)):
self.statement_bytes += len(text.encode())
values = params.values() if isinstance(params, dict) else (params or ())
for value in values:
if isinstance(value, bytes):
self.parameter_bytes += len(value)
elif value is not None:
encoded = (_json(value) if isinstance(value, (dict, list, tuple)) else
value.isoformat() if isinstance(value, datetime) else str(value))
self.parameter_bytes += len(encoded.encode())
self.statements += 1
def execute(self, sql, params=None, **kwargs):
self._record(sql, params)
return self.connection.execute(sql, params, **kwargs)
def cursor(self, *args, **kwargs):
meter = self
class Cursor:
def __init__(self, cursor):
self.cursor = cursor
def __getattr__(self, name):
return getattr(self.cursor, name)
def __enter__(self):
self.cursor.__enter__()
return self
def __exit__(self, *exc):
return self.cursor.__exit__(*exc)
def __iter__(self):
return iter(self.cursor)
def execute(self, sql, params=None, **options):
meter._record(sql, params)
self.cursor.execute(sql, params, **options)
return self
def executemany(self, sql, params_seq, **options):
params_seq = list(params_seq)
for params in params_seq:
meter._record(sql, params)
self.cursor.executemany(sql, params_seq, **options)
return self
def copy(self, *args, **options):
raise ValueError('COPY byte measurement is unsupported.')
return Cursor(self.connection.cursor(*args, **kwargs))
def report(self):
return {'metric': 'sql_write_input_utf8_bytes', 'bytes': self.statement_bytes + self.parameter_bytes,
'statement_bytes': self.statement_bytes, 'parameter_bytes': self.parameter_bytes,
'write_statements': self.statements,
'excludes': ['PostgreSQL protocol and TLS', 'WAL', 'indexes', 'storage page writes'],
'parameter_encoding': 'UTF-8 strings, ISO datetimes, canonical JSON collections, str scalars, raw bytes, zero for NULL'}
def measure_save(tenant_slug, table_key, row_id, values, *, actor,
statement_timeout_s=STATEMENT_TIMEOUT_S):
"""Measure a staging native-table grid event, then roll back its cell, history and Activity writes."""
import uuid
from core import grid_events, store_pg, table_store, user_tables
st = store_pg.PgStore(tenant_slug)
_migration_guard(st.tenant_slug)
if not isinstance(values, dict) or len(values) != 1 or not actor:
raise ValueError('Measure exactly one cell value.')
if not str(table_key).startswith('ut_'):
raise ValueError('Measure a native editable user-table record.')
pid = int(row_id)
with store_pg._pool().connection() as con:
with con.transaction(force_rollback=True):
_settings(con, statement_timeout_s)
measured = WriteMeter(con)
with st.transaction(measured):
stamp = _document(con, st.tenant_slug)
if not is_marker(stamp):
raise ValueError('Migrate history before measuring a cell save.')
if not user_tables.records_mutable(table_key, st=st):
raise ValueError('Measure a native editable user-table record.')
record = user_tables.get_record(table_key, pid, st=st) or {}
if str(pid) not in (record.get('rows') or {}):
raise ValueError('The measurement record does not exist.')
table = _table(st.tenant_slug)
count_sql = f'SELECT count(*) FROM {table} WHERE "table" = %s AND row_id = %s'
count_args = (str(table_key), str(pid))
before = con.execute(count_sql, count_args).fetchone()[0]
event_id = 'history-measure:' + uuid.uuid4().hex
ctx = grid_events.EventCtx(uname=actor, admin=True, allowed_pids=frozenset({pid}),
fields=record.get('fields') or [], st=st, scope_key=str(table_key), by=actor,
table=table_store.make(f'{table_key}_table_workspace', st=st))
grid_events.handle_one({'id': event_id, 'type': 'overlay_patch',
'pid': pid, 'updates': values}, ctx)
after = con.execute(count_sql, count_args).fetchone()[0]
if (not ctx.out.writes.get(event_id, {}).get('ok') or after != before + 1
or _document(con, st.tenant_slug) != stamp):
raise ValueError('The measurement did not save exactly one changed cell and history row.')
report = measured.report()
return dict(report, scope='staging native-table grid overlay_patch SQL write inputs', fixture=False,
request_overhead='HTTP handlers, receipt middleware and asynchronous work excluded',
rolled_back=True, ok=report['bytes'] < 65536, limit_bytes=65536)
def main(argv=None):
import argparse
parser = argparse.ArgumentParser(prog='python -m core.history_rows')
parser.add_argument('action', choices=('migrate', 'verify', 'rollback', 'measure-save'))
parser.add_argument('slug')
parser.add_argument('--dry-run', action='store_true')
parser.add_argument('--statement-timeout', type=float, default=STATEMENT_TIMEOUT_S)
parser.add_argument('--json', action='store_true', help='print value-free JSON (the default)')
parser.add_argument('--table')
parser.add_argument('--row')
parser.add_argument('--values-file', help='one changed cell as a JSON object, never printed')
parser.add_argument('--actor', default='history-byte-measurement')
args = parser.parse_args(argv)
try:
if args.action == 'measure-save':
if args.dry_run or not all((args.table, args.row, args.values_file)):
parser.error('measure-save requires --table, --row and --values-file, without --dry-run')
with open(args.values_file, encoding='utf-8') as source:
values = json.load(source)
result = measure_save(args.slug, args.table, args.row, values, actor=args.actor,
statement_timeout_s=args.statement_timeout)
else:
options = {'statement_timeout_s': args.statement_timeout}
if args.action != 'verify':
options['dry_run'] = args.dry_run
result = {'migrate': migrate, 'verify': verify, 'rollback': rollback}[args.action](args.slug, **options)
except (MigrationMismatch, ValueError, PermissionError) as exc:
result = {'ok': False, 'error': str(exc)}
except Exception as exc:
# Database exception text can contain rejected customer values.
result = {'ok': False, 'error': type(exc).__name__, 'detail': 'History operation failed without committing.'}
print(json.dumps(result, sort_keys=True))
return 0 if result['ok'] else 1
if __name__ == '__main__':
from core import history_rows as _module
raise SystemExit(_module.main())