#!/usr/bin/env python3 """Restore a disposable PostgreSQL snapshot and reconcile a simulated action. The configured administration database is never migrated. Both created databases are removed afterward. The private dump contains only synthetic test accounts. """ import argparse import base64 import hashlib import json import os from pathlib import Path import secrets import shutil import subprocess import tempfile import sys import time import uuid ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(ROOT)) from sqlalchemy import create_engine, select, text, func from sqlalchemy.engine import make_url from backend.flagship.store import Store, accounts, events, records, sessions from backend.flagship.identity import Identity from scripts.verify_runtime import checkout def run(output): base = make_url(os.environ['PG_TEST_URL']) if base.get_backend_name() != 'postgresql' or base.host not in {'localhost', '127.0.0.1'}: raise ValueError('Use an isolated localhost PostgreSQL administration server') names = ['hf_backup_' + uuid.uuid4().hex, 'hf_restore_' + uuid.uuid4().hex] admin = create_engine(base, isolation_level='AUTOCOMMIT') stores = [] created = [] output.parent.mkdir(parents=True, exist_ok=True) dump = output.parent / ('synthetic-backup-' + uuid.uuid4().hex + '.dump') os.environ['VAULT_SECRET_KEY'] = base64.b64encode(secrets.token_bytes(32)).decode() try: for name in names: with admin.connect() as c: c.execute(text('CREATE DATABASE "' + name + '"')) created.append(name) urls = [base.set(database=name).render_as_string(hide_password=False) for name in names] origin = Store(urls[0]); stores.append(origin); origin.migrate() identity = Identity(origin) actor = identity.provision('backup_buyer', 'synthetic-backup-password-123') identity.issue(actor) identity.set_provider_token(actor, 'synthetic-provider-credential-never-sent') commerce, action = checkout(origin, actor, 'lost_response') assert commerce.execute(actor, action['id'], 'lost')['data']['state'] == 'UNKNOWN' def counts(store): with store.engine.connect() as c: return {table.name: c.execute(select(func.count()).select_from(table)).scalar_one() for table in (accounts, sessions, records, events)} expected_counts = counts(origin) captured = time.time() subprocess.run(['pg_dump', '-Fc', '--no-owner', '--dbname', urls[0], '--file', str(dump)], check=True, capture_output=True) late = origin.create('backup_probe', actor['id'], {'source': 'simulation', 'after_snapshot': True}) started = time.perf_counter() subprocess.run(['pg_restore', '--no-owner', '--dbname', urls[1], str(dump)], check=True, capture_output=True) restored = Store(urls[1]); stores.append(restored) assert restored.ready() and counts(restored) == expected_counts assert Identity(restored).provider_token(actor) == 'synthetic-provider-credential-never-sent' try: restored.get('backup_probe', late['id'], actor['id']) except LookupError: pass else: raise AssertionError('Post-snapshot row unexpectedly restored') from backend.flagship.commerce import Commerce from backend.flagship.provider import Provider recovered = Commerce(restored, Provider(restored, Identity(restored))) assert recovered.execute(actor, action['id'], 'after-restore')['data']['state'] == 'UNKNOWN' assert recovered.reconcile(actor, action['id'])['data']['state'] == 'SUCCEEDED' report = {'schema_version': 1, 'source': 'controlled_simulation', 'passed': True, 'snapshot_at_unix': captured, 'restore_and_verification_seconds': time.perf_counter()-started, 'restored_table_counts_before_reconciliation': expected_counts, 'vault_decryption_with_same_key': True, 'retry_state': 'UNKNOWN', 'reconciled_state': 'SUCCEEDED', 'post_snapshot_write_absent': True, 'dump_sha256': hashlib.sha256(dump.read_bytes()).hexdigest(), 'source_sha256': hashlib.sha256(Path(__file__).read_bytes()).hexdigest(), 'limitations': ['Same-host pg_dump/pg_restore exercise, not off-host disaster recovery.', 'Synthetic accounts and simulated provider effects only.', 'Recovery point is the database snapshot; subsequent writes are not restored.', 'Model-registry backup and human setup/pilot are separate gates.']} output.write_text(json.dumps(report, indent=2)+'\n') print(json.dumps(report, indent=2)) finally: for store in stores: store.engine.dispose() for name in created: with admin.connect() as c: c.execute(text('DROP DATABASE "' + name + '" WITH (FORCE)')) admin.dispose() def run_offhost(source_url, output): """Read-only dump of a remote database restored into a disposable local one. Nothing is written to the source. The source URL is never printed or stored. """ base = make_url(os.environ['PG_TEST_URL']) if base.get_backend_name() != 'postgresql' or base.host not in {'localhost', '127.0.0.1'}: raise ValueError('Restore target must be an isolated localhost PostgreSQL server') name = 'hf_offhost_restore_' + uuid.uuid4().hex admin = create_engine(base, isolation_level='AUTOCOMMIT') source = create_engine(source_url) output.parent.mkdir(parents=True, exist_ok=True) workdir = Path(tempfile.mkdtemp(prefix='hf-offhost-')) # 0700, removed afterwards dump = workdir / 'backup.dump' def snapshot(engine): with engine.connect() as c: ids = sorted(r[0] for r in c.execute(select(accounts.c.id))) return {'counts': {t.name: c.execute(select(func.count()).select_from(t)).scalar_one() for t in (accounts, sessions, records)}, 'account_ids_sha256': hashlib.sha256('\n'.join(ids).encode()).hexdigest()} def event_prefix(engine, upto): # hf_events is append-only with increasing ids; a consistent dump holds exactly the rows up to its maximum. with engine.connect() as c: rows = c.execute(select(events.c.id, events.c.type).where(events.c.id <= upto).order_by(events.c.id)).all() return {'rows': len(rows), 'max_id': upto, 'sha256': hashlib.sha256(repr(rows).encode()).hexdigest()} restored = None try: with admin.connect() as c: c.execute(text('CREATE DATABASE "' + name + '"')) target = base.set(database=name).render_as_string(hide_password=False) before = snapshot(source) started = time.perf_counter() subprocess.run(['pg_dump', '-Fc', '--no-owner', '--no-privileges', '--dbname', source_url, '--file', str(dump)], check=True, capture_output=True) dumped = time.perf_counter() after = snapshot(source) subprocess.run(['pg_restore', '--no-owner', '--no-privileges', '--dbname', target, str(dump)], check=True, capture_output=True) restored = Store(target) result = snapshot(restored.engine) with restored.engine.connect() as c: restored_max = c.execute(select(func.max(events.c.id))).scalar() or 0 restored_events, source_events = event_prefix(restored.engine, restored_max), event_prefix(source, restored_max) # Production keeps writing during the dump: mutable tables must match the source just before or just after # it, and the event log must be exactly the source's prefix up to the dumped maximum id. consistent = result in (before, after) and restored_events == source_events report = {'schema_version': 1, 'source': 'production_neon_read_only', 'passed': bool(restored.ready() and consistent), 'source_host_suffix': make_url(source_url).host.split('.', 1)[1], 'restored_schema_ready': restored.ready(), 'restored': result, 'restored_event_log': restored_events, 'source_event_log_same_prefix': source_events, 'source_before_dump': before, 'source_after_dump': after, 'dump_seconds': round(dumped - started, 2), 'restore_seconds': round(time.perf_counter() - dumped, 2), 'dump_bytes': dump.stat().st_size, 'checked_at_unix': time.time(), 'source_sha256': hashlib.sha256(Path(__file__).read_bytes()).hexdigest(), 'limitations': ['Restore target is a disposable local PostgreSQL 18, not a second cloud region.', 'Vault-encrypted provider credentials are restored as ciphertext; the production key never ' 'leaves the deployment, so decryption is not exercised here.', 'Recovery point is the dump; there is no point-in-time recovery test.']} output.write_text(json.dumps(report, indent=2) + '\n') print(json.dumps(report, indent=2)) if not report['passed']: raise SystemExit(1) finally: shutil.rmtree(workdir, ignore_errors=True) # production data never stays on disk if restored: restored.engine.dispose() source.dispose() with admin.connect() as c: c.execute(text('DROP DATABASE IF EXISTS "' + name + '" WITH (FORCE)')) admin.dispose() if __name__ == '__main__': parser = argparse.ArgumentParser(description=__doc__) parser.add_argument('--output', type=Path, required=True) parser.add_argument('--source-url-file', type=Path, help='File holding a remote database URL to dump read-only') args = parser.parse_args() if args.source_url_file: run_offhost(args.source_url_file.read_text().strip(), args.output) else: run(args.output)