HyperFlow / scripts /verify_backup_restore.py
Gaurav711's picture
Deploy ba6e7c6
27e707f verified
Raw History Blame Contribute Delete
10 kB
#!/usr/bin/env python3
"""Restore a disposable PostgreSQL snapshot and reconcile a simulated action.
The configured administration database is never migrated. Both created databases
are removed afterward. The private dump contains only synthetic test accounts.
"""
import argparse
import base64
import hashlib
import json
import os
from pathlib import Path
import secrets
import shutil
import subprocess
import tempfile
import sys
import time
import uuid
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT))
from sqlalchemy import create_engine, select, text, func
from sqlalchemy.engine import make_url
from backend.flagship.store import Store, accounts, events, records, sessions
from backend.flagship.identity import Identity
from scripts.verify_runtime import checkout
def run(output):
base = make_url(os.environ['PG_TEST_URL'])
if base.get_backend_name() != 'postgresql' or base.host not in {'localhost', '127.0.0.1'}:
raise ValueError('Use an isolated localhost PostgreSQL administration server')
names = ['hf_backup_' + uuid.uuid4().hex, 'hf_restore_' + uuid.uuid4().hex]
admin = create_engine(base, isolation_level='AUTOCOMMIT')
stores = []
created = []
output.parent.mkdir(parents=True, exist_ok=True)
dump = output.parent / ('synthetic-backup-' + uuid.uuid4().hex + '.dump')
os.environ['VAULT_SECRET_KEY'] = base64.b64encode(secrets.token_bytes(32)).decode()
try:
for name in names:
with admin.connect() as c:
c.execute(text('CREATE DATABASE "' + name + '"'))
created.append(name)
urls = [base.set(database=name).render_as_string(hide_password=False) for name in names]
origin = Store(urls[0]); stores.append(origin); origin.migrate()
identity = Identity(origin)
actor = identity.provision('backup_buyer', 'synthetic-backup-password-123')
identity.issue(actor)
identity.set_provider_token(actor, 'synthetic-provider-credential-never-sent')
commerce, action = checkout(origin, actor, 'lost_response')
assert commerce.execute(actor, action['id'], 'lost')['data']['state'] == 'UNKNOWN'
def counts(store):
with store.engine.connect() as c:
return {table.name: c.execute(select(func.count()).select_from(table)).scalar_one()
for table in (accounts, sessions, records, events)}
expected_counts = counts(origin)
captured = time.time()
subprocess.run(['pg_dump', '-Fc', '--no-owner', '--dbname', urls[0], '--file', str(dump)], check=True, capture_output=True)
late = origin.create('backup_probe', actor['id'], {'source': 'simulation', 'after_snapshot': True})
started = time.perf_counter()
subprocess.run(['pg_restore', '--no-owner', '--dbname', urls[1], str(dump)], check=True, capture_output=True)
restored = Store(urls[1]); stores.append(restored)
assert restored.ready() and counts(restored) == expected_counts
assert Identity(restored).provider_token(actor) == 'synthetic-provider-credential-never-sent'
try:
restored.get('backup_probe', late['id'], actor['id'])
except LookupError:
pass
else:
raise AssertionError('Post-snapshot row unexpectedly restored')
from backend.flagship.commerce import Commerce
from backend.flagship.provider import Provider
recovered = Commerce(restored, Provider(restored, Identity(restored)))
assert recovered.execute(actor, action['id'], 'after-restore')['data']['state'] == 'UNKNOWN'
assert recovered.reconcile(actor, action['id'])['data']['state'] == 'SUCCEEDED'
report = {'schema_version': 1, 'source': 'controlled_simulation', 'passed': True,
'snapshot_at_unix': captured, 'restore_and_verification_seconds': time.perf_counter()-started,
'restored_table_counts_before_reconciliation': expected_counts,
'vault_decryption_with_same_key': True, 'retry_state': 'UNKNOWN', 'reconciled_state': 'SUCCEEDED',
'post_snapshot_write_absent': True, 'dump_sha256': hashlib.sha256(dump.read_bytes()).hexdigest(),
'source_sha256': hashlib.sha256(Path(__file__).read_bytes()).hexdigest(),
'limitations': ['Same-host pg_dump/pg_restore exercise, not off-host disaster recovery.',
'Synthetic accounts and simulated provider effects only.',
'Recovery point is the database snapshot; subsequent writes are not restored.',
'Model-registry backup and human setup/pilot are separate gates.']}
output.write_text(json.dumps(report, indent=2)+'\n')
print(json.dumps(report, indent=2))
finally:
for store in stores:
store.engine.dispose()
for name in created:
with admin.connect() as c:
c.execute(text('DROP DATABASE "' + name + '" WITH (FORCE)'))
admin.dispose()
def run_offhost(source_url, output):
"""Read-only dump of a remote database restored into a disposable local one.
Nothing is written to the source. The source URL is never printed or stored.
"""
base = make_url(os.environ['PG_TEST_URL'])
if base.get_backend_name() != 'postgresql' or base.host not in {'localhost', '127.0.0.1'}:
raise ValueError('Restore target must be an isolated localhost PostgreSQL server')
name = 'hf_offhost_restore_' + uuid.uuid4().hex
admin = create_engine(base, isolation_level='AUTOCOMMIT')
source = create_engine(source_url)
output.parent.mkdir(parents=True, exist_ok=True)
workdir = Path(tempfile.mkdtemp(prefix='hf-offhost-')) # 0700, removed afterwards
dump = workdir / 'backup.dump'
def snapshot(engine):
with engine.connect() as c:
ids = sorted(r[0] for r in c.execute(select(accounts.c.id)))
return {'counts': {t.name: c.execute(select(func.count()).select_from(t)).scalar_one()
for t in (accounts, sessions, records)},
'account_ids_sha256': hashlib.sha256('\n'.join(ids).encode()).hexdigest()}
def event_prefix(engine, upto):
# hf_events is append-only with increasing ids; a consistent dump holds exactly the rows up to its maximum.
with engine.connect() as c:
rows = c.execute(select(events.c.id, events.c.type).where(events.c.id <= upto).order_by(events.c.id)).all()
return {'rows': len(rows), 'max_id': upto, 'sha256': hashlib.sha256(repr(rows).encode()).hexdigest()}
restored = None
try:
with admin.connect() as c:
c.execute(text('CREATE DATABASE "' + name + '"'))
target = base.set(database=name).render_as_string(hide_password=False)
before = snapshot(source)
started = time.perf_counter()
subprocess.run(['pg_dump', '-Fc', '--no-owner', '--no-privileges', '--dbname', source_url, '--file', str(dump)],
check=True, capture_output=True)
dumped = time.perf_counter()
after = snapshot(source)
subprocess.run(['pg_restore', '--no-owner', '--no-privileges', '--dbname', target, str(dump)], check=True, capture_output=True)
restored = Store(target)
result = snapshot(restored.engine)
with restored.engine.connect() as c:
restored_max = c.execute(select(func.max(events.c.id))).scalar() or 0
restored_events, source_events = event_prefix(restored.engine, restored_max), event_prefix(source, restored_max)
# Production keeps writing during the dump: mutable tables must match the source just before or just after
# it, and the event log must be exactly the source's prefix up to the dumped maximum id.
consistent = result in (before, after) and restored_events == source_events
report = {'schema_version': 1, 'source': 'production_neon_read_only', 'passed': bool(restored.ready() and consistent),
'source_host_suffix': make_url(source_url).host.split('.', 1)[1],
'restored_schema_ready': restored.ready(), 'restored': result, 'restored_event_log': restored_events,
'source_event_log_same_prefix': source_events,
'source_before_dump': before, 'source_after_dump': after,
'dump_seconds': round(dumped - started, 2), 'restore_seconds': round(time.perf_counter() - dumped, 2),
'dump_bytes': dump.stat().st_size, 'checked_at_unix': time.time(),
'source_sha256': hashlib.sha256(Path(__file__).read_bytes()).hexdigest(),
'limitations': ['Restore target is a disposable local PostgreSQL 18, not a second cloud region.',
'Vault-encrypted provider credentials are restored as ciphertext; the production key never '
'leaves the deployment, so decryption is not exercised here.',
'Recovery point is the dump; there is no point-in-time recovery test.']}
output.write_text(json.dumps(report, indent=2) + '\n')
print(json.dumps(report, indent=2))
if not report['passed']:
raise SystemExit(1)
finally:
shutil.rmtree(workdir, ignore_errors=True) # production data never stays on disk
if restored:
restored.engine.dispose()
source.dispose()
with admin.connect() as c:
c.execute(text('DROP DATABASE IF EXISTS "' + name + '" WITH (FORCE)'))
admin.dispose()
if __name__ == '__main__':
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--output', type=Path, required=True)
parser.add_argument('--source-url-file', type=Path, help='File holding a remote database URL to dump read-only')
args = parser.parse_args()
if args.source_url_file:
run_offhost(args.source_url_file.read_text().strip(), args.output)
else:
run(args.output)