File size: 10,033 Bytes
2b86c60 27e707f 2b86c60 27e707f 2b86c60 27e707f 2b86c60 27e707f 2b86c60 27e707f | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 | #!/usr/bin/env python3
"""Restore a disposable PostgreSQL snapshot and reconcile a simulated action.
The configured administration database is never migrated. Both created databases
are removed afterward. The private dump contains only synthetic test accounts.
"""
import argparse
import base64
import hashlib
import json
import os
from pathlib import Path
import secrets
import shutil
import subprocess
import tempfile
import sys
import time
import uuid
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT))
from sqlalchemy import create_engine, select, text, func
from sqlalchemy.engine import make_url
from backend.flagship.store import Store, accounts, events, records, sessions
from backend.flagship.identity import Identity
from scripts.verify_runtime import checkout
def run(output):
base = make_url(os.environ['PG_TEST_URL'])
if base.get_backend_name() != 'postgresql' or base.host not in {'localhost', '127.0.0.1'}:
raise ValueError('Use an isolated localhost PostgreSQL administration server')
names = ['hf_backup_' + uuid.uuid4().hex, 'hf_restore_' + uuid.uuid4().hex]
admin = create_engine(base, isolation_level='AUTOCOMMIT')
stores = []
created = []
output.parent.mkdir(parents=True, exist_ok=True)
dump = output.parent / ('synthetic-backup-' + uuid.uuid4().hex + '.dump')
os.environ['VAULT_SECRET_KEY'] = base64.b64encode(secrets.token_bytes(32)).decode()
try:
for name in names:
with admin.connect() as c:
c.execute(text('CREATE DATABASE "' + name + '"'))
created.append(name)
urls = [base.set(database=name).render_as_string(hide_password=False) for name in names]
origin = Store(urls[0]); stores.append(origin); origin.migrate()
identity = Identity(origin)
actor = identity.provision('backup_buyer', 'synthetic-backup-password-123')
identity.issue(actor)
identity.set_provider_token(actor, 'synthetic-provider-credential-never-sent')
commerce, action = checkout(origin, actor, 'lost_response')
assert commerce.execute(actor, action['id'], 'lost')['data']['state'] == 'UNKNOWN'
def counts(store):
with store.engine.connect() as c:
return {table.name: c.execute(select(func.count()).select_from(table)).scalar_one()
for table in (accounts, sessions, records, events)}
expected_counts = counts(origin)
captured = time.time()
subprocess.run(['pg_dump', '-Fc', '--no-owner', '--dbname', urls[0], '--file', str(dump)], check=True, capture_output=True)
late = origin.create('backup_probe', actor['id'], {'source': 'simulation', 'after_snapshot': True})
started = time.perf_counter()
subprocess.run(['pg_restore', '--no-owner', '--dbname', urls[1], str(dump)], check=True, capture_output=True)
restored = Store(urls[1]); stores.append(restored)
assert restored.ready() and counts(restored) == expected_counts
assert Identity(restored).provider_token(actor) == 'synthetic-provider-credential-never-sent'
try:
restored.get('backup_probe', late['id'], actor['id'])
except LookupError:
pass
else:
raise AssertionError('Post-snapshot row unexpectedly restored')
from backend.flagship.commerce import Commerce
from backend.flagship.provider import Provider
recovered = Commerce(restored, Provider(restored, Identity(restored)))
assert recovered.execute(actor, action['id'], 'after-restore')['data']['state'] == 'UNKNOWN'
assert recovered.reconcile(actor, action['id'])['data']['state'] == 'SUCCEEDED'
report = {'schema_version': 1, 'source': 'controlled_simulation', 'passed': True,
'snapshot_at_unix': captured, 'restore_and_verification_seconds': time.perf_counter()-started,
'restored_table_counts_before_reconciliation': expected_counts,
'vault_decryption_with_same_key': True, 'retry_state': 'UNKNOWN', 'reconciled_state': 'SUCCEEDED',
'post_snapshot_write_absent': True, 'dump_sha256': hashlib.sha256(dump.read_bytes()).hexdigest(),
'source_sha256': hashlib.sha256(Path(__file__).read_bytes()).hexdigest(),
'limitations': ['Same-host pg_dump/pg_restore exercise, not off-host disaster recovery.',
'Synthetic accounts and simulated provider effects only.',
'Recovery point is the database snapshot; subsequent writes are not restored.',
'Model-registry backup and human setup/pilot are separate gates.']}
output.write_text(json.dumps(report, indent=2)+'\n')
print(json.dumps(report, indent=2))
finally:
for store in stores:
store.engine.dispose()
for name in created:
with admin.connect() as c:
c.execute(text('DROP DATABASE "' + name + '" WITH (FORCE)'))
admin.dispose()
def run_offhost(source_url, output):
"""Read-only dump of a remote database restored into a disposable local one.
Nothing is written to the source. The source URL is never printed or stored.
"""
base = make_url(os.environ['PG_TEST_URL'])
if base.get_backend_name() != 'postgresql' or base.host not in {'localhost', '127.0.0.1'}:
raise ValueError('Restore target must be an isolated localhost PostgreSQL server')
name = 'hf_offhost_restore_' + uuid.uuid4().hex
admin = create_engine(base, isolation_level='AUTOCOMMIT')
source = create_engine(source_url)
output.parent.mkdir(parents=True, exist_ok=True)
workdir = Path(tempfile.mkdtemp(prefix='hf-offhost-')) # 0700, removed afterwards
dump = workdir / 'backup.dump'
def snapshot(engine):
with engine.connect() as c:
ids = sorted(r[0] for r in c.execute(select(accounts.c.id)))
return {'counts': {t.name: c.execute(select(func.count()).select_from(t)).scalar_one()
for t in (accounts, sessions, records)},
'account_ids_sha256': hashlib.sha256('\n'.join(ids).encode()).hexdigest()}
def event_prefix(engine, upto):
# hf_events is append-only with increasing ids; a consistent dump holds exactly the rows up to its maximum.
with engine.connect() as c:
rows = c.execute(select(events.c.id, events.c.type).where(events.c.id <= upto).order_by(events.c.id)).all()
return {'rows': len(rows), 'max_id': upto, 'sha256': hashlib.sha256(repr(rows).encode()).hexdigest()}
restored = None
try:
with admin.connect() as c:
c.execute(text('CREATE DATABASE "' + name + '"'))
target = base.set(database=name).render_as_string(hide_password=False)
before = snapshot(source)
started = time.perf_counter()
subprocess.run(['pg_dump', '-Fc', '--no-owner', '--no-privileges', '--dbname', source_url, '--file', str(dump)],
check=True, capture_output=True)
dumped = time.perf_counter()
after = snapshot(source)
subprocess.run(['pg_restore', '--no-owner', '--no-privileges', '--dbname', target, str(dump)], check=True, capture_output=True)
restored = Store(target)
result = snapshot(restored.engine)
with restored.engine.connect() as c:
restored_max = c.execute(select(func.max(events.c.id))).scalar() or 0
restored_events, source_events = event_prefix(restored.engine, restored_max), event_prefix(source, restored_max)
# Production keeps writing during the dump: mutable tables must match the source just before or just after
# it, and the event log must be exactly the source's prefix up to the dumped maximum id.
consistent = result in (before, after) and restored_events == source_events
report = {'schema_version': 1, 'source': 'production_neon_read_only', 'passed': bool(restored.ready() and consistent),
'source_host_suffix': make_url(source_url).host.split('.', 1)[1],
'restored_schema_ready': restored.ready(), 'restored': result, 'restored_event_log': restored_events,
'source_event_log_same_prefix': source_events,
'source_before_dump': before, 'source_after_dump': after,
'dump_seconds': round(dumped - started, 2), 'restore_seconds': round(time.perf_counter() - dumped, 2),
'dump_bytes': dump.stat().st_size, 'checked_at_unix': time.time(),
'source_sha256': hashlib.sha256(Path(__file__).read_bytes()).hexdigest(),
'limitations': ['Restore target is a disposable local PostgreSQL 18, not a second cloud region.',
'Vault-encrypted provider credentials are restored as ciphertext; the production key never '
'leaves the deployment, so decryption is not exercised here.',
'Recovery point is the dump; there is no point-in-time recovery test.']}
output.write_text(json.dumps(report, indent=2) + '\n')
print(json.dumps(report, indent=2))
if not report['passed']:
raise SystemExit(1)
finally:
shutil.rmtree(workdir, ignore_errors=True) # production data never stays on disk
if restored:
restored.engine.dispose()
source.dispose()
with admin.connect() as c:
c.execute(text('DROP DATABASE IF EXISTS "' + name + '" WITH (FORCE)'))
admin.dispose()
if __name__ == '__main__':
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--output', type=Path, required=True)
parser.add_argument('--source-url-file', type=Path, help='File holding a remote database URL to dump read-only')
args = parser.parse_args()
if args.source_url_file:
run_offhost(args.source_url_file.read_text().strip(), args.output)
else:
run(args.output)
|