Download scripts/verify_backup_restore.py from Gaurav711/HyperFlow: direct link, hf CLI and curl.
- Browser
- Download file 10 kB
-
https://huggingface.co/spaces/Gaurav711/HyperFlow/resolve/main/scripts/verify_backup_restore.py
- Command line
-
hf download hf://spaces/Gaurav711/HyperFlow/scripts/verify_backup_restore.py
-
curl -L -o verify_backup_restore.py https://huggingface.co/spaces/Gaurav711/HyperFlow/resolve/main/scripts/verify_backup_restore.py
10 kB
| #!/usr/bin/env python3 | |
| """Restore a disposable PostgreSQL snapshot and reconcile a simulated action. | |
| The configured administration database is never migrated. Both created databases | |
| are removed afterward. The private dump contains only synthetic test accounts. | |
| """ | |
| import argparse | |
| import base64 | |
| import hashlib | |
| import json | |
| import os | |
| from pathlib import Path | |
| import secrets | |
| import shutil | |
| import subprocess | |
| import tempfile | |
| import sys | |
| import time | |
| import uuid | |
| ROOT = Path(__file__).resolve().parents[1] | |
| sys.path.insert(0, str(ROOT)) | |
| from sqlalchemy import create_engine, select, text, func | |
| from sqlalchemy.engine import make_url | |
| from backend.flagship.store import Store, accounts, events, records, sessions | |
| from backend.flagship.identity import Identity | |
| from scripts.verify_runtime import checkout | |
| def run(output): | |
| base = make_url(os.environ['PG_TEST_URL']) | |
| if base.get_backend_name() != 'postgresql' or base.host not in {'localhost', '127.0.0.1'}: | |
| raise ValueError('Use an isolated localhost PostgreSQL administration server') | |
| names = ['hf_backup_' + uuid.uuid4().hex, 'hf_restore_' + uuid.uuid4().hex] | |
| admin = create_engine(base, isolation_level='AUTOCOMMIT') | |
| stores = [] | |
| created = [] | |
| output.parent.mkdir(parents=True, exist_ok=True) | |
| dump = output.parent / ('synthetic-backup-' + uuid.uuid4().hex + '.dump') | |
| os.environ['VAULT_SECRET_KEY'] = base64.b64encode(secrets.token_bytes(32)).decode() | |
| try: | |
| for name in names: | |
| with admin.connect() as c: | |
| c.execute(text('CREATE DATABASE "' + name + '"')) | |
| created.append(name) | |
| urls = [base.set(database=name).render_as_string(hide_password=False) for name in names] | |
| origin = Store(urls[0]); stores.append(origin); origin.migrate() | |
| identity = Identity(origin) | |
| actor = identity.provision('backup_buyer', 'synthetic-backup-password-123') | |
| identity.issue(actor) | |
| identity.set_provider_token(actor, 'synthetic-provider-credential-never-sent') | |
| commerce, action = checkout(origin, actor, 'lost_response') | |
| assert commerce.execute(actor, action['id'], 'lost')['data']['state'] == 'UNKNOWN' | |
| def counts(store): | |
| with store.engine.connect() as c: | |
| return {table.name: c.execute(select(func.count()).select_from(table)).scalar_one() | |
| for table in (accounts, sessions, records, events)} | |
| expected_counts = counts(origin) | |
| captured = time.time() | |
| subprocess.run(['pg_dump', '-Fc', '--no-owner', '--dbname', urls[0], '--file', str(dump)], check=True, capture_output=True) | |
| late = origin.create('backup_probe', actor['id'], {'source': 'simulation', 'after_snapshot': True}) | |
| started = time.perf_counter() | |
| subprocess.run(['pg_restore', '--no-owner', '--dbname', urls[1], str(dump)], check=True, capture_output=True) | |
| restored = Store(urls[1]); stores.append(restored) | |
| assert restored.ready() and counts(restored) == expected_counts | |
| assert Identity(restored).provider_token(actor) == 'synthetic-provider-credential-never-sent' | |
| try: | |
| restored.get('backup_probe', late['id'], actor['id']) | |
| except LookupError: | |
| pass | |
| else: | |
| raise AssertionError('Post-snapshot row unexpectedly restored') | |
| from backend.flagship.commerce import Commerce | |
| from backend.flagship.provider import Provider | |
| recovered = Commerce(restored, Provider(restored, Identity(restored))) | |
| assert recovered.execute(actor, action['id'], 'after-restore')['data']['state'] == 'UNKNOWN' | |
| assert recovered.reconcile(actor, action['id'])['data']['state'] == 'SUCCEEDED' | |
| report = {'schema_version': 1, 'source': 'controlled_simulation', 'passed': True, | |
| 'snapshot_at_unix': captured, 'restore_and_verification_seconds': time.perf_counter()-started, | |
| 'restored_table_counts_before_reconciliation': expected_counts, | |
| 'vault_decryption_with_same_key': True, 'retry_state': 'UNKNOWN', 'reconciled_state': 'SUCCEEDED', | |
| 'post_snapshot_write_absent': True, 'dump_sha256': hashlib.sha256(dump.read_bytes()).hexdigest(), | |
| 'source_sha256': hashlib.sha256(Path(__file__).read_bytes()).hexdigest(), | |
| 'limitations': ['Same-host pg_dump/pg_restore exercise, not off-host disaster recovery.', | |
| 'Synthetic accounts and simulated provider effects only.', | |
| 'Recovery point is the database snapshot; subsequent writes are not restored.', | |
| 'Model-registry backup and human setup/pilot are separate gates.']} | |
| output.write_text(json.dumps(report, indent=2)+'\n') | |
| print(json.dumps(report, indent=2)) | |
| finally: | |
| for store in stores: | |
| store.engine.dispose() | |
| for name in created: | |
| with admin.connect() as c: | |
| c.execute(text('DROP DATABASE "' + name + '" WITH (FORCE)')) | |
| admin.dispose() | |
| def run_offhost(source_url, output): | |
| """Read-only dump of a remote database restored into a disposable local one. | |
| Nothing is written to the source. The source URL is never printed or stored. | |
| """ | |
| base = make_url(os.environ['PG_TEST_URL']) | |
| if base.get_backend_name() != 'postgresql' or base.host not in {'localhost', '127.0.0.1'}: | |
| raise ValueError('Restore target must be an isolated localhost PostgreSQL server') | |
| name = 'hf_offhost_restore_' + uuid.uuid4().hex | |
| admin = create_engine(base, isolation_level='AUTOCOMMIT') | |
| source = create_engine(source_url) | |
| output.parent.mkdir(parents=True, exist_ok=True) | |
| workdir = Path(tempfile.mkdtemp(prefix='hf-offhost-')) # 0700, removed afterwards | |
| dump = workdir / 'backup.dump' | |
| def snapshot(engine): | |
| with engine.connect() as c: | |
| ids = sorted(r[0] for r in c.execute(select(accounts.c.id))) | |
| return {'counts': {t.name: c.execute(select(func.count()).select_from(t)).scalar_one() | |
| for t in (accounts, sessions, records)}, | |
| 'account_ids_sha256': hashlib.sha256('\n'.join(ids).encode()).hexdigest()} | |
| def event_prefix(engine, upto): | |
| # hf_events is append-only with increasing ids; a consistent dump holds exactly the rows up to its maximum. | |
| with engine.connect() as c: | |
| rows = c.execute(select(events.c.id, events.c.type).where(events.c.id <= upto).order_by(events.c.id)).all() | |
| return {'rows': len(rows), 'max_id': upto, 'sha256': hashlib.sha256(repr(rows).encode()).hexdigest()} | |
| restored = None | |
| try: | |
| with admin.connect() as c: | |
| c.execute(text('CREATE DATABASE "' + name + '"')) | |
| target = base.set(database=name).render_as_string(hide_password=False) | |
| before = snapshot(source) | |
| started = time.perf_counter() | |
| subprocess.run(['pg_dump', '-Fc', '--no-owner', '--no-privileges', '--dbname', source_url, '--file', str(dump)], | |
| check=True, capture_output=True) | |
| dumped = time.perf_counter() | |
| after = snapshot(source) | |
| subprocess.run(['pg_restore', '--no-owner', '--no-privileges', '--dbname', target, str(dump)], check=True, capture_output=True) | |
| restored = Store(target) | |
| result = snapshot(restored.engine) | |
| with restored.engine.connect() as c: | |
| restored_max = c.execute(select(func.max(events.c.id))).scalar() or 0 | |
| restored_events, source_events = event_prefix(restored.engine, restored_max), event_prefix(source, restored_max) | |
| # Production keeps writing during the dump: mutable tables must match the source just before or just after | |
| # it, and the event log must be exactly the source's prefix up to the dumped maximum id. | |
| consistent = result in (before, after) and restored_events == source_events | |
| report = {'schema_version': 1, 'source': 'production_neon_read_only', 'passed': bool(restored.ready() and consistent), | |
| 'source_host_suffix': make_url(source_url).host.split('.', 1)[1], | |
| 'restored_schema_ready': restored.ready(), 'restored': result, 'restored_event_log': restored_events, | |
| 'source_event_log_same_prefix': source_events, | |
| 'source_before_dump': before, 'source_after_dump': after, | |
| 'dump_seconds': round(dumped - started, 2), 'restore_seconds': round(time.perf_counter() - dumped, 2), | |
| 'dump_bytes': dump.stat().st_size, 'checked_at_unix': time.time(), | |
| 'source_sha256': hashlib.sha256(Path(__file__).read_bytes()).hexdigest(), | |
| 'limitations': ['Restore target is a disposable local PostgreSQL 18, not a second cloud region.', | |
| 'Vault-encrypted provider credentials are restored as ciphertext; the production key never ' | |
| 'leaves the deployment, so decryption is not exercised here.', | |
| 'Recovery point is the dump; there is no point-in-time recovery test.']} | |
| output.write_text(json.dumps(report, indent=2) + '\n') | |
| print(json.dumps(report, indent=2)) | |
| if not report['passed']: | |
| raise SystemExit(1) | |
| finally: | |
| shutil.rmtree(workdir, ignore_errors=True) # production data never stays on disk | |
| if restored: | |
| restored.engine.dispose() | |
| source.dispose() | |
| with admin.connect() as c: | |
| c.execute(text('DROP DATABASE IF EXISTS "' + name + '" WITH (FORCE)')) | |
| admin.dispose() | |
| if __name__ == '__main__': | |
| parser = argparse.ArgumentParser(description=__doc__) | |
| parser.add_argument('--output', type=Path, required=True) | |
| parser.add_argument('--source-url-file', type=Path, help='File holding a remote database URL to dump read-only') | |
| args = parser.parse_args() | |
| if args.source_url_file: | |
| run_offhost(args.source_url_file.read_text().strip(), args.output) | |
| else: | |
| run(args.output) | |