File size: 10,033 Bytes
2b86c60
 
 
 
 
 
 
 
 
 
 
 
 
27e707f
2b86c60
27e707f
2b86c60
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
27e707f
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2b86c60
 
 
27e707f
2b86c60
27e707f
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
#!/usr/bin/env python3
"""Restore a disposable PostgreSQL snapshot and reconcile a simulated action.

The configured administration database is never migrated. Both created databases
are removed afterward. The private dump contains only synthetic test accounts.
"""
import argparse
import base64
import hashlib
import json
import os
from pathlib import Path
import secrets
import shutil
import subprocess
import tempfile
import sys
import time
import uuid
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT))
from sqlalchemy import create_engine, select, text, func
from sqlalchemy.engine import make_url
from backend.flagship.store import Store, accounts, events, records, sessions
from backend.flagship.identity import Identity
from scripts.verify_runtime import checkout


def run(output):
    base = make_url(os.environ['PG_TEST_URL'])
    if base.get_backend_name() != 'postgresql' or base.host not in {'localhost', '127.0.0.1'}:
        raise ValueError('Use an isolated localhost PostgreSQL administration server')
    names = ['hf_backup_' + uuid.uuid4().hex, 'hf_restore_' + uuid.uuid4().hex]
    admin = create_engine(base, isolation_level='AUTOCOMMIT')
    stores = []
    created = []
    output.parent.mkdir(parents=True, exist_ok=True)
    dump = output.parent / ('synthetic-backup-' + uuid.uuid4().hex + '.dump')
    os.environ['VAULT_SECRET_KEY'] = base64.b64encode(secrets.token_bytes(32)).decode()
    try:
        for name in names:
            with admin.connect() as c:
                c.execute(text('CREATE DATABASE "' + name + '"'))
            created.append(name)
        urls = [base.set(database=name).render_as_string(hide_password=False) for name in names]
        origin = Store(urls[0]); stores.append(origin); origin.migrate()
        identity = Identity(origin)
        actor = identity.provision('backup_buyer', 'synthetic-backup-password-123')
        identity.issue(actor)
        identity.set_provider_token(actor, 'synthetic-provider-credential-never-sent')
        commerce, action = checkout(origin, actor, 'lost_response')
        assert commerce.execute(actor, action['id'], 'lost')['data']['state'] == 'UNKNOWN'
        def counts(store):
            with store.engine.connect() as c:
                return {table.name: c.execute(select(func.count()).select_from(table)).scalar_one()
                        for table in (accounts, sessions, records, events)}
        expected_counts = counts(origin)
        captured = time.time()
        subprocess.run(['pg_dump', '-Fc', '--no-owner', '--dbname', urls[0], '--file', str(dump)], check=True, capture_output=True)
        late = origin.create('backup_probe', actor['id'], {'source': 'simulation', 'after_snapshot': True})
        started = time.perf_counter()
        subprocess.run(['pg_restore', '--no-owner', '--dbname', urls[1], str(dump)], check=True, capture_output=True)
        restored = Store(urls[1]); stores.append(restored)
        assert restored.ready() and counts(restored) == expected_counts
        assert Identity(restored).provider_token(actor) == 'synthetic-provider-credential-never-sent'
        try:
            restored.get('backup_probe', late['id'], actor['id'])
        except LookupError:
            pass
        else:
            raise AssertionError('Post-snapshot row unexpectedly restored')
        from backend.flagship.commerce import Commerce
        from backend.flagship.provider import Provider
        recovered = Commerce(restored, Provider(restored, Identity(restored)))
        assert recovered.execute(actor, action['id'], 'after-restore')['data']['state'] == 'UNKNOWN'
        assert recovered.reconcile(actor, action['id'])['data']['state'] == 'SUCCEEDED'
        report = {'schema_version': 1, 'source': 'controlled_simulation', 'passed': True,
            'snapshot_at_unix': captured, 'restore_and_verification_seconds': time.perf_counter()-started,
            'restored_table_counts_before_reconciliation': expected_counts,
            'vault_decryption_with_same_key': True, 'retry_state': 'UNKNOWN', 'reconciled_state': 'SUCCEEDED',
            'post_snapshot_write_absent': True, 'dump_sha256': hashlib.sha256(dump.read_bytes()).hexdigest(),
            'source_sha256': hashlib.sha256(Path(__file__).read_bytes()).hexdigest(),
            'limitations': ['Same-host pg_dump/pg_restore exercise, not off-host disaster recovery.',
                'Synthetic accounts and simulated provider effects only.',
                'Recovery point is the database snapshot; subsequent writes are not restored.',
                'Model-registry backup and human setup/pilot are separate gates.']}
        output.write_text(json.dumps(report, indent=2)+'\n')
        print(json.dumps(report, indent=2))
    finally:
        for store in stores:
            store.engine.dispose()
        for name in created:
            with admin.connect() as c:
                c.execute(text('DROP DATABASE "' + name + '" WITH (FORCE)'))
        admin.dispose()


def run_offhost(source_url, output):
    """Read-only dump of a remote database restored into a disposable local one.

    Nothing is written to the source. The source URL is never printed or stored.
    """
    base = make_url(os.environ['PG_TEST_URL'])
    if base.get_backend_name() != 'postgresql' or base.host not in {'localhost', '127.0.0.1'}:
        raise ValueError('Restore target must be an isolated localhost PostgreSQL server')
    name = 'hf_offhost_restore_' + uuid.uuid4().hex
    admin = create_engine(base, isolation_level='AUTOCOMMIT')
    source = create_engine(source_url)
    output.parent.mkdir(parents=True, exist_ok=True)
    workdir = Path(tempfile.mkdtemp(prefix='hf-offhost-'))  # 0700, removed afterwards
    dump = workdir / 'backup.dump'

    def snapshot(engine):
        with engine.connect() as c:
            ids = sorted(r[0] for r in c.execute(select(accounts.c.id)))
            return {'counts': {t.name: c.execute(select(func.count()).select_from(t)).scalar_one()
                               for t in (accounts, sessions, records)},
                    'account_ids_sha256': hashlib.sha256('\n'.join(ids).encode()).hexdigest()}

    def event_prefix(engine, upto):
        # hf_events is append-only with increasing ids; a consistent dump holds exactly the rows up to its maximum.
        with engine.connect() as c:
            rows = c.execute(select(events.c.id, events.c.type).where(events.c.id <= upto).order_by(events.c.id)).all()
        return {'rows': len(rows), 'max_id': upto, 'sha256': hashlib.sha256(repr(rows).encode()).hexdigest()}

    restored = None
    try:
        with admin.connect() as c:
            c.execute(text('CREATE DATABASE "' + name + '"'))
        target = base.set(database=name).render_as_string(hide_password=False)
        before = snapshot(source)
        started = time.perf_counter()
        subprocess.run(['pg_dump', '-Fc', '--no-owner', '--no-privileges', '--dbname', source_url, '--file', str(dump)],
                       check=True, capture_output=True)
        dumped = time.perf_counter()
        after = snapshot(source)
        subprocess.run(['pg_restore', '--no-owner', '--no-privileges', '--dbname', target, str(dump)], check=True, capture_output=True)
        restored = Store(target)
        result = snapshot(restored.engine)
        with restored.engine.connect() as c:
            restored_max = c.execute(select(func.max(events.c.id))).scalar() or 0
        restored_events, source_events = event_prefix(restored.engine, restored_max), event_prefix(source, restored_max)
        # Production keeps writing during the dump: mutable tables must match the source just before or just after
        # it, and the event log must be exactly the source's prefix up to the dumped maximum id.
        consistent = result in (before, after) and restored_events == source_events
        report = {'schema_version': 1, 'source': 'production_neon_read_only', 'passed': bool(restored.ready() and consistent),
                  'source_host_suffix': make_url(source_url).host.split('.', 1)[1],
                  'restored_schema_ready': restored.ready(), 'restored': result, 'restored_event_log': restored_events,
                  'source_event_log_same_prefix': source_events,
                  'source_before_dump': before, 'source_after_dump': after,
                  'dump_seconds': round(dumped - started, 2), 'restore_seconds': round(time.perf_counter() - dumped, 2),
                  'dump_bytes': dump.stat().st_size, 'checked_at_unix': time.time(),
                  'source_sha256': hashlib.sha256(Path(__file__).read_bytes()).hexdigest(),
                  'limitations': ['Restore target is a disposable local PostgreSQL 18, not a second cloud region.',
                                  'Vault-encrypted provider credentials are restored as ciphertext; the production key never '
                                  'leaves the deployment, so decryption is not exercised here.',
                                  'Recovery point is the dump; there is no point-in-time recovery test.']}
        output.write_text(json.dumps(report, indent=2) + '\n')
        print(json.dumps(report, indent=2))
        if not report['passed']:
            raise SystemExit(1)
    finally:
        shutil.rmtree(workdir, ignore_errors=True)  # production data never stays on disk
        if restored:
            restored.engine.dispose()
        source.dispose()
        with admin.connect() as c:
            c.execute(text('DROP DATABASE IF EXISTS "' + name + '" WITH (FORCE)'))
        admin.dispose()


if __name__ == '__main__':
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument('--output', type=Path, required=True)
    parser.add_argument('--source-url-file', type=Path, help='File holding a remote database URL to dump read-only')
    args = parser.parse_args()
    if args.source_url_file:
        run_offhost(args.source_url_file.read_text().strip(), args.output)
    else:
        run(args.output)