File size: 12,631 Bytes
c8ba9c6 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 | """CPU-only checks for transfer integrity, interruption, and safe publication."""
import concurrent.futures
import hashlib
import importlib.util
import inspect
import json
from pathlib import Path
import subprocess
import sys
from types import SimpleNamespace
import pytest
SPEC = importlib.util.spec_from_file_location(
'transfer_artifacts', Path(__file__).parent / 'ops' / 'transfer_artifacts.py')
transfer = importlib.util.module_from_spec(SPEC)
SPEC.loader.exec_module(transfer)
RECEIVER = inspect.getsource(transfer.remote_main) + '\nremote_main()\n'
LOCAL_SSH = [sys.executable, '-c',
'import os, shlex, sys; command = shlex.split(sys.argv[1]); '
'os.execvp(command[0], command)']
def receive(action, target, manifest, data=b'', index=None, success=True):
command = [sys.executable, '-c', RECEIVER, action, str(target), manifest['sha256']]
if index is not None:
command.append(str(index))
result = subprocess.run(command, input=data, capture_output=True)
if not success:
assert result.returncode != 0
return result.stderr.decode()
assert result.returncode == 0, result.stderr.decode()
return json.loads(result.stdout)
@pytest.fixture
def artifact(tmp_path):
# Tests must be invoked with --basetemp below the workspace.
transfer.workspace_path(tmp_path)
data = bytes(range(251)) * 100
source = tmp_path / 'source.bin'
source.write_bytes(data)
manifest, _ = transfer.source_manifest(source, 8192)
return source, tmp_path / 'remote' / 'target.bin', data, manifest
def prepare(target, manifest):
return receive('prepare', target, manifest, json.dumps(manifest).encode())
def upload(index, target, data, manifest):
size = manifest['chunk_size']
return receive('chunk', target, manifest, data[index * size:(index + 1) * size], index)
def test_parallel_upload_verifies_before_atomic_publication(artifact):
source, target, data, manifest = artifact
assert manifest['sha256'] == hashlib.sha256(data).hexdigest()
assert prepare(target, manifest)['completed'] == []
with concurrent.futures.ThreadPoolExecutor(max_workers=3) as pool:
results = list(pool.map(lambda i: upload(i, target, data, manifest),
range(len(manifest['chunk_sha256']))))
assert sum(row['bytes'] for row in results) == len(data)
assert not target.exists()
result = receive('finalize', target, manifest)
assert result['complete'] and result['sha256'] == manifest['sha256']
assert target.read_bytes() == source.read_bytes()
assert not Path(str(target) + '.uploading').exists()
assert prepare(target, manifest)['complete']
def test_resume_rechecks_receipts_and_retransmits_corrupt_chunks(artifact):
_, target, data, manifest = artifact
prepare(target, manifest)
upload(0, target, data, manifest)
assert prepare(target, manifest)['completed'] == [0]
partial = Path(str(target) + '.uploading')
with partial.open('r+b') as stream:
stream.write(b'corrupt')
assert prepare(target, manifest)['completed'] == []
error = receive('chunk', target, manifest, b'interrupted', 0, success=False)
assert 'ended early' in error
assert prepare(target, manifest)['completed'] == []
for i in range(len(manifest['chunk_sha256'])):
upload(i, target, data, manifest)
receive('finalize', target, manifest)
assert target.read_bytes() == data
def test_wrong_chunk_hash_and_final_corruption_never_publish(artifact):
_, target, data, manifest = artifact
prepare(target, manifest)
error = receive('chunk', target, manifest, b'x' * manifest['chunk_size'], 0, success=False)
assert 'Chunk SHA256 mismatch' in error
for i in range(len(manifest['chunk_sha256'])):
upload(i, target, data, manifest)
with Path(str(target) + '.uploading').open('r+b') as stream:
stream.write(b'corrupt after receiving')
error = receive('finalize', target, manifest, success=False)
assert 'Full destination SHA256 mismatch' in error
assert not target.exists()
def test_existing_destination_and_foreign_partial_are_protected(artifact):
_, target, _, manifest = artifact
target.parent.mkdir()
target.write_bytes(b'existing content')
error = receive('prepare', target, manifest, json.dumps(manifest).encode(), success=False)
assert 'refusing overwrite' in error
assert target.read_bytes() == b'existing content'
target.unlink()
partial = Path(str(target) + '.uploading')
partial.write_bytes(b'foreign partial')
error = receive('prepare', target, manifest, json.dumps(manifest).encode(), success=False)
assert 'Unclaimed upload sidecar' in error
assert partial.read_bytes() == b'foreign partial'
def test_paths_and_symlinks_are_rejected(artifact):
source, target, _, manifest = artifact
with pytest.raises(ValueError, match='below'):
transfer.workspace_path('/tmp/outside.bin')
target.parent.symlink_to(source.parent, target_is_directory=True)
with pytest.raises(ValueError, match='Symlink'):
transfer.workspace_path(target)
error = receive('prepare', target, manifest, json.dumps(manifest).encode(), success=False)
assert 'Symlink is not allowed' in error
def test_destination_created_before_finalize_is_preserved(artifact):
_, target, data, manifest = artifact
prepare(target, manifest)
for i in range(len(manifest['chunk_sha256'])):
upload(i, target, data, manifest)
target.write_bytes(b'created independently')
error = receive('finalize', target, manifest, success=False)
assert 'refusing overwrite' in error
assert target.read_bytes() == b'created independently'
def test_client_streaming_without_an_ssh_server(artifact):
source, target, data, manifest = artifact
args = SimpleNamespace(port=22, key=source, known_hosts=source,
host='unused', compress=False)
receiver = transfer.SSHReceiver(args, target, manifest)
# Replace just the SSH transport with a local command dispatcher. The client
# still serializes the remote command and streams the actual binary input.
receiver.ssh = [sys.executable, '-c',
'import os, shlex, sys; command = shlex.split(sys.argv[1]); '
'os.execvp(command[0], command)']
assert receiver.call('prepare', json.dumps(manifest).encode())['completed'] == []
for index in range(len(manifest['chunk_sha256'])):
assert receiver.call('chunk', source=source, index=index)['index'] == index
assert receiver.call('finalize')['sha256'] == hashlib.sha256(data).hexdigest()
assert target.read_bytes() == data
def test_expected_source_hash_rejected_before_ssh(artifact):
source, target, _, _ = artifact
command = [sys.executable, str(Path(transfer.__file__)), str(source), str(target),
'--host', 'unused', '--key', str(source), '--known-hosts', str(source),
'--expected-sha256', '0' * 64]
result = subprocess.run(command, capture_output=True)
assert result.returncode != 0
assert b'Source SHA256 does not match --expected-sha256' in result.stderr
assert not target.exists()
def pull_client(source, target, chunk_size=8192):
args = SimpleNamespace(port=22, key=source, known_hosts=source,
host='unused', compress=False)
remote = transfer.SSHSource(args, source)
remote.ssh = LOCAL_SSH
manifest, initial = remote.manifest(chunk_size)
receiver = transfer.PullReceiver(args, target, manifest, remote, initial)
return remote, receiver, manifest, initial
def test_parallel_pull_resumes_with_verified_receipts(artifact):
source, target, data, expected = artifact
remote, receiver, manifest, initial = pull_client(source, target)
assert manifest == expected
assert receiver.call('prepare', json.dumps(manifest).encode())['completed'] == []
receiver.call('chunk', index=0)
assert receiver.call('prepare', json.dumps(manifest).encode())['completed'] == [0]
with concurrent.futures.ThreadPoolExecutor(max_workers=3) as pool:
results = list(pool.map(lambda i: receiver.call('chunk', index=i),
range(1, len(manifest['chunk_sha256']))))
assert len(results) == len(manifest['chunk_sha256']) - 1
assert not target.exists()
remote.check_identity(initial)
assert receiver.call('finalize')['sha256'] == manifest['sha256']
assert target.read_bytes() == data
assert receiver.processes == set()
def test_pull_changed_source_is_rejected_before_streaming(artifact):
source, target, data, _ = artifact
remote, receiver, manifest, initial = pull_client(source, target)
receiver.call('prepare', json.dumps(manifest).encode())
source.write_bytes(data + b'changed')
with pytest.raises(RuntimeError, match='Source changed during transfer'):
receiver.call('chunk', index=0)
with pytest.raises(RuntimeError, match='Source changed during transfer'):
remote.check_identity(initial)
assert not target.exists()
assert receiver.call('prepare', json.dumps(manifest).encode())['completed'] == []
assert receiver.processes == set()
@pytest.mark.parametrize('failure', ['corrupt', 'exit'])
def test_pull_transport_corruption_or_failure_never_publishes(artifact, failure):
source, target, _, _ = artifact
remote, receiver, manifest, _ = pull_client(source, target)
receiver.call('prepare', json.dumps(manifest).encode())
# The fake transport either corrupts a byte or reports failure after sending
# all bytes. Both endpoints' exit status must succeed before acknowledgment.
code = ('import shlex, subprocess, sys; '
'result = subprocess.run(shlex.split(sys.argv[1]), capture_output=True); '
'data = result.stdout; '
+ ('data = bytes([data[0] ^ 1]) + data[1:]; ' if failure == 'corrupt' else '')
+ 'sys.stdout.buffer.write(data); sys.stderr.buffer.write(result.stderr); '
+ ('sys.exit(result.returncode)' if failure == 'corrupt' else 'sys.exit(42)'))
remote.ssh = [sys.executable, '-c', code]
expected = 'Chunk SHA256 mismatch' if failure == 'corrupt' else 'SSH source failed'
with pytest.raises(RuntimeError, match=expected):
receiver.call('chunk', index=0)
assert not target.exists()
assert receiver.processes == set()
def test_pull_source_symlinks_are_rejected_remotely(artifact):
source, target, _, _ = artifact
link = source.parent / 'linked.bin'
link.symlink_to(source)
args = SimpleNamespace(port=22, key=source, known_hosts=source,
host='unused', compress=False)
remote = transfer.SSHSource(args, link)
remote.ssh = LOCAL_SSH
with pytest.raises(RuntimeError, match='Symlink is not allowed'):
remote.manifest(8192)
assert not target.exists()
@pytest.mark.parametrize('failure', [None, 'expected_hash', 'changed_before_finalize'])
def test_pull_cli_verifies_remote_source_and_final_publication(artifact, monkeypatch, failure):
source, target, data, _ = artifact
digest = hashlib.sha256(data).hexdigest()
monkeypatch.setattr(transfer, 'ssh_command', lambda args: LOCAL_SSH)
argv = [str(Path(transfer.__file__)), str(source), str(target), '--pull',
'--host', 'unused', '--key', str(source), '--known-hosts', str(source),
'--chunk-mib', '1', '--connections', '3', '--retries', '1',
'--manifest-dir', str(source.parent / 'manifests'),
'--expected-sha256', '0' * 64 if failure == 'expected_hash' else digest]
monkeypatch.setattr(sys, 'argv', argv)
if failure == 'changed_before_finalize':
original = transfer.SSHSource.check_identity
def change_then_check(remote, expected):
source.write_bytes(data + b'changed after the last chunk')
original(remote, expected)
monkeypatch.setattr(transfer.SSHSource, 'check_identity', change_then_check)
if failure:
error = ValueError if failure == 'expected_hash' else RuntimeError
expected = '--expected-sha256' if failure == 'expected_hash' else 'Source changed'
with pytest.raises(error, match=expected):
transfer.main()
assert not target.exists()
if failure == 'expected_hash':
assert not target.parent.exists()
else:
transfer.main()
assert target.read_bytes() == data
# An already complete destination is independently hashed on a rerun.
transfer.main()
|