File size: 3,560 Bytes
aa02c28 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 | import json
from pathlib import Path
import pytest
from psi import dlc_storage as storage
def checkpoint(run, step, world=8):
path = run/'checkpoints'/f'ckpt_{step}'
path.mkdir(parents=True)
for name in ['model.safetensors', 'optimizer.bin', 'scheduler.bin', 'custom_checkpoint_0.pkl'] + [f'random_states_{i}.pkl' for i in range(world)]:
(path/name).write_bytes(b'complete-state-'+str(step).encode())
(path/'hfd_fused_delta_manifest.json').write_text(json.dumps({'contract': {'version': 1}}))
(path/'full_duplex_source_boundary.json').write_text(json.dumps(dict(
global_step=step, source_group_complete=True, train_infer_contract={'version': 1})))
return path
def run_files(run):
run.mkdir(exist_ok=True)
for name in ['run_config.json', 'argv.txt', 'architecture_manifest.json', 'control_token_contract.json']:
(run/name).write_text('{}')
(run/'control_tokenizer').mkdir(exist_ok=True)
(run/'control_tokenizer/tokenizer.json').write_text('{}')
def test_eight_rank_roundtrip_and_transactional_retention(tmp_path):
run, backup, restored = [tmp_path/n for n in ['run', 'backup', 'restored']]
run_files(run)
for step in (10, 20, 30):
checkpoint(run, step)
assert storage.synchronize(run, backup, 8) == step
assert not (backup/'checkpoints/ckpt_10').exists()
assert (backup/'checkpoints/ckpt_20').exists()
restored_ckpt = storage.restore(backup, restored, 8)
assert storage.ready(restored_ckpt, 8)
assert (restored/'control_tokenizer/tokenizer.json').is_file()
assert storage.digest(restored_ckpt/'optimizer.bin') == storage.digest(run/'checkpoints/ckpt_30/optimizer.bin')
def test_incomplete_rank_rng_is_never_published(tmp_path):
run, backup = tmp_path/'run', tmp_path/'backup'
run_files(run)
ckpt = checkpoint(run, 10)
(ckpt/'random_states_7.pkl').unlink()
assert storage.synchronize(run, backup, 8) is None
assert not (backup/'latest_backup.json').exists()
def test_copy_failure_keeps_previous_generation_and_retries(tmp_path, monkeypatch):
run, backup = tmp_path/'run', tmp_path/'backup'
run_files(run)
checkpoint(run, 10)
storage.synchronize(run, backup, 8)
checkpoint(run, 20)
original = storage.copy_verified
def failed(source, target):
if source.name == 'optimizer.bin':
raise OSError('No space left on device')
return original(source, target)
monkeypatch.setattr(storage, 'copy_verified', failed)
with pytest.raises(OSError):
storage.synchronize(run, backup, 8)
assert json.loads((backup/'latest_backup.json').read_text())['step'] == 10
assert not (backup/'checkpoints/ckpt_20').exists()
monkeypatch.setattr(storage, 'copy_verified', original)
assert storage.synchronize(run, backup, 8) == 20
assert storage.ready(backup/'checkpoints/ckpt_20', 8)
def test_restore_rejects_corrupted_bytes(tmp_path):
run, backup = tmp_path/'run', tmp_path/'backup'
run_files(run)
checkpoint(run, 10)
storage.synchronize(run, backup, 8)
(backup/'checkpoints/ckpt_10/optimizer.bin').write_bytes(b'corrupted')
with pytest.raises(OSError, match='checksum'):
storage.restore(backup, tmp_path/'restore', 8)
def test_restore_rejects_world_size_change(tmp_path):
run, backup = tmp_path/'run', tmp_path/'backup'
run_files(run)
checkpoint(run,10)
storage.synchronize(run,backup,8)
with pytest.raises(ValueError,match='rank count'):
storage.restore(backup,tmp_path/'restore',1)
|