import json from pathlib import Path import pytest from psi import dlc_storage as storage def checkpoint(run, step, world=8): path = run/'checkpoints'/f'ckpt_{step}' path.mkdir(parents=True) for name in ['model.safetensors', 'optimizer.bin', 'scheduler.bin', 'custom_checkpoint_0.pkl'] + [f'random_states_{i}.pkl' for i in range(world)]: (path/name).write_bytes(b'complete-state-'+str(step).encode()) (path/'hfd_fused_delta_manifest.json').write_text(json.dumps({'contract': {'version': 1}})) (path/'full_duplex_source_boundary.json').write_text(json.dumps(dict( global_step=step, source_group_complete=True, train_infer_contract={'version': 1}))) return path def run_files(run): run.mkdir(exist_ok=True) for name in ['run_config.json', 'argv.txt', 'architecture_manifest.json', 'control_token_contract.json']: (run/name).write_text('{}') (run/'control_tokenizer').mkdir(exist_ok=True) (run/'control_tokenizer/tokenizer.json').write_text('{}') def test_eight_rank_roundtrip_and_transactional_retention(tmp_path): run, backup, restored = [tmp_path/n for n in ['run', 'backup', 'restored']] run_files(run) for step in (10, 20, 30): checkpoint(run, step) assert storage.synchronize(run, backup, 8) == step assert not (backup/'checkpoints/ckpt_10').exists() assert (backup/'checkpoints/ckpt_20').exists() restored_ckpt = storage.restore(backup, restored, 8) assert storage.ready(restored_ckpt, 8) assert (restored/'control_tokenizer/tokenizer.json').is_file() assert storage.digest(restored_ckpt/'optimizer.bin') == storage.digest(run/'checkpoints/ckpt_30/optimizer.bin') def test_incomplete_rank_rng_is_never_published(tmp_path): run, backup = tmp_path/'run', tmp_path/'backup' run_files(run) ckpt = checkpoint(run, 10) (ckpt/'random_states_7.pkl').unlink() assert storage.synchronize(run, backup, 8) is None assert not (backup/'latest_backup.json').exists() def test_copy_failure_keeps_previous_generation_and_retries(tmp_path, monkeypatch): run, backup = tmp_path/'run', tmp_path/'backup' run_files(run) checkpoint(run, 10) storage.synchronize(run, backup, 8) checkpoint(run, 20) original = storage.copy_verified def failed(source, target): if source.name == 'optimizer.bin': raise OSError('No space left on device') return original(source, target) monkeypatch.setattr(storage, 'copy_verified', failed) with pytest.raises(OSError): storage.synchronize(run, backup, 8) assert json.loads((backup/'latest_backup.json').read_text())['step'] == 10 assert not (backup/'checkpoints/ckpt_20').exists() monkeypatch.setattr(storage, 'copy_verified', original) assert storage.synchronize(run, backup, 8) == 20 assert storage.ready(backup/'checkpoints/ckpt_20', 8) def test_restore_rejects_corrupted_bytes(tmp_path): run, backup = tmp_path/'run', tmp_path/'backup' run_files(run) checkpoint(run, 10) storage.synchronize(run, backup, 8) (backup/'checkpoints/ckpt_10/optimizer.bin').write_bytes(b'corrupted') with pytest.raises(OSError, match='checksum'): storage.restore(backup, tmp_path/'restore', 8) def test_restore_rejects_world_size_change(tmp_path): run, backup = tmp_path/'run', tmp_path/'backup' run_files(run) checkpoint(run,10) storage.synchronize(run,backup,8) with pytest.raises(ValueError,match='rank count'): storage.restore(backup,tmp_path/'restore',1)