Download tests/test_dlc_storage.py from Tinghong-Ye22/code-HFD: direct link, hf CLI and curl.
- Browser
- Download file 3.56 kB
-
https://huggingface.co/Tinghong-Ye22/code-HFD/resolve/main/tests/test_dlc_storage.py
- Command line
-
hf download hf://Tinghong-Ye22/code-HFD/tests/test_dlc_storage.py
-
curl -L -o test_dlc_storage.py https://huggingface.co/Tinghong-Ye22/code-HFD/resolve/main/tests/test_dlc_storage.py
3.56 kB
| import json | |
| from pathlib import Path | |
| import pytest | |
| from psi import dlc_storage as storage | |
| def checkpoint(run, step, world=8): | |
| path = run/'checkpoints'/f'ckpt_{step}' | |
| path.mkdir(parents=True) | |
| for name in ['model.safetensors', 'optimizer.bin', 'scheduler.bin', 'custom_checkpoint_0.pkl'] + [f'random_states_{i}.pkl' for i in range(world)]: | |
| (path/name).write_bytes(b'complete-state-'+str(step).encode()) | |
| (path/'hfd_fused_delta_manifest.json').write_text(json.dumps({'contract': {'version': 1}})) | |
| (path/'full_duplex_source_boundary.json').write_text(json.dumps(dict( | |
| global_step=step, source_group_complete=True, train_infer_contract={'version': 1}))) | |
| return path | |
| def run_files(run): | |
| run.mkdir(exist_ok=True) | |
| for name in ['run_config.json', 'argv.txt', 'architecture_manifest.json', 'control_token_contract.json']: | |
| (run/name).write_text('{}') | |
| (run/'control_tokenizer').mkdir(exist_ok=True) | |
| (run/'control_tokenizer/tokenizer.json').write_text('{}') | |
| def test_eight_rank_roundtrip_and_transactional_retention(tmp_path): | |
| run, backup, restored = [tmp_path/n for n in ['run', 'backup', 'restored']] | |
| run_files(run) | |
| for step in (10, 20, 30): | |
| checkpoint(run, step) | |
| assert storage.synchronize(run, backup, 8) == step | |
| assert not (backup/'checkpoints/ckpt_10').exists() | |
| assert (backup/'checkpoints/ckpt_20').exists() | |
| restored_ckpt = storage.restore(backup, restored, 8) | |
| assert storage.ready(restored_ckpt, 8) | |
| assert (restored/'control_tokenizer/tokenizer.json').is_file() | |
| assert storage.digest(restored_ckpt/'optimizer.bin') == storage.digest(run/'checkpoints/ckpt_30/optimizer.bin') | |
| def test_incomplete_rank_rng_is_never_published(tmp_path): | |
| run, backup = tmp_path/'run', tmp_path/'backup' | |
| run_files(run) | |
| ckpt = checkpoint(run, 10) | |
| (ckpt/'random_states_7.pkl').unlink() | |
| assert storage.synchronize(run, backup, 8) is None | |
| assert not (backup/'latest_backup.json').exists() | |
| def test_copy_failure_keeps_previous_generation_and_retries(tmp_path, monkeypatch): | |
| run, backup = tmp_path/'run', tmp_path/'backup' | |
| run_files(run) | |
| checkpoint(run, 10) | |
| storage.synchronize(run, backup, 8) | |
| checkpoint(run, 20) | |
| original = storage.copy_verified | |
| def failed(source, target): | |
| if source.name == 'optimizer.bin': | |
| raise OSError('No space left on device') | |
| return original(source, target) | |
| monkeypatch.setattr(storage, 'copy_verified', failed) | |
| with pytest.raises(OSError): | |
| storage.synchronize(run, backup, 8) | |
| assert json.loads((backup/'latest_backup.json').read_text())['step'] == 10 | |
| assert not (backup/'checkpoints/ckpt_20').exists() | |
| monkeypatch.setattr(storage, 'copy_verified', original) | |
| assert storage.synchronize(run, backup, 8) == 20 | |
| assert storage.ready(backup/'checkpoints/ckpt_20', 8) | |
| def test_restore_rejects_corrupted_bytes(tmp_path): | |
| run, backup = tmp_path/'run', tmp_path/'backup' | |
| run_files(run) | |
| checkpoint(run, 10) | |
| storage.synchronize(run, backup, 8) | |
| (backup/'checkpoints/ckpt_10/optimizer.bin').write_bytes(b'corrupted') | |
| with pytest.raises(OSError, match='checksum'): | |
| storage.restore(backup, tmp_path/'restore', 8) | |
| def test_restore_rejects_world_size_change(tmp_path): | |
| run, backup = tmp_path/'run', tmp_path/'backup' | |
| run_files(run) | |
| checkpoint(run,10) | |
| storage.synchronize(run,backup,8) | |
| with pytest.raises(ValueError,match='rank count'): | |
| storage.restore(backup,tmp_path/'restore',1) | |