File size: 3,560 Bytes
aa02c28
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
import json
from pathlib import Path
import pytest
from psi import dlc_storage as storage


def checkpoint(run, step, world=8):
    path = run/'checkpoints'/f'ckpt_{step}'
    path.mkdir(parents=True)
    for name in ['model.safetensors', 'optimizer.bin', 'scheduler.bin', 'custom_checkpoint_0.pkl'] + [f'random_states_{i}.pkl' for i in range(world)]:
        (path/name).write_bytes(b'complete-state-'+str(step).encode())
    (path/'hfd_fused_delta_manifest.json').write_text(json.dumps({'contract': {'version': 1}}))
    (path/'full_duplex_source_boundary.json').write_text(json.dumps(dict(
        global_step=step, source_group_complete=True, train_infer_contract={'version': 1})))
    return path


def run_files(run):
    run.mkdir(exist_ok=True)
    for name in ['run_config.json', 'argv.txt', 'architecture_manifest.json', 'control_token_contract.json']:
        (run/name).write_text('{}')
    (run/'control_tokenizer').mkdir(exist_ok=True)
    (run/'control_tokenizer/tokenizer.json').write_text('{}')


def test_eight_rank_roundtrip_and_transactional_retention(tmp_path):
    run, backup, restored = [tmp_path/n for n in ['run', 'backup', 'restored']]
    run_files(run)
    for step in (10, 20, 30):
        checkpoint(run, step)
        assert storage.synchronize(run, backup, 8) == step
    assert not (backup/'checkpoints/ckpt_10').exists()
    assert (backup/'checkpoints/ckpt_20').exists()
    restored_ckpt = storage.restore(backup, restored, 8)
    assert storage.ready(restored_ckpt, 8)
    assert (restored/'control_tokenizer/tokenizer.json').is_file()
    assert storage.digest(restored_ckpt/'optimizer.bin') == storage.digest(run/'checkpoints/ckpt_30/optimizer.bin')


def test_incomplete_rank_rng_is_never_published(tmp_path):
    run, backup = tmp_path/'run', tmp_path/'backup'
    run_files(run)
    ckpt = checkpoint(run, 10)
    (ckpt/'random_states_7.pkl').unlink()
    assert storage.synchronize(run, backup, 8) is None
    assert not (backup/'latest_backup.json').exists()


def test_copy_failure_keeps_previous_generation_and_retries(tmp_path, monkeypatch):
    run, backup = tmp_path/'run', tmp_path/'backup'
    run_files(run)
    checkpoint(run, 10)
    storage.synchronize(run, backup, 8)
    checkpoint(run, 20)
    original = storage.copy_verified
    def failed(source, target):
        if source.name == 'optimizer.bin':
            raise OSError('No space left on device')
        return original(source, target)
    monkeypatch.setattr(storage, 'copy_verified', failed)
    with pytest.raises(OSError):
        storage.synchronize(run, backup, 8)
    assert json.loads((backup/'latest_backup.json').read_text())['step'] == 10
    assert not (backup/'checkpoints/ckpt_20').exists()
    monkeypatch.setattr(storage, 'copy_verified', original)
    assert storage.synchronize(run, backup, 8) == 20
    assert storage.ready(backup/'checkpoints/ckpt_20', 8)


def test_restore_rejects_corrupted_bytes(tmp_path):
    run, backup = tmp_path/'run', tmp_path/'backup'
    run_files(run)
    checkpoint(run, 10)
    storage.synchronize(run, backup, 8)
    (backup/'checkpoints/ckpt_10/optimizer.bin').write_bytes(b'corrupted')
    with pytest.raises(OSError, match='checksum'):
        storage.restore(backup, tmp_path/'restore', 8)


def test_restore_rejects_world_size_change(tmp_path):
    run, backup = tmp_path/'run', tmp_path/'backup'
    run_files(run)
    checkpoint(run,10)
    storage.synchronize(run,backup,8)
    with pytest.raises(ValueError,match='rank count'):
        storage.restore(backup,tmp_path/'restore',1)