File size: 11,014 Bytes
c8ba9c6 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 | """CPU tests for the unattended base -> immutable archive -> separate SFT gate."""
import copy
import json
from pathlib import Path
import stat
import pytest
import torch
from ops import posttrain_handoff as handoff
@pytest.fixture
def fixture(tmp_path):
base_work = tmp_path / 'base'
(base_work / 'run').mkdir(parents=True)
(base_work / 'control').mkdir()
tokenizer = tmp_path / 'tokenizer.model'
tokenizer.write_bytes(b'original required tokenizer fixture')
base = dict(step=76294, tokens_seen=10000007168, tokenizer_sha256=handoff.sha256(tokenizer))
checkpoint = dict(format='warpstate-server-v1', config=dict(d_model=2),
step=base['step'], tokens_seen=base['tokens_seen'],
val_step=base['step'], val_tokens=base['tokens_seen'], val_loss=2.3,
signature=dict(data=dict(signature=dict(tokenizer_sha256=base['tokenizer_sha256']))),
model={'weights': torch.arange(6, dtype=torch.float32).reshape(2, 3)},
optimizer={'unmodified': torch.tensor([12.0])})
source = base_work / 'run/latest.pt'
torch.save(checkpoint, source)
plan = dict(format='warpstate-posttrain-plan-v1', stage='sft', after_completion='pause_for_evaluation',
base=base, handoff=dict(base_work_dir=str(base_work), base_program='warpstate-train',
supervisor_socket=str(base_work / 'control/supervisor.sock'),
archive_dir=str(tmp_path / 'archives/step-00076294'),
work_dir=str(tmp_path / 'posttrain'), data_dir=str(tmp_path / 'data'),
tokenizer=str(tokenizer), processes=2, poll_seconds=1),
training=dict(sequence_length=2048))
config = tmp_path / 'plan.json'
config.write_text(json.dumps(plan))
return dict(root=tmp_path, source=source, tokenizer=tokenizer, base=base, checkpoint=checkpoint,
archive=tmp_path / 'archives/step-00076294', plan=plan, config=config)
@pytest.mark.parametrize('state,exitcode,expected', [
('RUNNING', 0, False), ('STOPPED', 0, False), ('STOPPING', 0, False),
('EXITED', 1, False), ('FATAL', 0, False), ('EXITED', 0, True)])
def test_only_clean_supervisor_exit_opens_gate(state, exitcode, expected):
assert handoff.clean_exit(dict(statename=state, exitstatus=exitcode)) is expected
@pytest.mark.parametrize('field,value', [('step', 76293), ('step', 76295),
('tokens_seen', 10000000000), ('val_step', 76200), ('val_tokens', 0),
('val_loss', float('nan')), ('format', 'warpstate-chat-v1')])
def test_wrong_or_stale_boundary_rejected(fixture, field, value):
checkpoint = copy.deepcopy(fixture['checkpoint'])
checkpoint[field] = value
with pytest.raises(ValueError):
handoff.validate_boundary(checkpoint, fixture['base'], fixture['base']['tokenizer_sha256'])
def test_archive_and_copy_preserve_exact_raw_tensors_and_bytes(fixture):
source_hash = handoff.sha256(fixture['source'])
metadata = handoff.archive_checkpoint(fixture['source'], fixture['tokenizer'], fixture['archive'],
fixture['base'], 'test-revision')
original = torch.load(fixture['source'], weights_only=False, mmap=True)
exported = torch.load(fixture['archive'] / 'model.pt', weights_only=False, mmap=True)
assert torch.equal(original['model']['weights'], exported['model']['weights'])
assert 'optimizer' not in exported and 'rng_by_rank' not in exported
assert handoff.sha256(fixture['source']) == source_hash
hashes = {p.name: handoff.sha256(p) for p in fixture['archive'].iterdir()}
assert all(not stat.S_IMODE(p.stat().st_mode) & 0o222 for p in fixture['archive'].iterdir())
model = handoff.independent_initialization(fixture['archive'], fixture['root'] / 'init',
fixture['base'], metadata['snapshot_sha256'])
assert handoff.sha256(model) == metadata['snapshot_sha256']
assert model.stat().st_ino != (fixture['archive'] / 'model.pt').stat().st_ino
# Repeated handoff reuses the original archive/copy without changing bytes.
inode = model.stat().st_ino
assert handoff.archive_checkpoint(fixture['source'], fixture['tokenizer'], fixture['archive'],
fixture['base'], 'different-revision') == metadata
handoff.independent_initialization(fixture['archive'], model.parent, fixture['base'],
metadata['snapshot_sha256'])
assert model.stat().st_ino == inode
assert {p.name: handoff.sha256(p) for p in fixture['archive'].iterdir()} == hashes
def test_incomplete_existing_archive_is_not_replaced(fixture):
fixture['archive'].mkdir(parents=True)
marker = fixture['archive'] / 'user-file'
marker.write_text('keep')
with pytest.raises(ValueError, match='checksum'):
handoff.archive_checkpoint(fixture['source'], fixture['tokenizer'], fixture['archive'],
fixture['base'], 'test')
assert marker.read_text() == 'keep'
assert list(fixture['archive'].iterdir()) == [marker]
def test_publishing_does_not_replace_even_an_empty_directory(tmp_path):
staging, target = tmp_path / 'staging', tmp_path / 'target'
staging.mkdir()
target.mkdir()
(staging / 'payload').write_text('complete')
inode = target.stat().st_ino
with pytest.raises(FileExistsError):
handoff.publish_directory(staging, target)
assert target.stat().st_ino == inode and (staging / 'payload').exists()
def prepare_data(fixture):
directory = Path(fixture['plan']['handoff']['data_dir'])
directory.mkdir()
outputs = {}
for name in ('sft_train.jsonl', 'sft_val.jsonl'):
(directory / name).write_text('{}\n')
outputs[name] = dict(rows=1, sha256=handoff.sha256(directory / name))
manifest = dict(format='warpstate-posttrain-data-v1', complete=True,
tokenizer_sha256=fixture['base']['tokenizer_sha256'], max_length=2048, outputs=outputs)
(directory / 'manifest.json').write_text(json.dumps(manifest))
return directory
def test_data_hashes_must_match_before_launch(fixture):
directory = prepare_data(fixture)
assert handoff.data_readiness(directory, fixture['plan'], verify=True)[0]
(directory / 'sft_train.jsonl').write_text('changed\n')
with pytest.raises(ValueError, match='checksum'):
handoff.data_readiness(directory, fixture['plan'], verify=True)
def test_corrupted_raw_tensor_cannot_reuse_existing_archive(fixture):
handoff.archive_checkpoint(fixture['source'], fixture['tokenizer'], fixture['archive'], fixture['base'], 'test')
fixture['checkpoint']['model']['weights'][0, 0] = 99
torch.save(fixture['checkpoint'], fixture['source'])
with pytest.raises(ValueError, match='tensor'):
handoff.archive_checkpoint(fixture['source'], fixture['tokenizer'], fixture['archive'], fixture['base'], 'test')
def test_complete_handoff_waits_archives_and_passes_separate_output_paths(fixture, monkeypatch):
prepare_data(fixture)
monkeypatch.setattr(handoff, 'STOPPED', False)
monkeypatch.setattr(handoff, 'source_identity', lambda config: {'fixture': 'immutable'})
monkeypatch.setattr(handoff, 'verify_preflight', lambda *args: {'accepted': True})
statuses = iter([dict(statename='RUNNING', exitstatus=0), dict(statename='EXITED', exitstatus=0)])
monkeypatch.setattr(handoff, 'process_status', lambda *args: next(statuses, dict(statename='EXITED', exitstatus=0)))
monkeypatch.setattr(handoff.time, 'sleep', lambda seconds: None)
commands = []
base_work = Path(fixture['plan']['handoff']['base_work_dir'])
def fake_train(command, stop, env, *, pass_fds):
commands.append(command)
assert len(pass_fds) == 2
with pytest.raises(BlockingIOError):
with handoff.exclusive_lock(base_work / 'control/training-launcher.lock'):
pass
# Released precisely before worker SH acquisition.
with handoff.exclusive_lock(base_work / 'pipeline.lock'):
pass
if 'posttrain_preflight.py' in ' '.join(command):
return 0
work = Path(command[command.index('--work-dir') + 1])
weights = Path(command[command.index('--weights') + 1])
assert not weights.is_relative_to(work)
assert Path(command[command.index('--stop-file') + 1]) == stop
assert work.name == 'run'
work.mkdir(parents=True)
model_hash = handoff.sha256(weights)
completion = dict(format='warpstate-posttrain-completion-v1', stage='sft',
completed=True, paused_for_evaluation=True, step=1,
total_steps=1, base_step=fixture['base']['step'])
checkpoint = dict(format='warpstate-posttrain-v1', completed=True, step=1, total_steps=1,
val_step=1, val_loss=2., signature=dict(plan=fixture['plan'], base_sha256=model_hash),
origin=dict(step=fixture['base']['step']))
torch.save(checkpoint, work / 'latest.pt')
(work / 'completion.json').write_text(json.dumps(completion))
(work / 'evaluation.json').write_text(json.dumps(dict(stage='sft', step=1,
base_sha256=model_hash, val_loss=2.)))
return 0
monkeypatch.setattr(handoff, 'supervise', fake_train)
assert handoff.run_handoff(fixture['config']) == 0
state = json.loads((fixture['root'] / 'posttrain/control/handoff.json').read_text())
assert state['phase'] == 'complete'
count = len(commands)
assert handoff.run_handoff(fixture['config']) == 0
assert len(commands) == count # Never retrain after terminal completion.
def test_overshoot_never_archives_or_trains(fixture, monkeypatch):
prepare_data(fixture)
fixture['checkpoint']['step'] = 76295
torch.save(fixture['checkpoint'], fixture['source'])
monkeypatch.setattr(handoff, 'STOPPED', False)
monkeypatch.setattr(handoff, 'source_identity', lambda config: {'fixture': 'immutable'})
monkeypatch.setattr(handoff, 'process_status', lambda *args: dict(statename='EXITED', exitstatus=0))
monkeypatch.setattr(handoff, 'supervise', lambda *a, **kw: pytest.fail('Must not start SFT'))
with pytest.raises(ValueError, match='step must equal'):
handoff.run_handoff(fixture['config'])
assert not fixture['archive'].exists()
def test_sft_completion_requires_matching_durable_checkpoint(fixture, tmp_path):
(tmp_path / 'completion.json').write_text(json.dumps(dict(completed=True)))
torch.save(dict(format='warpstate-posttrain-v1', completed=False, step=1, total_steps=2,
val_step=1, val_loss=2.), tmp_path / 'latest.pt')
(tmp_path / 'evaluation.json').write_text('{}')
with pytest.raises(ValueError, match='fully evaluated'):
handoff.verify_completion(tmp_path, fixture['plan'], 'hash')
|