WarpStateAI / test_posttrain_handoff.py
zerdovzad's picture
Upload 139 files
c8ba9c6 verified
Raw History Blame Contribute Delete
11 kB
"""CPU tests for the unattended base -> immutable archive -> separate SFT gate."""
import copy
import json
from pathlib import Path
import stat
import pytest
import torch
from ops import posttrain_handoff as handoff
@pytest.fixture
def fixture(tmp_path):
base_work = tmp_path / 'base'
(base_work / 'run').mkdir(parents=True)
(base_work / 'control').mkdir()
tokenizer = tmp_path / 'tokenizer.model'
tokenizer.write_bytes(b'original required tokenizer fixture')
base = dict(step=76294, tokens_seen=10000007168, tokenizer_sha256=handoff.sha256(tokenizer))
checkpoint = dict(format='warpstate-server-v1', config=dict(d_model=2),
step=base['step'], tokens_seen=base['tokens_seen'],
val_step=base['step'], val_tokens=base['tokens_seen'], val_loss=2.3,
signature=dict(data=dict(signature=dict(tokenizer_sha256=base['tokenizer_sha256']))),
model={'weights': torch.arange(6, dtype=torch.float32).reshape(2, 3)},
optimizer={'unmodified': torch.tensor([12.0])})
source = base_work / 'run/latest.pt'
torch.save(checkpoint, source)
plan = dict(format='warpstate-posttrain-plan-v1', stage='sft', after_completion='pause_for_evaluation',
base=base, handoff=dict(base_work_dir=str(base_work), base_program='warpstate-train',
supervisor_socket=str(base_work / 'control/supervisor.sock'),
archive_dir=str(tmp_path / 'archives/step-00076294'),
work_dir=str(tmp_path / 'posttrain'), data_dir=str(tmp_path / 'data'),
tokenizer=str(tokenizer), processes=2, poll_seconds=1),
training=dict(sequence_length=2048))
config = tmp_path / 'plan.json'
config.write_text(json.dumps(plan))
return dict(root=tmp_path, source=source, tokenizer=tokenizer, base=base, checkpoint=checkpoint,
archive=tmp_path / 'archives/step-00076294', plan=plan, config=config)
@pytest.mark.parametrize('state,exitcode,expected', [
('RUNNING', 0, False), ('STOPPED', 0, False), ('STOPPING', 0, False),
('EXITED', 1, False), ('FATAL', 0, False), ('EXITED', 0, True)])
def test_only_clean_supervisor_exit_opens_gate(state, exitcode, expected):
assert handoff.clean_exit(dict(statename=state, exitstatus=exitcode)) is expected
@pytest.mark.parametrize('field,value', [('step', 76293), ('step', 76295),
('tokens_seen', 10000000000), ('val_step', 76200), ('val_tokens', 0),
('val_loss', float('nan')), ('format', 'warpstate-chat-v1')])
def test_wrong_or_stale_boundary_rejected(fixture, field, value):
checkpoint = copy.deepcopy(fixture['checkpoint'])
checkpoint[field] = value
with pytest.raises(ValueError):
handoff.validate_boundary(checkpoint, fixture['base'], fixture['base']['tokenizer_sha256'])
def test_archive_and_copy_preserve_exact_raw_tensors_and_bytes(fixture):
source_hash = handoff.sha256(fixture['source'])
metadata = handoff.archive_checkpoint(fixture['source'], fixture['tokenizer'], fixture['archive'],
fixture['base'], 'test-revision')
original = torch.load(fixture['source'], weights_only=False, mmap=True)
exported = torch.load(fixture['archive'] / 'model.pt', weights_only=False, mmap=True)
assert torch.equal(original['model']['weights'], exported['model']['weights'])
assert 'optimizer' not in exported and 'rng_by_rank' not in exported
assert handoff.sha256(fixture['source']) == source_hash
hashes = {p.name: handoff.sha256(p) for p in fixture['archive'].iterdir()}
assert all(not stat.S_IMODE(p.stat().st_mode) & 0o222 for p in fixture['archive'].iterdir())
model = handoff.independent_initialization(fixture['archive'], fixture['root'] / 'init',
fixture['base'], metadata['snapshot_sha256'])
assert handoff.sha256(model) == metadata['snapshot_sha256']
assert model.stat().st_ino != (fixture['archive'] / 'model.pt').stat().st_ino
# Repeated handoff reuses the original archive/copy without changing bytes.
inode = model.stat().st_ino
assert handoff.archive_checkpoint(fixture['source'], fixture['tokenizer'], fixture['archive'],
fixture['base'], 'different-revision') == metadata
handoff.independent_initialization(fixture['archive'], model.parent, fixture['base'],
metadata['snapshot_sha256'])
assert model.stat().st_ino == inode
assert {p.name: handoff.sha256(p) for p in fixture['archive'].iterdir()} == hashes
def test_incomplete_existing_archive_is_not_replaced(fixture):
fixture['archive'].mkdir(parents=True)
marker = fixture['archive'] / 'user-file'
marker.write_text('keep')
with pytest.raises(ValueError, match='checksum'):
handoff.archive_checkpoint(fixture['source'], fixture['tokenizer'], fixture['archive'],
fixture['base'], 'test')
assert marker.read_text() == 'keep'
assert list(fixture['archive'].iterdir()) == [marker]
def test_publishing_does_not_replace_even_an_empty_directory(tmp_path):
staging, target = tmp_path / 'staging', tmp_path / 'target'
staging.mkdir()
target.mkdir()
(staging / 'payload').write_text('complete')
inode = target.stat().st_ino
with pytest.raises(FileExistsError):
handoff.publish_directory(staging, target)
assert target.stat().st_ino == inode and (staging / 'payload').exists()
def prepare_data(fixture):
directory = Path(fixture['plan']['handoff']['data_dir'])
directory.mkdir()
outputs = {}
for name in ('sft_train.jsonl', 'sft_val.jsonl'):
(directory / name).write_text('{}\n')
outputs[name] = dict(rows=1, sha256=handoff.sha256(directory / name))
manifest = dict(format='warpstate-posttrain-data-v1', complete=True,
tokenizer_sha256=fixture['base']['tokenizer_sha256'], max_length=2048, outputs=outputs)
(directory / 'manifest.json').write_text(json.dumps(manifest))
return directory
def test_data_hashes_must_match_before_launch(fixture):
directory = prepare_data(fixture)
assert handoff.data_readiness(directory, fixture['plan'], verify=True)[0]
(directory / 'sft_train.jsonl').write_text('changed\n')
with pytest.raises(ValueError, match='checksum'):
handoff.data_readiness(directory, fixture['plan'], verify=True)
def test_corrupted_raw_tensor_cannot_reuse_existing_archive(fixture):
handoff.archive_checkpoint(fixture['source'], fixture['tokenizer'], fixture['archive'], fixture['base'], 'test')
fixture['checkpoint']['model']['weights'][0, 0] = 99
torch.save(fixture['checkpoint'], fixture['source'])
with pytest.raises(ValueError, match='tensor'):
handoff.archive_checkpoint(fixture['source'], fixture['tokenizer'], fixture['archive'], fixture['base'], 'test')
def test_complete_handoff_waits_archives_and_passes_separate_output_paths(fixture, monkeypatch):
prepare_data(fixture)
monkeypatch.setattr(handoff, 'STOPPED', False)
monkeypatch.setattr(handoff, 'source_identity', lambda config: {'fixture': 'immutable'})
monkeypatch.setattr(handoff, 'verify_preflight', lambda *args: {'accepted': True})
statuses = iter([dict(statename='RUNNING', exitstatus=0), dict(statename='EXITED', exitstatus=0)])
monkeypatch.setattr(handoff, 'process_status', lambda *args: next(statuses, dict(statename='EXITED', exitstatus=0)))
monkeypatch.setattr(handoff.time, 'sleep', lambda seconds: None)
commands = []
base_work = Path(fixture['plan']['handoff']['base_work_dir'])
def fake_train(command, stop, env, *, pass_fds):
commands.append(command)
assert len(pass_fds) == 2
with pytest.raises(BlockingIOError):
with handoff.exclusive_lock(base_work / 'control/training-launcher.lock'):
pass
# Released precisely before worker SH acquisition.
with handoff.exclusive_lock(base_work / 'pipeline.lock'):
pass
if 'posttrain_preflight.py' in ' '.join(command):
return 0
work = Path(command[command.index('--work-dir') + 1])
weights = Path(command[command.index('--weights') + 1])
assert not weights.is_relative_to(work)
assert Path(command[command.index('--stop-file') + 1]) == stop
assert work.name == 'run'
work.mkdir(parents=True)
model_hash = handoff.sha256(weights)
completion = dict(format='warpstate-posttrain-completion-v1', stage='sft',
completed=True, paused_for_evaluation=True, step=1,
total_steps=1, base_step=fixture['base']['step'])
checkpoint = dict(format='warpstate-posttrain-v1', completed=True, step=1, total_steps=1,
val_step=1, val_loss=2., signature=dict(plan=fixture['plan'], base_sha256=model_hash),
origin=dict(step=fixture['base']['step']))
torch.save(checkpoint, work / 'latest.pt')
(work / 'completion.json').write_text(json.dumps(completion))
(work / 'evaluation.json').write_text(json.dumps(dict(stage='sft', step=1,
base_sha256=model_hash, val_loss=2.)))
return 0
monkeypatch.setattr(handoff, 'supervise', fake_train)
assert handoff.run_handoff(fixture['config']) == 0
state = json.loads((fixture['root'] / 'posttrain/control/handoff.json').read_text())
assert state['phase'] == 'complete'
count = len(commands)
assert handoff.run_handoff(fixture['config']) == 0
assert len(commands) == count # Never retrain after terminal completion.
def test_overshoot_never_archives_or_trains(fixture, monkeypatch):
prepare_data(fixture)
fixture['checkpoint']['step'] = 76295
torch.save(fixture['checkpoint'], fixture['source'])
monkeypatch.setattr(handoff, 'STOPPED', False)
monkeypatch.setattr(handoff, 'source_identity', lambda config: {'fixture': 'immutable'})
monkeypatch.setattr(handoff, 'process_status', lambda *args: dict(statename='EXITED', exitstatus=0))
monkeypatch.setattr(handoff, 'supervise', lambda *a, **kw: pytest.fail('Must not start SFT'))
with pytest.raises(ValueError, match='step must equal'):
handoff.run_handoff(fixture['config'])
assert not fixture['archive'].exists()
def test_sft_completion_requires_matching_durable_checkpoint(fixture, tmp_path):
(tmp_path / 'completion.json').write_text(json.dumps(dict(completed=True)))
torch.save(dict(format='warpstate-posttrain-v1', completed=False, step=1, total_steps=2,
val_step=1, val_loss=2.), tmp_path / 'latest.pt')
(tmp_path / 'evaluation.json').write_text('{}')
with pytest.raises(ValueError, match='fully evaluated'):
handoff.verify_completion(tmp_path, fixture['plan'], 'hash')