Download test_dpo.py from zerdovzad/WarpStateAI: direct link, hf CLI and curl.
- Browser
- Download file 9.75 kB
-
https://huggingface.co/zerdovzad/WarpStateAI/resolve/main/test_dpo.py
- Command line
-
hf download hf://zerdovzad/WarpStateAI/test_dpo.py
-
curl -L -o test_dpo.py https://huggingface.co/zerdovzad/WarpStateAI/resolve/main/test_dpo.py
9.75 kB
| """CPU DPO arithmetic, frozen-reference and exact continuation checks.""" | |
| import copy | |
| import json | |
| import math | |
| import shutil | |
| import numpy as np | |
| import pytest | |
| import torch | |
| import torch.nn.functional as F | |
| import dpo | |
| from ops.dpo_data import collate_pairs, encode_pair, load_data | |
| from ops.dpo_preflight import make_fixture | |
| from ops.posttrain_data import encode_messages | |
| from test_posttrain import CharacterTokenizer, assert_tree_equal | |
| def pair(): | |
| return dict(prompt=[{'role': 'user', 'content': 'hello'}, | |
| {'role': 'assistant', 'content': 'old answer'}, | |
| {'role': 'user', 'content': 'two plus two?'}], | |
| chosen=[{'role': 'assistant', 'content': 'four'}], | |
| rejected=[{'role': 'assistant', 'content': 'five, perhaps six'}]) | |
| def test_multiturn_masks_only_final_response_and_eos(): | |
| tokenizer = CharacterTokenizer() | |
| row = pair() | |
| encoded = encode_pair(row, tokenizer, 512) | |
| for branch in ('chosen', 'rejected'): | |
| assert encoded[branch]['input_ids'] == encode_messages(row['prompt'] + row[branch], tokenizer)['input_ids'] | |
| assert [value for value in encoded[branch]['labels'] if value != -100] == [ord(c) for c in row[branch][0]['content']] + [2] | |
| x, y, active = collate_pairs([encoded, None], 512, 0) | |
| assert active.tolist() == [True, False] | |
| assert not y[2:].ne(-100).any() | |
| assert x.shape == y.shape == (4, 512) | |
| def test_invalid_preference_pairs_rejected(mutation): | |
| row = pair() | |
| mutation(row) | |
| with pytest.raises(ValueError): | |
| encode_pair(row, CharacterTokenizer(), 512) | |
| def test_length_normalized_and_summed_objectives_match_explicit_formula(): | |
| policy = torch.tensor([[-10., -19.], [-4., -9.]], dtype=torch.float64, requires_grad=True) | |
| reference = torch.tensor([[-12., -20.], [-4., -8.]], dtype=torch.float64, requires_grad=True) | |
| counts = torch.tensor([[2, 10], [1, 3]]) | |
| loss, margin, accuracy = dpo.preference_metrics(policy, reference, counts, 5., 'length-normalized-dpo') | |
| expected = 5 * ((policy[:, 0] - reference.detach()[:, 0]) / counts[:, 0] - | |
| (policy[:, 1] - reference.detach()[:, 1]) / counts[:, 1]) | |
| torch.testing.assert_close(loss, -F.logsigmoid(expected)) | |
| torch.testing.assert_close(margin, expected) | |
| assert accuracy.tolist() == [1., 1.] | |
| loss.sum().backward() | |
| assert reference.grad is None | |
| assert (policy.grad[:, 0] < 0).all() and (policy.grad[:, 1] > 0).all() | |
| summed, _, _ = dpo.preference_metrics(policy.detach(), reference, counts, .1, 'summed-dpo') | |
| expected = .1 * ((policy[:, 0] - reference[:, 0]) - (policy[:, 1] - reference[:, 1])) | |
| torch.testing.assert_close(summed, -F.logsigmoid(expected)) | |
| def test_global_pair_mean_handles_empty_rank_and_unequal_lengths(): | |
| parameter = torch.tensor(.2, dtype=torch.float64, requires_grad=True) | |
| counts = torch.tensor([[2, 10], [4, 2], [1, 6]]) | |
| base = torch.tensor([[-10., -19.], [-5., -3.], [-2., -8.]], dtype=torch.float64) | |
| direction = torch.tensor([[1., 2.], [3., 1.], [2., 5.]], dtype=torch.float64) | |
| gradients = [] | |
| for ids in ([0, 1, 2], []): | |
| parameter.grad = None | |
| if ids: | |
| loss, _, _ = dpo.preference_metrics(base[ids] + parameter * direction[ids], base[ids], counts[ids], 5., 'length-normalized-dpo') | |
| (loss.sum() * 2 / 3).backward() | |
| else: | |
| (parameter * 0.).backward() | |
| gradients.append(parameter.grad.clone()) | |
| parameter.grad = None | |
| loss, _, _ = dpo.preference_metrics(base + parameter * direction, base, counts, 5., 'length-normalized-dpo') | |
| loss.mean().backward() | |
| torch.testing.assert_close(sum(gradients) / 2, parameter.grad) | |
| def test_chunked_sequence_logps_and_gradients_match_full_logits(): | |
| from warpstate import WarpStateConfig, WarpStateLM | |
| cfg = WarpStateConfig(vocab_size=32, d_model=16, n_heads=4, n_cores=1, | |
| logical_depth=1, ffn_hidden=32, chunk_size=4, parallel_memory=True) | |
| torch.manual_seed(19) | |
| original = WarpStateLM(cfg) | |
| chunked = copy.deepcopy(original) | |
| dpo.apply_sequence_logps(chunked, 3) | |
| x = torch.randint(0, 32, (4, 8)) | |
| y = torch.randint(0, 32, (4, 8)) | |
| y[:, :2] = -100 | |
| y[3] = -100 | |
| logits = original(x) | |
| expected = -F.cross_entropy(logits.reshape(-1, 32), y.reshape(-1), reduction='none').reshape(4, 8).sum(1) | |
| _, actual = chunked(x, y, return_logits=False) | |
| torch.testing.assert_close(actual, expected, rtol=1e-6, atol=1e-5) | |
| expected.sum().backward() | |
| actual.sum().backward() | |
| for a, b in zip(original.parameters(), chunked.parameters()): | |
| torch.testing.assert_close(a.grad, b.grad, rtol=1e-5, atol=2e-6) | |
| assert actual[3] == 0 | |
| def test_exact_bf16_resume_rebuilt_reference_and_immutable_sft(tmp_path): | |
| args = make_fixture(tmp_path / 'fixture') | |
| original_hash = dpo.digest(args.weights) | |
| original = torch.load(args.weights, weights_only=False, mmap=True) | |
| try: | |
| dpo.train(args) | |
| uninterrupted = torch.load(args.work_dir / 'latest.pt', weights_only=False, mmap=True) | |
| original_cache = dpo.digest(args.reference_cache_dir / 'manifest.json') | |
| args.work_dir = tmp_path / 'split' | |
| args.max_steps = 1 | |
| dpo.train(args) | |
| partial = torch.load(args.work_dir / 'latest.pt', weights_only=False, mmap=True) | |
| assert partial['step'] == 1 and not partial['completed'] | |
| # A deterministic external-data cache can be rebuilt while resuming a | |
| # changed policy; the source remains the frozen SFT, never latest.pt. | |
| shutil.rmtree(args.reference_cache_dir) | |
| args.max_steps = None | |
| dpo.train(args) | |
| resumed = torch.load(args.work_dir / 'latest.pt', weights_only=False, mmap=True) | |
| assert dpo.digest(args.reference_cache_dir / 'manifest.json') == original_cache | |
| assert resumed['step'] == 3 and resumed['sample_cursor'] == 5 and resumed['completed'] | |
| for key in ('model', 'optimizer', 'sample_cursor', 'tokens_seen', 'response_tokens_seen', 'validation'): | |
| assert_tree_equal(uninterrupted[key], resumed[key]) | |
| assert torch.equal(uninterrupted['rng_by_rank'][0]['torch'], resumed['rng_by_rank'][0]['torch']) | |
| assert all(v.dtype == torch.float32 for v in resumed['model'].values()) | |
| assert any(not torch.equal(original['model'][key], value) for key, value in resumed['model'].items()) | |
| assert dpo.digest(args.weights) == original_hash | |
| metrics = [json.loads(line) for line in (tmp_path / 'fixture/run/metrics.jsonl').read_text().splitlines()] | |
| assert metrics[0]['val_loss'] == pytest.approx(math.log(2), abs=1e-7) | |
| report = json.loads((args.work_dir / 'evaluation.json').read_text()) | |
| assert report['full_holdout'] and report['evaluated_samples'] == 2 | |
| completion = json.loads((args.work_dir / 'completion.json').read_text()) | |
| assert completion['paused_for_evaluation'] and completion['completed'] | |
| finally: | |
| torch.use_deterministic_algorithms(False) | |
| def test_reference_identity_tampering_and_incomplete_data_fail_closed(tmp_path): | |
| import sentencepiece as spm | |
| args = make_fixture(tmp_path / 'fixture') | |
| args.reference_only = True | |
| try: | |
| dpo.train(args) | |
| manifest_path = args.reference_cache_dir / 'manifest.json' | |
| manifest = json.loads(manifest_path.read_text()) | |
| identity = manifest['identity'] | |
| with pytest.raises(ValueError, match='identity'): | |
| dpo.validate_cache(args.reference_cache_dir, identity | {'weights_sha256': 'changed'}, (5, 2)) | |
| array = args.reference_cache_dir / 'train.npy' | |
| original = array.read_bytes() | |
| array.write_bytes(original + b'changed') | |
| with pytest.raises(ValueError, match='changed'): | |
| dpo.validate_cache(args.reference_cache_dir, identity, (5, 2)) | |
| array.write_bytes(original) | |
| manifest['complete'] = False | |
| manifest_path.write_text(json.dumps(manifest)) | |
| with pytest.raises(ValueError, match='completion'): | |
| dpo.validate_cache(args.reference_cache_dir, identity, (5, 2)) | |
| data_manifest_path = args.data_dir / 'manifest.json' | |
| data_manifest = json.loads(data_manifest_path.read_text()) | |
| data_manifest['renderer_sha256'] = 'changed' | |
| data_manifest_path.write_text(json.dumps(data_manifest)) | |
| with pytest.raises(ValueError, match='renderer'): | |
| load_data(args.data_dir, spm.SentencePieceProcessor(model_file=str(args.tokenizer)), dpo.digest(args.tokenizer), 128) | |
| finally: | |
| torch.use_deterministic_algorithms(False) | |
| def test_schedule_is_linear_and_pause_independent(): | |
| settings = dpo.DEFAULTS | dict(lr=1., min_lr=0., warmup_ratio=.1) | |
| assert dpo.learning_rate(0, 100, settings) == .1 | |
| assert dpo.learning_rate(9, 100, settings) == 1. | |
| assert dpo.learning_rate(10, 100, settings) == 1. | |
| assert dpo.learning_rate(54, 100, settings) == pytest.approx(45 / 89) | |
| assert dpo.learning_rate(99, 100, settings) == 0. | |
| def test_invalid_plan_rejected(tmp_path, settings): | |
| path = tmp_path / 'plan.json' | |
| path.write_text(json.dumps(dict(format='warpstate-dpo-plan-v1', stage='dpo', training=settings))) | |
| with pytest.raises(ValueError): | |
| dpo.load_plan(path) | |