File size: 9,255 Bytes
0eeed4c | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 | """Regression for company Fixed1/GRASS/ALIGN invalid-response crashes."""
import hashlib
import copy
import json
import pathlib
import unittest
import test_mp_opd_port as fixtures
ROOT = pathlib.Path(__file__).resolve().parents[1]
if fixtures.HAS_TORCH:
import torch
class ReplacementTokenizer(fixtures.Tokenizer):
pieces = {**fixtures.Tokenizer.pieces, 5: '\ufffd'}
class TransientPrefixTokenizer(fixtures.Tokenizer):
def decode(self, ids, **kwargs):
if list(ids) == [1]:
return '\ufffd' # Incomplete byte prefix which recovers in full decode.
return super().decode(ids, **kwargs)
@unittest.skipUnless(fixtures.HAS_TORCH, 'requires existing PyTorch; no bootstrap')
class InvalidResponseTests(unittest.TestCase):
def fixture(self, mode='mp_opd_fixed'):
helper = fixtures.MPPortTests()
algo, student, teacher = helper.fixture(mode)
algo.kd.mp_opd_fixed_span_length = 1
algo.atomizer.student_tokenizer = ReplacementTokenizer()
return helper, algo, student, teacher
def test_fixed1_mixed_batch_loss_gradients_offsets_and_denominator(self):
helper, algo, student, _ = self.fixture()
batch = helper.batch(3)
batch['stu_input_ids'][0,1] = 5
# Give each response distinct teacher states to catch offset errors.
width = int(batch['tea_loss_mask'][0].sum())
batch['teacher_hiddens'] = (batch['teacher_hiddens'].detach()
* torch.tensor([1.,2.,3.]).repeat_interleave(width)[:,None]).requires_grad_()
actual = algo.training_step(batch)
clean = helper.batch(2)
clean['avg_micro_batch_token_num'] = batch['avg_micro_batch_token_num']
clean['teacher_hiddens'] = batch['teacher_hiddens'][width:]
expected = algo.training_step(clean)['train/loss']
torch.testing.assert_close(actual['train/loss'], expected)
actual_grads=torch.autograd.grad(actual['train/loss'],tuple(student.parameters()))
expected_grads=torch.autograd.grad(expected,tuple(student.parameters()))
for a,b in zip(actual_grads,expected_grads):
torch.testing.assert_close(a,b)
self.assertEqual(float(actual['distill/mp_opd_valid_responses']),2.)
self.assertEqual(float(actual['distill/mp_opd_attempted_responses']),3.)
self.assertEqual(float(actual['distill/mp_opd_skipped_replacement_responses']),1.)
self.assertAlmostEqual(float(actual['distill/mp_opd_skipped_replacement_fraction']),1/3,places=6)
self.assertEqual(float(actual['distill/mp_opd_skipped_student_loss_tokens']),10.)
self.assertEqual(float(actual['distill/mp_opd_skipped_teacher_loss_tokens']),7.)
self.assertAlmostEqual(float(actual['distill/mp_opd_skipped_student_token_fraction']),1/3,places=6)
# A removed response has no surrogate gradient, including its unique token.
self.assertFalse(bool(actual_grads[0][5].any()))
def test_all_replacement_responses_produce_finite_zero_and_backward(self):
helper,algo,student,_ = self.fixture()
batch=helper.batch(2)
batch['stu_input_ids'][:,1]=5
result=algo.training_step(batch)
self.assertEqual(float(result['train/loss'].detach()),0.)
self.assertEqual(float(result['distill/mp_opd_valid_responses']),0.)
self.assertEqual(float(result['distill/mp_opd_skipped_replacement_fraction']),1.)
result['train/loss'].backward()
self.assertTrue(all(p.grad is None or not bool(p.grad.any()) for p in student.parameters()))
def test_existing_mp_modes_skip_but_other_alignment_errors_stay_fatal(self):
for mode in fixtures.registry:
helper,algo,student,_=self.fixture(mode)
batch=helper.batch(2)
batch['stu_input_ids'][0,1]=5
result=algo.training_step(batch)
self.assertEqual(float(result['distill/mp_opd_skipped_replacement_responses']),1.)
self.assertTrue(all(bool(torch.isfinite(value)) for value in result.values()))
result['train/loss'].backward()
self.assertTrue(any(p.grad is not None for p in student.parameters()))
helper,algo,*_=self.fixture()
bad=helper.batch()
bad['tea_input_ids'][0,1]=7 # Unsupported added token remains fatal.
with self.assertRaisesRegex(ValueError,'unsupported_added_token'):
algo.training_step(bad)
def test_normalization_mismatch_loss_and_gradients_for_all_mp_modes(self):
import test_align_new
for mode in (*fixtures.registry, 'align_new'):
with self.subTest(mode=mode):
if mode == 'align_new':
_,algo,_,_,_,_=test_align_new.AlignNewTests().setup_pair()
helper=fixtures.MPPortTests()
student=algo.student
else:
helper,algo,student,_=self.fixture(mode)
batch=helper.batch(2)
batch['tea_input_ids'][0,1]=6
# Distinct hidden rows ensure a rejected first row cannot shift
# the teacher states used by the next valid response.
width=int(batch['tea_loss_mask'][0].sum())
batch['teacher_hiddens']=batch['teacher_hiddens'].detach().clone()
batch['teacher_hiddens'][:width]*=3
state=copy.deepcopy(algo.state_dict())
actual=algo.training_step(batch)
clean=helper.batch()
clean['avg_micro_batch_token_num']=batch['avg_micro_batch_token_num']
clean['teacher_hiddens']=batch['teacher_hiddens'][width:]
algo.load_state_dict(state)
expected=algo.training_step(clean)['train/loss']
torch.testing.assert_close(actual['train/loss'],expected)
for a,b in zip(torch.autograd.grad(actual['train/loss'],tuple(student.parameters())),
torch.autograd.grad(expected,tuple(student.parameters()))):
torch.testing.assert_close(a,b)
self.assertEqual(float(actual['distill/mp_opd_skipped_normalization_responses']),1.)
self.assertEqual(float(actual['distill/mp_opd_skipped_replacement_responses']),0.)
self.assertEqual(float(actual['distill/mp_opd_skipped_normalization_fraction']),.5)
self.assertEqual(float(actual['distill/mp_opd_skipped_responses']),1.)
def test_transient_replacement_prefix_is_not_excluded(self):
helper,algo,*_=self.fixture()
algo.atomizer.student_tokenizer=TransientPrefixTokenizer()
result=algo.training_step(helper.batch())
self.assertEqual(float(result['distill/mp_opd_valid_responses']),1.)
self.assertEqual(float(result['distill/mp_opd_skipped_responses']),0.)
def test_align_new_and_trust_full_window_use_same_exclusion(self):
import test_align_new
import test_trust_paper
_,align,_,_,_,_=test_align_new.AlignNewTests().setup_pair()
align.atomizer.student_tokenizer=ReplacementTokenizer()
batch=fixtures.MPPortTests().batch(2)
batch['stu_input_ids'][0,1]=5
result=align.training_step(batch)
self.assertEqual(float(result['distill/mp_opd_skipped_replacement_responses']),1.)
for mode,reason in ((m,r) for m in ('trust_r','trust_b','trust_p')
for r in ('replacement','normalization')):
helper,algo,student,_=test_trust_paper.TrustTests().fixture(mode)
algo.atomizer.student_tokenizer=ReplacementTokenizer()
bad,good=helper.batch(),helper.batch()
if reason == 'replacement':
bad['stu_input_ids'][0,1]=5
else:
bad['tea_input_ids'][0,1]=6
for b in (bad,good): b['avg_micro_batch_token_num']*=2
if algo.requires_optimizer_batch:
algo.prepare_optimizer_batch([bad,good])
self.assertEqual(float(algo._batch_metrics['mp_opd_'+mode+'_calibration_response_count']),1.)
result=algo.training_step(bad)
self.assertEqual(float(result['distill/mp_opd_skipped_'+reason+'_responses']),1.)
loss=result['train/loss']+algo.training_step(good)['train/loss']
full_grad=torch.autograd.grad(loss,tuple(student.parameters()))
algo.finish_optimizer_batch()
if algo.requires_optimizer_batch:
algo.strategy.accumulated_gradient=1
algo.prepare_optimizer_batch([good])
expected=algo.training_step(good)['train/loss']
torch.testing.assert_close(loss,expected,atol=1e-6,rtol=1e-5)
for a,b in zip(full_grad,torch.autograd.grad(expected,tuple(student.parameters()))):
torch.testing.assert_close(a,b,atol=1e-6,rtol=1e-5)
algo.finish_optimizer_batch()
class InvalidResponseSourceTests(unittest.TestCase):
def test_adapter_patch_hash(self):
policy=json.loads((ROOT/'docs/MP_OPD_INVALID_RESPONSES.json').read_text())
self.assertEqual(hashlib.sha256((ROOT/policy['path']).read_bytes()).hexdigest(),policy['updated_sha256'])
if __name__=='__main__':
unittest.main()
|