ctopd / tests /test_invalid_response.py
codemaivanngu's picture
fix(mp-opd): exclude unalignable responses with reason metrics
0eeed4c unverified
Raw History Blame Contribute Delete
9.26 kB
"""Regression for company Fixed1/GRASS/ALIGN invalid-response crashes."""
import hashlib
import copy
import json
import pathlib
import unittest
import test_mp_opd_port as fixtures
ROOT = pathlib.Path(__file__).resolve().parents[1]
if fixtures.HAS_TORCH:
import torch
class ReplacementTokenizer(fixtures.Tokenizer):
pieces = {**fixtures.Tokenizer.pieces, 5: '\ufffd'}
class TransientPrefixTokenizer(fixtures.Tokenizer):
def decode(self, ids, **kwargs):
if list(ids) == [1]:
return '\ufffd' # Incomplete byte prefix which recovers in full decode.
return super().decode(ids, **kwargs)
@unittest.skipUnless(fixtures.HAS_TORCH, 'requires existing PyTorch; no bootstrap')
class InvalidResponseTests(unittest.TestCase):
def fixture(self, mode='mp_opd_fixed'):
helper = fixtures.MPPortTests()
algo, student, teacher = helper.fixture(mode)
algo.kd.mp_opd_fixed_span_length = 1
algo.atomizer.student_tokenizer = ReplacementTokenizer()
return helper, algo, student, teacher
def test_fixed1_mixed_batch_loss_gradients_offsets_and_denominator(self):
helper, algo, student, _ = self.fixture()
batch = helper.batch(3)
batch['stu_input_ids'][0,1] = 5
# Give each response distinct teacher states to catch offset errors.
width = int(batch['tea_loss_mask'][0].sum())
batch['teacher_hiddens'] = (batch['teacher_hiddens'].detach()
* torch.tensor([1.,2.,3.]).repeat_interleave(width)[:,None]).requires_grad_()
actual = algo.training_step(batch)
clean = helper.batch(2)
clean['avg_micro_batch_token_num'] = batch['avg_micro_batch_token_num']
clean['teacher_hiddens'] = batch['teacher_hiddens'][width:]
expected = algo.training_step(clean)['train/loss']
torch.testing.assert_close(actual['train/loss'], expected)
actual_grads=torch.autograd.grad(actual['train/loss'],tuple(student.parameters()))
expected_grads=torch.autograd.grad(expected,tuple(student.parameters()))
for a,b in zip(actual_grads,expected_grads):
torch.testing.assert_close(a,b)
self.assertEqual(float(actual['distill/mp_opd_valid_responses']),2.)
self.assertEqual(float(actual['distill/mp_opd_attempted_responses']),3.)
self.assertEqual(float(actual['distill/mp_opd_skipped_replacement_responses']),1.)
self.assertAlmostEqual(float(actual['distill/mp_opd_skipped_replacement_fraction']),1/3,places=6)
self.assertEqual(float(actual['distill/mp_opd_skipped_student_loss_tokens']),10.)
self.assertEqual(float(actual['distill/mp_opd_skipped_teacher_loss_tokens']),7.)
self.assertAlmostEqual(float(actual['distill/mp_opd_skipped_student_token_fraction']),1/3,places=6)
# A removed response has no surrogate gradient, including its unique token.
self.assertFalse(bool(actual_grads[0][5].any()))
def test_all_replacement_responses_produce_finite_zero_and_backward(self):
helper,algo,student,_ = self.fixture()
batch=helper.batch(2)
batch['stu_input_ids'][:,1]=5
result=algo.training_step(batch)
self.assertEqual(float(result['train/loss'].detach()),0.)
self.assertEqual(float(result['distill/mp_opd_valid_responses']),0.)
self.assertEqual(float(result['distill/mp_opd_skipped_replacement_fraction']),1.)
result['train/loss'].backward()
self.assertTrue(all(p.grad is None or not bool(p.grad.any()) for p in student.parameters()))
def test_existing_mp_modes_skip_but_other_alignment_errors_stay_fatal(self):
for mode in fixtures.registry:
helper,algo,student,_=self.fixture(mode)
batch=helper.batch(2)
batch['stu_input_ids'][0,1]=5
result=algo.training_step(batch)
self.assertEqual(float(result['distill/mp_opd_skipped_replacement_responses']),1.)
self.assertTrue(all(bool(torch.isfinite(value)) for value in result.values()))
result['train/loss'].backward()
self.assertTrue(any(p.grad is not None for p in student.parameters()))
helper,algo,*_=self.fixture()
bad=helper.batch()
bad['tea_input_ids'][0,1]=7 # Unsupported added token remains fatal.
with self.assertRaisesRegex(ValueError,'unsupported_added_token'):
algo.training_step(bad)
def test_normalization_mismatch_loss_and_gradients_for_all_mp_modes(self):
import test_align_new
for mode in (*fixtures.registry, 'align_new'):
with self.subTest(mode=mode):
if mode == 'align_new':
_,algo,_,_,_,_=test_align_new.AlignNewTests().setup_pair()
helper=fixtures.MPPortTests()
student=algo.student
else:
helper,algo,student,_=self.fixture(mode)
batch=helper.batch(2)
batch['tea_input_ids'][0,1]=6
# Distinct hidden rows ensure a rejected first row cannot shift
# the teacher states used by the next valid response.
width=int(batch['tea_loss_mask'][0].sum())
batch['teacher_hiddens']=batch['teacher_hiddens'].detach().clone()
batch['teacher_hiddens'][:width]*=3
state=copy.deepcopy(algo.state_dict())
actual=algo.training_step(batch)
clean=helper.batch()
clean['avg_micro_batch_token_num']=batch['avg_micro_batch_token_num']
clean['teacher_hiddens']=batch['teacher_hiddens'][width:]
algo.load_state_dict(state)
expected=algo.training_step(clean)['train/loss']
torch.testing.assert_close(actual['train/loss'],expected)
for a,b in zip(torch.autograd.grad(actual['train/loss'],tuple(student.parameters())),
torch.autograd.grad(expected,tuple(student.parameters()))):
torch.testing.assert_close(a,b)
self.assertEqual(float(actual['distill/mp_opd_skipped_normalization_responses']),1.)
self.assertEqual(float(actual['distill/mp_opd_skipped_replacement_responses']),0.)
self.assertEqual(float(actual['distill/mp_opd_skipped_normalization_fraction']),.5)
self.assertEqual(float(actual['distill/mp_opd_skipped_responses']),1.)
def test_transient_replacement_prefix_is_not_excluded(self):
helper,algo,*_=self.fixture()
algo.atomizer.student_tokenizer=TransientPrefixTokenizer()
result=algo.training_step(helper.batch())
self.assertEqual(float(result['distill/mp_opd_valid_responses']),1.)
self.assertEqual(float(result['distill/mp_opd_skipped_responses']),0.)
def test_align_new_and_trust_full_window_use_same_exclusion(self):
import test_align_new
import test_trust_paper
_,align,_,_,_,_=test_align_new.AlignNewTests().setup_pair()
align.atomizer.student_tokenizer=ReplacementTokenizer()
batch=fixtures.MPPortTests().batch(2)
batch['stu_input_ids'][0,1]=5
result=align.training_step(batch)
self.assertEqual(float(result['distill/mp_opd_skipped_replacement_responses']),1.)
for mode,reason in ((m,r) for m in ('trust_r','trust_b','trust_p')
for r in ('replacement','normalization')):
helper,algo,student,_=test_trust_paper.TrustTests().fixture(mode)
algo.atomizer.student_tokenizer=ReplacementTokenizer()
bad,good=helper.batch(),helper.batch()
if reason == 'replacement':
bad['stu_input_ids'][0,1]=5
else:
bad['tea_input_ids'][0,1]=6
for b in (bad,good): b['avg_micro_batch_token_num']*=2
if algo.requires_optimizer_batch:
algo.prepare_optimizer_batch([bad,good])
self.assertEqual(float(algo._batch_metrics['mp_opd_'+mode+'_calibration_response_count']),1.)
result=algo.training_step(bad)
self.assertEqual(float(result['distill/mp_opd_skipped_'+reason+'_responses']),1.)
loss=result['train/loss']+algo.training_step(good)['train/loss']
full_grad=torch.autograd.grad(loss,tuple(student.parameters()))
algo.finish_optimizer_batch()
if algo.requires_optimizer_batch:
algo.strategy.accumulated_gradient=1
algo.prepare_optimizer_batch([good])
expected=algo.training_step(good)['train/loss']
torch.testing.assert_close(loss,expected,atol=1e-6,rtol=1e-5)
for a,b in zip(full_grad,torch.autograd.grad(expected,tuple(student.parameters()))):
torch.testing.assert_close(a,b,atol=1e-6,rtol=1e-5)
algo.finish_optimizer_batch()
class InvalidResponseSourceTests(unittest.TestCase):
def test_adapter_patch_hash(self):
policy=json.loads((ROOT/'docs/MP_OPD_INVALID_RESPONSES.json').read_text())
self.assertEqual(hashlib.sha256((ROOT/policy['path']).read_bytes()).hexdigest(),policy['updated_sha256'])
if __name__=='__main__':
unittest.main()