ctopd / tests /test_h1b.py
codemaivanngu's picture
fix(h1b): exclude non-EOS specials in offline row_groups
a1a9de3
Raw History Blame Contribute Delete
21.7 kB
"""Offline H1b contracts. Real tokenizer/rollout/B200 gates are separate tools."""
import copy
import ast
from collections import defaultdict
import hashlib
import importlib.util
import json
import math
from pathlib import Path
import random
import sys
import tempfile
import types
import unittest
import test_mp_opd_port as fixtures
ROOT=Path(__file__).resolve().parents[1]
if fixtures.HAS_TORCH:
import torch
# Test-only namespace: keep the real numerical/logging modules, bypass the
# utility package's eager tokenizer factory and optional HF dependencies.
if 'kdflow.utils' not in sys.modules:
package=types.ModuleType('kdflow.utils'); package.__path__=[str(ROOT/'kdflow/utils')]
sys.modules['kdflow.utils']=package
@unittest.skipUnless(fixtures.HAS_TORCH,'requires existing torch; no bootstrap')
class H1bTests(unittest.TestCase):
def fixture(self,arm='a'):
from kdflow.algorithms.h1b_span import H1bSpanCTKD
helper=fixtures.MPPortTests(); mp,student,teacher=helper.fixture()
kd=mp.args.kd
kd.h1b_arm=arm; kd.h1b_span_lambda=.5; kd.h1b_log_sample_rate=.05; kd.h1b_dump_every='0,20,60,100'
mp.args.train.save_path=None
mp.strategy.step=0; mp.strategy.accumulated_gradient=1
algo=H1bSpanCTKD(mp.strategy,student,teacher,fixtures.Tokenizer(),fixtures.Tokenizer())
return helper,algo,student,teacher
def test_T2_cover_matches_bruteforce_and_ff_edges(self):
from kdflow.h1b.vocab_index import VocabIndex,next_prefix
vi=VocabIndex(fixtures.Tokenizer())
rng=random.Random(42)
for _ in range(2000):
rem=bytes(rng.choice([97,98,99,100,255]) for _ in range(rng.randint(1,5)))
self.assertEqual(set(vi.cover(rem)),{i for i,b in vi._bytes.items() if b.startswith(rem) and b})
self.assertIsNone(next_prefix(b'\xff\xff'))
self.assertEqual(next_prefix(b'a\xff'),b'b')
with self.assertRaises(ValueError): vi.cover(b'')
def test_T3_strict_groups_match_author_fixture_eos_and_offsets(self):
from kdflow.h1b.groups import align_byte_span_groups_batch
from kdflow.token_alignment import align_byte_span_batch_1to1,get_tokenizer_byte_codec
tok=fixtures.Tokenizer(); codec=get_tokenizer_byte_codec(tok).piece_to_bytes
for tea,stu in [([['ab','c','<eos>'],['a','b']],[['a','b','c','<eos>'],['ab']]),
([['a'],[]],[['a'],[]]),([['<eos>','ab','<eos>']],[['<eos>','a','b','<eos>']])]:
rows=align_byte_span_groups_batch(tea,stu,tok.eos_token,tok.eos_token,codec,codec)
groups=[g for r in rows for g in r.groups if g.kind=='strict']
a=([g.t_idx[0] for g in groups],[g.s_idx[0] for g in groups])
self.assertEqual(a,align_byte_span_batch_1to1(tea,stu,tok.eos_token,tok.eos_token,codec,codec))
def test_T9b_row_groups_rejects_unsupported_special_whole_response(self):
import torch
from tools.h1b.common import row_groups
# A sampled reserved special must invalidate the whole response with
# reason 'unsupported_special', matching the training_step post-filter,
# instead of reaching the byte vocabulary index (KeyError there).
class Tok(fixtures.Tokenizer):
pieces={0:'<eos>',1:'a',2:'b',7:'<pad>',8:'<rsvd>'}
all_special_ids=[0,7,8]
tok=Tok()
ids=torch.tensor([[1,2,0],[1,8,0]])
mask=torch.tensor([[True,True,False],[True,True,False]])
batch={'stu_input_ids':ids,'stu_loss_mask':mask,'stu_attn_mask':torch.ones_like(ids),
'tea_input_ids':ids.clone(),'tea_loss_mask':mask.clone(),'tea_attn_mask':torch.ones_like(ids)}
rows=row_groups(batch,tok,tok)
self.assertTrue(rows[0].valid)
self.assertFalse(rows[1].valid)
self.assertEqual(rows[1].reason,'unsupported_special')
def test_T4_trunk_and_exact_enumeration_with_duplicate_byte_ids(self):
from kdflow.h1b.vocab_index import VocabIndex
from kdflow.h1b.strprob import chunk_logprobs
from kdflow.h1b.exact import exact_chunk_logprob
# 'A' and 'a' map to the same bytes. Exact proper-prefix enumeration
# must branch on both IDs rather than taking only id_of(...).
class Tok(fixtures.Tokenizer):
pieces={0:'<eos>',1:'a',2:'b',3:'ab',4:'A',5:'abc',6:'ba',7:'<pad>'}
codec=types.SimpleNamespace(piece_to_bytes=lambda s:s.lower().encode())
vi=VocabIndex(Tok(),codec)
torch.manual_seed(19)
table={():torch.randn(8).log_softmax(0),(1,):torch.randn(8).log_softmax(0),(4,):torch.randn(8).log_softmax(0)}
lp=chunk_logprobs(torch.stack([table[()],table[(1,)]]),[1,2],b'ab',vi)
expected=torch.logsumexp(torch.stack([table[()][3],table[()][5],table[()][1]+torch.logsumexp(table[(1,)][[2,6]],0)]),0)
torch.testing.assert_close(lp.log_p_trunk,expected)
result=exact_chunk_logprob(lambda ids:table[ids],b'ab',vi)
exact=math.exp(float(expected))+math.exp(float(table[()][4]+torch.logsumexp(table[(4,)][[2,6]],0)))
self.assertAlmostEqual(result['log_probability_lower'],math.log(exact),places=6)
self.assertTrue(result['exact'])
self.assertGreaterEqual(result['log_probability_lower']+1e-6,float(lp.log_p_trunk))
bounded=exact_chunk_logprob(lambda ids:table[ids],b'ab',vi,max_nodes=1)
self.assertLessEqual(bounded['log_probability_lower'],math.log(exact)+1e-6)
self.assertGreaterEqual(bounded['log_probability_upper']+1e-6,math.log(exact))
def test_T4_thirty_token_lm_fifty_strings(self):
import itertools
from kdflow.h1b.vocab_index import VocabIndex
from kdflow.h1b.strprob import chunk_logprobs
from kdflow.h1b.exact import exact_chunk_logprob
words=[''.join(x) for size in range(1,5) for x in itertools.product('ab',repeat=size)]
class Tok(fixtures.Tokenizer):
pieces={0:'<eos>',31:'<pad>',**{i+1:w for i,w in enumerate(words)}}
all_special_ids=[0,31]; pad_token_id=31
vi=VocabIndex(Tok()); rng=random.Random(17)
def oracle(path):
seed=int.from_bytes(hashlib.sha256(bytes(path)).digest()[:4],'big')
return torch.randn(32,generator=torch.Generator().manual_seed(seed),dtype=torch.float64).log_softmax(0)
for _ in range(50):
chunk=''.join(rng.choice('ab') for _ in range(rng.randint(1,4))).encode()
path=[vi.id_of(bytes([b])) for b in chunk]
lp=chunk_logprobs(torch.stack([oracle(tuple(path[:j])) for j in range(len(path))]),path,chunk,vi)
acc=1.; brute=0.
for j,token in enumerate(path):
row=oracle(tuple(path[:j])).exp(); rem=chunk[j:]
brute+=acc*sum(float(row[i]) for i,b in vi._bytes.items() if b.startswith(rem))
acc*=float(row[token])
self.assertAlmostEqual(float(lp.log_p_trunk),math.log(brute),places=6)
exact=exact_chunk_logprob(oracle,chunk,vi)
self.assertGreaterEqual(exact['log_probability_lower']+1e-6,float(lp.log_p_trunk))
def test_T5_same_tokenizer_all_arms_have_zero_span(self):
reference=None
for arm in ('a','b','b_bl','c_str','c'):
helper,algo,_,_=self.fixture(arm); batch=helper.batch()
batch['tea_input_ids']=batch['stu_input_ids'].clone()
batch['tea_loss_mask']=batch['stu_loss_mask'].clone()
batch['tea_attn_mask']=batch['stu_attn_mask'].clone()
batch['teacher_hiddens']=torch.randn(int(batch['tea_loss_mask'].sum()),4)
info=algo.training_step(batch)
self.assertEqual(float(info['train/h1b_span_loss'].detach()),0.)
self.assertEqual(float(info['distill/h1b_mis_groups']),0.)
value=info['train/loss'].detach()
if reference is None: reference=value
else: torch.testing.assert_close(value,reference)
def test_T6_baseline_previous_window_and_evaluation_immutability(self):
helper,algo,student,_=self.fixture('c')
before=algo.baseline.prev.clone()
algo.training_step(helper.batch())
torch.testing.assert_close(algo.baseline.prev,before)
algo.on_optimizer_window_end()
self.assertTrue(bool(algo.baseline.prev.any()))
state=copy.deepcopy(algo.state_dict()); counts=algo.baseline.counts.clone()
student.eval()
with torch.no_grad(): algo.training_step(helper.batch())
torch.testing.assert_close(algo.baseline.prev,state['baseline']['prev'])
torch.testing.assert_close(algo.baseline.counts,counts)
self.assertEqual(algo.chunk_logger.rows,0)
algo.load_state_dict(state)
def test_T7_span_surrogate_gradient_equals_mse(self):
helper,algo,student,_=self.fixture('b')
algo.replay_enabled=True
info=algo.training_step(helper.batch())
expected=sum(algo.lam*(r['log_q']-(r['log_q'].detach()-r['path']))**2 for r in algo.replay_records)/helper.batch()['avg_micro_batch_token_num']
a=torch.autograd.grad(info['train/h1b_span_loss'],tuple(student.parameters()),retain_graph=True)
b=torch.autograd.grad(expected,tuple(student.parameters()))
for x,y in zip(a,b): torch.testing.assert_close(x,y,atol=1e-6,rtol=1e-5)
def test_T8_arm_a_loss_and_all_gradients_equal_strict(self):
from kdflow.algorithms.byte_align_ctkd import ByteAlignCrossTokenizerKD
helper,algo,student,teacher=self.fixture('a')
strict=ByteAlignCrossTokenizerKD(algo.strategy,student,teacher,fixtures.Tokenizer(),fixtures.Tokenizer())
batch=helper.batch(2)
a=algo.training_step(batch)['train/loss']; b=strict.training_step(batch)['train/loss']
torch.testing.assert_close(a,b,atol=1e-6,rtol=1e-6)
for x,y in zip(torch.autograd.grad(a,tuple(student.parameters())),torch.autograd.grad(b,tuple(student.parameters()))):
torch.testing.assert_close(x,y,atol=1e-6,rtol=1e-6)
self.assertIsNone(teacher.weight.grad)
def test_byte_mismatch_drops_complete_response_including_eos(self):
helper,algo,student,_=self.fixture('b')
batch=helper.batch(); batch['tea_input_ids'][0,1]=5
info=algo.training_step(batch)
self.assertEqual(float(info['train/loss'].detach()),0.)
self.assertEqual(float(info['distill/h1b_dropped_responses']),1.)
info['train/loss'].backward()
self.assertTrue(all(p.grad is None or not bool(p.grad.any()) for p in student.parameters()))
def test_tile_recomputation_preserves_path_gradient_and_statistics(self):
helper,algo,student,_=self.fixture('c_str')
batch=helper.batch(); a=algo.training_step(batch)
ga=torch.autograd.grad(a['train/loss'],tuple(student.parameters()))
algo.args.train.chunked_loss_size=1
b=algo.training_step(batch); gb=torch.autograd.grad(b['train/loss'],tuple(student.parameters()))
torch.testing.assert_close(a['train/loss'],b['train/loss'])
for x,y in zip(ga,gb): torch.testing.assert_close(x,y,atol=1e-6,rtol=1e-5)
def test_window_statistics_use_chunk_counts_not_microbatch_means(self):
from kdflow.h1b.chunklog import ChunkLogger
log=ChunkLogger(sample_rate=0,dump_steps=())
for v in (1.,2.,9.):
row=dict(path=v,str=v/2,path_bl=v-1,str_bl=v/2-2,gap=v/2,slack_stu=1.,slack_tea=1+v/2,
overhang_stu=0.,overhang_tea=0.,bucket=0,response_hash=str(v),b0=0,b1=1)
log.add(row)
out=log.end_window(0)
self.assertEqual(out['distill/h1b_gap_mean'],2.)
self.assertAlmostEqual(out['distill/h1b_var_path'],(1+4+81)/3-16)
def test_actual_actor_hook_runs_after_one_update_and_not_during_eval(self):
from kdflow.utils.logging_utils import normalize_eval_metrics
helper,algo,student,_=self.fixture('c')
tree=ast.parse((ROOT/'kdflow/ray/train/student_actor.py').read_text())
scope=dict(torch=torch,defaultdict=defaultdict,normalize_eval_metrics=normalize_eval_metrics)
methods={}
for name in ('fit','evaluate'):
node=next(n for n in ast.walk(tree) if isinstance(n,ast.FunctionDef) and n.name==name)
node.decorator_list=[]
exec(compile(ast.Module(body=[node],type_ignores=[]),'actual_h1b_actor','exec'),scope)
methods[name]=scope[name]
strategy=algo.strategy; strategy.accumulated_gradient=2
algo.args.train.use_dynamic_bsz=False; algo.args.kd.use_ema_teacher=False
optimizer=torch.optim.SGD(student.parameters(),lr=.01); updates=[]
def backward(loss,*_):
strategy.step=(strategy.step+1)%2
(loss/2).backward()
def step(*_):
if strategy.step==0: optimizer.step(); optimizer.zero_grad(); updates.append(1)
strategy.backward=backward; strategy.optimizer_step=step
strategy.all_reduce=lambda x,**_:x
actor=types.SimpleNamespace(student=student,kd_algorithm=algo,args=algo.args,strategy=strategy,
optim=optimizer,scheduler=types.SimpleNamespace(get_last_lr=lambda:[.01]),_prepare_micro_batch=lambda b:dict(b))
original=algo.on_optimizer_window_end
def end():
self.assertEqual(len(updates),1)
return original()
algo.on_optimizer_window_end=end
batches=[helper.batch(),helper.batch(2)]
result=methods['fit'](actor,batches)
self.assertEqual(result['distill/h1b_window_responses'],3.)
self.assertEqual(algo.window,1)
previous=algo.baseline.prev.clone()
with torch.no_grad(): methods['evaluate'](actor,batches)
self.assertEqual(algo.window,1)
torch.testing.assert_close(algo.baseline.prev,previous)
def test_math_eval_override_never_changes_rollout_top_p(self):
from kdflow.arguments.rollout_args import RolloutArguments
tree=ast.parse((ROOT/'kdflow/evaluation/math_evaluator.py').read_text())
method=next(n for n in ast.walk(tree) if isinstance(n,ast.FunctionDef) and n.name=='evaluate')
assignments=[n for n in method.body if isinstance(n,ast.Assign) and any(isinstance(t,ast.Name) and t.id in {'eval_top_p','sampling_params'} for t in n.targets)]
for override,expected in ((None,1.),(.95,.95)):
actor=types.SimpleNamespace(args=types.SimpleNamespace(rollout=RolloutArguments(top_p=1.,eval_math_top_p=override)),
max_tokens=8192,n_samples=32,generate_kwargs={'top_p':1.,'temperature':1.})
scope={'self':actor}
exec(compile(ast.Module(body=assignments,type_ignores=[]),'actual_math_sampling','exec'),scope)
self.assertEqual(scope['sampling_params']['top_p'],expected)
self.assertEqual(actor.generate_kwargs['top_p'],1.)
with self.assertRaises(ValueError): RolloutArguments(eval_math_top_p=0)
def test_fit_curve_and_analysis_handle_known_direction_and_zero_vectors(self):
import numpy as np
from tools.h1b.fit_cb import fit_curve,analyze_matrices
sizes=[64,128,256,512,1024,2048,4096]
fit=fit_curve(sizes,[.7/math.sqrt(1+300/b) for b in sizes])
self.assertAlmostEqual(fit['c_infinity'],.7,places=2)
x=np.tile(np.array([1.,2.,3.]),(8,1))
report=analyze_matrices(x,x,x,x)
self.assertTrue(all(abs(r['cosine']-1)<1e-9 for r in report['curve']))
zero=analyze_matrices(x,x*0,x,x)
self.assertTrue(all(r['cosine'] is None for r in zero['curve']))
def test_replay_analysis_cli_on_complete_small_saved_sketches(self):
from tools.h1b.fit_cb import main
torch.manual_seed(3)
keys=['strict','path','str','base_path','base_str','strict_0','strict_1']
with tempfile.TemporaryDirectory() as tmp:
root=Path(tmp); rows=[]
for i in range(8):
files={}
for key in keys:
file=f'{i}_{key}.pt'; torch.save(torch.randn(16),root/file); files[key]=file
rows.append(dict(files=files,tokens=20+i))
(root/'manifest.json').write_text(json.dumps(dict(records=rows,groups=8,group_size=64,bucket_classes=[])))
main(['--replay',tmp,'--output',str(root/'curves.json'),'--bootstrap','2'])
result=json.loads((root/'curves.json').read_text())
self.assertEqual(set(result['results']),{'path','str','path_bl','str_bl'})
self.assertIsNotNone(result['cos_mean_path_str'])
def test_h1b_rollout_preserves_eos_policy_and_reports_raw_reencoding(self):
from typing import Any,Dict,List,Optional
tree=ast.parse((ROOT/'kdflow/trainer/data_processor/rollout_data_processor.py').read_text())
node=next(n for n in tree.body if isinstance(n,ast.ClassDef))
scope=dict(torch=torch,Any=Any,Dict=Dict,List=List,Optional=Optional)
exec(compile(ast.Module(body=[node],type_ignores=[]),'actual_h1b_processor','exec'),scope)
class Tok:
eos_token_id=0
def __call__(self,text,return_tensors=None,**kwargs):
ids=[{'p':6,'a':1,'b':2,'c':3}[c] for c in text]
return {'input_ids':torch.tensor([ids]),'attention_mask':torch.ones(1,len(ids),dtype=torch.long)} if return_tensors else {'input_ids':ids}
obj=object.__new__(scope['RolloutDataProcessor'])
obj.args=types.SimpleNamespace(kd=types.SimpleNamespace(kd_algorithm='h1b_span'))
obj.student_processor=Tok(); obj.teacher_processors={}; obj.is_same_tokenizer=True
sample=obj._build_rollout_sample('p','p',dict(text='abc',output_ids=[1,2,3]),None,append_eos=False)
self.assertEqual(float(sample['stu_reencode_match']),1.)
self.assertEqual(sample['stu_input_ids'].tolist(),[6,1,2,3])
sample=obj._build_rollout_sample('p','p',dict(text='abc',output_ids=[2,3]),None,append_eos=True)
self.assertEqual(float(sample['stu_reencode_match']),0.)
self.assertEqual(sample['stu_input_ids'][-1],0)
def test_KV_oracle_keeps_sibling_prefix_cache_immutable(self):
from tools.h1b.common import HFPrefixOracle
class Base(torch.nn.Module):
def forward(self,ids,past_key_values=None,use_cache=False,**kwargs):
cache={'ids':[]} if past_key_values is None else past_key_values
cache['ids'].extend(ids[0].tolist()) # Deliberate in-place cache mutation.
x=torch.tensor([sum(cache['ids']),len(cache['ids']),cache['ids'][-1],1.],dtype=torch.float32)
return types.SimpleNamespace(last_hidden_state=x[None,None,:],past_key_values=cache if use_cache else None)
class Model(torch.nn.Module):
def __init__(self): super().__init__(); self.base_model=Base(); self.head=torch.nn.Linear(4,8)
def get_output_embeddings(self): return self.head
model=Model(); cached=HFPrefixOracle(model,[6,1,2],cache_depth=2); direct=HFPrefixOracle(model,[6,1,2],use_cache=False)
for branch in ((),(1,),(1,2),(1,2,3),(2,),(2,3),(1,)):
torch.testing.assert_close(cached(branch),direct(branch))
self.assertEqual(cached.stack[0][2]['ids'],[6,1,2])
def test_old_archive_prompt_recovery_requires_exact_unambiguous_match(self):
from tools.h1b.enrich_archive import recover
records=[dict(prompt='S:x',output='abc',finish_reason='stop')]
rows=[dict(s='S:x',t='T:x')]
result=recover(records,rows,lambda r:r['s'],lambda r:r['t'])
self.assertEqual(result[0]['tea_prompt'],'T:x')
self.assertEqual(result[0]['output'],'abc')
self.assertNotIn('output_ids',result[0])
with self.assertRaises(ValueError): recover(records,[],lambda r:r['s'],lambda r:r['t'])
with self.assertRaises(ValueError): recover(records,rows+[dict(s='S:x',t='T:y')],lambda r:r['s'],lambda r:r['t'])
def test_T11_sketch_cosine_norm_and_chunk_invariance(self):
from tools.h1b.sketch import count_sketch
torch.manual_seed(9); a=torch.randn(2000); b=a+.2*torch.randn_like(a)
sa,na=count_sketch([('x',a)],chunk_size=17)
sb,nb=count_sketch([('x',b)],chunk_size=99)
sa2,_=count_sketch([('x',a)],chunk_size=33)
torch.testing.assert_close(sa,sa2)
self.assertLess(abs(float(sa.norm())/na-1),.01)
self.assertLess(abs(float(sb.norm())/nb-1),.01)
self.assertLess(abs(float(torch.nn.functional.cosine_similarity(sa,sb,dim=0)-torch.nn.functional.cosine_similarity(a,b,dim=0))),.01)
def test_presets_share_full_workload_and_separate_eval_top_p(self):
from tools.launch_single_b200 import parser,build_plan,H1B_ARMS
original=build_plan(parser().parse_args(['--algorithm','strict-full','--dry-run']))
allowed={'kd_algorithm','h1b_arm','h1b_span_lambda','h1b_log_sample_rate','h1b_dump_every','top_p','eval_math_top_p','save_path','ckpt_path'}
for name,arm in H1B_ARMS.items():
plan=build_plan(parser().parse_args(['--algorithm',name,'--dry-run']))
self.assertEqual(plan['flags']['h1b_arm'],arm)
self.assertEqual(plan['flags']['top_p'],1.)
self.assertEqual(plan['flags']['eval_math_top_p'],.95)
for key,value in original['flags'].items():
if key not in allowed: self.assertEqual(plan['flags'][key],value)
def test_source_manifest_frozen_strict(self):
manifest=json.loads((ROOT/'docs/H1B_SOURCE.json').read_text())
for row in manifest['files']+manifest['frozen_strict']:
self.assertEqual(hashlib.sha256((ROOT/row['path']).read_bytes()).hexdigest(),row['sha256'])
if __name__=='__main__': unittest.main()