Commit ·
c15dbb0
1
Parent(s): a1a9de3
feat(h1b): add resume-capable grad replay copy
Browse files- tools/h1b/grad_replay_resume.py +151 -0
tools/h1b/grad_replay_resume.py
ADDED
|
@@ -0,0 +1,151 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Replay fixed archived responses through H1b; resume-capable copy; never generate new responses."""
|
| 2 |
+
import argparse
|
| 3 |
+
import hashlib
|
| 4 |
+
import json
|
| 5 |
+
from pathlib import Path
|
| 6 |
+
import types
|
| 7 |
+
import torch
|
| 8 |
+
from kdflow.algorithms.h1b_span import H1bSpanCTKD
|
| 9 |
+
from kdflow.arguments.distillation_args import DistillationArguments
|
| 10 |
+
from .common import archive_rows,load_pair,tokenize_record,forward_hidden,StudentWrapper,snapshot_identity,source_identity
|
| 11 |
+
from .sketch import count_sketch
|
| 12 |
+
|
| 13 |
+
|
| 14 |
+
def collate(batches):
|
| 15 |
+
from torch.nn.utils.rnn import pad_sequence
|
| 16 |
+
return {k:pad_sequence([b[k][0] for b in batches],batch_first=True) for k in batches[0]}
|
| 17 |
+
|
| 18 |
+
|
| 19 |
+
def main(argv=None):
|
| 20 |
+
p=argparse.ArgumentParser(description=__doc__)
|
| 21 |
+
for name in ('archive','student','teacher','output'): p.add_argument('--'+name,required=True)
|
| 22 |
+
p.add_argument('--limit',type=int,default=4096); p.add_argument('--group-size',type=int,default=64)
|
| 23 |
+
p.add_argument('--micro-batch',type=int,default=4); p.add_argument('--tile-size',type=int,default=128)
|
| 24 |
+
p.add_argument('--dimension',type=int,default=2**22); p.add_argument('--seed',type=int,default=42)
|
| 25 |
+
p.add_argument('--device',default='cuda'); p.add_argument('--buffer-device',default='cuda')
|
| 26 |
+
p.add_argument('--exact-groups',type=int,default=2)
|
| 27 |
+
p.add_argument('--buckets',default='',help='Optional byte class IDs 0=digits,3=letters; adds gradient buffers')
|
| 28 |
+
p.add_argument('--assume-shared-prompt',action='store_true')
|
| 29 |
+
p.add_argument('--resume',action='store_true',help='Resume into existing output dir; skip groups already done')
|
| 30 |
+
p.add_argument('--start-group',type=int,default=0)
|
| 31 |
+
p.add_argument('--end-group',type=int,default=None)
|
| 32 |
+
args=p.parse_args(argv)
|
| 33 |
+
if min(args.limit,args.group_size,args.micro_batch,args.tile_size,args.dimension)<1:
|
| 34 |
+
p.error('Sizes must be positive')
|
| 35 |
+
if args.limit%args.group_size or args.group_size%args.micro_batch:
|
| 36 |
+
p.error('Require complete equal groups and microbatches')
|
| 37 |
+
root=Path(args.output)
|
| 38 |
+
loaded=None
|
| 39 |
+
if args.resume:
|
| 40 |
+
if not root.is_dir(): p.error('Resume requires existing output dir')
|
| 41 |
+
mfile=root/'manifest.json'
|
| 42 |
+
if not mfile.exists(): p.error('Resume requires manifest.json')
|
| 43 |
+
loaded=json.loads(mfile.read_text())
|
| 44 |
+
else:
|
| 45 |
+
if root.exists(): raise FileExistsError(root)
|
| 46 |
+
records=list(archive_rows(args.archive,args.limit))
|
| 47 |
+
if len(records)!=args.limit: raise ValueError('Archive has fewer responses than requested')
|
| 48 |
+
selected=[int(s) for s in args.buckets.split(',') if s.strip()]
|
| 49 |
+
if any(not 0<=s<=5 for s in selected): raise ValueError('Byte class must be in 0..5')
|
| 50 |
+
ngroups=args.limit//args.group_size
|
| 51 |
+
end=args.end_group if args.end_group is not None else ngroups
|
| 52 |
+
if not 0<=args.start_group<=end<=ngroups: p.error('Require 0 <= start <= end <= ngroups')
|
| 53 |
+
if loaded is not None:
|
| 54 |
+
if loaded.get('groups')!=ngroups or loaded.get('group_size')!=args.group_size:
|
| 55 |
+
raise ValueError('Resume manifest groups/group_size mismatch')
|
| 56 |
+
if loaded.get('dimension')!=args.dimension or loaded.get('seed')!=args.seed:
|
| 57 |
+
raise ValueError('Resume manifest dimension/seed mismatch')
|
| 58 |
+
if list(loaded.get('bucket_classes',[]))!=selected:
|
| 59 |
+
raise ValueError('Resume manifest buckets mismatch')
|
| 60 |
+
models,toks=load_pair(args.student,args.teacher,args.device)
|
| 61 |
+
student,teacher=models
|
| 62 |
+
teacher.requires_grad_(False)
|
| 63 |
+
wrapper=StudentWrapper(student)
|
| 64 |
+
strategy=types.SimpleNamespace(args=types.SimpleNamespace(
|
| 65 |
+
kd=DistillationArguments(kd_algorithm='h1b_span',kd_ratio=1,kd_temperature=1,kd_loss_fn='rkl'),
|
| 66 |
+
model=types.SimpleNamespace(ring_attn_size=1),
|
| 67 |
+
train=types.SimpleNamespace(chunked_loss_size=args.tile_size,num_nodes=1,num_gpus_per_node=1,save_path=None),
|
| 68 |
+
scenario='on_policy_kd'),ring_attn_group=None)
|
| 69 |
+
algo=H1bSpanCTKD(strategy,wrapper,teacher.get_output_embeddings(),*toks)
|
| 70 |
+
algo.replay_enabled=True
|
| 71 |
+
algo.replay_control=False
|
| 72 |
+
wrapper.eval()
|
| 73 |
+
config=student.config
|
| 74 |
+
if any(getattr(config,k,0) for k in ('attention_dropout','hidden_dropout','hidden_dropout_prob')):
|
| 75 |
+
raise ValueError('Deterministic checkpointed replay requires zero model dropout')
|
| 76 |
+
student.gradient_checkpointing_enable(gradient_checkpointing_kwargs={'use_reentrant':False})
|
| 77 |
+
student.base_model.train()
|
| 78 |
+
encoded=[tokenize_record(r,*toks,args.assume_shared_prompt) for r in records]
|
| 79 |
+
sums=torch.zeros(ngroups,2,24,dtype=torch.float64); counts=torch.zeros_like(sums)
|
| 80 |
+
def batch_at(start,denominator):
|
| 81 |
+
batch=collate(encoded[start:start+args.micro_batch])
|
| 82 |
+
with torch.no_grad(): batch['teacher_hiddens']=forward_hidden(teacher,batch,'tea')
|
| 83 |
+
for k,v in batch.items(): batch[k]=v.to(args.device)
|
| 84 |
+
batch['avg_micro_batch_token_num']=denominator
|
| 85 |
+
return batch
|
| 86 |
+
with torch.no_grad():
|
| 87 |
+
for i in range(0,args.limit,args.micro_batch):
|
| 88 |
+
algo.training_step(batch_at(i,1.))
|
| 89 |
+
gi=i//args.group_size
|
| 90 |
+
for record in algo.replay_records:
|
| 91 |
+
b=record['bucket']
|
| 92 |
+
for kind,key in enumerate(('path','str')):
|
| 93 |
+
sums[gi,kind,b]+=float(record[key]); counts[gi,kind,b]+=1
|
| 94 |
+
if ngroups<2: raise ValueError('Leave-group-out replay needs at least two groups')
|
| 95 |
+
algo.replay_control=True
|
| 96 |
+
root.mkdir(parents=True,exist_ok=bool(args.resume))
|
| 97 |
+
named=list(wrapper.named_parameters()); params=tuple(v for _,v in named)
|
| 98 |
+
offsets=[0]
|
| 99 |
+
for _,param in named: offsets.append(offsets[-1]+param.numel())
|
| 100 |
+
keys=['strict','path','str','base_path','base_str','strict_0','strict_1']
|
| 101 |
+
for b in selected:
|
| 102 |
+
keys.extend(f'class{b}_{term}' for term in ('path','str','base_path','base_str'))
|
| 103 |
+
if loaded is not None:
|
| 104 |
+
manifest=loaded
|
| 105 |
+
else:
|
| 106 |
+
manifest=dict(version=1,source=source_identity(),archive_sha256=hashlib.sha256(Path(args.archive).read_bytes()).hexdigest(),
|
| 107 |
+
student=snapshot_identity(args.student),teacher=snapshot_identity(args.teacher),group_size=args.group_size,groups=ngroups,
|
| 108 |
+
dimension=args.dimension,seed=args.seed,normalization='sum token-normalized microbatch gradients per group',
|
| 109 |
+
baseline='leave-group-out bucket means; outcome-dependent bucket caveat applies',
|
| 110 |
+
preconditioner='raw gradients only',bucket_classes=selected,records=[])
|
| 111 |
+
done={r['group']:r for r in manifest.get('records',[])}
|
| 112 |
+
for gi in range(args.start_group,end):
|
| 113 |
+
if gi in done and all((root/f).exists() for f in done[gi].get('files',{}).values()):
|
| 114 |
+
continue
|
| 115 |
+
manifest['records']=[r for r in manifest['records'] if r['group']!=gi]
|
| 116 |
+
other_s=sums.sum(0)-sums[gi]; other_n=counts.sum(0)-counts[gi]
|
| 117 |
+
algo.baseline.prev=torch.where(other_n>0,other_s/other_n.clamp_min(1),torch.zeros_like(other_s))
|
| 118 |
+
buffers={k:torch.zeros(offsets[-1],dtype=torch.float32,device=args.buffer_device) for k in keys}
|
| 119 |
+
lo=gi*args.group_size
|
| 120 |
+
tokens=sum(int(b['stu_loss_mask'].sum()) for b in encoded[lo:lo+args.group_size])
|
| 121 |
+
for i in range(lo,lo+args.group_size,args.micro_batch):
|
| 122 |
+
algo.training_step(batch_at(i,max(tokens,1)))
|
| 123 |
+
terms=dict(algo.replay_terms)
|
| 124 |
+
for b in selected:
|
| 125 |
+
chunk_records=[r for r in algo.replay_records if r['bucket']//4==b]
|
| 126 |
+
for term in ('path','str','base_path','base_str'):
|
| 127 |
+
zero=terms['strict']*0
|
| 128 |
+
key=term.removeprefix('base_')
|
| 129 |
+
terms[f'class{b}_{term}']=sum((2*algo.lam*(algo.baseline.lookup(r['bucket'],key)
|
| 130 |
+
if term.startswith('base_') else r[term])*r['log_q']/max(tokens,1) for r in chunk_records),zero)
|
| 131 |
+
for ki,key in enumerate(keys):
|
| 132 |
+
grads=torch.autograd.grad(terms[key],params,retain_graph=ki<len(keys)-1,allow_unused=True)
|
| 133 |
+
for pi,grad in enumerate(grads):
|
| 134 |
+
if grad is not None: buffers[key][offsets[pi]:offsets[pi+1]].add_(grad.detach().flatten().to(args.buffer_device))
|
| 135 |
+
files={}; norms={}
|
| 136 |
+
for key,flat in buffers.items():
|
| 137 |
+
tensors=((name,flat[offsets[j]:offsets[j+1]]) for j,(name,_) in enumerate(named))
|
| 138 |
+
sketch,norm=count_sketch(tensors,args.dimension,args.seed)
|
| 139 |
+
file=f'group_{gi:03d}_{key}.pt'
|
| 140 |
+
torch.save(sketch,root/file); files[key]=file; norms[key]=norm
|
| 141 |
+
if gi<args.exact_groups:
|
| 142 |
+
torch.save(flat.cpu(),root/f'group_{gi:03d}_{key}_exact.pt')
|
| 143 |
+
manifest['records'].append(dict(group=gi,tokens=tokens,files=files,exact_norms=norms,
|
| 144 |
+
bucket_sums=sums[gi].tolist(),bucket_counts=counts[gi].tolist()))
|
| 145 |
+
manifest['records'].sort(key=lambda r: r['group'])
|
| 146 |
+
(root/'manifest.json').write_text(json.dumps(manifest,indent=2))
|
| 147 |
+
del buffers
|
| 148 |
+
print(json.dumps(dict(groups=ngroups,output=str(root),qualification='raw fixed-text replay; GPU execution must be independently qualified')))
|
| 149 |
+
|
| 150 |
+
|
| 151 |
+
if __name__=='__main__': main()
|