File size: 10,747 Bytes
2bf0f47
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
"""H1b Strict + detached path/string span coefficients, opt-in only."""
import hashlib
from dataclasses import replace
import math
import torch
from kdflow.algorithms import register_algorithm
from kdflow.algorithms.byte_align_ctkd import ByteAlignCrossTokenizerKD
from kdflow.loss.chunked_loss import chunked_loss
from kdflow.h1b.groups import align_byte_span_groups_batch
from kdflow.h1b.vocab_index import VocabIndex
from kdflow.h1b.strprob import score_hidden_chunks
from kdflow.h1b.baseline import BucketBaseline
from kdflow.h1b.buckets import bucket_of
from kdflow.h1b.chunklog import ChunkLogger


@register_algorithm('h1b_span')
class H1bSpanCTKD(ByteAlignCrossTokenizerKD):
    def __init__(self,*args,**kwargs):
        super().__init__(*args,**kwargs)
        kd=self.args.kd
        self.arm=kd.h1b_arm; self.lam=kd.h1b_span_lambda
        if self.arm not in {'a','b','b_bl','c_str','c'} or not math.isfinite(self.lam) or self.lam<0:
            raise ValueError('Invalid H1b arm or span lambda')
        if kd.kd_ratio!=1 or kd.kd_temperature!=1 or kd.kd_loss_fn!='rkl':
            raise ValueError('H1b requires kd_ratio=1, temperature=1 and rkl')
        if (self.args.train.num_nodes*self.args.train.num_gpus_per_node!=1
                or self.args.model.ring_attn_size!=1):
            raise ValueError('H1b supports one student rank and ring size 1')
        config=getattr(self.student.model,'config',None)
        config=getattr(config,'text_config',None) or config
        if getattr(config,'final_logit_softcapping',None) or getattr(self.teacher_lm_head,'_mp_opd_softcap',None):
            raise ValueError('H1b softcapped heads are not qualified')
        self.svi=VocabIndex(self.student_tokenizer,self.student_byte_codec)
        self.tvi=VocabIndex(self.teacher_tokenizer,self.teacher_byte_codec)
        self.baseline=BucketBaseline()
        self.window=0
        self.chunk_logger=ChunkLogger(getattr(self.args.train,'save_path',None),
            kd.h1b_log_sample_rate,tuple(int(s) for s in kd.h1b_dump_every.split(',') if s.strip()))

    def validate_optimizer_window(self,batches):
        if self.strategy.step!=0 or len(batches)!=self.strategy.accumulated_gradient:
            raise ValueError('H1b requires one complete optimizer window per fit')

    def state_dict(self):
        return dict(version=1,arm=self.arm,span_lambda=self.lam,window=self.window,baseline=self.baseline.state_dict())

    def load_state_dict(self,state):
        if state['version']!=1 or state['arm']!=self.arm or state['span_lambda']!=self.lam:
            raise ValueError('Changed H1b state contract')
        self.baseline.load_state_dict(state['baseline']); self.window=int(state['window'])
        self.chunk_logger.reset(); self.chunk_logger.step=self.window

    def on_optimizer_window_end(self):
        if self.strategy.step!=0: raise ValueError('Cannot close partial H1b optimizer window')
        metrics=self.chunk_logger.end_window(self.window)
        self.baseline.end_window(); self.window+=1
        return metrics

    def training_step(self,batch):
        replay=getattr(self,'replay_enabled',False)
        # Offline replay intentionally retains graphs; normal training must not
        # keep the previous microbatch graph alive during the next forward.
        self.replay_terms={}; self.replay_records=[]
        sm,tm=batch['stu_loss_mask'].bool(),batch['tea_loss_mask'].bool()
        sy,ty=batch['stu_input_ids'].roll(-1,1),batch['tea_input_ids'].roll(-1,1)
        st=[self.student_tokenizer.convert_ids_to_tokens(sy[i][sm[i]].cpu().tolist()) for i in range(len(sm))]
        tt=[self.teacher_tokenizer.convert_ids_to_tokens(ty[i][tm[i]].cpu().tolist()) for i in range(len(tm))]
        rows=align_byte_span_groups_batch(tt,st,self.teacher_tokenizer.eos_token,self.student_tokenizer.eos_token,
            self.teacher_byte_codec.piece_to_bytes,self.student_byte_codec.piece_to_bytes)
        for i,row in enumerate(rows):
            sids=sy[i][sm[i]].cpu().tolist(); tids=ty[i][tm[i]].cpu().tolist()
            if (any(t in self.svi.special and t!=self.student_tokenizer.eos_token_id for t in sids)
                    or any(t in self.tvi.special and t!=self.teacher_tokenizer.eos_token_id for t in tids)):
                rows[i]=replace(row,groups=(),valid=False,reason='unsupported_special')
        sh=self.student(batch['stu_input_ids'],attention_mask=batch['stu_attn_mask'],allgather_logits=True,
            ring_attn_group=self.strategy.ring_attn_group,**(batch.get('stu_multi_modal_inputs') or {}))['hidden_states'][-1][sm]
        th=batch['teacher_hiddens']
        if isinstance(th,list): th=torch.cat(th,0)
        if len(th)!=int(tm.sum()): raise ValueError('H1b teacher hidden/mask mismatch')
        th=th.detach().to(self.teacher_lm_head.weight)
        sl,tl=sy[sm],ty[tm]
        groups=[g for row in rows for g in row.groups]
        strict=[g for g in groups if g.kind=='strict']
        mis=[g for g in groups if g.kind=='mis']
        si=[g.s_idx[0] for g in strict]; ti=[g.t_idx[0] for g in strict]
        size=self.args.train.chunked_loss_size or max(len(sh),1)
        denom=float(batch['avg_micro_batch_token_num'])
        if not math.isfinite(denom) or denom<=0: raise ValueError('Positive finite common normalizer required')
        zero=sh.sum()*0.
        strict_sum=zero; metric_sums={}
        if si:
            ash,ath=sh[si],th[ti]
            def slog(h,skip=False): return self.student.model.lm_head(h,skip=skip)[:,self.student_overlap_token_ids]
            def tlog(a,b):
                with torch.no_grad():
                    return self.teacher_lm_head(ath[a:b])[:,self.teacher_overlap_token_ids]
            strict_sum,metric_sums=chunked_loss(ash,self.student.model.lm_head,self.loss_fn,
                student_logits_fn=slog,teacher_logits_fn=tlog,chunk_size=size,reduction='sum',
                metric_fns=self.metric_fns,return_metrics=True)
        sprob=score_hidden_chunks(self.student.model.lm_head,sh,sl,[(g.s_idx,g.content) for g in mis],self.svi,size,student=True)
        with torch.no_grad():
            tprob=score_hidden_chunks(self.teacher_lm_head,th,tl,[(g.t_idx,g.content) for g in mis],self.tvi,size,student=False)
        terms={k:zero for k in ('path','str','base_path','base_str')}
        training=self.student.training and torch.is_grad_enabled()
        log=self.chunk_logger
        if training:
            log.rows+=len(rows); log.dropped+=sum(not r.valid for r in rows)
            log.byte_dropped+=sum(r.reason=='byte_mismatch' for r in rows)
            log.stu+=sum(r.student_tokens for r in rows); log.tea+=sum(r.teacher_tokens for r in rows)
            log.strict_stu+=sum(len(g.s_idx) for g in strict if not g.eos)
            log.strict_tea+=sum(len(g.t_idx) for g in strict if not g.eos)
            log.mis_stu+=sum(len(g.s_idx) for g in mis)
            if 'stu_reencode_match' in batch:
                values=torch.as_tensor(batch['stu_reencode_match']).detach()
                log.reencode_sum+=float(values.sum()); log.reencode_n+=values.numel()
        # Response identity excludes prompt text and does not retain raw content.
        response_hashes={i:hashlib.sha256(bytes(str((st[i],tt[i])),'utf8')).hexdigest() for i in range(len(rows))}
        owner={id(g):i for i,r in enumerate(rows) for g in r.groups}
        for g,s,t in zip(mis,sprob,tprob):
            dp=(s.log_q_path-t.log_q_path).detach(); ds=(s.log_p_trunk-t.log_p_trunk).detach()
            bucket=bucket_of(g.content)
            bp=self.baseline.lookup(bucket,'path'); bs=self.baseline.lookup(bucket,'str')
            if replay:
                self.replay_records.append(dict(bucket=bucket,path=dp,str=ds,log_q=s.log_q_path))
            for k,v in (('path',dp),('str',ds),('base_path',bp),('base_str',bs)):
                terms[k]=terms[k]+2*self.lam*v*s.log_q_path
            if training:
                p,v=float(dp),float(ds)
                coefficient=0. if self.arm=='a' else 2*self.lam*(p-bp if self.arm=='b_bl' else v-bs if self.arm=='c' else p if self.arm=='b' else v)
                log.add(dict(response_hash=response_hashes[owner[id(g)]],step=self.window,bucket=bucket,
                    b0=g.b0,b1=g.b1,chunk_hex=g.content.hex(),s_tokens=len(g.s_idx),t_tokens=len(g.t_idx),
                    path=p,str=v,path_bl=p-bp,str_bl=v-bs,gap=p-v,baseline_path=bp,baseline_str=bs,coefficient=coefficient,
                    stu_log_q_path=float(s.log_q_path.detach()),tea_log_q_path=float(t.log_q_path),
                    stu_log_p_trunk=float(s.log_p_trunk),tea_log_p_trunk=float(t.log_p_trunk),
                    stu_log_overhang=float(s.log_overhang),tea_log_overhang=float(t.log_overhang),
                    slack_stu=float(s.log_p_trunk-s.log_q_path.detach()),slack_tea=float(t.log_p_trunk-t.log_q_path),
                    overhang_stu=float((s.log_overhang-s.log_p_trunk).exp()),overhang_tea=float((t.log_overhang-t.log_p_trunk).exp())))
                self.baseline.accumulate(bucket,p,'path'); self.baseline.accumulate(bucket,v,'str')
        span=zero if self.arm=='a' else (terms['path'] if self.arm=='b' else terms['path']-terms['base_path'] if self.arm=='b_bl'
                                       else terms['str'] if self.arm=='c_str' else terms['str']-terms['base_str'])
        loss=(strict_sum+span)/denom
        if not bool(torch.isfinite(loss)): raise FloatingPointError('Nonfinite H1b loss')
        if replay:
            self.replay_terms={k:v/denom for k,v in terms.items()}
            self.replay_terms['strict']=strict_sum/denom
        if getattr(self,'replay_control',False):
            for parity in (0,1):
                x,y=sh[si[parity::2]],th[ti[parity::2]]
                if len(x):
                    def replay_teacher(a,b,y=y):
                        with torch.no_grad():
                            return self.teacher_lm_head(y[a:b])[:,self.teacher_overlap_token_ids]
                    val=chunked_loss(x,self.student.model.lm_head,self.loss_fn,
                        student_logits_fn=slog,
                        teacher_logits_fn=replay_teacher,
                        chunk_size=size,reduction='sum')
                else: val=zero
                self.replay_terms['strict_'+str(parity)]=val/denom
        info={'train/loss':loss,'train/kd_loss':loss,'train/h1b_span_loss':span/denom,
              'distill/alignment_ratio':torch.tensor(len(si)/max(len(sl),1)),
              'distill/h1b_mis_groups':loss.detach().new_tensor(float(len(mis))),
              'distill/h1b_dropped_responses':loss.detach().new_tensor(float(sum(not r.valid for r in rows)))}
        info.update({k:v/denom for k,v in metric_sums.items()})
        return info