File size: 3,383 Bytes
619cff6
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
"""Single-rank actor integration for complete-window TRUST-B/P calibration."""
from collections import defaultdict

import torch

from .mp_opd_trust import restore_rng


def fit_trust_batch(actor, train_data):
    actor.student.train()
    if actor.args.train.use_dynamic_bsz:
        actor.strategy.accumulated_gradient = len(train_data)
        actor.strategy.step = 0
    algo = actor.kd_algorithm
    prepared = algo.prepare_optimizer_batch(train_data,batch_loader=actor._prepare_micro_batch)
    status = defaultdict(list)
    try:
        for index,batch in enumerate(train_data):
            restore_rng(prepared['rng_states'][index])
            micro = actor._prepare_micro_batch(batch)
            info = algo.training_step(micro)
            for key,value in info.items():
                status[key].append(value.item())
            actor.strategy.backward(info['train/loss'],actor.student,actor.optim)
            status['train/grad_norm'].append(torch.nn.utils.clip_grad_norm_(actor.student.parameters(),float('inf')).item())
            actor.strategy.optimizer_step(actor.optim,actor.student,actor.scheduler)
            if actor.args.kd.use_ema_teacher and actor.strategy.step == 0:
                actor.ema_update()
            for name in ('response_length','total_length'):
                if name in micro:
                    status['rollout/'+name].append(micro[name].mean().item())
            del micro
        restore_rng(prepared['after_rng'])
    finally:
        algo.finish_optimizer_batch()
    status = {k:sum(v)/len(v) for k,v in status.items()}
    status['train/lr'] = actor.scheduler.get_last_lr()[0]
    return {k:actor.strategy.all_reduce(v,op='mean') for k,v in status.items()}


@torch.no_grad()
def evaluate_trust_batch(actor, eval_data):
    """One evaluation-set calibration; no training-state reuse or optimizer step."""
    algo = actor.kd_algorithm
    was_training = actor.student.training
    actor.student.eval()
    batches = list(eval_data)

    def load(batch):
        micro = actor._prepare_micro_batch(batch)
        weight = micro.pop('_eval_weight',1.)
        if float(weight) != 1.:
            raise ValueError('Single-rank TRUST evaluation requires unit sample weights')
        micro['avg_micro_batch_token_num'] = 1.
        return micro

    sums,total = defaultdict(float),0.
    try:
        if not batches:
            return {'eval/num_tokens':0.}
        prepared = algo.prepare_optimizer_batch(batches,batch_loader=load,check_window=False)
        for index,batch in enumerate(batches):
            restore_rng(prepared['rng_states'][index])
            micro = load(batch)
            count = float(micro['stu_loss_mask'].sum())
            metrics = algo.training_step(micro)
            for key,value in metrics.items():
                # Loss already sums tokens; diagnostic values are response averages.
                sums[key] += float(value) * (1. if key in {'train/loss','train/kd_loss'} else count)
            total += count
        restore_rng(prepared['after_rng'])
    finally:
        algo.finish_optimizer_batch()
        actor.student.train(was_training)
    from kdflow.utils.logging_utils import normalize_eval_metrics
    result = {k:actor.strategy.all_reduce(v,op='sum')/max(total,1.) for k,v in sums.items()}
    result['eval/num_tokens'] = total
    return normalize_eval_metrics(result)