ctopd / kdflow /algorithms /trust_training.py
codemaivanngu's picture
feat(kd): add TRUST R B P and separate SimCT paper scores
619cff6 unverified
Raw History Blame Contribute Delete
3.38 kB
"""Single-rank actor integration for complete-window TRUST-B/P calibration."""
from collections import defaultdict
import torch
from .mp_opd_trust import restore_rng
def fit_trust_batch(actor, train_data):
actor.student.train()
if actor.args.train.use_dynamic_bsz:
actor.strategy.accumulated_gradient = len(train_data)
actor.strategy.step = 0
algo = actor.kd_algorithm
prepared = algo.prepare_optimizer_batch(train_data,batch_loader=actor._prepare_micro_batch)
status = defaultdict(list)
try:
for index,batch in enumerate(train_data):
restore_rng(prepared['rng_states'][index])
micro = actor._prepare_micro_batch(batch)
info = algo.training_step(micro)
for key,value in info.items():
status[key].append(value.item())
actor.strategy.backward(info['train/loss'],actor.student,actor.optim)
status['train/grad_norm'].append(torch.nn.utils.clip_grad_norm_(actor.student.parameters(),float('inf')).item())
actor.strategy.optimizer_step(actor.optim,actor.student,actor.scheduler)
if actor.args.kd.use_ema_teacher and actor.strategy.step == 0:
actor.ema_update()
for name in ('response_length','total_length'):
if name in micro:
status['rollout/'+name].append(micro[name].mean().item())
del micro
restore_rng(prepared['after_rng'])
finally:
algo.finish_optimizer_batch()
status = {k:sum(v)/len(v) for k,v in status.items()}
status['train/lr'] = actor.scheduler.get_last_lr()[0]
return {k:actor.strategy.all_reduce(v,op='mean') for k,v in status.items()}
@torch.no_grad()
def evaluate_trust_batch(actor, eval_data):
"""One evaluation-set calibration; no training-state reuse or optimizer step."""
algo = actor.kd_algorithm
was_training = actor.student.training
actor.student.eval()
batches = list(eval_data)
def load(batch):
micro = actor._prepare_micro_batch(batch)
weight = micro.pop('_eval_weight',1.)
if float(weight) != 1.:
raise ValueError('Single-rank TRUST evaluation requires unit sample weights')
micro['avg_micro_batch_token_num'] = 1.
return micro
sums,total = defaultdict(float),0.
try:
if not batches:
return {'eval/num_tokens':0.}
prepared = algo.prepare_optimizer_batch(batches,batch_loader=load,check_window=False)
for index,batch in enumerate(batches):
restore_rng(prepared['rng_states'][index])
micro = load(batch)
count = float(micro['stu_loss_mask'].sum())
metrics = algo.training_step(micro)
for key,value in metrics.items():
# Loss already sums tokens; diagnostic values are response averages.
sums[key] += float(value) * (1. if key in {'train/loss','train/kd_loss'} else count)
total += count
restore_rng(prepared['after_rng'])
finally:
algo.finish_optimizer_batch()
actor.student.train(was_training)
from kdflow.utils.logging_utils import normalize_eval_metrics
result = {k:actor.strategy.all_reduce(v,op='sum')/max(total,1.) for k,v in sums.items()}
result['eval/num_tokens'] = total
return normalize_eval_metrics(result)