Download kdflow/algorithms/trust_training.py from codemaivanngu/ctopd: direct link, hf CLI and curl.
- Browser
- Download file 3.38 kB
-
https://huggingface.co/codemaivanngu/ctopd/resolve/main/kdflow/algorithms/trust_training.py
- Command line
-
hf download hf://codemaivanngu/ctopd/kdflow/algorithms/trust_training.py
-
curl -L -o trust_training.py https://huggingface.co/codemaivanngu/ctopd/resolve/main/kdflow/algorithms/trust_training.py
3.38 kB
| """Single-rank actor integration for complete-window TRUST-B/P calibration.""" | |
| from collections import defaultdict | |
| import torch | |
| from .mp_opd_trust import restore_rng | |
| def fit_trust_batch(actor, train_data): | |
| actor.student.train() | |
| if actor.args.train.use_dynamic_bsz: | |
| actor.strategy.accumulated_gradient = len(train_data) | |
| actor.strategy.step = 0 | |
| algo = actor.kd_algorithm | |
| prepared = algo.prepare_optimizer_batch(train_data,batch_loader=actor._prepare_micro_batch) | |
| status = defaultdict(list) | |
| try: | |
| for index,batch in enumerate(train_data): | |
| restore_rng(prepared['rng_states'][index]) | |
| micro = actor._prepare_micro_batch(batch) | |
| info = algo.training_step(micro) | |
| for key,value in info.items(): | |
| status[key].append(value.item()) | |
| actor.strategy.backward(info['train/loss'],actor.student,actor.optim) | |
| status['train/grad_norm'].append(torch.nn.utils.clip_grad_norm_(actor.student.parameters(),float('inf')).item()) | |
| actor.strategy.optimizer_step(actor.optim,actor.student,actor.scheduler) | |
| if actor.args.kd.use_ema_teacher and actor.strategy.step == 0: | |
| actor.ema_update() | |
| for name in ('response_length','total_length'): | |
| if name in micro: | |
| status['rollout/'+name].append(micro[name].mean().item()) | |
| del micro | |
| restore_rng(prepared['after_rng']) | |
| finally: | |
| algo.finish_optimizer_batch() | |
| status = {k:sum(v)/len(v) for k,v in status.items()} | |
| status['train/lr'] = actor.scheduler.get_last_lr()[0] | |
| return {k:actor.strategy.all_reduce(v,op='mean') for k,v in status.items()} | |
| def evaluate_trust_batch(actor, eval_data): | |
| """One evaluation-set calibration; no training-state reuse or optimizer step.""" | |
| algo = actor.kd_algorithm | |
| was_training = actor.student.training | |
| actor.student.eval() | |
| batches = list(eval_data) | |
| def load(batch): | |
| micro = actor._prepare_micro_batch(batch) | |
| weight = micro.pop('_eval_weight',1.) | |
| if float(weight) != 1.: | |
| raise ValueError('Single-rank TRUST evaluation requires unit sample weights') | |
| micro['avg_micro_batch_token_num'] = 1. | |
| return micro | |
| sums,total = defaultdict(float),0. | |
| try: | |
| if not batches: | |
| return {'eval/num_tokens':0.} | |
| prepared = algo.prepare_optimizer_batch(batches,batch_loader=load,check_window=False) | |
| for index,batch in enumerate(batches): | |
| restore_rng(prepared['rng_states'][index]) | |
| micro = load(batch) | |
| count = float(micro['stu_loss_mask'].sum()) | |
| metrics = algo.training_step(micro) | |
| for key,value in metrics.items(): | |
| # Loss already sums tokens; diagnostic values are response averages. | |
| sums[key] += float(value) * (1. if key in {'train/loss','train/kd_loss'} else count) | |
| total += count | |
| restore_rng(prepared['after_rng']) | |
| finally: | |
| algo.finish_optimizer_batch() | |
| actor.student.train(was_training) | |
| from kdflow.utils.logging_utils import normalize_eval_metrics | |
| result = {k:actor.strategy.all_reduce(v,op='sum')/max(total,1.) for k,v in sums.items()} | |
| result['eval/num_tokens'] = total | |
| return normalize_eval_metrics(result) | |