File size: 8,225 Bytes
619cff6 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 | """Isolated TRUST-R/B/P adapters; B/P require a complete optimizer window."""
import math
import random
import numpy as np
import torch
from . import register_algorithm
from .mp_opd import MPOnPolicyKD, fixed_partition
def capture_rng():
return (random.getstate(), np.random.get_state(), torch.get_rng_state(),
torch.cuda.get_rng_state_all() if torch.cuda.is_initialized() else None)
def restore_rng(state):
random.setstate(state[0])
np.random.set_state(state[1])
torch.set_rng_state(state[2])
if state[3] is not None:
torch.cuda.set_rng_state_all(state[3])
class TrustMPOnPolicyKD(MPOnPolicyKD):
mode = 'atomic'
trust_mode = None
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
from ._mp_opd_port._mp_opd_grass_span import _env_flag
self.mode = self.trust_mode
self.requires_optimizer_batch = self.mode in {'trust_b', 'trust_p'}
self.eps_g = float(self.kd.mp_opd_trust_eps_g)
if not math.isfinite(self.eps_g) or self.eps_g < 0:
raise ValueError('TRUST eps_g must be finite and nonnegative')
if self.kd.mp_opd_grass_geometry != 'exact_head' or not _env_flag('MP_OPD_GRASS_EXACT_FP32', True):
raise ValueError('TRUST requires exact_head with exact FP32 geometry enabled')
if self.kd.mp_opd_trust_vocab_chunk < 1:
raise ValueError('TRUST vocabulary tile must be positive')
self._collector = self._result = None
self._normalizer = None
self._batch_metrics = {}
def _partition(self, atoms, student_ids, teacher_ids):
return fixed_partition(len(atoms), 1), None, 0
def training_step(self, batch):
if self.requires_optimizer_batch:
if self._collector is None and self._result is None:
raise RuntimeError('TRUST-B/P requires prepare_optimizer_batch before scoring/backward')
normalizer = float(batch['avg_micro_batch_token_num'])
if not math.isfinite(normalizer) or normalizer <= 0:
raise ValueError('TRUST-B/P requires a finite positive common normalizer')
if self._normalizer is not None and normalizer != self._normalizer:
raise ValueError('TRUST-B/P requires one common normalizer across the window')
self._normalizer = normalizer
return super().training_step(batch)
def _effective_rates(self, credits, atoms, partition, hidden, labels, **_kwargs):
from ._mp_opd_port._mp_opd_trust import (strict_mask, trust_calibrate,
trust_response_metrics, gram_affordable)
from ._mp_opd_port._mp_opd_grass_chunk import chunk_head_gram
strict = strict_mask([a.boundary_type for a in atoms]).to(credits.rate.device)
ranges = tuple((a.student_start, a.student_end) for a in atoms)
if self._collector is not None:
with torch.no_grad():
logits = self._head_logits(self.student.model.lm_head, hidden, self.student_cap, True)
self._collector.add_response(logits, hidden.detach(), labels, ranges,
credits.rate, strict, -credits.student_token_nll)
return torch.zeros_like(credits.rate), {}
if self.requires_optimizer_batch:
result, metrics = self._result, self._batch_metrics
else:
with torch.no_grad():
logits = self._head_logits(self.student.model.lm_head, hidden, self.student_cap, True)
if not gram_affordable(labels.numel(), logits.shape[-1], logits.device):
raise RuntimeError('TRUST-R exact Gram exceeds memory guard; no silent Atomic fallback')
head = chunk_head_gram(logits, hidden.detach(), labels, ranges, ((0,len(atoms)),),
selected_log_prob=-credits.student_token_nll, softcap=self.student_cap,
head_bias=getattr(self.student.model.lm_head,'bias',None) is not None)
q, h = credits.rate.detach().float(), head.gram.detach().float()
result = trust_calibrate(q[strict], q[~strict], h[strict][:,strict],
h[~strict][:,~strict], h[strict][:,~strict], eps_g=self.eps_g)
raw = trust_response_metrics(result,strict,credits.weight,scope='response')
metrics = {'mp_opd_trust_r_'+k: credits.rate.new_tensor(v) for k,v in raw.items()}
if self.mode == 'trust_p':
coefficients = torch.where(strict, float(result.strict_scale), 1.).to(credits.rate)
else:
coefficients = torch.where(strict, 1., float(result.lam)).to(credits.rate)
return credits.rate.detach()*coefficients, metrics
def prepare_optimizer_batch(self, batches, *, batch_loader=lambda x:x, check_window=True):
from ._mp_opd_port._mp_opd_trust_batch import TrustHeadAccumulator
from ._mp_opd_port._mp_opd_trust import trust_batch_metrics
from ._mp_opd_port._mp_opd_trust_p import trust_p_calibrate_scalars, trust_p_batch_metrics
if not self.requires_optimizer_batch:
return None
if self._collector is not None or self._result is not None:
raise RuntimeError('TRUST optimizer window is already active')
if not batches:
raise ValueError('TRUST requires a nonempty optimizer window')
if check_window and (len(batches) != self.strategy.accumulated_gradient or self.strategy.step != 0):
raise ValueError('TRUST-B/P requires exactly one complete optimizer accumulation window at step0')
before = capture_rng()
states = []
self._normalizer = None
collector = TrustHeadAccumulator(softcap=self.student_cap,
head_bias=getattr(self.student.model.lm_head,'bias',None) is not None,
vocab_chunk=self.kd.mp_opd_trust_vocab_chunk, force_fp32=self.mode == 'trust_p')
self._collector = collector
try:
with torch.no_grad():
for batch in batches:
states.append(capture_rng())
self.training_step(batch_loader(batch))
after = capture_rng()
options = {'calibrate':trust_p_calibrate_scalars} if self.mode == 'trust_p' else {}
self._result = collector.finalize(eps_g=self.eps_g, distributed=False, **options)
records = collector.responses
ns = sum(r['strict_atoms'] for r in records)
nm = sum(r['mismatch_atoms'] for r in records)
counts = dict(strict_span_count=ns,mismatch_span_count=nm,strict_atom_count=ns,mismatch_atom_count=nm,
strict_token_count=sum(r['strict_tokens'] for r in records),
mismatch_token_count=sum(r['mismatch_tokens'] for r in records),
strict_span_fraction=ns/max(ns+nm,1),mismatch_span_fraction=nm/max(ns+nm,1))
metric_fn = trust_p_batch_metrics if self.mode == 'trust_p' else trust_batch_metrics
self._batch_metrics = {f'mp_opd_{self.mode}_'+k: torch.tensor(float(v),device=self.teacher_head.weight.device,dtype=torch.float32)
for k,v in metric_fn(self._result,counts).items()}
prefix = 'mp_opd_'+self.mode+'_'
self._batch_metrics.update({prefix+'full_batch':torch.tensor(1.,device=self.teacher_head.weight.device),
prefix+'calibration_response_count':torch.tensor(float(len(records)),device=self.teacher_head.weight.device)})
return {'rng_states':states,'after_rng':after}
except BaseException:
self.finish_optimizer_batch()
raise
finally:
self._collector = None
restore_rng(before)
def finish_optimizer_batch(self):
self._collector = self._result = None
self._normalizer = None
self._batch_metrics = {}
@register_algorithm('mp_opd_trust_r')
class TrustRMPOnPolicyKD(TrustMPOnPolicyKD):
trust_mode = 'trust_r'
@register_algorithm('mp_opd_trust_b')
class TrustBMPOnPolicyKD(TrustMPOnPolicyKD):
trust_mode = 'trust_b'
@register_algorithm('mp_opd_trust_p')
class TrustPMPOnPolicyKD(TrustMPOnPolicyKD):
trust_mode = 'trust_p'
|