File size: 8,225 Bytes
619cff6
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
"""Isolated TRUST-R/B/P adapters; B/P require a complete optimizer window."""
import math
import random

import numpy as np
import torch

from . import register_algorithm
from .mp_opd import MPOnPolicyKD, fixed_partition


def capture_rng():
    return (random.getstate(), np.random.get_state(), torch.get_rng_state(),
            torch.cuda.get_rng_state_all() if torch.cuda.is_initialized() else None)


def restore_rng(state):
    random.setstate(state[0])
    np.random.set_state(state[1])
    torch.set_rng_state(state[2])
    if state[3] is not None:
        torch.cuda.set_rng_state_all(state[3])


class TrustMPOnPolicyKD(MPOnPolicyKD):
    mode = 'atomic'
    trust_mode = None

    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        from ._mp_opd_port._mp_opd_grass_span import _env_flag
        self.mode = self.trust_mode
        self.requires_optimizer_batch = self.mode in {'trust_b', 'trust_p'}
        self.eps_g = float(self.kd.mp_opd_trust_eps_g)
        if not math.isfinite(self.eps_g) or self.eps_g < 0:
            raise ValueError('TRUST eps_g must be finite and nonnegative')
        if self.kd.mp_opd_grass_geometry != 'exact_head' or not _env_flag('MP_OPD_GRASS_EXACT_FP32', True):
            raise ValueError('TRUST requires exact_head with exact FP32 geometry enabled')
        if self.kd.mp_opd_trust_vocab_chunk < 1:
            raise ValueError('TRUST vocabulary tile must be positive')
        self._collector = self._result = None
        self._normalizer = None
        self._batch_metrics = {}

    def _partition(self, atoms, student_ids, teacher_ids):
        return fixed_partition(len(atoms), 1), None, 0

    def training_step(self, batch):
        if self.requires_optimizer_batch:
            if self._collector is None and self._result is None:
                raise RuntimeError('TRUST-B/P requires prepare_optimizer_batch before scoring/backward')
            normalizer = float(batch['avg_micro_batch_token_num'])
            if not math.isfinite(normalizer) or normalizer <= 0:
                raise ValueError('TRUST-B/P requires a finite positive common normalizer')
            if self._normalizer is not None and normalizer != self._normalizer:
                raise ValueError('TRUST-B/P requires one common normalizer across the window')
            self._normalizer = normalizer
        return super().training_step(batch)

    def _effective_rates(self, credits, atoms, partition, hidden, labels, **_kwargs):
        from ._mp_opd_port._mp_opd_trust import (strict_mask, trust_calibrate,
            trust_response_metrics, gram_affordable)
        from ._mp_opd_port._mp_opd_grass_chunk import chunk_head_gram
        strict = strict_mask([a.boundary_type for a in atoms]).to(credits.rate.device)
        ranges = tuple((a.student_start, a.student_end) for a in atoms)
        if self._collector is not None:
            with torch.no_grad():
                logits = self._head_logits(self.student.model.lm_head, hidden, self.student_cap, True)
                self._collector.add_response(logits, hidden.detach(), labels, ranges,
                    credits.rate, strict, -credits.student_token_nll)
            return torch.zeros_like(credits.rate), {}
        if self.requires_optimizer_batch:
            result, metrics = self._result, self._batch_metrics
        else:
            with torch.no_grad():
                logits = self._head_logits(self.student.model.lm_head, hidden, self.student_cap, True)
                if not gram_affordable(labels.numel(), logits.shape[-1], logits.device):
                    raise RuntimeError('TRUST-R exact Gram exceeds memory guard; no silent Atomic fallback')
                head = chunk_head_gram(logits, hidden.detach(), labels, ranges, ((0,len(atoms)),),
                    selected_log_prob=-credits.student_token_nll, softcap=self.student_cap,
                    head_bias=getattr(self.student.model.lm_head,'bias',None) is not None)
                q, h = credits.rate.detach().float(), head.gram.detach().float()
                result = trust_calibrate(q[strict], q[~strict], h[strict][:,strict],
                    h[~strict][:,~strict], h[strict][:,~strict], eps_g=self.eps_g)
                raw = trust_response_metrics(result,strict,credits.weight,scope='response')
                metrics = {'mp_opd_trust_r_'+k: credits.rate.new_tensor(v) for k,v in raw.items()}
        if self.mode == 'trust_p':
            coefficients = torch.where(strict, float(result.strict_scale), 1.).to(credits.rate)
        else:
            coefficients = torch.where(strict, 1., float(result.lam)).to(credits.rate)
        return credits.rate.detach()*coefficients, metrics

    def prepare_optimizer_batch(self, batches, *, batch_loader=lambda x:x, check_window=True):
        from ._mp_opd_port._mp_opd_trust_batch import TrustHeadAccumulator
        from ._mp_opd_port._mp_opd_trust import trust_batch_metrics
        from ._mp_opd_port._mp_opd_trust_p import trust_p_calibrate_scalars, trust_p_batch_metrics
        if not self.requires_optimizer_batch:
            return None
        if self._collector is not None or self._result is not None:
            raise RuntimeError('TRUST optimizer window is already active')
        if not batches:
            raise ValueError('TRUST requires a nonempty optimizer window')
        if check_window and (len(batches) != self.strategy.accumulated_gradient or self.strategy.step != 0):
            raise ValueError('TRUST-B/P requires exactly one complete optimizer accumulation window at step0')
        before = capture_rng()
        states = []
        self._normalizer = None
        collector = TrustHeadAccumulator(softcap=self.student_cap,
            head_bias=getattr(self.student.model.lm_head,'bias',None) is not None,
            vocab_chunk=self.kd.mp_opd_trust_vocab_chunk, force_fp32=self.mode == 'trust_p')
        self._collector = collector
        try:
            with torch.no_grad():
                for batch in batches:
                    states.append(capture_rng())
                    self.training_step(batch_loader(batch))
            after = capture_rng()
            options = {'calibrate':trust_p_calibrate_scalars} if self.mode == 'trust_p' else {}
            self._result = collector.finalize(eps_g=self.eps_g, distributed=False, **options)
            records = collector.responses
            ns = sum(r['strict_atoms'] for r in records)
            nm = sum(r['mismatch_atoms'] for r in records)
            counts = dict(strict_span_count=ns,mismatch_span_count=nm,strict_atom_count=ns,mismatch_atom_count=nm,
                strict_token_count=sum(r['strict_tokens'] for r in records),
                mismatch_token_count=sum(r['mismatch_tokens'] for r in records),
                strict_span_fraction=ns/max(ns+nm,1),mismatch_span_fraction=nm/max(ns+nm,1))
            metric_fn = trust_p_batch_metrics if self.mode == 'trust_p' else trust_batch_metrics
            self._batch_metrics = {f'mp_opd_{self.mode}_'+k: torch.tensor(float(v),device=self.teacher_head.weight.device,dtype=torch.float32)
                                   for k,v in metric_fn(self._result,counts).items()}
            prefix = 'mp_opd_'+self.mode+'_'
            self._batch_metrics.update({prefix+'full_batch':torch.tensor(1.,device=self.teacher_head.weight.device),
                prefix+'calibration_response_count':torch.tensor(float(len(records)),device=self.teacher_head.weight.device)})
            return {'rng_states':states,'after_rng':after}
        except BaseException:
            self.finish_optimizer_batch()
            raise
        finally:
            self._collector = None
            restore_rng(before)

    def finish_optimizer_batch(self):
        self._collector = self._result = None
        self._normalizer = None
        self._batch_metrics = {}


@register_algorithm('mp_opd_trust_r')
class TrustRMPOnPolicyKD(TrustMPOnPolicyKD):
    trust_mode = 'trust_r'


@register_algorithm('mp_opd_trust_b')
class TrustBMPOnPolicyKD(TrustMPOnPolicyKD):
    trust_mode = 'trust_b'


@register_algorithm('mp_opd_trust_p')
class TrustPMPOnPolicyKD(TrustMPOnPolicyKD):
    trust_mode = 'trust_p'