"""Opt-in SimCT MP-OPD adapter; existing CTOPD algorithms are not subclasses. Numerical kernels live in _mp_opd_port and load only when a new mode is selected. The contract is sampled path-credit times student NLL, not shared-vocab KL. """ from __future__ import annotations import hashlib import json import random import torch from torch.utils.checkpoint import checkpoint from . import register_algorithm def fixed_partition(n, length): if n < 0 or length < 1: raise ValueError('n must be nonnegative and length positive') return tuple((start, min(start + length, n)) for start in range(0, n, length)) def random_partition(n, minimum, maximum, seed): """Exact-cover donor tiling; only the final tail may be shorter than minimum.""" if n < 0 or not 1 <= minimum <= maximum: raise ValueError('Require n >= 0 and 1 <= minimum <= maximum') rng, result, cursor = random.Random(int(seed)), [], 0 while cursor < n: high = min(maximum, n - cursor) length = rng.randint(min(minimum, high), high) result.append((cursor, cursor + length)) cursor += length return tuple(result) def response_seed(seed, student_ids, teacher_ids): # No row/rank/microbatch index: regrouping the same samples cannot retile them. payload = json.dumps([int(seed), student_ids, teacher_ids], separators=(',', ':')).encode() return int.from_bytes(hashlib.sha256(payload).digest()[:8], 'big') def final_softcap(config): config = getattr(config, 'text_config', None) or config cap = getattr(config, 'final_logit_softcapping', None) if cap is not None and float(cap) <= 0: raise ValueError('final_logit_softcapping must be positive') return None if cap is None else float(cap) class MPOnPolicyKD: mode = None def __init__(self, strategy, student_model, teacher_lm_head, student_tokenizer, teacher_tokenizer, **_kwargs): # Lazy imports keep all donor helpers off the existing Strict import path. from ._mp_opd_port._mp_opd_atoms import SimCTAtomizer from ._mp_opd_port._mp_opd_grass_span import GrassNoiseEstimator, _env_flag self.strategy, self.args = strategy, strategy.args self.student, self.teacher_head = student_model, teacher_lm_head self.kd = self.args.kd if self.kd.kd_ratio != 1.0 or self.kd.kd_temperature != 1.0 or self.kd.kd_loss_fn != 'rkl': raise ValueError('MP-OPD requires kd_ratio=1, kd_temperature=1, kd_loss_fn=rkl') if getattr(self.args, 'scenario', 'on_policy_kd') != 'on_policy_kd': raise ValueError('The MP-OPD port supports on-policy KD only') world = getattr(self.args.train, 'num_nodes', 1) * getattr(self.args.train, 'num_gpus_per_node', 1) if world != 1 or getattr(self.args.model, 'ring_attn_size', 1) != 1: raise ValueError('The initial MP-OPD port requires one student rank and ring_attn_size=1') self.atomizer = SimCTAtomizer(student_tokenizer, teacher_tokenizer) self.student_cap = final_softcap(self.student.model.config) self.teacher_cap = getattr(teacher_lm_head, '_mp_opd_softcap', None) if self.mode in {'grass_chunk', 'align'} and not _env_flag('MP_OPD_GRASS_EXACT_FP32', True): raise ValueError('The port requires MP_OPD_GRASS_EXACT_FP32 enabled for head geometry') self.noise = GrassNoiseEstimator( rho=self.kd.mp_opd_grass_sigma_rho, min_adjacent_pairs=self.kd.mp_opd_grass_sigma_min_pairs, ) if self.mode == 'grass_chunk' else None self.aligner = None if self.mode in {'grass_chunk', 'align'} and self.kd.mp_opd_chunk_source == 'xtoken': from ._mp_opd_port.xtoken_upstream_token_aligner import TokenAligner self.aligner = TokenAligner(student_tokenizer, teacher_tokenizer, '', max_comb_len=self.kd.mp_opd_alignment_max_comb_len) def state_dict(self): return {'version': 1, 'mode': self.mode, 'contract': {name: value for name, value in vars(self.kd).items() if name.startswith('mp_opd_')}, 'noise': self.noise.state_dict() if self.noise is not None else None} def load_state_dict(self, state): if (state.get('version') != 1 or state.get('mode') != self.mode or state.get('contract') != self.state_dict()['contract']): raise ValueError('Incompatible MP-OPD state') if self.noise is not None: self.noise.load_state_dict(state['noise']) @staticmethod def _head_logits(head, hidden, softcap, student): logits = head(hidden, skip=False) if student else head(hidden) return logits if softcap is None else torch.tanh(logits / softcap) * softcap def _selected_logps(self, head, hidden, labels, softcap, *, student): """Checkpoint token tiles so backward never retains response-wide logits.""" width = self.args.train.chunked_loss_size or 2048 if width < 1: raise ValueError('chunked_loss_size must be positive') pieces = [] def compute(h, y): logits = self._head_logits(head, h, softcap, student) return torch.log_softmax(logits.float(), -1).gather(1, y[:, None]).squeeze(1) for start in range(0, labels.numel(), width): h, y = hidden[start:start + width], labels[start:start + width] if student and torch.is_grad_enabled(): pieces.append(checkpoint(compute, h, y, use_reentrant=False)) else: pieces.append(compute(h, y)) return torch.cat(pieces) if pieces else hidden.sum(-1)[:0] def _partition(self, atoms, student_ids, teacher_ids): from ._mp_opd_port._mp_opd_grass_chunk import ( atom_chunk_ids_from_tokens, chunk_partition, run_chunk_assignment, ) if self.mode == 'atomic': return fixed_partition(len(atoms), 1), None, 0 if self.mode == 'fixed': return fixed_partition(len(atoms), self.kd.mp_opd_fixed_span_length), None, 0 if self.mode == 'random': return random_partition(len(atoms), self.kd.mp_opd_random_min_length, self.kd.mp_opd_random_max_length, response_seed(self.kd.mp_opd_random_seed, student_ids, teacher_ids)), None, 0 if self.kd.mp_opd_chunk_source == 'run': assignment = run_chunk_assignment(len(atoms), self.kd.mp_opd_chunk_run_length) else: aligned = self.aligner.align(torch.tensor([student_ids]), torch.tensor([teacher_ids])) assignment = atom_chunk_ids_from_tokens( aligned.student_chunk_id[0].tolist(), [(atom.student_start, atom.student_end) for atom in atoms], policy=self.kd.mp_opd_chunk_straddle) partition, noncontiguous = chunk_partition(assignment) return partition, assignment, noncontiguous def _effective_rates(self, credits, atoms, partition, hidden, labels, *, assignment=None, noncontiguous=0): from ._mp_opd_port._mp_opd_grass_span import hard_pooled_credits if self.mode == 'atomic': return credits.rate, {} if self.mode in {'fixed', 'random'}: return hard_pooled_credits(credits.base_credit, credits.weight, partition).to(credits.rate), {} from ._mp_opd_port._mp_opd_grass_chunk import chunk_head_gram with torch.no_grad(): logits = self._head_logits(self.student.model.lm_head, hidden, self.student_cap, True) head = chunk_head_gram( logits, hidden.detach(), labels, [(atom.student_start, atom.student_end) for atom in atoms], partition, selected_log_prob=-credits.student_token_nll, softcap=self.student_cap, head_bias=getattr(self.student.model.lm_head, 'bias', None) is not None, diagonal_only=self.kd.mp_opd_grass_geometry == 'diag') if self.mode == 'align': from ._mp_opd_port._mp_opd_align import align_chunks, align_metrics stats = align_chunks(credits.rate, head.gram, partition, eps_h=self.kd.mp_opd_align_eps_h) effective = stats['rate'].to(credits.rate.dtype) metrics = align_metrics(credits.rate, effective, head.gram, stats) else: from ._mp_opd_port._mp_opd_grass_chunk import ( apply_chunk_shrinkage, grass_chunk_metrics, grass_chunk_tables, ) from ._mp_opd_port._mp_opd_grass_span import grass_span_ids, grass_credit_residual ids = grass_span_ids(partition, len(atoms), credits.rate.device) if self.student.training: self.noise.update(credits.rate, credits.weight, same_group=ids[:-1] == ids[1:]) tables = grass_chunk_tables(credits.rate, credits.weight, head.gram, partition, self.noise.sigma2) effective, _ = apply_chunk_shrinkage(credits.rate, credits.weight, tables) effective = effective.to(credits.rate.dtype) residual = grass_credit_residual(credits.rate, credits.weight, effective) if float(residual) > 1e-6 * max(float(credits.base_credit.abs().sum()), 1.0): raise RuntimeError('GRASS-Chunk violated weighted credit conservation') metrics = grass_chunk_metrics(tables, credits.rate, credits.weight, head.gram, effective, residual, assignment=assignment, noncontiguous_splits=noncontiguous, source=self.kd.mp_opd_chunk_source) return effective, metrics def training_step(self, micro_batch): from ._mp_opd_port._mp_opd_credit import AtomCreditTensors, soft_partition_loss student_mask, teacher_mask = micro_batch['stu_loss_mask'].bool(), micro_batch['tea_loss_mask'].bool() if student_mask.shape[0] != teacher_mask.shape[0]: raise ValueError('MP-OPD teacher/student response counts differ') output = self.student(micro_batch['stu_input_ids'], attention_mask=micro_batch['stu_attn_mask'], allgather_logits=True, ring_attn_group=self.strategy.ring_attn_group, **(micro_batch.get('stu_multi_modal_inputs') or {})) hidden = output['hidden_states'][-1][student_mask] teacher_hidden = micro_batch['teacher_hiddens'] if isinstance(teacher_hidden, list): teacher_hidden = torch.cat(teacher_hidden, dim=0) if teacher_hidden.shape[0] != int(teacher_mask.sum()): raise ValueError('Teacher hidden rows do not match the response loss mask') teacher_hidden = teacher_hidden.to(self.teacher_head.weight) student_labels = micro_batch['stu_input_ids'].roll(-1, dims=1) teacher_labels = micro_batch['tea_input_ids'].roll(-1, dims=1) total, metrics, responses, atom_count = hidden.sum() * 0.0, {}, 0, 0 skipped_replacement = skipped_normalization = skipped_empty = 0 skipped_student_tokens = skipped_teacher_tokens = 0 student_offset = teacher_offset = 0 for row in range(student_mask.shape[0]): sy, ty = student_labels[row][student_mask[row]], teacher_labels[row][teacher_mask[row]] sh = hidden[student_offset:student_offset + sy.numel()] th = teacher_hidden[teacher_offset:teacher_offset + ty.numel()] student_offset += sy.numel() teacher_offset += ty.numel() sid, tid = sy.detach().cpu().tolist(), ty.detach().cpu().tolist() atomized = self.atomizer.atomize(sid, tid, sample_id=str(response_seed(0, sid, tid))) if not atomized.valid: if atomized.failure_reason in {'empty_response', 'replacement_character', 'normalization_mismatch'}: # Exclude the whole unalignable response, retaining the shared # optimizer-window denominator. Never sanitize or relabel tokens. skipped_replacement += int(atomized.failure_reason == 'replacement_character') skipped_normalization += int(atomized.failure_reason == 'normalization_mismatch') skipped_empty += int(atomized.failure_reason == 'empty_response') skipped_student_tokens += sy.numel() skipped_teacher_tokens += ty.numel() continue raise ValueError('MP-OPD atomization failed: ' + atomized.failure_reason) atoms = atomized.atoms ns, nt = atomized.covered_student_events, atomized.covered_teacher_events sy, ty, sh, th = sy[:ns], ty[:nt], sh[:ns], th[:nt] slp = self._selected_logps(self.student.model.lm_head, sh, sy, self.student_cap, student=True) with torch.no_grad(): tlp = self._selected_logps(self.teacher_head, th, ty, self.teacher_cap, student=False) ss = torch.stack([slp[a.student_start:a.student_end].sum() for a in atoms]) ts = torch.stack([tlp[a.teacher_start:a.teacher_end].sum() for a in atoms]).detach() weight = ss.new_tensor([a.student_token_count for a in atoms]) base = (ts - ss.detach()).detach() credits = AtomCreditTensors(ts, ss.detach(), base, weight, (base / weight).detach(), -ss, -slp) partition, assignment, noncontiguous = self._partition(atoms, sid[:ns], tid[:nt]) effective, extras = self._effective_rates(credits, atoms, partition, sh, sy, assignment=assignment, noncontiguous=noncontiguous) loss = soft_partition_loss(credits.current_nll, effective) if not bool(torch.isfinite(loss)): raise FloatingPointError('MP-OPD produced non-finite loss; no Atomic fallback') total = total + loss for key, value in extras.items(): key = 'distill/' + key metrics[key] = metrics.get(key, total.detach().new_zeros(())) + value.detach() responses += 1 atom_count += len(atoms) denominator = micro_batch['avg_micro_batch_token_num'] if float(denominator) <= 0: raise ValueError('MP-OPD requires a positive shared token normalizer') loss = total / denominator if not bool(torch.isfinite(loss)): raise FloatingPointError('MP-OPD produced non-finite normalized loss') info = {'train/loss': loss, 'train/kd_loss': loss, 'distill/mp_opd_valid_responses': loss.detach().new_tensor(float(responses)), 'distill/mp_opd_atoms': loss.detach().new_tensor(float(atom_count))} attempted = student_mask.shape[0] exclusions = { 'attempted_responses': attempted, 'skipped_replacement_responses': skipped_replacement, 'skipped_normalization_responses': skipped_normalization, 'skipped_empty_responses': skipped_empty, 'skipped_responses': skipped_replacement + skipped_normalization + skipped_empty, 'skipped_replacement_fraction': skipped_replacement / max(attempted, 1), 'skipped_normalization_fraction': skipped_normalization / max(attempted, 1), 'skipped_student_loss_tokens': skipped_student_tokens, 'skipped_teacher_loss_tokens': skipped_teacher_tokens, 'skipped_student_token_fraction': skipped_student_tokens / max(int(student_mask.sum()), 1), } info.update({'distill/mp_opd_' + key: loss.detach().new_tensor(float(value)) for key, value in exclusions.items()}) info.update({key: value / max(responses, 1) for key, value in metrics.items()}) return info @register_algorithm('mp_opd_atomic') class AtomicMPOnPolicyKD(MPOnPolicyKD): mode = 'atomic' @register_algorithm('mp_opd_fixed') class FixedMPOnPolicyKD(MPOnPolicyKD): mode = 'fixed' @register_algorithm('mp_opd_random') class RandomMPOnPolicyKD(MPOnPolicyKD): mode = 'random' @register_algorithm('mp_opd_grass_chunk') class GrassChunkMPOnPolicyKD(MPOnPolicyKD): mode = 'grass_chunk' @register_algorithm('mp_opd_align') class AlignMPOnPolicyKD(MPOnPolicyKD): mode = 'align'