Download kdflow/algorithms/mp_opd.py from codemaivanngu/ctopd: direct link, hf CLI and curl.
- Browser
- Download file 16.6 kB
-
https://huggingface.co/codemaivanngu/ctopd/resolve/main/kdflow/algorithms/mp_opd.py
- Command line
-
hf download hf://codemaivanngu/ctopd/kdflow/algorithms/mp_opd.py
-
curl -L -o mp_opd.py https://huggingface.co/codemaivanngu/ctopd/resolve/main/kdflow/algorithms/mp_opd.py
16.6 kB
| """Opt-in SimCT MP-OPD adapter; existing CTOPD algorithms are not subclasses. | |
| Numerical kernels live in _mp_opd_port and load only when a new mode is selected. | |
| The contract is sampled path-credit times student NLL, not shared-vocab KL. | |
| """ | |
| from __future__ import annotations | |
| import hashlib | |
| import json | |
| import random | |
| import torch | |
| from torch.utils.checkpoint import checkpoint | |
| from . import register_algorithm | |
| def fixed_partition(n, length): | |
| if n < 0 or length < 1: | |
| raise ValueError('n must be nonnegative and length positive') | |
| return tuple((start, min(start + length, n)) for start in range(0, n, length)) | |
| def random_partition(n, minimum, maximum, seed): | |
| """Exact-cover donor tiling; only the final tail may be shorter than minimum.""" | |
| if n < 0 or not 1 <= minimum <= maximum: | |
| raise ValueError('Require n >= 0 and 1 <= minimum <= maximum') | |
| rng, result, cursor = random.Random(int(seed)), [], 0 | |
| while cursor < n: | |
| high = min(maximum, n - cursor) | |
| length = rng.randint(min(minimum, high), high) | |
| result.append((cursor, cursor + length)) | |
| cursor += length | |
| return tuple(result) | |
| def response_seed(seed, student_ids, teacher_ids): | |
| # No row/rank/microbatch index: regrouping the same samples cannot retile them. | |
| payload = json.dumps([int(seed), student_ids, teacher_ids], separators=(',', ':')).encode() | |
| return int.from_bytes(hashlib.sha256(payload).digest()[:8], 'big') | |
| def final_softcap(config): | |
| config = getattr(config, 'text_config', None) or config | |
| cap = getattr(config, 'final_logit_softcapping', None) | |
| if cap is not None and float(cap) <= 0: | |
| raise ValueError('final_logit_softcapping must be positive') | |
| return None if cap is None else float(cap) | |
| class MPOnPolicyKD: | |
| mode = None | |
| def __init__(self, strategy, student_model, teacher_lm_head, | |
| student_tokenizer, teacher_tokenizer, **_kwargs): | |
| # Lazy imports keep all donor helpers off the existing Strict import path. | |
| from ._mp_opd_port._mp_opd_atoms import SimCTAtomizer | |
| from ._mp_opd_port._mp_opd_grass_span import GrassNoiseEstimator, _env_flag | |
| self.strategy, self.args = strategy, strategy.args | |
| self.student, self.teacher_head = student_model, teacher_lm_head | |
| self.kd = self.args.kd | |
| if self.kd.kd_ratio != 1.0 or self.kd.kd_temperature != 1.0 or self.kd.kd_loss_fn != 'rkl': | |
| raise ValueError('MP-OPD requires kd_ratio=1, kd_temperature=1, kd_loss_fn=rkl') | |
| if getattr(self.args, 'scenario', 'on_policy_kd') != 'on_policy_kd': | |
| raise ValueError('The MP-OPD port supports on-policy KD only') | |
| world = getattr(self.args.train, 'num_nodes', 1) * getattr(self.args.train, 'num_gpus_per_node', 1) | |
| if world != 1 or getattr(self.args.model, 'ring_attn_size', 1) != 1: | |
| raise ValueError('The initial MP-OPD port requires one student rank and ring_attn_size=1') | |
| self.atomizer = SimCTAtomizer(student_tokenizer, teacher_tokenizer) | |
| self.student_cap = final_softcap(self.student.model.config) | |
| self.teacher_cap = getattr(teacher_lm_head, '_mp_opd_softcap', None) | |
| if self.mode in {'grass_chunk', 'align'} and not _env_flag('MP_OPD_GRASS_EXACT_FP32', True): | |
| raise ValueError('The port requires MP_OPD_GRASS_EXACT_FP32 enabled for head geometry') | |
| self.noise = GrassNoiseEstimator( | |
| rho=self.kd.mp_opd_grass_sigma_rho, | |
| min_adjacent_pairs=self.kd.mp_opd_grass_sigma_min_pairs, | |
| ) if self.mode == 'grass_chunk' else None | |
| self.aligner = None | |
| if self.mode in {'grass_chunk', 'align'} and self.kd.mp_opd_chunk_source == 'xtoken': | |
| from ._mp_opd_port.xtoken_upstream_token_aligner import TokenAligner | |
| self.aligner = TokenAligner(student_tokenizer, teacher_tokenizer, '', | |
| max_comb_len=self.kd.mp_opd_alignment_max_comb_len) | |
| def state_dict(self): | |
| return {'version': 1, 'mode': self.mode, | |
| 'contract': {name: value for name, value in vars(self.kd).items() if name.startswith('mp_opd_')}, | |
| 'noise': self.noise.state_dict() if self.noise is not None else None} | |
| def load_state_dict(self, state): | |
| if (state.get('version') != 1 or state.get('mode') != self.mode | |
| or state.get('contract') != self.state_dict()['contract']): | |
| raise ValueError('Incompatible MP-OPD state') | |
| if self.noise is not None: | |
| self.noise.load_state_dict(state['noise']) | |
| def _head_logits(head, hidden, softcap, student): | |
| logits = head(hidden, skip=False) if student else head(hidden) | |
| return logits if softcap is None else torch.tanh(logits / softcap) * softcap | |
| def _selected_logps(self, head, hidden, labels, softcap, *, student): | |
| """Checkpoint token tiles so backward never retains response-wide logits.""" | |
| width = self.args.train.chunked_loss_size or 2048 | |
| if width < 1: | |
| raise ValueError('chunked_loss_size must be positive') | |
| pieces = [] | |
| def compute(h, y): | |
| logits = self._head_logits(head, h, softcap, student) | |
| return torch.log_softmax(logits.float(), -1).gather(1, y[:, None]).squeeze(1) | |
| for start in range(0, labels.numel(), width): | |
| h, y = hidden[start:start + width], labels[start:start + width] | |
| if student and torch.is_grad_enabled(): | |
| pieces.append(checkpoint(compute, h, y, use_reentrant=False)) | |
| else: | |
| pieces.append(compute(h, y)) | |
| return torch.cat(pieces) if pieces else hidden.sum(-1)[:0] | |
| def _partition(self, atoms, student_ids, teacher_ids): | |
| from ._mp_opd_port._mp_opd_grass_chunk import ( | |
| atom_chunk_ids_from_tokens, chunk_partition, run_chunk_assignment, | |
| ) | |
| if self.mode == 'atomic': | |
| return fixed_partition(len(atoms), 1), None, 0 | |
| if self.mode == 'fixed': | |
| return fixed_partition(len(atoms), self.kd.mp_opd_fixed_span_length), None, 0 | |
| if self.mode == 'random': | |
| return random_partition(len(atoms), self.kd.mp_opd_random_min_length, | |
| self.kd.mp_opd_random_max_length, | |
| response_seed(self.kd.mp_opd_random_seed, student_ids, teacher_ids)), None, 0 | |
| if self.kd.mp_opd_chunk_source == 'run': | |
| assignment = run_chunk_assignment(len(atoms), self.kd.mp_opd_chunk_run_length) | |
| else: | |
| aligned = self.aligner.align(torch.tensor([student_ids]), torch.tensor([teacher_ids])) | |
| assignment = atom_chunk_ids_from_tokens( | |
| aligned.student_chunk_id[0].tolist(), | |
| [(atom.student_start, atom.student_end) for atom in atoms], | |
| policy=self.kd.mp_opd_chunk_straddle) | |
| partition, noncontiguous = chunk_partition(assignment) | |
| return partition, assignment, noncontiguous | |
| def _effective_rates(self, credits, atoms, partition, hidden, labels, *, assignment=None, noncontiguous=0): | |
| from ._mp_opd_port._mp_opd_grass_span import hard_pooled_credits | |
| if self.mode == 'atomic': | |
| return credits.rate, {} | |
| if self.mode in {'fixed', 'random'}: | |
| return hard_pooled_credits(credits.base_credit, credits.weight, partition).to(credits.rate), {} | |
| from ._mp_opd_port._mp_opd_grass_chunk import chunk_head_gram | |
| with torch.no_grad(): | |
| logits = self._head_logits(self.student.model.lm_head, hidden, | |
| self.student_cap, True) | |
| head = chunk_head_gram( | |
| logits, hidden.detach(), labels, | |
| [(atom.student_start, atom.student_end) for atom in atoms], partition, | |
| selected_log_prob=-credits.student_token_nll, | |
| softcap=self.student_cap, | |
| head_bias=getattr(self.student.model.lm_head, 'bias', None) is not None, | |
| diagonal_only=self.kd.mp_opd_grass_geometry == 'diag') | |
| if self.mode == 'align': | |
| from ._mp_opd_port._mp_opd_align import align_chunks, align_metrics | |
| stats = align_chunks(credits.rate, head.gram, partition, eps_h=self.kd.mp_opd_align_eps_h) | |
| effective = stats['rate'].to(credits.rate.dtype) | |
| metrics = align_metrics(credits.rate, effective, head.gram, stats) | |
| else: | |
| from ._mp_opd_port._mp_opd_grass_chunk import ( | |
| apply_chunk_shrinkage, grass_chunk_metrics, grass_chunk_tables, | |
| ) | |
| from ._mp_opd_port._mp_opd_grass_span import grass_span_ids, grass_credit_residual | |
| ids = grass_span_ids(partition, len(atoms), credits.rate.device) | |
| if self.student.training: | |
| self.noise.update(credits.rate, credits.weight, same_group=ids[:-1] == ids[1:]) | |
| tables = grass_chunk_tables(credits.rate, credits.weight, head.gram, | |
| partition, self.noise.sigma2) | |
| effective, _ = apply_chunk_shrinkage(credits.rate, credits.weight, tables) | |
| effective = effective.to(credits.rate.dtype) | |
| residual = grass_credit_residual(credits.rate, credits.weight, effective) | |
| if float(residual) > 1e-6 * max(float(credits.base_credit.abs().sum()), 1.0): | |
| raise RuntimeError('GRASS-Chunk violated weighted credit conservation') | |
| metrics = grass_chunk_metrics(tables, credits.rate, credits.weight, | |
| head.gram, effective, residual, assignment=assignment, | |
| noncontiguous_splits=noncontiguous, | |
| source=self.kd.mp_opd_chunk_source) | |
| return effective, metrics | |
| def training_step(self, micro_batch): | |
| from ._mp_opd_port._mp_opd_credit import AtomCreditTensors, soft_partition_loss | |
| student_mask, teacher_mask = micro_batch['stu_loss_mask'].bool(), micro_batch['tea_loss_mask'].bool() | |
| if student_mask.shape[0] != teacher_mask.shape[0]: | |
| raise ValueError('MP-OPD teacher/student response counts differ') | |
| output = self.student(micro_batch['stu_input_ids'], attention_mask=micro_batch['stu_attn_mask'], | |
| allgather_logits=True, ring_attn_group=self.strategy.ring_attn_group, | |
| **(micro_batch.get('stu_multi_modal_inputs') or {})) | |
| hidden = output['hidden_states'][-1][student_mask] | |
| teacher_hidden = micro_batch['teacher_hiddens'] | |
| if isinstance(teacher_hidden, list): | |
| teacher_hidden = torch.cat(teacher_hidden, dim=0) | |
| if teacher_hidden.shape[0] != int(teacher_mask.sum()): | |
| raise ValueError('Teacher hidden rows do not match the response loss mask') | |
| teacher_hidden = teacher_hidden.to(self.teacher_head.weight) | |
| student_labels = micro_batch['stu_input_ids'].roll(-1, dims=1) | |
| teacher_labels = micro_batch['tea_input_ids'].roll(-1, dims=1) | |
| total, metrics, responses, atom_count = hidden.sum() * 0.0, {}, 0, 0 | |
| skipped_replacement = skipped_normalization = skipped_empty = 0 | |
| skipped_student_tokens = skipped_teacher_tokens = 0 | |
| student_offset = teacher_offset = 0 | |
| for row in range(student_mask.shape[0]): | |
| sy, ty = student_labels[row][student_mask[row]], teacher_labels[row][teacher_mask[row]] | |
| sh = hidden[student_offset:student_offset + sy.numel()] | |
| th = teacher_hidden[teacher_offset:teacher_offset + ty.numel()] | |
| student_offset += sy.numel() | |
| teacher_offset += ty.numel() | |
| sid, tid = sy.detach().cpu().tolist(), ty.detach().cpu().tolist() | |
| atomized = self.atomizer.atomize(sid, tid, sample_id=str(response_seed(0, sid, tid))) | |
| if not atomized.valid: | |
| if atomized.failure_reason in {'empty_response', 'replacement_character', 'normalization_mismatch'}: | |
| # Exclude the whole unalignable response, retaining the shared | |
| # optimizer-window denominator. Never sanitize or relabel tokens. | |
| skipped_replacement += int(atomized.failure_reason == 'replacement_character') | |
| skipped_normalization += int(atomized.failure_reason == 'normalization_mismatch') | |
| skipped_empty += int(atomized.failure_reason == 'empty_response') | |
| skipped_student_tokens += sy.numel() | |
| skipped_teacher_tokens += ty.numel() | |
| continue | |
| raise ValueError('MP-OPD atomization failed: ' + atomized.failure_reason) | |
| atoms = atomized.atoms | |
| ns, nt = atomized.covered_student_events, atomized.covered_teacher_events | |
| sy, ty, sh, th = sy[:ns], ty[:nt], sh[:ns], th[:nt] | |
| slp = self._selected_logps(self.student.model.lm_head, sh, sy, self.student_cap, student=True) | |
| with torch.no_grad(): | |
| tlp = self._selected_logps(self.teacher_head, th, ty, self.teacher_cap, student=False) | |
| ss = torch.stack([slp[a.student_start:a.student_end].sum() for a in atoms]) | |
| ts = torch.stack([tlp[a.teacher_start:a.teacher_end].sum() for a in atoms]).detach() | |
| weight = ss.new_tensor([a.student_token_count for a in atoms]) | |
| base = (ts - ss.detach()).detach() | |
| credits = AtomCreditTensors(ts, ss.detach(), base, weight, (base / weight).detach(), -ss, -slp) | |
| partition, assignment, noncontiguous = self._partition(atoms, sid[:ns], tid[:nt]) | |
| effective, extras = self._effective_rates(credits, atoms, partition, sh, sy, | |
| assignment=assignment, noncontiguous=noncontiguous) | |
| loss = soft_partition_loss(credits.current_nll, effective) | |
| if not bool(torch.isfinite(loss)): | |
| raise FloatingPointError('MP-OPD produced non-finite loss; no Atomic fallback') | |
| total = total + loss | |
| for key, value in extras.items(): | |
| key = 'distill/' + key | |
| metrics[key] = metrics.get(key, total.detach().new_zeros(())) + value.detach() | |
| responses += 1 | |
| atom_count += len(atoms) | |
| denominator = micro_batch['avg_micro_batch_token_num'] | |
| if float(denominator) <= 0: | |
| raise ValueError('MP-OPD requires a positive shared token normalizer') | |
| loss = total / denominator | |
| if not bool(torch.isfinite(loss)): | |
| raise FloatingPointError('MP-OPD produced non-finite normalized loss') | |
| info = {'train/loss': loss, 'train/kd_loss': loss, | |
| 'distill/mp_opd_valid_responses': loss.detach().new_tensor(float(responses)), | |
| 'distill/mp_opd_atoms': loss.detach().new_tensor(float(atom_count))} | |
| attempted = student_mask.shape[0] | |
| exclusions = { | |
| 'attempted_responses': attempted, | |
| 'skipped_replacement_responses': skipped_replacement, | |
| 'skipped_normalization_responses': skipped_normalization, | |
| 'skipped_empty_responses': skipped_empty, | |
| 'skipped_responses': skipped_replacement + skipped_normalization + skipped_empty, | |
| 'skipped_replacement_fraction': skipped_replacement / max(attempted, 1), | |
| 'skipped_normalization_fraction': skipped_normalization / max(attempted, 1), | |
| 'skipped_student_loss_tokens': skipped_student_tokens, | |
| 'skipped_teacher_loss_tokens': skipped_teacher_tokens, | |
| 'skipped_student_token_fraction': skipped_student_tokens / max(int(student_mask.sum()), 1), | |
| } | |
| info.update({'distill/mp_opd_' + key: loss.detach().new_tensor(float(value)) | |
| for key, value in exclusions.items()}) | |
| info.update({key: value / max(responses, 1) for key, value in metrics.items()}) | |
| return info | |
| class AtomicMPOnPolicyKD(MPOnPolicyKD): | |
| mode = 'atomic' | |
| class FixedMPOnPolicyKD(MPOnPolicyKD): | |
| mode = 'fixed' | |
| class RandomMPOnPolicyKD(MPOnPolicyKD): | |
| mode = 'random' | |
| class GrassChunkMPOnPolicyKD(MPOnPolicyKD): | |
| mode = 'grass_chunk' | |
| class AlignMPOnPolicyKD(MPOnPolicyKD): | |
| mode = 'align' | |