"""Packet-level diagnostics only; never modify losses or teacher-force state.""" import json from pathlib import Path import torch def record_control(trainer, output, labels, update_mask, source_ids): tag = "train" if trainer.model.training else "eval" bank = getattr(trainer, "_token_control_meters", None) if bank is None: bank = trainer._token_control_meters = {} if tag not in bank: bank[tag] = dict(confusion=torch.zeros(9, device=labels.device, dtype=torch.long), paired=0, gt_switch=0, pred_switch=0, exact_switch=0, false_switch=0, triplets=0, flicker=0) active = torch.tensor(update_mask, device=labels.device, dtype=torch.bool) pred = output.control_logits.detach().argmax(-1) counts = torch.bincount((labels[active] * 3 + pred[active]), minlength=9) bank[tag]["confusion"] += counts state = getattr(trainer, "_token_control_sequences", None) if state is None: state = trainer._token_control_sequences = {} previous = state.setdefault(tag, {}) observed = torch.stack((labels, pred), -1).detach().cpu().tolist() windows = output.kv_cache.logical_window_counts meter = bank[tag] for i, (source, enabled, pair, window) in enumerate(zip(source_ids, update_mask, observed, windows)): if not enabled: continue # Bound memory to current slots, and never count source/epoch boundaries. old = previous.get(i) gt, prediction = pair if old is not None and old[0] == source and old[1] + 1 == window: old_gt, old_pred, older_pred = old[2:] gs, ps = gt != old_gt, prediction != old_pred meter["paired"] += 1 meter["gt_switch"] += gs meter["pred_switch"] += ps meter["exact_switch"] += int(gs and ps and old_pred == old_gt and prediction == gt) meter["false_switch"] += int(ps and not gs) if older_pred is not None: meter["triplets"] += 1 meter["flicker"] += int(older_pred == prediction and prediction != old_pred) else: old_pred = None previous[i] = (source, window, gt, prediction, old_pred) # Bounded interpretable cases, not an invented transition success score. samples = getattr(trainer, "_token_control_samples", []) if len(samples) < 64: samples.append(dict(split=tag, source_ids=list(source_ids), active=list(update_mask), gt=labels.detach().cpu().tolist(), pred=pred.cpu().tolist(), logits=output.control_logits.detach().float().cpu().tolist())) trainer._token_control_samples = samples def flush(trainer, tag): bank = getattr(trainer, "_token_control_meters", {}) names = ("paired", "gt_switch", "pred_switch", "exact_switch", "false_switch", "triplets", "flicker") meter = bank.pop(tag, None) if meter is None: meter = dict(confusion=torch.zeros(9, device=trainer.accelerator.device, dtype=torch.long), **{name: 0 for name in names}) totals = torch.cat((meter["confusion"], torch.tensor( [meter[name] for name in names], device=meter["confusion"].device, dtype=torch.long))) if torch.distributed.is_available() and torch.distributed.is_initialized(): torch.distributed.all_reduce(totals) meter["confusion"] = totals[:9] meter.update(zip(names, totals[9:].tolist())) cm = meter["confusion"].reshape(3,3).float() if cm.sum().item() == 0: return {} tp = cm.diag() precision = tp / cm.sum(0).clamp_min(1) recall = tp / cm.sum(1).clamp_min(1) f1 = 2 * precision * recall / (precision + recall).clamp_min(1e-12) result = {f"{tag}/token_control_macro_f1": f1.mean().item(), f"{tag}/token_control_accuracy": (tp.sum()/cm.sum()).item()} for name in ("paired", "gt_switch", "pred_switch", "exact_switch", "false_switch", "triplets", "flicker"): result[f"{tag}/token_{name}_count"] = meter[name] result[f"{tag}/token_exact_transition_recall"] = meter["exact_switch"] / max(meter["gt_switch"], 1) result[f"{tag}/token_false_transition_rate"] = meter["false_switch"] / max(meter["paired"], 1) result[f"{tag}/token_flicker_rate"] = meter["flicker"] / max(meter["triplets"], 1) model = trainer.accelerator.unwrap_model(trainer.model) result[f"{tag}/token_body_kv_physical_length"] = getattr(model, "control_token_last_kv", 0) if getattr(model, "control_token_arm", None) == "B" and hasattr(model, "control_token_last_ledger"): birth, mask = model.control_token_last_ledger lengths = [torch.unique(row[(valid.bool()) & (row >= 0)]).numel() for row, valid in zip(birth, mask)] result[f"{tag}/token_state_history_mean_length"] = sum(lengths) / max(len(lengths), 1) names = ("listen", "motion", "idle") for i, name in enumerate(names): result[f"{tag}/token_control_{name}_precision"] = precision[i].item() result[f"{tag}/token_control_{name}_recall"] = recall[i].item() for j, predicted in enumerate(names): result[f"{tag}/token_control_cm_{name}_{predicted}"] = cm[i,j].item() return result