Download src/psi/models/interaction_arch/control_metrics.py from Tinghong-Ye22/code-HFD: direct link, hf CLI and curl.
- Browser
- Download file 5.21 kB
-
https://huggingface.co/Tinghong-Ye22/code-HFD/resolve/main/src/psi/models/interaction_arch/control_metrics.py
- Command line
-
hf download hf://Tinghong-Ye22/code-HFD/src/psi/models/interaction_arch/control_metrics.py
-
curl -L -o control_metrics.py https://huggingface.co/Tinghong-Ye22/code-HFD/resolve/main/src/psi/models/interaction_arch/control_metrics.py
5.21 kB
| """Packet-level diagnostics only; never modify losses or teacher-force state.""" | |
| import json | |
| from pathlib import Path | |
| import torch | |
| def record_control(trainer, output, labels, update_mask, source_ids): | |
| tag = "train" if trainer.model.training else "eval" | |
| bank = getattr(trainer, "_token_control_meters", None) | |
| if bank is None: | |
| bank = trainer._token_control_meters = {} | |
| if tag not in bank: | |
| bank[tag] = dict(confusion=torch.zeros(9, device=labels.device, dtype=torch.long), | |
| paired=0, gt_switch=0, pred_switch=0, exact_switch=0, | |
| false_switch=0, triplets=0, flicker=0) | |
| active = torch.tensor(update_mask, device=labels.device, dtype=torch.bool) | |
| pred = output.control_logits.detach().argmax(-1) | |
| counts = torch.bincount((labels[active] * 3 + pred[active]), minlength=9) | |
| bank[tag]["confusion"] += counts | |
| state = getattr(trainer, "_token_control_sequences", None) | |
| if state is None: | |
| state = trainer._token_control_sequences = {} | |
| previous = state.setdefault(tag, {}) | |
| observed = torch.stack((labels, pred), -1).detach().cpu().tolist() | |
| windows = output.kv_cache.logical_window_counts | |
| meter = bank[tag] | |
| for i, (source, enabled, pair, window) in enumerate(zip(source_ids, update_mask, observed, windows)): | |
| if not enabled: | |
| continue | |
| # Bound memory to current slots, and never count source/epoch boundaries. | |
| old = previous.get(i) | |
| gt, prediction = pair | |
| if old is not None and old[0] == source and old[1] + 1 == window: | |
| old_gt, old_pred, older_pred = old[2:] | |
| gs, ps = gt != old_gt, prediction != old_pred | |
| meter["paired"] += 1 | |
| meter["gt_switch"] += gs | |
| meter["pred_switch"] += ps | |
| meter["exact_switch"] += int(gs and ps and old_pred == old_gt and prediction == gt) | |
| meter["false_switch"] += int(ps and not gs) | |
| if older_pred is not None: | |
| meter["triplets"] += 1 | |
| meter["flicker"] += int(older_pred == prediction and prediction != old_pred) | |
| else: | |
| old_pred = None | |
| previous[i] = (source, window, gt, prediction, old_pred) | |
| # Bounded interpretable cases, not an invented transition success score. | |
| samples = getattr(trainer, "_token_control_samples", []) | |
| if len(samples) < 64: | |
| samples.append(dict(split=tag, source_ids=list(source_ids), active=list(update_mask), | |
| gt=labels.detach().cpu().tolist(), pred=pred.cpu().tolist(), | |
| logits=output.control_logits.detach().float().cpu().tolist())) | |
| trainer._token_control_samples = samples | |
| def flush(trainer, tag): | |
| bank = getattr(trainer, "_token_control_meters", {}) | |
| names = ("paired", "gt_switch", "pred_switch", "exact_switch", "false_switch", "triplets", "flicker") | |
| meter = bank.pop(tag, None) | |
| if meter is None: | |
| meter = dict(confusion=torch.zeros(9, device=trainer.accelerator.device, dtype=torch.long), | |
| **{name: 0 for name in names}) | |
| totals = torch.cat((meter["confusion"], torch.tensor( | |
| [meter[name] for name in names], device=meter["confusion"].device, dtype=torch.long))) | |
| if torch.distributed.is_available() and torch.distributed.is_initialized(): | |
| torch.distributed.all_reduce(totals) | |
| meter["confusion"] = totals[:9] | |
| meter.update(zip(names, totals[9:].tolist())) | |
| cm = meter["confusion"].reshape(3,3).float() | |
| if cm.sum().item() == 0: | |
| return {} | |
| tp = cm.diag() | |
| precision = tp / cm.sum(0).clamp_min(1) | |
| recall = tp / cm.sum(1).clamp_min(1) | |
| f1 = 2 * precision * recall / (precision + recall).clamp_min(1e-12) | |
| result = {f"{tag}/token_control_macro_f1": f1.mean().item(), | |
| f"{tag}/token_control_accuracy": (tp.sum()/cm.sum()).item()} | |
| for name in ("paired", "gt_switch", "pred_switch", "exact_switch", "false_switch", "triplets", "flicker"): | |
| result[f"{tag}/token_{name}_count"] = meter[name] | |
| result[f"{tag}/token_exact_transition_recall"] = meter["exact_switch"] / max(meter["gt_switch"], 1) | |
| result[f"{tag}/token_false_transition_rate"] = meter["false_switch"] / max(meter["paired"], 1) | |
| result[f"{tag}/token_flicker_rate"] = meter["flicker"] / max(meter["triplets"], 1) | |
| model = trainer.accelerator.unwrap_model(trainer.model) | |
| result[f"{tag}/token_body_kv_physical_length"] = getattr(model, "control_token_last_kv", 0) | |
| if getattr(model, "control_token_arm", None) == "B" and hasattr(model, "control_token_last_ledger"): | |
| birth, mask = model.control_token_last_ledger | |
| lengths = [torch.unique(row[(valid.bool()) & (row >= 0)]).numel() for row, valid in zip(birth, mask)] | |
| result[f"{tag}/token_state_history_mean_length"] = sum(lengths) / max(len(lengths), 1) | |
| names = ("listen", "motion", "idle") | |
| for i, name in enumerate(names): | |
| result[f"{tag}/token_control_{name}_precision"] = precision[i].item() | |
| result[f"{tag}/token_control_{name}_recall"] = recall[i].item() | |
| for j, predicted in enumerate(names): | |
| result[f"{tag}/token_control_cm_{name}_{predicted}"] = cm[i,j].item() | |
| return result | |