code-HFD / src /psi /models /interaction_arch /control_metrics.py
Tinghong-Ye22's picture
Add files using upload-large-folder tool
06f9561 verified
Raw History Blame Contribute Delete
5.21 kB
"""Packet-level diagnostics only; never modify losses or teacher-force state."""
import json
from pathlib import Path
import torch
def record_control(trainer, output, labels, update_mask, source_ids):
tag = "train" if trainer.model.training else "eval"
bank = getattr(trainer, "_token_control_meters", None)
if bank is None:
bank = trainer._token_control_meters = {}
if tag not in bank:
bank[tag] = dict(confusion=torch.zeros(9, device=labels.device, dtype=torch.long),
paired=0, gt_switch=0, pred_switch=0, exact_switch=0,
false_switch=0, triplets=0, flicker=0)
active = torch.tensor(update_mask, device=labels.device, dtype=torch.bool)
pred = output.control_logits.detach().argmax(-1)
counts = torch.bincount((labels[active] * 3 + pred[active]), minlength=9)
bank[tag]["confusion"] += counts
state = getattr(trainer, "_token_control_sequences", None)
if state is None:
state = trainer._token_control_sequences = {}
previous = state.setdefault(tag, {})
observed = torch.stack((labels, pred), -1).detach().cpu().tolist()
windows = output.kv_cache.logical_window_counts
meter = bank[tag]
for i, (source, enabled, pair, window) in enumerate(zip(source_ids, update_mask, observed, windows)):
if not enabled:
continue
# Bound memory to current slots, and never count source/epoch boundaries.
old = previous.get(i)
gt, prediction = pair
if old is not None and old[0] == source and old[1] + 1 == window:
old_gt, old_pred, older_pred = old[2:]
gs, ps = gt != old_gt, prediction != old_pred
meter["paired"] += 1
meter["gt_switch"] += gs
meter["pred_switch"] += ps
meter["exact_switch"] += int(gs and ps and old_pred == old_gt and prediction == gt)
meter["false_switch"] += int(ps and not gs)
if older_pred is not None:
meter["triplets"] += 1
meter["flicker"] += int(older_pred == prediction and prediction != old_pred)
else:
old_pred = None
previous[i] = (source, window, gt, prediction, old_pred)
# Bounded interpretable cases, not an invented transition success score.
samples = getattr(trainer, "_token_control_samples", [])
if len(samples) < 64:
samples.append(dict(split=tag, source_ids=list(source_ids), active=list(update_mask),
gt=labels.detach().cpu().tolist(), pred=pred.cpu().tolist(),
logits=output.control_logits.detach().float().cpu().tolist()))
trainer._token_control_samples = samples
def flush(trainer, tag):
bank = getattr(trainer, "_token_control_meters", {})
names = ("paired", "gt_switch", "pred_switch", "exact_switch", "false_switch", "triplets", "flicker")
meter = bank.pop(tag, None)
if meter is None:
meter = dict(confusion=torch.zeros(9, device=trainer.accelerator.device, dtype=torch.long),
**{name: 0 for name in names})
totals = torch.cat((meter["confusion"], torch.tensor(
[meter[name] for name in names], device=meter["confusion"].device, dtype=torch.long)))
if torch.distributed.is_available() and torch.distributed.is_initialized():
torch.distributed.all_reduce(totals)
meter["confusion"] = totals[:9]
meter.update(zip(names, totals[9:].tolist()))
cm = meter["confusion"].reshape(3,3).float()
if cm.sum().item() == 0:
return {}
tp = cm.diag()
precision = tp / cm.sum(0).clamp_min(1)
recall = tp / cm.sum(1).clamp_min(1)
f1 = 2 * precision * recall / (precision + recall).clamp_min(1e-12)
result = {f"{tag}/token_control_macro_f1": f1.mean().item(),
f"{tag}/token_control_accuracy": (tp.sum()/cm.sum()).item()}
for name in ("paired", "gt_switch", "pred_switch", "exact_switch", "false_switch", "triplets", "flicker"):
result[f"{tag}/token_{name}_count"] = meter[name]
result[f"{tag}/token_exact_transition_recall"] = meter["exact_switch"] / max(meter["gt_switch"], 1)
result[f"{tag}/token_false_transition_rate"] = meter["false_switch"] / max(meter["paired"], 1)
result[f"{tag}/token_flicker_rate"] = meter["flicker"] / max(meter["triplets"], 1)
model = trainer.accelerator.unwrap_model(trainer.model)
result[f"{tag}/token_body_kv_physical_length"] = getattr(model, "control_token_last_kv", 0)
if getattr(model, "control_token_arm", None) == "B" and hasattr(model, "control_token_last_ledger"):
birth, mask = model.control_token_last_ledger
lengths = [torch.unique(row[(valid.bool()) & (row >= 0)]).numel() for row, valid in zip(birth, mask)]
result[f"{tag}/token_state_history_mean_length"] = sum(lengths) / max(len(lengths), 1)
names = ("listen", "motion", "idle")
for i, name in enumerate(names):
result[f"{tag}/token_control_{name}_precision"] = precision[i].item()
result[f"{tag}/token_control_{name}_recall"] = recall[i].item()
for j, predicted in enumerate(names):
result[f"{tag}/token_control_cm_{name}_{predicted}"] = cm[i,j].item()
return result