"""Observe the compressed MLA cache, never expanded attention K/V or RDA state.""" import torch class LatentCacheObserver: def __init__(self, attention): self.attention = attention self.maximum = 0.0 self.rows = 0 self.calls = 0 self._tail = None self.worst_row = None self.handles = [attention.kv_a_proj_with_mqa.register_forward_hook(self._projected), attention.kv_a_layernorm.register_forward_hook(self._normalized)] def _projected(self, module, inputs, output): assert output.shape[-1] == self.attention.kv_lora_rank + self.attention.qk_pe self._tail = output[..., self.attention.kv_lora_rank:] def _normalized(self, module, inputs, output): assert self._tail is not None, 'KV normalization without preceding KV projection' assert output.shape[-1] == self.attention.kv_lora_rank latent = torch.cat([output, self._tail], dim=-1) self._tail = None assert torch.isfinite(latent).all().item(), 'Nonfinite latent during calibration' flat = latent.reshape(-1, latent.shape[-1]) row_max = flat.float().abs().amax(dim=-1) value, index = row_max.max(dim=0) current = value.item() if self.worst_row is None or current > self.maximum: self.worst_row = flat[index].detach().cpu().clone() self.maximum = max(self.maximum, current) self.rows += flat.shape[0] self.calls += 1 def scale(self): assert self.calls and self._tail is None, 'Incomplete calibration observation' assert self.maximum > 0, 'No positive latent magnitude observed; refusing unit fallback' return self.maximum / torch.finfo(torch.float8_e4m3fn).max def numerical_evidence(self): scale = self.scale() original = self.worst_row.float() quantized = (original / scale).to(torch.float8_e4m3fn) restored = quantized.float() * scale assert torch.isfinite(restored).all().item() error = restored-original return {'rows': self.rows, 'calls': self.calls, 'amax': self.maximum, 'scale': scale, 'max_abs_error': error.abs().max().item(), 'mean_squared_error': error.square().mean().item(), 'clipped_values': int(((original/scale).abs() > torch.finfo(torch.float8_e4m3fn).max).sum().item())} def close(self): for handle in self.handles: handle.remove() self.handles.clear() self._tail = None