cbert33's picture
Upload calibrated FP8 KV model card and metadata
ff7679d verified
Raw History Blame Contribute Delete
2.55 kB
"""Observe the compressed MLA cache, never expanded attention K/V or RDA state."""
import torch
class LatentCacheObserver:
def __init__(self, attention):
self.attention = attention
self.maximum = 0.0
self.rows = 0
self.calls = 0
self._tail = None
self.worst_row = None
self.handles = [attention.kv_a_proj_with_mqa.register_forward_hook(self._projected),
attention.kv_a_layernorm.register_forward_hook(self._normalized)]
def _projected(self, module, inputs, output):
assert output.shape[-1] == self.attention.kv_lora_rank + self.attention.qk_pe
self._tail = output[..., self.attention.kv_lora_rank:]
def _normalized(self, module, inputs, output):
assert self._tail is not None, 'KV normalization without preceding KV projection'
assert output.shape[-1] == self.attention.kv_lora_rank
latent = torch.cat([output, self._tail], dim=-1)
self._tail = None
assert torch.isfinite(latent).all().item(), 'Nonfinite latent during calibration'
flat = latent.reshape(-1, latent.shape[-1])
row_max = flat.float().abs().amax(dim=-1)
value, index = row_max.max(dim=0)
current = value.item()
if self.worst_row is None or current > self.maximum:
self.worst_row = flat[index].detach().cpu().clone()
self.maximum = max(self.maximum, current)
self.rows += flat.shape[0]
self.calls += 1
def scale(self):
assert self.calls and self._tail is None, 'Incomplete calibration observation'
assert self.maximum > 0, 'No positive latent magnitude observed; refusing unit fallback'
return self.maximum / torch.finfo(torch.float8_e4m3fn).max
def numerical_evidence(self):
scale = self.scale()
original = self.worst_row.float()
quantized = (original / scale).to(torch.float8_e4m3fn)
restored = quantized.float() * scale
assert torch.isfinite(restored).all().item()
error = restored-original
return {'rows': self.rows, 'calls': self.calls, 'amax': self.maximum,
'scale': scale, 'max_abs_error': error.abs().max().item(),
'mean_squared_error': error.square().mean().item(),
'clipped_values': int(((original/scale).abs() > torch.finfo(torch.float8_e4m3fn).max).sum().item())}
def close(self):
for handle in self.handles:
handle.remove()
self.handles.clear()
self._tail = None