"""Scalar records for vector gates; no model mutation or loss change.""" import torch def module_statistics(modules): with torch.no_grad(): return {'gates':{k:float(m.alpha.detach().double().mean()) for k,m in modules.items()}, 'gate_max_abs':{k:float(m.alpha.detach().abs().max()) for k,m in modules.items()}, 'gate_norm':{k:float(m.alpha.detach().double().norm()) for k,m in modules.items()}, 'W0_norm':{k:sum(float(w.detach().double().square().sum()) for w in m.w0)**.5 for k,m in modules.items()}, 'inner_lr':{k:float(m.inner_lr) for k,m in modules.items()}}