revert the f32 router gemv: no end-to-end gain

torch-ext/topk/layers.py

@@ -34,10 +34,6 @@ class SoftmaxTopKRouter(nn.Module):
 
     def forward(self, hidden_states: torch.Tensor):
         hidden_states = hidden_states.reshape(-1, self.hidden_dim)
-        # The router's weight is one of the tensors a GGUF leaves dense, and ggml has a gemv for
-        # those -- 8.8us against 14.5 for `F.linear` at 256x2048. The loader attaches it when it can;
-        # without it this is the model's own spelling.
-        f32_matmul = getattr(self, "_ggml_f32_matmul", None)
-        router_logits = f32_matmul(hidden_states) if f32_matmul is not None else F.linear(hidden_states, self.weight)
+        router_logits = F.linear(hidden_states, self.weight)
         scores, indices = ops.top_k(router_logits, self.top_k, True)
         return router_logits, scores.to(router_logits.dtype), indices