QEV-9B / head_qev.py
querypieai's picture
Add merged inference-only model, multilingual cards and research license
96adfbe verified
Raw History Blame Contribute Delete
812 Bytes
from __future__ import annotations
import math
import torch
from torch import nn
class PointerHead(nn.Module):
"""Scores each option boundary against the question decision state."""
def __init__(self, hidden_size: int, pointer_dim: int = 256, dropout: float = 0.0):
super().__init__()
self.query = nn.Linear(hidden_size, pointer_dim)
self.key = nn.Linear(hidden_size, pointer_dim)
self.dropout = nn.Dropout(dropout)
self.scale = 1 / math.sqrt(pointer_dim)
self.temperature = 1.0
def forward(self, decision: torch.Tensor, options: torch.Tensor) -> torch.Tensor:
logits = (self.key(self.dropout(options)) * self.query(self.dropout(decision))).sum(-1) * self.scale
return logits if self.training else logits / float(self.temperature)