from __future__ import annotations import math import torch from torch import nn class PointerHead(nn.Module): """Scores each option boundary against the question decision state.""" def __init__(self, hidden_size: int, pointer_dim: int = 256, dropout: float = 0.0): super().__init__() self.query = nn.Linear(hidden_size, pointer_dim) self.key = nn.Linear(hidden_size, pointer_dim) self.dropout = nn.Dropout(dropout) self.scale = 1 / math.sqrt(pointer_dim) self.temperature = 1.0 def forward(self, decision: torch.Tensor, options: torch.Tensor) -> torch.Tensor: logits = (self.key(self.dropout(options)) * self.query(self.dropout(decision))).sum(-1) * self.scale return logits if self.training else logits / float(self.temperature)