nsaroiu's picture
Deploy 07d224d
8538f10 verified
Raw History Blame Contribute Delete
20.4 kB
"""
model.py
Deck-completion model (see README.md's "Architecture" section): a hybrid
ID+content card embedding table, a permutation-invariant
set encoder that pools a variable-size set of card embeddings into one
vector, and a scoring head tied to the same embedding table (word2vec-style)
that scores the entire vocabulary in one forward pass.
Two set encoders are implemented, selectable via build_encoder() /
train.py's --encoder-type: DeepSetsEncoder (mean-pool + MLP -- the original
baseline, cheap enough that a bug is almost certainly in the data/training
pipeline rather than the architecture) and SetTransformerEncoder (self-
attention over the set + a CLS pooling token, compared against DeepSets on
Recall@K/MRR before being adopted as the synergy specialist below).
DeckCompletionModel itself is architecture-agnostic -- it takes
a constructed encoder rather than building one, so both share every other
piece of the pipeline (HybridCardEmbedding, the tied scoring head, training/
eval code) unchanged.
PruneModel (bottom of this file) is a different shape of model entirely:
every encoder above is permutation-INVARIANT (pools a set down to one
vector, then scores the whole vocab from it). PruneModel is permutation-
EQUIVARIANT -- one score per input card, not one pooled vector -- because
its job is "does this specific card, already in this specific deck, belong"
rather than "what's missing". See its own docstring and training/
train_prune.py / training/prune_dataset.py for the full design.
"""
from __future__ import annotations
import numpy as np
import torch
import torch.nn as nn
# Canonical Commander deck size (99 nonland/nonbasic cards + 1-2
# commanders, rounded), used only to normalize the optional completeness
# feature below into a roughly [0, 1] ratio.
DECK_SIZE = 100
class HybridCardEmbedding(nn.Module):
"""Per-card embedding = learned id embedding + a small MLP over
structured features (CardTokenizer.feature_matrix()) + a small MLP over
semantic text features (CardTokenizer.text_embeddings()), summed. Gives
every card, even one with zero decklist occurrences, a meaningful
starting representation instead of a near-random one.
structured_features/text_features are registered as (non-trainable)
buffers -- they're fixed per-card content, only the projection layers
(and the id embedding) are learned.
"""
def __init__(
self,
vocab_size: int,
embed_dim: int,
structured_features: np.ndarray,
text_features: np.ndarray,
pad_id: int,
):
super().__init__()
self.vocab_size = vocab_size
self.embed_dim = embed_dim
self.id_embedding = nn.Embedding(vocab_size, embed_dim, padding_idx=pad_id)
nn.init.normal_(self.id_embedding.weight, mean=0.0, std=0.02)
# persistent=False: these are deterministically reproducible from the
# tokenizer file (feature_matrix()/text_embeddings()), not learned --
# excluding them from state_dict keeps checkpoints to just the
# trainable parameters instead of needlessly duplicating ~92MB of
# fixed content features into every saved checkpoint.
self.register_buffer("structured_features", torch.tensor(structured_features, dtype=torch.float32), persistent=False)
self.register_buffer("text_features", torch.tensor(text_features, dtype=torch.float32), persistent=False)
self.structured_proj = nn.Sequential(
nn.Linear(structured_features.shape[1], embed_dim),
nn.ReLU(),
nn.Linear(embed_dim, embed_dim),
)
self.text_proj = nn.Sequential(
nn.Linear(text_features.shape[1], embed_dim),
nn.ReLU(),
nn.Linear(embed_dim, embed_dim),
)
def forward(self, ids: torch.Tensor) -> torch.Tensor:
"""ids: any integer tensor shape (...) -> embeddings (..., embed_dim)."""
id_e = self.id_embedding(ids)
struct_e = self.structured_proj(self.structured_features[ids])
text_e = self.text_proj(self.text_features[ids])
return id_e + struct_e + text_e
def full_table(self) -> torch.Tensor:
"""(vocab_size, embed_dim) embedding for every card in the vocab --
used by the tied scoring head. Recomputed each call since the
projection layers are being trained; cheap at this vocab size."""
idx = torch.arange(self.vocab_size, device=self.id_embedding.weight.device)
return self.forward(idx)
class DeepSetsEncoder(nn.Module):
"""Permutation-invariant set encoder: encode each element independently
(already done by HybridCardEmbedding), mean-pool over the set, then an
MLP on the pooled vector. Can't directly represent pairwise/combo
interactions the way self-attention can (see SetTransformerEncoder
below) -- an acceptable tradeoff for a first baseline, and still the
stronger of the two at staple recognition specifically (see
DeckCompletionModel's ensemble usage).
"""
def __init__(self, embed_dim: int, hidden_dim: int):
super().__init__()
self.mlp = nn.Sequential(
nn.Linear(embed_dim, hidden_dim),
nn.ReLU(),
nn.Linear(hidden_dim, embed_dim),
)
def forward(self, card_embeds: torch.Tensor, mask: torch.Tensor) -> torch.Tensor:
"""card_embeds: (B, L, D), mask: (B, L) bool (True = real card, not
padding) -> pooled (B, D)."""
mask_f = mask.unsqueeze(-1).to(card_embeds.dtype)
summed = (card_embeds * mask_f).sum(dim=1)
counts = mask_f.sum(dim=1).clamp(min=1.0)
pooled = summed / counts
return self.mlp(pooled)
class SetTransformerEncoder(nn.Module):
"""Permutation-invariant set encoder via self-attention: a learned CLS
token attends over the (masked) set alongside every real element, and
its output is the pooled vector -- the "Set Transformer" alternative to
DeepSetsEncoder's mean-pool, named as the natural next architecture to
try in this module's docstring above. No positional encoding, since set
order carries no meaning.
CLS-token pooling (not PMA/attention-pooling) specifically because the
CLS token is always unmasked: a set that's entirely padding (e.g.
complete_deck.py's commander-only completion, an all-False ctx_mask)
still has one real, attendable token, so there's no all-masked-row edge
case to special-case the way DeepSetsEncoder's mean-pool needs a
denominator clamp for the same input.
"""
def __init__(
self,
embed_dim: int,
num_heads: int = 4,
num_layers: int = 3,
dim_feedforward: int = 512,
dropout: float = 0.1,
):
super().__init__()
self.cls_token = nn.Parameter(torch.zeros(1, 1, embed_dim))
nn.init.normal_(self.cls_token, std=0.02)
layer = nn.TransformerEncoderLayer(
d_model=embed_dim,
nhead=num_heads,
dim_feedforward=dim_feedforward,
dropout=dropout,
activation="gelu",
batch_first=True,
norm_first=True, # Pre-LN: trains more stably with no LR warmup
)
self.transformer = nn.TransformerEncoder(layer, num_layers=num_layers)
def forward(self, card_embeds: torch.Tensor, mask: torch.Tensor) -> torch.Tensor:
"""card_embeds: (B, L, D), mask: (B, L) bool (True = real card, not
padding) -> pooled (B, D), the transformer's CLS output."""
batch_size = card_embeds.size(0)
cls = self.cls_token.expand(batch_size, 1, -1)
x = torch.cat([cls, card_embeds], dim=1)
full_mask = torch.cat([mask.new_ones(batch_size, 1), mask], dim=1)
# nn.TransformerEncoder's key_padding_mask convention is inverted
# from this codebase's (True = ignore, vs. True = real card here).
out = self.transformer(x, src_key_padding_mask=~full_mask)
return out[:, 0, :]
def build_encoder(encoder_type: str, embed_dim: int, hidden_dim: int, **kwargs) -> nn.Module:
"""Constructs a set encoder by name, from a checkpoint's ("encoder_type",
"encoder_kwargs") or train.py's --encoder-type CLI flag -- the single
place both train.py and evaluate.py go to reconstruct an encoder, so
they can never disagree on how."""
if encoder_type == "deepsets":
return DeepSetsEncoder(embed_dim, hidden_dim)
if encoder_type == "set_transformer":
return SetTransformerEncoder(embed_dim, **kwargs)
raise ValueError(f"Unknown encoder_type: {encoder_type!r}")
class DeckCompletionModel(nn.Module):
"""Commander(s) and context cards are each pooled by the same set
encoder (a commander set is just a small card set), combined into one
deck vector, then scored against the full card vocabulary via a dot
product with the tied embedding table.
Takes an already-constructed encoder (DeepSetsEncoder or
SetTransformerEncoder, see build_encoder above) rather than building one
itself, so this class stays architecture-agnostic -- deck_proj's job
(project concatenated ctx+cmdr vectors to a deck vector) is identical
either way.
use_completeness_feature: appends one extra scalar (known-card count /
DECK_SIZE) to deck_proj's input. Motivated by a read-only probe finding
real signal: a linear regression of known_count against the raw pooled
ctx_vec (before deck_proj) reached held-out R^2=0.21 on a production
checkpoint that was never trained to preserve it, but R^2=0.05 against
the post-deck_proj deck_vec actually used for scoring -- i.e. deck_proj
already has a natural route to this information available (mean-
pooling's per-dimension variance shrinks as more cards are averaged in)
and mostly discards it, since nothing in training ever rewarded keeping
it. In practice a real training run found this hurt more than it
helped (regressed staple-tier MRR and held-out-commander recall with no
compensating gain) -- kept here, off by default (False, so every
existing checkpoint/caller is unaffected), for the record rather than
wired into any production checkpoint."""
def __init__(
self,
card_embedding: HybridCardEmbedding,
encoder: nn.Module,
hidden_dim: int,
use_completeness_feature: bool = False,
):
super().__init__()
self.card_embedding = card_embedding
embed_dim = card_embedding.embed_dim
self.encoder = encoder
self.use_completeness_feature = use_completeness_feature
extra_dim = 1 if use_completeness_feature else 0
self.deck_proj = nn.Sequential(
nn.Linear(embed_dim * 2 + extra_dim, hidden_dim),
nn.ReLU(),
nn.Linear(hidden_dim, embed_dim),
)
def deck_vector(
self,
ctx_ids: torch.Tensor,
ctx_mask: torch.Tensor,
cmdr_ids: torch.Tensor,
cmdr_mask: torch.Tensor,
) -> torch.Tensor:
ctx_vec = self.encoder(self.card_embedding(ctx_ids), ctx_mask)
cmdr_vec = self.encoder(self.card_embedding(cmdr_ids), cmdr_mask)
parts = [ctx_vec, cmdr_vec]
if self.use_completeness_feature:
known = (ctx_mask.sum(dim=1) + cmdr_mask.sum(dim=1)).to(ctx_vec.dtype)
known_ratio = (known / DECK_SIZE).clamp(max=1.0).unsqueeze(-1) # (B, 1)
parts.append(known_ratio)
return self.deck_proj(torch.cat(parts, dim=-1))
def forward(
self,
ctx_ids: torch.Tensor,
ctx_mask: torch.Tensor,
cmdr_ids: torch.Tensor,
cmdr_mask: torch.Tensor,
) -> torch.Tensor:
"""Returns (B, vocab_size) logits -- one per card in the vocab."""
deck_vec = self.deck_vector(ctx_ids, ctx_mask, cmdr_ids, cmdr_mask)
full_table = self.card_embedding.full_table() # (V, D)
return deck_vec @ full_table.T
class TwoHeadDeckCompletionModel(nn.Module):
"""Two specialized scoring heads sharing one trunk (HybridCardEmbedding
+ a set encoder), instead of DeckCompletionModel's single deck_proj --
built to replace training/ensemble.py's two-FULL-MODEL specialization
(a staple-precision checkpoint + a synergy-MRR checkpoint, fused at
generation time) with something that gets the same specialization at
roughly one model's cost: the expensive part (embedding lookups, set
encoding) is shared, only the small final projection-and-score step is
duplicated. See training/ensemble.py's docstring for why one model
under one loss couldn't hold both properties at once, and why this
two-head attempt was itself eventually rejected in favor of two fully
independent checkpoints (kept here for the record).
staple_proj is meant to be trained with a pointwise loss (BCE,
calibrated per-card precision); synergy_proj with a competitive/ranking
loss (multi_positive_softmax_loss, sharp relative ordering) -- see
training/train_two_head.py's build_loss_fn-equivalent. Both heads score
against the SAME tied full_table(), so a card's identity/content
embedding is shared between what the two heads say about it; only the
deck-context-to-score projection differs.
"""
def __init__(self, card_embedding: HybridCardEmbedding, encoder: nn.Module, hidden_dim: int):
super().__init__()
self.card_embedding = card_embedding
self.encoder = encoder
embed_dim = card_embedding.embed_dim
def make_head() -> nn.Module:
return nn.Sequential(
nn.Linear(embed_dim * 2, hidden_dim),
nn.ReLU(),
nn.Linear(hidden_dim, embed_dim),
)
self.staple_proj = make_head()
self.synergy_proj = make_head()
def _pool(
self,
ctx_ids: torch.Tensor,
ctx_mask: torch.Tensor,
cmdr_ids: torch.Tensor,
cmdr_mask: torch.Tensor,
) -> torch.Tensor:
ctx_vec = self.encoder(self.card_embedding(ctx_ids), ctx_mask)
cmdr_vec = self.encoder(self.card_embedding(cmdr_ids), cmdr_mask)
return torch.cat([ctx_vec, cmdr_vec], dim=-1)
def forward(
self,
ctx_ids: torch.Tensor,
ctx_mask: torch.Tensor,
cmdr_ids: torch.Tensor,
cmdr_mask: torch.Tensor,
) -> dict[str, torch.Tensor]:
"""Returns {"staple": (B, V) logits, "synergy": (B, V) logits}."""
pooled = self._pool(ctx_ids, ctx_mask, cmdr_ids, cmdr_mask)
full_table = self.card_embedding.full_table() # (V, D)
return {
"staple": self.staple_proj(pooled) @ full_table.T,
"synergy": self.synergy_proj(pooled) @ full_table.T,
}
class HeadView(nn.Module):
"""Adapts one head of a TwoHeadDeckCompletionModel to the single-tensor
`model(ctx_ids, ctx_mask, cmdr_ids, cmdr_mask) -> (B, V) logits`
interface training/metrics.py's recall_at_k/nearest_neighbors already
expect -- so a two-head
model's staple/synergy heads can be evaluated with the EXACT existing
machinery, unchanged, as if each were its own checkpoint, rather than
needing a parallel eval stack built just for this architecture."""
def __init__(self, two_head_model: TwoHeadDeckCompletionModel, head: str):
super().__init__()
if head not in ("staple", "synergy"):
raise ValueError(f"Unknown head: {head!r}")
self.model = two_head_model
self.head = head
self.card_embedding = two_head_model.card_embedding
def forward(self, *args) -> torch.Tensor:
return self.model(*args)[self.head]
class PruneEncoder(nn.Module):
"""Permutation-EQUIVARIANT self-attention encoder: every position in the
input set attends to every other position (commander(s) included) and
the encoder returns one refined embedding PER POSITION, not one pooled
vector -- see PruneModel below for why. Structurally identical to
SetTransformerEncoder's transformer stack, minus the CLS token (there's
nothing to pool to here) and minus the final [:, 0, :] slice (every
position's output is kept, not just one). No positional encoding, same
as every other set encoder in this file, since set order carries no
meaning.
"""
def __init__(
self,
embed_dim: int,
num_heads: int = 4,
num_layers: int = 3,
dim_feedforward: int = 512,
dropout: float = 0.1,
):
super().__init__()
layer = nn.TransformerEncoderLayer(
d_model=embed_dim,
nhead=num_heads,
dim_feedforward=dim_feedforward,
dropout=dropout,
activation="gelu",
batch_first=True,
norm_first=True,
)
self.transformer = nn.TransformerEncoder(layer, num_layers=num_layers)
def forward(self, embeds: torch.Tensor, mask: torch.Tensor) -> torch.Tensor:
"""embeds: (B, L, D), mask: (B, L) bool (True = real, not padding)
-> (B, L, D), one refined embedding per input position. Every
sequence passed in here always has at least one real (unmasked)
position -- the commander(s), concatenated in by PruneModel below
-- so there's no all-masked-row edge case to guard against, the same
reasoning SetTransformerEncoder's CLS token relies on."""
return self.transformer(embeds, src_key_padding_mask=~mask)
class PruneModel(nn.Module):
"""Scores each card in a candidate deck for whether it belongs, given
the commander(s) and the REST of the deck -- the deck-pruning model
(training/prune_dataset.py, training/train_prune.py). Trained as a
real-vs-corrupted discriminator: take a real deck, swap a few of its
cards for real cards pulled from OTHER decks, and predict per-card which
ones are the swaps. Motivation, in short: every generation model in
this file only ever ADDS cards; nothing evaluates a card already sitting in a
completed list, which is the one deckbuilding move a human does on a
final pass and this pipeline couldn't do at all before this model.
Structurally the mirror image of DeckCompletionModel: that model pools
a variable-size set down to ONE vector, then scores the whole vocab
against it (permutation-invariant, one output shared across every
candidate card). This model keeps one output PER input card
(permutation-equivariant, via PruneEncoder's self-attention) -- there's
no fixed vocab to score against here, only "does this specific card, in
this specific spot, fit the other 99". A learned segment embedding
distinguishes commander positions (context only, always real, never
scored) from deck-card positions (scored, possibly corrupted).
"""
def __init__(self, card_embedding: HybridCardEmbedding, encoder: PruneEncoder, hidden_dim: int):
super().__init__()
self.card_embedding = card_embedding
self.encoder = encoder
embed_dim = card_embedding.embed_dim
# 0 = deck-card position (scored), 1 = commander position (context only)
self.segment_embedding = nn.Embedding(2, embed_dim)
self.score_head = nn.Sequential(
nn.Linear(embed_dim, hidden_dim),
nn.ReLU(),
nn.Linear(hidden_dim, 1),
)
def forward(
self,
deck_ids: torch.Tensor,
deck_mask: torch.Tensor,
cmdr_ids: torch.Tensor,
cmdr_mask: torch.Tensor,
) -> torch.Tensor:
"""Returns (B, L_deck) real-vs-corrupted logits, one per deck_ids
position (higher = more confidently real). Padding positions'
logits are meaningless -- callers must mask with deck_mask, same
convention as every masked tensor elsewhere in this pipeline."""
deck_embeds = self.card_embedding(deck_ids) + self.segment_embedding(
torch.zeros_like(deck_ids)
)
cmdr_embeds = self.card_embedding(cmdr_ids) + self.segment_embedding(
torch.ones_like(cmdr_ids)
)
combined = torch.cat([deck_embeds, cmdr_embeds], dim=1)
combined_mask = torch.cat([deck_mask, cmdr_mask], dim=1)
out = self.encoder(combined, combined_mask)
deck_out = out[:, : deck_ids.shape[1], :]
return self.score_head(deck_out).squeeze(-1)