""" PNAConv-based Graph Metanetwork for MetaPruning. Paper: "Meta Pruning via Graph Metanetworks" (arXiv:2506.12041) Appendix B.1 describes the full architecture. """ import torch import torch.nn as nn import torch.nn.functional as F from typing import Optional class EdgeInvertor(nn.Module): """ EdgeInvertor from paper Eq. (16): [1,1,...,1 (hidden_dim/2), -1,-1,...,-1 (hidden_dim/2)] Used to encode directional edge features. """ def __init__(self, hidden_dim: int): super().__init__() half = hidden_dim // 2 self.register_buffer("invertor", torch.cat([ torch.ones(half), -torch.ones(hidden_dim - half) ])) def forward(self, edge_features: torch.Tensor) -> torch.Tensor: return edge_features * self.invertor.unsqueeze(0) class PNAConvLayer(nn.Module): """ A single message passing layer for the metanetwork. Implements Eqs. (12-19) from Appendix B.1. Each layer: 1. Computes messages m_ij and m'_ij (Eq. 14-15) 2. Aggregates messages using MEAN, STD, MAX, MIN (Eq. 17-18) 3. Updates node features (Eq. 17) 4. Updates edge features (Eq. 19) """ def __init__(self, hidden_dim: int, dropout: float = 0.0): super().__init__() self.hidden_dim = hidden_dim # MLPs for computing messages (Eq. 14-15) self.mlp_node_1 = nn.Sequential( nn.Linear(hidden_dim, hidden_dim), nn.SiLU(), nn.Linear(hidden_dim, hidden_dim), ) self.mlp_node_2 = nn.Sequential( nn.Linear(hidden_dim, hidden_dim), nn.SiLU(), nn.Linear(hidden_dim, hidden_dim), ) # Edge invertor self.edge_invertor = EdgeInvertor(hidden_dim) # MLP for aggregation (Eq. 18) # Input: concatenation of [MEAN, STD, MAX, MIN] -> 4 * hidden_dim self.mlp_aggr = nn.Sequential( nn.Linear(4 * hidden_dim, hidden_dim), nn.SiLU(), nn.Linear(hidden_dim, hidden_dim), ) # Edge update MLPs (Eq. 19) self.mlp_edge_1 = nn.Sequential( nn.Linear(hidden_dim, hidden_dim), nn.SiLU(), nn.Linear(hidden_dim, hidden_dim), ) self.mlp_edge_2 = nn.Sequential( nn.Linear(hidden_dim, hidden_dim), nn.SiLU(), nn.Linear(hidden_dim, hidden_dim), ) self.dropout = nn.Dropout(dropout) if dropout > 0 else nn.Identity() def forward( self, node_features: torch.Tensor, # [num_nodes, hidden_dim] edge_index: torch.Tensor, # [2, num_edges] edge_features: torch.Tensor, # [num_edges, hidden_dim] ) -> tuple: """ Returns: (updated_node_features, updated_edge_features) """ num_nodes = node_features.size(0) num_edges = edge_index.size(1) src = edge_index[0] # [num_edges] dst = edge_index[1] # [num_edges] # --- Node Feature Update --- # Compute messages m_ij (Eq. 14): MLP^1(v_i) ⊙ MLP^2(v_j) ⊙ e_ij mlp1_src = self.mlp_node_1(node_features[src]) # [num_edges, hidden_dim] mlp2_dst = self.mlp_node_2(node_features[dst]) # [num_edges, hidden_dim] messages_ij = mlp1_src * mlp2_dst * edge_features # Hadamard product, [num_edges, hidden_dim] # Compute messages m'_ij (Eq. 15): MLP^1(v_j) ⊙ MLP^2(v_i) ⊙ (e_ij ⊙ EdgeInvertor) mlp1_dst = self.mlp_node_1(node_features[dst]) mlp2_src = self.mlp_node_2(node_features[src]) messages_ji = mlp1_dst * mlp2_src * self.edge_invertor(edge_features) # Aggregate messages for each node (Eq. 17-18) # We use scatter operations for MEAN, STD, MAX, MIN updated_nodes = node_features.clone() for node_idx in range(num_nodes): # Find edges where this node is the destination mask = (dst == node_idx) if mask.sum() == 0: continue msgs_to_node = messages_ij[mask] # [num_incoming, hidden_dim] msgs_from_node = messages_ji[mask] # [num_outgoing, hidden_dim] # Aggregate: MEAN, STD, MAX, MIN (Eq. 18) mean_feat = msgs_to_node.mean(dim=0, keepdim=True) std_feat = msgs_to_node.std(dim=0, keepdim=True) if msgs_to_node.size(0) > 1 else torch.zeros_like(mean_feat) max_feat = msgs_to_node.max(dim=0, keepdim=True)[0] min_feat = msgs_to_node.min(dim=0, keepdim=True)[0] # PNA aggregation: MLP_Aggr([MEAN, STD, MAX, MIN]) aggr_input = torch.cat([mean_feat, std_feat, max_feat, min_feat], dim=-1) # [1, 4*hidden_dim] aggr_output = self.mlp_aggr(aggr_input) # [1, hidden_dim] # Update node (Eq. 17): v_i += PNA_Aggr(m_ij) + PNA_Aggr(m'_ij) mean_feat_ji = msgs_from_node.mean(dim=0, keepdim=True) std_feat_ji = msgs_from_node.std(dim=0, keepdim=True) if msgs_from_node.size(0) > 1 else torch.zeros_like(mean_feat_ji) max_feat_ji = msgs_from_node.max(dim=0, keepdim=True)[0] min_feat_ji = msgs_from_node.min(dim=0, keepdim=True)[0] aggr_input_ji = torch.cat([mean_feat_ji, std_feat_ji, max_feat_ji, min_feat_ji], dim=-1) aggr_output_ji = self.mlp_aggr(aggr_input_ji) updated_nodes[node_idx] = node_features[node_idx] + aggr_output.squeeze(0) + aggr_output_ji.squeeze(0) updated_nodes = self.dropout(updated_nodes) # --- Edge Feature Update (Eq. 19) --- # e_ij += MLP^1(v_i) ⊙ MLP^2(v_j) ⊙ e_ij + MLP^1(v_j) ⊙ MLP^2(v_i) ⊙ (e_ij ⊙ EdgeInvertor) mlp_edge1_src = self.mlp_edge_1(node_features[src]) mlp_edge2_dst = self.mlp_edge_2(node_features[dst]) edge_update_1 = mlp_edge1_src * mlp_edge2_dst * edge_features mlp_edge1_dst = self.mlp_edge_1(node_features[dst]) mlp_edge2_src = self.mlp_edge_2(node_features[src]) edge_update_2 = mlp_edge1_dst * mlp_edge2_src * self.edge_invertor(edge_features) updated_edges = edge_features + edge_update_1 + edge_update_2 updated_edges = self.dropout(updated_edges) return updated_nodes, updated_edges class Metanetwork(nn.Module): """ Graph Metanetwork for MetaPruning. Architecture (from Appendix B.1, Eqs. 12-23): 1. Node/Edge Encoders (MLP) -> hidden_dim 2. N PNAConv message passing layers 3. Node/Edge Decoders (MLP) -> original dimensions 4. Residual connection: v_out = alpha * v_pred + v_in, e_out = beta * e_pred + e_in Paper defaults: - hidden_dim: 32-64 (start with 32) - num_layers: 3 - alpha, beta: 0.01 """ def __init__( self, node_in_dim: int, edge_in_dim: int, node_out_dim: int, edge_out_dim: int, hidden_dim: int = 32, num_layers: int = 3, alpha: float = 0.01, beta: float = 0.01, dropout: float = 0.0, ): super().__init__() self.hidden_dim = hidden_dim self.num_layers = num_layers self.alpha = alpha self.beta = beta # Node/Edge Encoders (Eq. 12-13) self.node_encoder = nn.Sequential( nn.Linear(node_in_dim, hidden_dim), nn.SiLU(), nn.Linear(hidden_dim, hidden_dim), ) self.edge_encoder = nn.Sequential( nn.Linear(edge_in_dim, hidden_dim), nn.SiLU(), nn.Linear(hidden_dim, hidden_dim), ) # Message passing layers self.convs = nn.ModuleList([ PNAConvLayer(hidden_dim, dropout=dropout) for _ in range(num_layers) ]) # Node/Edge Decoders (Eq. 20-21) self.node_decoder = nn.Sequential( nn.Linear(hidden_dim, hidden_dim), nn.SiLU(), nn.Linear(hidden_dim, node_out_dim), ) self.edge_decoder = nn.Sequential( nn.Linear(hidden_dim, hidden_dim), nn.SiLU(), nn.Linear(hidden_dim, edge_out_dim), ) def forward( self, node_features: torch.Tensor, edge_index: torch.Tensor, edge_features: torch.Tensor, ) -> dict: """ Args: node_features: [num_nodes, node_in_dim] edge_index: [2, num_edges] edge_features: [num_edges, edge_in_dim] Returns: dict with keys 'node_pred', 'edge_pred', 'node_out', 'edge_out' """ # Encode (Eq. 12-13) v = self.node_encoder(node_features) # [num_nodes, hidden_dim] e = self.edge_encoder(edge_features) # [num_edges, hidden_dim] # Message passing (Eq. 17-19) for conv in self.convs: v, e = conv(v, edge_index, e) # Decode (Eq. 20-21) node_pred = self.node_decoder(v) # [num_nodes, node_out_dim] edge_pred = self.edge_decoder(e) # [num_edges, edge_out_dim] # Residual output (Eq. 22-23) node_out = self.alpha * node_pred + node_features edge_out = self.beta * edge_pred + edge_features return { 'node_pred': node_pred, 'edge_pred': edge_pred, 'node_out': node_out, 'edge_out': edge_out, }