ESPR3SS0 commited on
Commit
daa2bf5
Β·
verified Β·
1 Parent(s): 639f82d

Add metapruning/gnn.py

Browse files
Files changed (1) hide show
  1. metapruning/gnn.py +263 -0
metapruning/gnn.py ADDED
@@ -0,0 +1,263 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ PNAConv-based Graph Metanetwork for MetaPruning.
3
+
4
+ Paper: "Meta Pruning via Graph Metanetworks" (arXiv:2506.12041)
5
+ Appendix B.1 describes the full architecture.
6
+ """
7
+
8
+ import torch
9
+ import torch.nn as nn
10
+ import torch.nn.functional as F
11
+ from typing import Optional
12
+
13
+
14
+ class EdgeInvertor(nn.Module):
15
+ """
16
+ EdgeInvertor from paper Eq. (16):
17
+ [1,1,...,1 (hidden_dim/2), -1,-1,...,-1 (hidden_dim/2)]
18
+ Used to encode directional edge features.
19
+ """
20
+ def __init__(self, hidden_dim: int):
21
+ super().__init__()
22
+ half = hidden_dim // 2
23
+ self.register_buffer("invertor", torch.cat([
24
+ torch.ones(half),
25
+ -torch.ones(hidden_dim - half)
26
+ ]))
27
+
28
+ def forward(self, edge_features: torch.Tensor) -> torch.Tensor:
29
+ return edge_features * self.invertor.unsqueeze(0)
30
+
31
+
32
+ class PNAConvLayer(nn.Module):
33
+ """
34
+ A single message passing layer for the metanetwork.
35
+ Implements Eqs. (12-19) from Appendix B.1.
36
+
37
+ Each layer:
38
+ 1. Computes messages m_ij and m'_ij (Eq. 14-15)
39
+ 2. Aggregates messages using MEAN, STD, MAX, MIN (Eq. 17-18)
40
+ 3. Updates node features (Eq. 17)
41
+ 4. Updates edge features (Eq. 19)
42
+ """
43
+
44
+ def __init__(self, hidden_dim: int, dropout: float = 0.0):
45
+ super().__init__()
46
+ self.hidden_dim = hidden_dim
47
+
48
+ # MLPs for computing messages (Eq. 14-15)
49
+ self.mlp_node_1 = nn.Sequential(
50
+ nn.Linear(hidden_dim, hidden_dim),
51
+ nn.SiLU(),
52
+ nn.Linear(hidden_dim, hidden_dim),
53
+ )
54
+ self.mlp_node_2 = nn.Sequential(
55
+ nn.Linear(hidden_dim, hidden_dim),
56
+ nn.SiLU(),
57
+ nn.Linear(hidden_dim, hidden_dim),
58
+ )
59
+
60
+ # Edge invertor
61
+ self.edge_invertor = EdgeInvertor(hidden_dim)
62
+
63
+ # MLP for aggregation (Eq. 18)
64
+ # Input: concatenation of [MEAN, STD, MAX, MIN] -> 4 * hidden_dim
65
+ self.mlp_aggr = nn.Sequential(
66
+ nn.Linear(4 * hidden_dim, hidden_dim),
67
+ nn.SiLU(),
68
+ nn.Linear(hidden_dim, hidden_dim),
69
+ )
70
+
71
+ # Edge update MLPs (Eq. 19)
72
+ self.mlp_edge_1 = nn.Sequential(
73
+ nn.Linear(hidden_dim, hidden_dim),
74
+ nn.SiLU(),
75
+ nn.Linear(hidden_dim, hidden_dim),
76
+ )
77
+ self.mlp_edge_2 = nn.Sequential(
78
+ nn.Linear(hidden_dim, hidden_dim),
79
+ nn.SiLU(),
80
+ nn.Linear(hidden_dim, hidden_dim),
81
+ )
82
+
83
+ self.dropout = nn.Dropout(dropout) if dropout > 0 else nn.Identity()
84
+
85
+ def forward(
86
+ self,
87
+ node_features: torch.Tensor, # [num_nodes, hidden_dim]
88
+ edge_index: torch.Tensor, # [2, num_edges]
89
+ edge_features: torch.Tensor, # [num_edges, hidden_dim]
90
+ ) -> tuple:
91
+ """
92
+ Returns: (updated_node_features, updated_edge_features)
93
+ """
94
+ num_nodes = node_features.size(0)
95
+ num_edges = edge_index.size(1)
96
+
97
+ src = edge_index[0] # [num_edges]
98
+ dst = edge_index[1] # [num_edges]
99
+
100
+ # --- Node Feature Update ---
101
+ # Compute messages m_ij (Eq. 14): MLP^1(v_i) βŠ™ MLP^2(v_j) βŠ™ e_ij
102
+ mlp1_src = self.mlp_node_1(node_features[src]) # [num_edges, hidden_dim]
103
+ mlp2_dst = self.mlp_node_2(node_features[dst]) # [num_edges, hidden_dim]
104
+ messages_ij = mlp1_src * mlp2_dst * edge_features # Hadamard product, [num_edges, hidden_dim]
105
+
106
+ # Compute messages m'_ij (Eq. 15): MLP^1(v_j) βŠ™ MLP^2(v_i) βŠ™ (e_ij βŠ™ EdgeInvertor)
107
+ mlp1_dst = self.mlp_node_1(node_features[dst])
108
+ mlp2_src = self.mlp_node_2(node_features[src])
109
+ messages_ji = mlp1_dst * mlp2_src * self.edge_invertor(edge_features)
110
+
111
+ # Aggregate messages for each node (Eq. 17-18)
112
+ # We use scatter operations for MEAN, STD, MAX, MIN
113
+ updated_nodes = node_features.clone()
114
+
115
+ for node_idx in range(num_nodes):
116
+ # Find edges where this node is the destination
117
+ mask = (dst == node_idx)
118
+ if mask.sum() == 0:
119
+ continue
120
+
121
+ msgs_to_node = messages_ij[mask] # [num_incoming, hidden_dim]
122
+ msgs_from_node = messages_ji[mask] # [num_outgoing, hidden_dim]
123
+
124
+ # Aggregate: MEAN, STD, MAX, MIN (Eq. 18)
125
+ mean_feat = msgs_to_node.mean(dim=0, keepdim=True)
126
+ std_feat = msgs_to_node.std(dim=0, keepdim=True) if msgs_to_node.size(0) > 1 else torch.zeros_like(mean_feat)
127
+ max_feat = msgs_to_node.max(dim=0, keepdim=True)[0]
128
+ min_feat = msgs_to_node.min(dim=0, keepdim=True)[0]
129
+
130
+ # PNA aggregation: MLP_Aggr([MEAN, STD, MAX, MIN])
131
+ aggr_input = torch.cat([mean_feat, std_feat, max_feat, min_feat], dim=-1) # [1, 4*hidden_dim]
132
+ aggr_output = self.mlp_aggr(aggr_input) # [1, hidden_dim]
133
+
134
+ # Update node (Eq. 17): v_i += PNA_Aggr(m_ij) + PNA_Aggr(m'_ij)
135
+ mean_feat_ji = msgs_from_node.mean(dim=0, keepdim=True)
136
+ std_feat_ji = msgs_from_node.std(dim=0, keepdim=True) if msgs_from_node.size(0) > 1 else torch.zeros_like(mean_feat_ji)
137
+ max_feat_ji = msgs_from_node.max(dim=0, keepdim=True)[0]
138
+ min_feat_ji = msgs_from_node.min(dim=0, keepdim=True)[0]
139
+ aggr_input_ji = torch.cat([mean_feat_ji, std_feat_ji, max_feat_ji, min_feat_ji], dim=-1)
140
+ aggr_output_ji = self.mlp_aggr(aggr_input_ji)
141
+
142
+ updated_nodes[node_idx] = node_features[node_idx] + aggr_output.squeeze(0) + aggr_output_ji.squeeze(0)
143
+
144
+ updated_nodes = self.dropout(updated_nodes)
145
+
146
+ # --- Edge Feature Update (Eq. 19) ---
147
+ # e_ij += MLP^1(v_i) βŠ™ MLP^2(v_j) βŠ™ e_ij + MLP^1(v_j) βŠ™ MLP^2(v_i) βŠ™ (e_ij βŠ™ EdgeInvertor)
148
+ mlp_edge1_src = self.mlp_edge_1(node_features[src])
149
+ mlp_edge2_dst = self.mlp_edge_2(node_features[dst])
150
+ edge_update_1 = mlp_edge1_src * mlp_edge2_dst * edge_features
151
+
152
+ mlp_edge1_dst = self.mlp_edge_1(node_features[dst])
153
+ mlp_edge2_src = self.mlp_edge_2(node_features[src])
154
+ edge_update_2 = mlp_edge1_dst * mlp_edge2_src * self.edge_invertor(edge_features)
155
+
156
+ updated_edges = edge_features + edge_update_1 + edge_update_2
157
+ updated_edges = self.dropout(updated_edges)
158
+
159
+ return updated_nodes, updated_edges
160
+
161
+
162
+ class Metanetwork(nn.Module):
163
+ """
164
+ Graph Metanetwork for MetaPruning.
165
+
166
+ Architecture (from Appendix B.1, Eqs. 12-23):
167
+ 1. Node/Edge Encoders (MLP) -> hidden_dim
168
+ 2. N PNAConv message passing layers
169
+ 3. Node/Edge Decoders (MLP) -> original dimensions
170
+ 4. Residual connection: v_out = alpha * v_pred + v_in, e_out = beta * e_pred + e_in
171
+
172
+ Paper defaults:
173
+ - hidden_dim: 32-64 (start with 32)
174
+ - num_layers: 3
175
+ - alpha, beta: 0.01
176
+ """
177
+
178
+ def __init__(
179
+ self,
180
+ node_in_dim: int,
181
+ edge_in_dim: int,
182
+ node_out_dim: int,
183
+ edge_out_dim: int,
184
+ hidden_dim: int = 32,
185
+ num_layers: int = 3,
186
+ alpha: float = 0.01,
187
+ beta: float = 0.01,
188
+ dropout: float = 0.0,
189
+ ):
190
+ super().__init__()
191
+
192
+ self.hidden_dim = hidden_dim
193
+ self.num_layers = num_layers
194
+ self.alpha = alpha
195
+ self.beta = beta
196
+
197
+ # Node/Edge Encoders (Eq. 12-13)
198
+ self.node_encoder = nn.Sequential(
199
+ nn.Linear(node_in_dim, hidden_dim),
200
+ nn.SiLU(),
201
+ nn.Linear(hidden_dim, hidden_dim),
202
+ )
203
+ self.edge_encoder = nn.Sequential(
204
+ nn.Linear(edge_in_dim, hidden_dim),
205
+ nn.SiLU(),
206
+ nn.Linear(hidden_dim, hidden_dim),
207
+ )
208
+
209
+ # Message passing layers
210
+ self.convs = nn.ModuleList([
211
+ PNAConvLayer(hidden_dim, dropout=dropout)
212
+ for _ in range(num_layers)
213
+ ])
214
+
215
+ # Node/Edge Decoders (Eq. 20-21)
216
+ self.node_decoder = nn.Sequential(
217
+ nn.Linear(hidden_dim, hidden_dim),
218
+ nn.SiLU(),
219
+ nn.Linear(hidden_dim, node_out_dim),
220
+ )
221
+ self.edge_decoder = nn.Sequential(
222
+ nn.Linear(hidden_dim, hidden_dim),
223
+ nn.SiLU(),
224
+ nn.Linear(hidden_dim, edge_out_dim),
225
+ )
226
+
227
+ def forward(
228
+ self,
229
+ node_features: torch.Tensor,
230
+ edge_index: torch.Tensor,
231
+ edge_features: torch.Tensor,
232
+ ) -> dict:
233
+ """
234
+ Args:
235
+ node_features: [num_nodes, node_in_dim]
236
+ edge_index: [2, num_edges]
237
+ edge_features: [num_edges, edge_in_dim]
238
+
239
+ Returns:
240
+ dict with keys 'node_pred', 'edge_pred', 'node_out', 'edge_out'
241
+ """
242
+ # Encode (Eq. 12-13)
243
+ v = self.node_encoder(node_features) # [num_nodes, hidden_dim]
244
+ e = self.edge_encoder(edge_features) # [num_edges, hidden_dim]
245
+
246
+ # Message passing (Eq. 17-19)
247
+ for conv in self.convs:
248
+ v, e = conv(v, edge_index, e)
249
+
250
+ # Decode (Eq. 20-21)
251
+ node_pred = self.node_decoder(v) # [num_nodes, node_out_dim]
252
+ edge_pred = self.edge_decoder(e) # [num_edges, edge_out_dim]
253
+
254
+ # Residual output (Eq. 22-23)
255
+ node_out = self.alpha * node_pred + node_features
256
+ edge_out = self.beta * edge_pred + edge_features
257
+
258
+ return {
259
+ 'node_pred': node_pred,
260
+ 'edge_pred': edge_pred,
261
+ 'node_out': node_out,
262
+ 'edge_out': edge_out,
263
+ }