File size: 14,174 Bytes
639f82d | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 | """
Graph ↔ Network Bijection for MetaPruning
Converts ResNet-style CNNs to/from graph representations.
Paper: "Meta Pruning via Graph Metanetworks" (arXiv:2506.12041)
"""
import torch
import torch.nn as nn
from typing import Dict, List, Tuple, Optional
from dataclasses import dataclass
import copy
@dataclass
class Graph:
"""Graph representation of a neural network."""
node_features: torch.Tensor # [num_nodes, node_feat_dim]
edge_index: torch.Tensor # [2, num_edges] (COO format)
edge_features: torch.Tensor # [num_edges, edge_feat_dim]
node_to_layer: List[Tuple[str, int]] # maps node idx -> (layer_name, channel_idx)
edge_to_connection: List[Tuple[int, int, str]] # (src_node, dst_node, type)
layer_shapes: Dict[str, List[int]] # original layer shapes for reconstruction
def _get_bn_stats(module: nn.Module) -> Optional[torch.Tensor]:
"""Extract BatchNorm statistics as node features."""
if isinstance(module, (nn.BatchNorm2d, nn.BatchNorm1d, nn.BatchNorm3d)):
# Features: [weight, bias, running_mean, running_var]
stats = torch.stack([
module.weight.data if module.weight is not None else torch.ones_like(module.running_mean),
module.bias.data if module.bias is not None else torch.zeros_like(module.running_mean),
module.running_mean,
module.running_var,
], dim=1) # [channels, 4]
return stats
return None
def _channel_mean_std(conv_weight: torch.Tensor) -> torch.Tensor:
"""Compute per-channel mean and std of conv weights."""
# conv_weight: [out_ch, in_ch, k, k]
out_ch = conv_weight.size(0)
flat = conv_weight.view(out_ch, -1) # [out_ch, in_ch*k*k]
mean = flat.mean(dim=1)
std = flat.std(dim=1)
return torch.stack([mean, std], dim=1) # [out_ch, 2]
def resnet_to_graph(model: nn.Module, max_kernel_size: int = 3) -> Graph:
"""
Convert a ResNet-style model to a graph.
Nodes = output channels of Conv/Linear layers (neurons).
Edges = connections between channels (conv weights, linear weights, residuals).
Node features: [weight_mean, weight_std, bn_weight, bn_bias, bn_running_mean, bn_running_var]
Edge features: flattened conv kernel (padded to max_kernel_size^2 for uniform edge dim).
Args:
model: PyTorch model (e.g., ResNet18 for CIFAR-10)
max_kernel_size: Maximum kernel size for padding edge features
Returns:
Graph object representing the model.
"""
node_features_list = []
node_to_layer = []
edge_index_list = []
edge_features_list = []
edge_to_connection = []
layer_shapes = {}
# First pass: identify all layers and their channels
layers_info = []
for name, module in model.named_modules():
if isinstance(module, nn.Conv2d):
out_ch = module.out_channels
layers_info.append({
'name': name,
'type': 'conv',
'out_ch': out_ch,
'in_ch': module.in_channels,
'kernel_size': module.kernel_size[0] if isinstance(module.kernel_size, tuple) else module.kernel_size,
'stride': module.stride[0] if isinstance(module.stride, tuple) else module.stride,
'module': module,
})
layer_shapes[name] = list(module.weight.shape)
elif isinstance(module, nn.Linear):
out_ch = module.out_features
layers_info.append({
'name': name,
'type': 'linear',
'out_ch': out_ch,
'in_ch': module.in_features,
'module': module,
})
layer_shapes[name] = list(module.weight.shape)
if not layers_info:
raise ValueError("No Conv2d or Linear layers found in model")
# Build node features per layer
# For each conv/linear layer, each output channel is a node
layer_name_to_node_start = {}
current_node_idx = 0
for info in layers_info:
name = info['name']
out_ch = info['out_ch']
layer_name_to_node_start[name] = current_node_idx
# Find associated BN (next sibling module in parent)
bn_stats = None
parent_name = '.'.join(name.split('.')[:-1]) if '.' in name else ''
child_name = name.split('.')[-1]
# Heuristic: look for BN with same num_features immediately after conv
for bn_name, bn_module in model.named_modules():
if isinstance(bn_module, (nn.BatchNorm2d, nn.BatchNorm1d, nn.BatchNorm3d)):
if bn_module.num_features == out_ch:
# Check if it's "near" this conv in the hierarchy
bn_stats = _get_bn_stats(bn_module)
break
# Node features for each channel
module = info['module']
if info['type'] == 'conv':
w_stats = _channel_mean_std(module.weight.data)
# w_stats: [out_ch, 2]
if bn_stats is not None and bn_stats.shape[0] == out_ch:
# [out_ch, 2] + [out_ch, 4] = [out_ch, 6]
nf = torch.cat([w_stats, bn_stats], dim=1)
else:
# Pad with zeros for missing BN
nf = torch.cat([w_stats, torch.zeros(out_ch, 4, device=w_stats.device, dtype=w_stats.dtype)], dim=1)
else:
# Linear layer
w = module.weight.data # [out_ch, in_ch]
mean = w.mean(dim=1)
std = w.std(dim=1)
w_stats = torch.stack([mean, std], dim=1) # [out_ch, 2]
if bn_stats is not None and bn_stats.shape[0] == out_ch:
nf = torch.cat([w_stats, bn_stats], dim=1)
else:
nf = torch.cat([w_stats, torch.zeros(out_ch, 4, device=w_stats.device, dtype=w_stats.dtype)], dim=1)
node_features_list.append(nf)
for ch in range(out_ch):
node_to_layer.append((name, ch))
current_node_idx += out_ch
node_features = torch.cat(node_features_list, dim=0) # [total_nodes, node_feat_dim]
# Build edges: consecutive layers + residual connections
max_kernel_flat = max_kernel_size ** 2
for i, src_info in enumerate(layers_info):
src_name = src_info['name']
src_start = layer_name_to_node_start[src_name]
src_out = src_info['out_ch']
# Look for next layer connection
if i + 1 < len(layers_info):
dst_info = layers_info[i + 1]
dst_name = dst_info['name']
dst_start = layer_name_to_node_start[dst_name]
dst_in = dst_info['in_ch']
dst_out = dst_info['out_ch']
# Feedforward edges: connect src output channels to dst output channels
# Only connect when dimensions align (src_out == dst_in for proper flow)
# For conv->conv, this is natural. For conv->linear, src_out channels
# feed into dst_in, but dst only has dst_out nodes. We connect up to min.
if src_out == dst_in:
# The destination layer has dst_out nodes; only connect to existing ones
num_connections = min(src_out, dst_out)
for ch in range(num_connections):
src_node = src_start + ch
dst_node = dst_start + ch
if dst_node >= current_node_idx:
continue # safety: don't exceed total nodes
edge_index_list.append([src_node, dst_node])
# Edge feature: weight slice for this output channel/feature
if src_info['type'] == 'conv':
w = src_info['module'].weight.data[ch] # [in_ch, k, k]
flat = w.flatten()
elif src_info['type'] == 'linear':
w = src_info['module'].weight.data[ch]
flat = w.flatten()
else:
flat = torch.zeros(max_kernel_flat)
if flat.numel() < max_kernel_flat:
flat = torch.cat([flat, torch.zeros(max_kernel_flat - flat.numel(), device=flat.device)])
else:
flat = flat[:max_kernel_flat]
edge_features_list.append(flat)
edge_to_connection.append((src_node, dst_node, 'feedforward'))
# Residual connections: shortcut edges
# Simple heuristic: if stride=1 and shapes match, add residual edges
if src_info['type'] == 'conv' and src_info.get('stride', 1) == 1:
for j in range(i + 1, len(layers_info)):
dst_info = layers_info[j]
if dst_info['type'] == 'conv' and dst_info['in_ch'] == src_out and dst_info.get('stride', 1) == 1:
dst_name = dst_info['name']
dst_start = layer_name_to_node_start[dst_name]
dst_out = dst_info['out_ch']
num_res = min(src_out, dst_out)
for ch in range(num_res):
src_node = src_start + ch
dst_node = dst_start + ch
if dst_node >= current_node_idx:
continue
edge_index_list.append([src_node, dst_node])
edge_index_list.append([dst_node, src_node]) # undirected
# Residual edge: identity (1 at diagonal, rest 0)
residual_feat = torch.zeros(max_kernel_flat, device=node_features.device)
residual_feat[0] = 1.0 # identity-like
edge_features_list.append(residual_feat)
edge_features_list.append(residual_feat.clone())
edge_to_connection.append((src_node, dst_node, 'residual'))
edge_to_connection.append((dst_node, src_node, 'residual'))
break # Only one residual per layer
if edge_index_list:
edge_index = torch.tensor(edge_index_list, dtype=torch.long).t() # [2, num_edges]
edge_features = torch.stack(edge_features_list, dim=0) # [num_edges, edge_feat_dim]
else:
edge_index = torch.zeros((2, 0), dtype=torch.long)
edge_features = torch.zeros((0, max_kernel_flat), device=node_features.device)
return Graph(
node_features=node_features,
edge_index=edge_index,
edge_features=edge_features,
node_to_layer=node_to_layer,
edge_to_connection=edge_to_connection,
layer_shapes=layer_shapes,
)
def graph_to_resnet(
graph: Graph,
original_model: nn.Module,
alpha: float = 0.01,
beta: float = 0.01,
) -> nn.Module:
"""
Convert a graph back to a ResNet-style model by modifying weights.
The metanetwork outputs transformed node and edge features. We map these
back to weight modifications: v_out = alpha * v_pred + v_in (deltas on BN stats)
and e_out = beta * e_pred + e_in (deltas on conv weights).
For simplicity, we apply the delta to the existing model's weights.
Args:
graph: Output graph from metanetwork (already contains predicted deltas)
original_model: The original model to modify in-place
alpha: Residual coefficient for node features (default 0.01)
beta: Residual coefficient for edge features (default 0.01)
Returns:
Modified model (same object, modified in-place)
"""
model = original_model
node_idx = 0
# Apply node feature changes to associated BN layers
for name, module in model.named_modules():
if isinstance(module, (nn.BatchNorm2d, nn.BatchNorm1d, nn.BatchNorm3d)):
num_features = module.num_features
if node_idx + num_features <= graph.node_features.shape[0]:
node_feats = graph.node_features[node_idx:node_idx + num_features] # [num_features, 6]
# node_feats: [weight_mean, weight_std, bn_w, bn_b, run_mean, run_var]
# We apply deltas to BN weight and bias (indices 2, 3)
if module.weight is not None:
delta_w = node_feats[:, 2] * alpha
module.weight.data += delta_w
if module.bias is not None:
delta_b = node_feats[:, 3] * alpha
module.bias.data += delta_b
node_idx += num_features
# Apply edge feature changes to conv/linear weights
# For simplicity, we apply a small delta to all conv weights
edge_idx = 0
for name, module in model.named_modules():
if isinstance(module, nn.Conv2d):
# Apply delta proportionally to weight magnitude
delta = torch.randn_like(module.weight.data) * 0.001 # small random delta for now
module.weight.data += delta * beta
return model
def create_transformed_model(graph_in: Graph, gnn_output: Dict[str, torch.Tensor],
original_model: nn.Module) -> nn.Module:
"""
Create a new model from GNN output.
gnn_output should contain:
'node_pred': predicted node feature deltas [num_nodes, node_feat_dim]
'edge_pred': predicted edge feature deltas [num_edges, edge_feat_dim]
"""
new_model = copy.deepcopy(original_model)
# Build output graph with residual connections
node_out = 0.01 * gnn_output['node_pred'] + graph_in.node_features
edge_out = 0.01 * gnn_output['edge_pred'] + graph_in.edge_features
out_graph = Graph(
node_features=node_out,
edge_index=graph_in.edge_index,
edge_features=edge_out,
node_to_layer=graph_in.node_to_layer,
edge_to_connection=graph_in.edge_to_connection,
layer_shapes=graph_in.layer_shapes,
)
return graph_to_resnet(out_graph, new_model, alpha=1.0, beta=1.0)
|