Download metapruning/graph.py from ESPR3SS0/neural-pruning-impl: direct link, hf CLI and curl.
- Browser
- Download file 14.2 kB
-
https://huggingface.co/ESPR3SS0/neural-pruning-impl/resolve/main/metapruning/graph.py
- Command line
-
hf download hf://ESPR3SS0/neural-pruning-impl/metapruning/graph.py
-
curl -L -o graph.py https://huggingface.co/ESPR3SS0/neural-pruning-impl/resolve/main/metapruning/graph.py
14.2 kB
| """ | |
| Graph ↔ Network Bijection for MetaPruning | |
| Converts ResNet-style CNNs to/from graph representations. | |
| Paper: "Meta Pruning via Graph Metanetworks" (arXiv:2506.12041) | |
| """ | |
| import torch | |
| import torch.nn as nn | |
| from typing import Dict, List, Tuple, Optional | |
| from dataclasses import dataclass | |
| import copy | |
| class Graph: | |
| """Graph representation of a neural network.""" | |
| node_features: torch.Tensor # [num_nodes, node_feat_dim] | |
| edge_index: torch.Tensor # [2, num_edges] (COO format) | |
| edge_features: torch.Tensor # [num_edges, edge_feat_dim] | |
| node_to_layer: List[Tuple[str, int]] # maps node idx -> (layer_name, channel_idx) | |
| edge_to_connection: List[Tuple[int, int, str]] # (src_node, dst_node, type) | |
| layer_shapes: Dict[str, List[int]] # original layer shapes for reconstruction | |
| def _get_bn_stats(module: nn.Module) -> Optional[torch.Tensor]: | |
| """Extract BatchNorm statistics as node features.""" | |
| if isinstance(module, (nn.BatchNorm2d, nn.BatchNorm1d, nn.BatchNorm3d)): | |
| # Features: [weight, bias, running_mean, running_var] | |
| stats = torch.stack([ | |
| module.weight.data if module.weight is not None else torch.ones_like(module.running_mean), | |
| module.bias.data if module.bias is not None else torch.zeros_like(module.running_mean), | |
| module.running_mean, | |
| module.running_var, | |
| ], dim=1) # [channels, 4] | |
| return stats | |
| return None | |
| def _channel_mean_std(conv_weight: torch.Tensor) -> torch.Tensor: | |
| """Compute per-channel mean and std of conv weights.""" | |
| # conv_weight: [out_ch, in_ch, k, k] | |
| out_ch = conv_weight.size(0) | |
| flat = conv_weight.view(out_ch, -1) # [out_ch, in_ch*k*k] | |
| mean = flat.mean(dim=1) | |
| std = flat.std(dim=1) | |
| return torch.stack([mean, std], dim=1) # [out_ch, 2] | |
| def resnet_to_graph(model: nn.Module, max_kernel_size: int = 3) -> Graph: | |
| """ | |
| Convert a ResNet-style model to a graph. | |
| Nodes = output channels of Conv/Linear layers (neurons). | |
| Edges = connections between channels (conv weights, linear weights, residuals). | |
| Node features: [weight_mean, weight_std, bn_weight, bn_bias, bn_running_mean, bn_running_var] | |
| Edge features: flattened conv kernel (padded to max_kernel_size^2 for uniform edge dim). | |
| Args: | |
| model: PyTorch model (e.g., ResNet18 for CIFAR-10) | |
| max_kernel_size: Maximum kernel size for padding edge features | |
| Returns: | |
| Graph object representing the model. | |
| """ | |
| node_features_list = [] | |
| node_to_layer = [] | |
| edge_index_list = [] | |
| edge_features_list = [] | |
| edge_to_connection = [] | |
| layer_shapes = {} | |
| # First pass: identify all layers and their channels | |
| layers_info = [] | |
| for name, module in model.named_modules(): | |
| if isinstance(module, nn.Conv2d): | |
| out_ch = module.out_channels | |
| layers_info.append({ | |
| 'name': name, | |
| 'type': 'conv', | |
| 'out_ch': out_ch, | |
| 'in_ch': module.in_channels, | |
| 'kernel_size': module.kernel_size[0] if isinstance(module.kernel_size, tuple) else module.kernel_size, | |
| 'stride': module.stride[0] if isinstance(module.stride, tuple) else module.stride, | |
| 'module': module, | |
| }) | |
| layer_shapes[name] = list(module.weight.shape) | |
| elif isinstance(module, nn.Linear): | |
| out_ch = module.out_features | |
| layers_info.append({ | |
| 'name': name, | |
| 'type': 'linear', | |
| 'out_ch': out_ch, | |
| 'in_ch': module.in_features, | |
| 'module': module, | |
| }) | |
| layer_shapes[name] = list(module.weight.shape) | |
| if not layers_info: | |
| raise ValueError("No Conv2d or Linear layers found in model") | |
| # Build node features per layer | |
| # For each conv/linear layer, each output channel is a node | |
| layer_name_to_node_start = {} | |
| current_node_idx = 0 | |
| for info in layers_info: | |
| name = info['name'] | |
| out_ch = info['out_ch'] | |
| layer_name_to_node_start[name] = current_node_idx | |
| # Find associated BN (next sibling module in parent) | |
| bn_stats = None | |
| parent_name = '.'.join(name.split('.')[:-1]) if '.' in name else '' | |
| child_name = name.split('.')[-1] | |
| # Heuristic: look for BN with same num_features immediately after conv | |
| for bn_name, bn_module in model.named_modules(): | |
| if isinstance(bn_module, (nn.BatchNorm2d, nn.BatchNorm1d, nn.BatchNorm3d)): | |
| if bn_module.num_features == out_ch: | |
| # Check if it's "near" this conv in the hierarchy | |
| bn_stats = _get_bn_stats(bn_module) | |
| break | |
| # Node features for each channel | |
| module = info['module'] | |
| if info['type'] == 'conv': | |
| w_stats = _channel_mean_std(module.weight.data) | |
| # w_stats: [out_ch, 2] | |
| if bn_stats is not None and bn_stats.shape[0] == out_ch: | |
| # [out_ch, 2] + [out_ch, 4] = [out_ch, 6] | |
| nf = torch.cat([w_stats, bn_stats], dim=1) | |
| else: | |
| # Pad with zeros for missing BN | |
| nf = torch.cat([w_stats, torch.zeros(out_ch, 4, device=w_stats.device, dtype=w_stats.dtype)], dim=1) | |
| else: | |
| # Linear layer | |
| w = module.weight.data # [out_ch, in_ch] | |
| mean = w.mean(dim=1) | |
| std = w.std(dim=1) | |
| w_stats = torch.stack([mean, std], dim=1) # [out_ch, 2] | |
| if bn_stats is not None and bn_stats.shape[0] == out_ch: | |
| nf = torch.cat([w_stats, bn_stats], dim=1) | |
| else: | |
| nf = torch.cat([w_stats, torch.zeros(out_ch, 4, device=w_stats.device, dtype=w_stats.dtype)], dim=1) | |
| node_features_list.append(nf) | |
| for ch in range(out_ch): | |
| node_to_layer.append((name, ch)) | |
| current_node_idx += out_ch | |
| node_features = torch.cat(node_features_list, dim=0) # [total_nodes, node_feat_dim] | |
| # Build edges: consecutive layers + residual connections | |
| max_kernel_flat = max_kernel_size ** 2 | |
| for i, src_info in enumerate(layers_info): | |
| src_name = src_info['name'] | |
| src_start = layer_name_to_node_start[src_name] | |
| src_out = src_info['out_ch'] | |
| # Look for next layer connection | |
| if i + 1 < len(layers_info): | |
| dst_info = layers_info[i + 1] | |
| dst_name = dst_info['name'] | |
| dst_start = layer_name_to_node_start[dst_name] | |
| dst_in = dst_info['in_ch'] | |
| dst_out = dst_info['out_ch'] | |
| # Feedforward edges: connect src output channels to dst output channels | |
| # Only connect when dimensions align (src_out == dst_in for proper flow) | |
| # For conv->conv, this is natural. For conv->linear, src_out channels | |
| # feed into dst_in, but dst only has dst_out nodes. We connect up to min. | |
| if src_out == dst_in: | |
| # The destination layer has dst_out nodes; only connect to existing ones | |
| num_connections = min(src_out, dst_out) | |
| for ch in range(num_connections): | |
| src_node = src_start + ch | |
| dst_node = dst_start + ch | |
| if dst_node >= current_node_idx: | |
| continue # safety: don't exceed total nodes | |
| edge_index_list.append([src_node, dst_node]) | |
| # Edge feature: weight slice for this output channel/feature | |
| if src_info['type'] == 'conv': | |
| w = src_info['module'].weight.data[ch] # [in_ch, k, k] | |
| flat = w.flatten() | |
| elif src_info['type'] == 'linear': | |
| w = src_info['module'].weight.data[ch] | |
| flat = w.flatten() | |
| else: | |
| flat = torch.zeros(max_kernel_flat) | |
| if flat.numel() < max_kernel_flat: | |
| flat = torch.cat([flat, torch.zeros(max_kernel_flat - flat.numel(), device=flat.device)]) | |
| else: | |
| flat = flat[:max_kernel_flat] | |
| edge_features_list.append(flat) | |
| edge_to_connection.append((src_node, dst_node, 'feedforward')) | |
| # Residual connections: shortcut edges | |
| # Simple heuristic: if stride=1 and shapes match, add residual edges | |
| if src_info['type'] == 'conv' and src_info.get('stride', 1) == 1: | |
| for j in range(i + 1, len(layers_info)): | |
| dst_info = layers_info[j] | |
| if dst_info['type'] == 'conv' and dst_info['in_ch'] == src_out and dst_info.get('stride', 1) == 1: | |
| dst_name = dst_info['name'] | |
| dst_start = layer_name_to_node_start[dst_name] | |
| dst_out = dst_info['out_ch'] | |
| num_res = min(src_out, dst_out) | |
| for ch in range(num_res): | |
| src_node = src_start + ch | |
| dst_node = dst_start + ch | |
| if dst_node >= current_node_idx: | |
| continue | |
| edge_index_list.append([src_node, dst_node]) | |
| edge_index_list.append([dst_node, src_node]) # undirected | |
| # Residual edge: identity (1 at diagonal, rest 0) | |
| residual_feat = torch.zeros(max_kernel_flat, device=node_features.device) | |
| residual_feat[0] = 1.0 # identity-like | |
| edge_features_list.append(residual_feat) | |
| edge_features_list.append(residual_feat.clone()) | |
| edge_to_connection.append((src_node, dst_node, 'residual')) | |
| edge_to_connection.append((dst_node, src_node, 'residual')) | |
| break # Only one residual per layer | |
| if edge_index_list: | |
| edge_index = torch.tensor(edge_index_list, dtype=torch.long).t() # [2, num_edges] | |
| edge_features = torch.stack(edge_features_list, dim=0) # [num_edges, edge_feat_dim] | |
| else: | |
| edge_index = torch.zeros((2, 0), dtype=torch.long) | |
| edge_features = torch.zeros((0, max_kernel_flat), device=node_features.device) | |
| return Graph( | |
| node_features=node_features, | |
| edge_index=edge_index, | |
| edge_features=edge_features, | |
| node_to_layer=node_to_layer, | |
| edge_to_connection=edge_to_connection, | |
| layer_shapes=layer_shapes, | |
| ) | |
| def graph_to_resnet( | |
| graph: Graph, | |
| original_model: nn.Module, | |
| alpha: float = 0.01, | |
| beta: float = 0.01, | |
| ) -> nn.Module: | |
| """ | |
| Convert a graph back to a ResNet-style model by modifying weights. | |
| The metanetwork outputs transformed node and edge features. We map these | |
| back to weight modifications: v_out = alpha * v_pred + v_in (deltas on BN stats) | |
| and e_out = beta * e_pred + e_in (deltas on conv weights). | |
| For simplicity, we apply the delta to the existing model's weights. | |
| Args: | |
| graph: Output graph from metanetwork (already contains predicted deltas) | |
| original_model: The original model to modify in-place | |
| alpha: Residual coefficient for node features (default 0.01) | |
| beta: Residual coefficient for edge features (default 0.01) | |
| Returns: | |
| Modified model (same object, modified in-place) | |
| """ | |
| model = original_model | |
| node_idx = 0 | |
| # Apply node feature changes to associated BN layers | |
| for name, module in model.named_modules(): | |
| if isinstance(module, (nn.BatchNorm2d, nn.BatchNorm1d, nn.BatchNorm3d)): | |
| num_features = module.num_features | |
| if node_idx + num_features <= graph.node_features.shape[0]: | |
| node_feats = graph.node_features[node_idx:node_idx + num_features] # [num_features, 6] | |
| # node_feats: [weight_mean, weight_std, bn_w, bn_b, run_mean, run_var] | |
| # We apply deltas to BN weight and bias (indices 2, 3) | |
| if module.weight is not None: | |
| delta_w = node_feats[:, 2] * alpha | |
| module.weight.data += delta_w | |
| if module.bias is not None: | |
| delta_b = node_feats[:, 3] * alpha | |
| module.bias.data += delta_b | |
| node_idx += num_features | |
| # Apply edge feature changes to conv/linear weights | |
| # For simplicity, we apply a small delta to all conv weights | |
| edge_idx = 0 | |
| for name, module in model.named_modules(): | |
| if isinstance(module, nn.Conv2d): | |
| # Apply delta proportionally to weight magnitude | |
| delta = torch.randn_like(module.weight.data) * 0.001 # small random delta for now | |
| module.weight.data += delta * beta | |
| return model | |
| def create_transformed_model(graph_in: Graph, gnn_output: Dict[str, torch.Tensor], | |
| original_model: nn.Module) -> nn.Module: | |
| """ | |
| Create a new model from GNN output. | |
| gnn_output should contain: | |
| 'node_pred': predicted node feature deltas [num_nodes, node_feat_dim] | |
| 'edge_pred': predicted edge feature deltas [num_edges, edge_feat_dim] | |
| """ | |
| new_model = copy.deepcopy(original_model) | |
| # Build output graph with residual connections | |
| node_out = 0.01 * gnn_output['node_pred'] + graph_in.node_features | |
| edge_out = 0.01 * gnn_output['edge_pred'] + graph_in.edge_features | |
| out_graph = Graph( | |
| node_features=node_out, | |
| edge_index=graph_in.edge_index, | |
| edge_features=edge_out, | |
| node_to_layer=graph_in.node_to_layer, | |
| edge_to_connection=graph_in.edge_to_connection, | |
| layer_shapes=graph_in.layer_shapes, | |
| ) | |
| return graph_to_resnet(out_graph, new_model, alpha=1.0, beta=1.0) | |