Image Classification
Keras
computer-vision
efficientnet
self-attention
weight-sharing
hybrid-vision-backbone
tensorflow
Eval Results (legacy)
Instructions to use meatloafer/efficientnet-b0-stage-attention with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- Keras
How to use meatloafer/efficientnet-b0-stage-attention with Keras:
# !pip install -U keras tensorflow huggingface_hub # Keras needs TensorFlow installed to read "hf://" paths, so the tensorflow backend is selected here; # "jax" and "torch" also work for computation once TensorFlow is installed. import os os.environ["KERAS_BACKEND"] = "tensorflow" import keras model = keras.saving.load_model("hf://meatloafer/efficientnet-b0-stage-attention") - Notebooks
- Google Colab
- Kaggle
Upload verified model weights, LiteRT FlatBuffers, DP profiles, code, and model card
587d427 verified Download code/attention_conv_model.py from meatloafer/efficientnet-b0-stage-attention: direct link, hf CLI and curl.
- Browser
- Download file 14.4 kB
-
https://huggingface.co/meatloafer/efficientnet-b0-stage-attention/resolve/main/code/attention_conv_model.py
- Command line
-
hf download hf://meatloafer/efficientnet-b0-stage-attention/code/attention_conv_model.py
-
curl -L -o attention_conv_model.py https://huggingface.co/meatloafer/efficientnet-b0-stage-attention/resolve/main/code/attention_conv_model.py
14.4 kB
| """Sparse multiscale attention adapters on an EfficientNet convolutional trunk. | |
| This builder is intentionally kept separate from ``models.py``. It preserves | |
| every MBConv stage and | |
| adds three residual attention adapters: windowed attention at P3 and global | |
| attention at P4 and P5. Zero-initialized LayerScale gates make the inserted | |
| blocks an identity at initialization, so a transferred EfficientNet checkpoint | |
| starts with unchanged logits. | |
| The strict training launcher supports the ``attention_conv`` architecture | |
| name. A run still requires a committed source-pinned registration and queue | |
| admission after the existing priority sequence. | |
| """ | |
| from __future__ import annotations | |
| import tensorflow as tf | |
| from imagenet_research.models import ( | |
| STAGE_SPECS, | |
| VARIANTS, | |
| _conv_bn_relu6, | |
| _mbconv, | |
| round_filters, | |
| round_repeats, | |
| ) | |
| ATTENTION_STAGE_INDICES = (2, 4, 6) | |
| EXPECTED_ATTENTION_LAYER_NAMES = ( | |
| "attnconv_stage3", | |
| "attnconv_stage5", | |
| "attnconv_stage7", | |
| ) | |
| P3_WINDOW_SIZE = 7 | |
| ATTENTION_HEADS = 4 | |
| ATTENTION_MLP_RATIO = 2 | |
| class ConvolutionalAttentionAdapter(tf.keras.layers.Layer): | |
| """Residual spatial attention with convolutional positional mixing. | |
| ``window_size=None`` uses global attention. A positive window size | |
| partitions a feature map into non-overlapping windows, padding only the | |
| bottom and right edges when needed. The depthwise positional convolution | |
| is applied before partitioning, so nearby windows still receive local | |
| convolutional context. | |
| The attention and MLP branches have independent per-channel LayerScale | |
| gates initialized to zero. At the initial checkpoint, the adapter is | |
| exactly the identity; training can first open the gates and then update | |
| each branch. | |
| """ | |
| def __init__( | |
| self, | |
| channels: int, | |
| heads: int = ATTENTION_HEADS, | |
| mlp_ratio: int = ATTENTION_MLP_RATIO, | |
| window_size: int | None = None, | |
| **kwargs, | |
| ): | |
| super().__init__(**kwargs) | |
| if channels < 1 or heads < 1 or channels % heads: | |
| raise ValueError("channels must be positive and divisible by heads") | |
| if mlp_ratio < 1: | |
| raise ValueError("mlp_ratio must be positive") | |
| if window_size is not None and window_size < 1: | |
| raise ValueError("window_size must be positive or None") | |
| self.channels = int(channels) | |
| self.heads = int(heads) | |
| self.mlp_ratio = int(mlp_ratio) | |
| self.window_size = None if window_size is None else int(window_size) | |
| self.position_conv = tf.keras.layers.DepthwiseConv2D( | |
| 3, | |
| padding="same", | |
| depthwise_initializer="he_normal", | |
| name="position_conv", | |
| ) | |
| self.attention_norm = tf.keras.layers.LayerNormalization( | |
| epsilon=1e-6, name="attention_norm" | |
| ) | |
| self.attention = tf.keras.layers.MultiHeadAttention( | |
| num_heads=self.heads, | |
| key_dim=self.channels // self.heads, | |
| output_shape=self.channels, | |
| name="mhsa", | |
| ) | |
| self.mlp_norm = tf.keras.layers.LayerNormalization( | |
| epsilon=1e-6, name="mlp_norm" | |
| ) | |
| self.mlp_expand = tf.keras.layers.Dense( | |
| self.channels * self.mlp_ratio, | |
| activation=tf.nn.relu6, | |
| name="mlp_expand", | |
| ) | |
| self.mlp_project = tf.keras.layers.Dense( | |
| self.channels, name="mlp_project" | |
| ) | |
| def build(self, input_shape): | |
| if input_shape[-1] is not None and int(input_shape[-1]) != self.channels: | |
| raise ValueError( | |
| f"channels={self.channels} does not match input channels={input_shape[-1]}" | |
| ) | |
| self.attention_scale = self.add_weight( | |
| name="attention_scale", | |
| shape=(self.channels,), | |
| initializer="zeros", | |
| trainable=True, | |
| ) | |
| self.mlp_scale = self.add_weight( | |
| name="mlp_scale", | |
| shape=(self.channels,), | |
| initializer="zeros", | |
| trainable=True, | |
| ) | |
| super().build(input_shape) | |
| def _window_attention(self, tokens, training): | |
| window = self.window_size | |
| shape = tf.shape(tokens) | |
| batch, height, width = shape[0], shape[1], shape[2] | |
| pad_height = tf.math.floormod(-height, window) | |
| pad_width = tf.math.floormod(-width, window) | |
| paddings = tf.stack( | |
| [ | |
| tf.constant([0, 0], dtype=tf.int32), | |
| tf.stack([tf.constant(0, tf.int32), pad_height]), | |
| tf.stack([tf.constant(0, tf.int32), pad_width]), | |
| tf.constant([0, 0], dtype=tf.int32), | |
| ] | |
| ) | |
| padded = tf.pad(tokens, paddings) | |
| padded_shape = tf.shape(padded) | |
| windows_h = padded_shape[1] // window | |
| windows_w = padded_shape[2] // window | |
| windows = tf.reshape( | |
| padded, | |
| [batch, windows_h, window, windows_w, window, self.channels], | |
| ) | |
| windows = tf.transpose(windows, [0, 1, 3, 2, 4, 5]) | |
| windows = tf.reshape(windows, [-1, window * window, self.channels]) | |
| windows = self.attention_norm(windows) | |
| attended = self.attention( | |
| windows, windows, training=training | |
| ) | |
| attended = tf.reshape( | |
| attended, | |
| [batch, windows_h, windows_w, window, window, self.channels], | |
| ) | |
| attended = tf.transpose(attended, [0, 1, 3, 2, 4, 5]) | |
| attended = tf.reshape( | |
| attended, | |
| [batch, padded_shape[1], padded_shape[2], self.channels], | |
| ) | |
| return attended[:, :height, :width, :] | |
| def _global_attention(self, tokens, training): | |
| shape = tf.shape(tokens) | |
| flattened = tf.reshape(tokens, [shape[0], shape[1] * shape[2], self.channels]) | |
| normalized = self.attention_norm(flattened) | |
| attended = self.attention( | |
| normalized, normalized, training=training | |
| ) | |
| return tf.reshape(attended, shape) | |
| def call(self, inputs, training=None): | |
| inputs = tf.cast(inputs, self.compute_dtype) | |
| position = self.position_conv(inputs) | |
| positioned = inputs + tf.cast(position, inputs.dtype) | |
| if self.window_size is None: | |
| attended = self._global_attention(positioned, training) | |
| else: | |
| attended = self._window_attention(positioned, training) | |
| attended = tf.cast(attended, inputs.dtype) | |
| x = inputs + tf.cast(self.attention_scale, inputs.dtype) * attended | |
| shape = tf.shape(x) | |
| tokens = tf.reshape(x, [shape[0], shape[1] * shape[2], self.channels]) | |
| mlp = self.mlp_norm(tokens) | |
| mlp = self.mlp_expand(mlp) | |
| mlp = self.mlp_project(mlp) | |
| tokens = tokens + tf.cast(self.mlp_scale, tokens.dtype) * tf.cast( | |
| mlp, tokens.dtype | |
| ) | |
| output = tf.reshape(tokens, shape) | |
| return tf.ensure_shape(output, inputs.shape) | |
| def get_config(self): | |
| config = super().get_config() | |
| config.update( | |
| { | |
| "channels": self.channels, | |
| "heads": self.heads, | |
| "mlp_ratio": self.mlp_ratio, | |
| "window_size": self.window_size, | |
| } | |
| ) | |
| return config | |
| def build_efficientnet_attention_conv( | |
| variant: str, | |
| num_classes: int = 1_000, | |
| ) -> tf.keras.Model: | |
| """Build EfficientNet with P3-windowed and P4/P5-global attention adapters.""" | |
| if variant not in VARIANTS: | |
| raise ValueError(f"Unknown EfficientNet scale {variant!r}; use {tuple(VARIANTS)}") | |
| scale = VARIANTS[variant] | |
| inputs = tf.keras.Input( | |
| shape=(scale.image_size, scale.image_size, 3), name="image" | |
| ) | |
| stem_filters = round_filters(32, scale.width) | |
| x = _conv_bn_relu6(inputs, stem_filters, 3, 2, "stem") | |
| repeats_by_stage = [round_repeats(spec[3], scale.depth) for spec in STAGE_SPECS] | |
| total_blocks = sum(repeats_by_stage) | |
| filters_in = stem_filters | |
| block_index = 0 | |
| for stage_index, (kernel, expansion, base_filters, _, first_stride) in enumerate( | |
| STAGE_SPECS | |
| ): | |
| filters_out = round_filters(base_filters, scale.width) | |
| repeats = repeats_by_stage[stage_index] | |
| for block in range(repeats): | |
| stride = first_stride if block == 0 else 1 | |
| drop_rate = 0.1 * block_index / max(total_blocks - 1, 1) | |
| x = _mbconv( | |
| x, | |
| filters_in=filters_in, | |
| filters_out=filters_out, | |
| kernel=kernel, | |
| stride=stride, | |
| expansion=expansion, | |
| se_ratio=0.25, | |
| drop_rate=drop_rate, | |
| name=f"stage{stage_index + 1}_block{block + 1}", | |
| ) | |
| filters_in = filters_out | |
| block_index += 1 | |
| if stage_index in ATTENTION_STAGE_INDICES: | |
| channels = filters_out | |
| window_size = P3_WINDOW_SIZE if stage_index == 2 else None | |
| x = ConvolutionalAttentionAdapter( | |
| channels=channels, | |
| heads=ATTENTION_HEADS, | |
| mlp_ratio=ATTENTION_MLP_RATIO, | |
| window_size=window_size, | |
| name=f"attnconv_stage{stage_index + 1}", | |
| )(x) | |
| head_filters = round_filters(1280, scale.width) | |
| x = _conv_bn_relu6(x, head_filters, 1, 1, "head") | |
| x = tf.keras.layers.GlobalAveragePooling2D(name="global_pool")(x) | |
| x = tf.keras.layers.Dropout(scale.dropout, name="classifier_dropout")(x) | |
| logits = tf.keras.layers.Dense(num_classes, dtype="float32", name="logits")(x) | |
| return tf.keras.Model( | |
| inputs, | |
| logits, | |
| name=f"efficientnet_{variant}_sparse_attnconv_relu6", | |
| ) | |
| def transfer_matching_backbone_weights( | |
| source_model: tf.keras.Model, | |
| target_model: tf.keras.Model, | |
| ) -> dict[str, object]: | |
| """Copy every exact name-and-shape match and report new target weights. | |
| Fails closed if any weighted source layer is missing or has a different | |
| weight layout, or if the target-only weighted layers are not exactly the | |
| three planned attention adapters. Call this before the parity audit so a | |
| transferred EfficientNet function is the actual audit baseline. | |
| """ | |
| source_layers = {layer.name: layer for layer in source_model.layers} | |
| target_layers = {layer.name: layer for layer in target_model.layers} | |
| copied_layers: list[str] = [] | |
| missing_layers: list[str] = [] | |
| shape_mismatches: dict[str, dict[str, list[tuple[int, ...]]]] = {} | |
| for name, source_layer in source_layers.items(): | |
| source_weights = source_layer.get_weights() | |
| if not source_weights: | |
| continue | |
| target_layer = target_layers.get(name) | |
| if target_layer is None: | |
| missing_layers.append(name) | |
| continue | |
| target_weights = target_layer.get_weights() | |
| source_shapes = [tuple(array.shape) for array in source_weights] | |
| target_shapes = [tuple(array.shape) for array in target_weights] | |
| if source_shapes != target_shapes: | |
| shape_mismatches[name] = { | |
| "source": source_shapes, | |
| "target": target_shapes, | |
| } | |
| continue | |
| target_layer.set_weights(source_weights) | |
| copied_layers.append(name) | |
| source_weighted_names = { | |
| layer.name for layer in source_model.layers if layer.get_weights() | |
| } | |
| target_only_weighted = sorted( | |
| layer.name | |
| for layer in target_model.layers | |
| if layer.get_weights() and layer.name not in source_weighted_names | |
| ) | |
| expected_target_only = sorted(EXPECTED_ATTENTION_LAYER_NAMES) | |
| target_only_set_matches_expected = target_only_weighted == expected_target_only | |
| report: dict[str, object] = { | |
| "copied_weighted_layer_count": len(copied_layers), | |
| "copied_weighted_layers": copied_layers, | |
| "missing_source_layers": missing_layers, | |
| "shape_mismatches": shape_mismatches, | |
| "target_only_weighted_layers": target_only_weighted, | |
| "expected_target_only_weighted_layers": expected_target_only, | |
| "target_only_set_matches_expected": target_only_set_matches_expected, | |
| "complete_source_transfer": ( | |
| not missing_layers | |
| and not shape_mismatches | |
| and target_only_set_matches_expected | |
| ), | |
| } | |
| if missing_layers or shape_mismatches or not target_only_set_matches_expected: | |
| raise ValueError(f"Incomplete EfficientNet weight transfer: {report}") | |
| return report | |
| def audit_initial_logits_parity( | |
| source_model: tf.keras.Model, | |
| target_model: tf.keras.Model, | |
| images: tf.Tensor, | |
| *, | |
| atol: float = 1e-6, | |
| rtol: float = 1e-6, | |
| ) -> dict[str, object]: | |
| """Measure inference-logit parity on a deterministic input batch. | |
| Use validation examples with ``training=False`` after weight transfer. | |
| The audit returns metrics instead of silently accepting a mismatch; the | |
| caller should record the report and require ``within_tolerance`` before | |
| training the attention-convolution arm. | |
| """ | |
| images = tf.convert_to_tensor(images) | |
| if images.shape.rank != 4: | |
| raise ValueError("images must be a rank-4 NHWC batch") | |
| if images.shape[0] == 0: | |
| raise ValueError("images must contain at least one example") | |
| reference_logits = tf.cast(source_model(images, training=False), tf.float64) | |
| target_logits = tf.cast(target_model(images, training=False), tf.float64) | |
| difference = tf.abs(reference_logits - target_logits) | |
| finite = tf.reduce_all(tf.math.is_finite(difference)) | |
| within_tolerance = tf.reduce_all( | |
| difference <= tf.cast(atol, tf.float64) | |
| + tf.cast(rtol, tf.float64) * tf.abs(reference_logits) | |
| ) | |
| return { | |
| "example_count": int(tf.shape(images)[0].numpy()), | |
| "logit_count": int(tf.size(difference).numpy()), | |
| "max_absolute_difference": float(tf.reduce_max(difference).numpy()), | |
| "mean_absolute_difference": float(tf.reduce_mean(difference).numpy()), | |
| "finite": bool(finite.numpy()), | |
| "within_tolerance": bool(tf.logical_and(finite, within_tolerance).numpy()), | |
| "atol": float(atol), | |
| "rtol": float(rtol), | |
| } | |