Image Classification
Keras
computer-vision
efficientnet
self-attention
weight-sharing
hybrid-vision-backbone
tensorflow
Eval Results (legacy)
Instructions to use meatloafer/efficientnet-b0-stage-attention with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- Keras
How to use meatloafer/efficientnet-b0-stage-attention with Keras:
# !pip install -U keras tensorflow huggingface_hub # Keras needs TensorFlow installed to read "hf://" paths, so the tensorflow backend is selected here; # "jax" and "torch" also work for computation once TensorFlow is installed. import os os.environ["KERAS_BACKEND"] = "tensorflow" import keras model = keras.saving.load_model("hf://meatloafer/efficientnet-b0-stage-attention") - Notebooks
- Google Colab
- Kaggle
File size: 14,364 Bytes
587d427 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 | """Sparse multiscale attention adapters on an EfficientNet convolutional trunk.
This builder is intentionally kept separate from ``models.py``. It preserves
every MBConv stage and
adds three residual attention adapters: windowed attention at P3 and global
attention at P4 and P5. Zero-initialized LayerScale gates make the inserted
blocks an identity at initialization, so a transferred EfficientNet checkpoint
starts with unchanged logits.
The strict training launcher supports the ``attention_conv`` architecture
name. A run still requires a committed source-pinned registration and queue
admission after the existing priority sequence.
"""
from __future__ import annotations
import tensorflow as tf
from imagenet_research.models import (
STAGE_SPECS,
VARIANTS,
_conv_bn_relu6,
_mbconv,
round_filters,
round_repeats,
)
ATTENTION_STAGE_INDICES = (2, 4, 6)
EXPECTED_ATTENTION_LAYER_NAMES = (
"attnconv_stage3",
"attnconv_stage5",
"attnconv_stage7",
)
P3_WINDOW_SIZE = 7
ATTENTION_HEADS = 4
ATTENTION_MLP_RATIO = 2
@tf.keras.utils.register_keras_serializable(package="imagenet_research")
class ConvolutionalAttentionAdapter(tf.keras.layers.Layer):
"""Residual spatial attention with convolutional positional mixing.
``window_size=None`` uses global attention. A positive window size
partitions a feature map into non-overlapping windows, padding only the
bottom and right edges when needed. The depthwise positional convolution
is applied before partitioning, so nearby windows still receive local
convolutional context.
The attention and MLP branches have independent per-channel LayerScale
gates initialized to zero. At the initial checkpoint, the adapter is
exactly the identity; training can first open the gates and then update
each branch.
"""
def __init__(
self,
channels: int,
heads: int = ATTENTION_HEADS,
mlp_ratio: int = ATTENTION_MLP_RATIO,
window_size: int | None = None,
**kwargs,
):
super().__init__(**kwargs)
if channels < 1 or heads < 1 or channels % heads:
raise ValueError("channels must be positive and divisible by heads")
if mlp_ratio < 1:
raise ValueError("mlp_ratio must be positive")
if window_size is not None and window_size < 1:
raise ValueError("window_size must be positive or None")
self.channels = int(channels)
self.heads = int(heads)
self.mlp_ratio = int(mlp_ratio)
self.window_size = None if window_size is None else int(window_size)
self.position_conv = tf.keras.layers.DepthwiseConv2D(
3,
padding="same",
depthwise_initializer="he_normal",
name="position_conv",
)
self.attention_norm = tf.keras.layers.LayerNormalization(
epsilon=1e-6, name="attention_norm"
)
self.attention = tf.keras.layers.MultiHeadAttention(
num_heads=self.heads,
key_dim=self.channels // self.heads,
output_shape=self.channels,
name="mhsa",
)
self.mlp_norm = tf.keras.layers.LayerNormalization(
epsilon=1e-6, name="mlp_norm"
)
self.mlp_expand = tf.keras.layers.Dense(
self.channels * self.mlp_ratio,
activation=tf.nn.relu6,
name="mlp_expand",
)
self.mlp_project = tf.keras.layers.Dense(
self.channels, name="mlp_project"
)
def build(self, input_shape):
if input_shape[-1] is not None and int(input_shape[-1]) != self.channels:
raise ValueError(
f"channels={self.channels} does not match input channels={input_shape[-1]}"
)
self.attention_scale = self.add_weight(
name="attention_scale",
shape=(self.channels,),
initializer="zeros",
trainable=True,
)
self.mlp_scale = self.add_weight(
name="mlp_scale",
shape=(self.channels,),
initializer="zeros",
trainable=True,
)
super().build(input_shape)
def _window_attention(self, tokens, training):
window = self.window_size
shape = tf.shape(tokens)
batch, height, width = shape[0], shape[1], shape[2]
pad_height = tf.math.floormod(-height, window)
pad_width = tf.math.floormod(-width, window)
paddings = tf.stack(
[
tf.constant([0, 0], dtype=tf.int32),
tf.stack([tf.constant(0, tf.int32), pad_height]),
tf.stack([tf.constant(0, tf.int32), pad_width]),
tf.constant([0, 0], dtype=tf.int32),
]
)
padded = tf.pad(tokens, paddings)
padded_shape = tf.shape(padded)
windows_h = padded_shape[1] // window
windows_w = padded_shape[2] // window
windows = tf.reshape(
padded,
[batch, windows_h, window, windows_w, window, self.channels],
)
windows = tf.transpose(windows, [0, 1, 3, 2, 4, 5])
windows = tf.reshape(windows, [-1, window * window, self.channels])
windows = self.attention_norm(windows)
attended = self.attention(
windows, windows, training=training
)
attended = tf.reshape(
attended,
[batch, windows_h, windows_w, window, window, self.channels],
)
attended = tf.transpose(attended, [0, 1, 3, 2, 4, 5])
attended = tf.reshape(
attended,
[batch, padded_shape[1], padded_shape[2], self.channels],
)
return attended[:, :height, :width, :]
def _global_attention(self, tokens, training):
shape = tf.shape(tokens)
flattened = tf.reshape(tokens, [shape[0], shape[1] * shape[2], self.channels])
normalized = self.attention_norm(flattened)
attended = self.attention(
normalized, normalized, training=training
)
return tf.reshape(attended, shape)
def call(self, inputs, training=None):
inputs = tf.cast(inputs, self.compute_dtype)
position = self.position_conv(inputs)
positioned = inputs + tf.cast(position, inputs.dtype)
if self.window_size is None:
attended = self._global_attention(positioned, training)
else:
attended = self._window_attention(positioned, training)
attended = tf.cast(attended, inputs.dtype)
x = inputs + tf.cast(self.attention_scale, inputs.dtype) * attended
shape = tf.shape(x)
tokens = tf.reshape(x, [shape[0], shape[1] * shape[2], self.channels])
mlp = self.mlp_norm(tokens)
mlp = self.mlp_expand(mlp)
mlp = self.mlp_project(mlp)
tokens = tokens + tf.cast(self.mlp_scale, tokens.dtype) * tf.cast(
mlp, tokens.dtype
)
output = tf.reshape(tokens, shape)
return tf.ensure_shape(output, inputs.shape)
def get_config(self):
config = super().get_config()
config.update(
{
"channels": self.channels,
"heads": self.heads,
"mlp_ratio": self.mlp_ratio,
"window_size": self.window_size,
}
)
return config
def build_efficientnet_attention_conv(
variant: str,
num_classes: int = 1_000,
) -> tf.keras.Model:
"""Build EfficientNet with P3-windowed and P4/P5-global attention adapters."""
if variant not in VARIANTS:
raise ValueError(f"Unknown EfficientNet scale {variant!r}; use {tuple(VARIANTS)}")
scale = VARIANTS[variant]
inputs = tf.keras.Input(
shape=(scale.image_size, scale.image_size, 3), name="image"
)
stem_filters = round_filters(32, scale.width)
x = _conv_bn_relu6(inputs, stem_filters, 3, 2, "stem")
repeats_by_stage = [round_repeats(spec[3], scale.depth) for spec in STAGE_SPECS]
total_blocks = sum(repeats_by_stage)
filters_in = stem_filters
block_index = 0
for stage_index, (kernel, expansion, base_filters, _, first_stride) in enumerate(
STAGE_SPECS
):
filters_out = round_filters(base_filters, scale.width)
repeats = repeats_by_stage[stage_index]
for block in range(repeats):
stride = first_stride if block == 0 else 1
drop_rate = 0.1 * block_index / max(total_blocks - 1, 1)
x = _mbconv(
x,
filters_in=filters_in,
filters_out=filters_out,
kernel=kernel,
stride=stride,
expansion=expansion,
se_ratio=0.25,
drop_rate=drop_rate,
name=f"stage{stage_index + 1}_block{block + 1}",
)
filters_in = filters_out
block_index += 1
if stage_index in ATTENTION_STAGE_INDICES:
channels = filters_out
window_size = P3_WINDOW_SIZE if stage_index == 2 else None
x = ConvolutionalAttentionAdapter(
channels=channels,
heads=ATTENTION_HEADS,
mlp_ratio=ATTENTION_MLP_RATIO,
window_size=window_size,
name=f"attnconv_stage{stage_index + 1}",
)(x)
head_filters = round_filters(1280, scale.width)
x = _conv_bn_relu6(x, head_filters, 1, 1, "head")
x = tf.keras.layers.GlobalAveragePooling2D(name="global_pool")(x)
x = tf.keras.layers.Dropout(scale.dropout, name="classifier_dropout")(x)
logits = tf.keras.layers.Dense(num_classes, dtype="float32", name="logits")(x)
return tf.keras.Model(
inputs,
logits,
name=f"efficientnet_{variant}_sparse_attnconv_relu6",
)
def transfer_matching_backbone_weights(
source_model: tf.keras.Model,
target_model: tf.keras.Model,
) -> dict[str, object]:
"""Copy every exact name-and-shape match and report new target weights.
Fails closed if any weighted source layer is missing or has a different
weight layout, or if the target-only weighted layers are not exactly the
three planned attention adapters. Call this before the parity audit so a
transferred EfficientNet function is the actual audit baseline.
"""
source_layers = {layer.name: layer for layer in source_model.layers}
target_layers = {layer.name: layer for layer in target_model.layers}
copied_layers: list[str] = []
missing_layers: list[str] = []
shape_mismatches: dict[str, dict[str, list[tuple[int, ...]]]] = {}
for name, source_layer in source_layers.items():
source_weights = source_layer.get_weights()
if not source_weights:
continue
target_layer = target_layers.get(name)
if target_layer is None:
missing_layers.append(name)
continue
target_weights = target_layer.get_weights()
source_shapes = [tuple(array.shape) for array in source_weights]
target_shapes = [tuple(array.shape) for array in target_weights]
if source_shapes != target_shapes:
shape_mismatches[name] = {
"source": source_shapes,
"target": target_shapes,
}
continue
target_layer.set_weights(source_weights)
copied_layers.append(name)
source_weighted_names = {
layer.name for layer in source_model.layers if layer.get_weights()
}
target_only_weighted = sorted(
layer.name
for layer in target_model.layers
if layer.get_weights() and layer.name not in source_weighted_names
)
expected_target_only = sorted(EXPECTED_ATTENTION_LAYER_NAMES)
target_only_set_matches_expected = target_only_weighted == expected_target_only
report: dict[str, object] = {
"copied_weighted_layer_count": len(copied_layers),
"copied_weighted_layers": copied_layers,
"missing_source_layers": missing_layers,
"shape_mismatches": shape_mismatches,
"target_only_weighted_layers": target_only_weighted,
"expected_target_only_weighted_layers": expected_target_only,
"target_only_set_matches_expected": target_only_set_matches_expected,
"complete_source_transfer": (
not missing_layers
and not shape_mismatches
and target_only_set_matches_expected
),
}
if missing_layers or shape_mismatches or not target_only_set_matches_expected:
raise ValueError(f"Incomplete EfficientNet weight transfer: {report}")
return report
def audit_initial_logits_parity(
source_model: tf.keras.Model,
target_model: tf.keras.Model,
images: tf.Tensor,
*,
atol: float = 1e-6,
rtol: float = 1e-6,
) -> dict[str, object]:
"""Measure inference-logit parity on a deterministic input batch.
Use validation examples with ``training=False`` after weight transfer.
The audit returns metrics instead of silently accepting a mismatch; the
caller should record the report and require ``within_tolerance`` before
training the attention-convolution arm.
"""
images = tf.convert_to_tensor(images)
if images.shape.rank != 4:
raise ValueError("images must be a rank-4 NHWC batch")
if images.shape[0] == 0:
raise ValueError("images must contain at least one example")
reference_logits = tf.cast(source_model(images, training=False), tf.float64)
target_logits = tf.cast(target_model(images, training=False), tf.float64)
difference = tf.abs(reference_logits - target_logits)
finite = tf.reduce_all(tf.math.is_finite(difference))
within_tolerance = tf.reduce_all(
difference <= tf.cast(atol, tf.float64)
+ tf.cast(rtol, tf.float64) * tf.abs(reference_logits)
)
return {
"example_count": int(tf.shape(images)[0].numpy()),
"logit_count": int(tf.size(difference).numpy()),
"max_absolute_difference": float(tf.reduce_max(difference).numpy()),
"mean_absolute_difference": float(tf.reduce_mean(difference).numpy()),
"finite": bool(finite.numpy()),
"within_tolerance": bool(tf.logical_and(finite, within_tolerance).numpy()),
"atol": float(atol),
"rtol": float(rtol),
}
|