"""Sparse multiscale attention adapters on an EfficientNet convolutional trunk. This builder is intentionally kept separate from ``models.py``. It preserves every MBConv stage and adds three residual attention adapters: windowed attention at P3 and global attention at P4 and P5. Zero-initialized LayerScale gates make the inserted blocks an identity at initialization, so a transferred EfficientNet checkpoint starts with unchanged logits. The strict training launcher supports the ``attention_conv`` architecture name. A run still requires a committed source-pinned registration and queue admission after the existing priority sequence. """ from __future__ import annotations import tensorflow as tf from imagenet_research.models import ( STAGE_SPECS, VARIANTS, _conv_bn_relu6, _mbconv, round_filters, round_repeats, ) ATTENTION_STAGE_INDICES = (2, 4, 6) EXPECTED_ATTENTION_LAYER_NAMES = ( "attnconv_stage3", "attnconv_stage5", "attnconv_stage7", ) P3_WINDOW_SIZE = 7 ATTENTION_HEADS = 4 ATTENTION_MLP_RATIO = 2 @tf.keras.utils.register_keras_serializable(package="imagenet_research") class ConvolutionalAttentionAdapter(tf.keras.layers.Layer): """Residual spatial attention with convolutional positional mixing. ``window_size=None`` uses global attention. A positive window size partitions a feature map into non-overlapping windows, padding only the bottom and right edges when needed. The depthwise positional convolution is applied before partitioning, so nearby windows still receive local convolutional context. The attention and MLP branches have independent per-channel LayerScale gates initialized to zero. At the initial checkpoint, the adapter is exactly the identity; training can first open the gates and then update each branch. """ def __init__( self, channels: int, heads: int = ATTENTION_HEADS, mlp_ratio: int = ATTENTION_MLP_RATIO, window_size: int | None = None, **kwargs, ): super().__init__(**kwargs) if channels < 1 or heads < 1 or channels % heads: raise ValueError("channels must be positive and divisible by heads") if mlp_ratio < 1: raise ValueError("mlp_ratio must be positive") if window_size is not None and window_size < 1: raise ValueError("window_size must be positive or None") self.channels = int(channels) self.heads = int(heads) self.mlp_ratio = int(mlp_ratio) self.window_size = None if window_size is None else int(window_size) self.position_conv = tf.keras.layers.DepthwiseConv2D( 3, padding="same", depthwise_initializer="he_normal", name="position_conv", ) self.attention_norm = tf.keras.layers.LayerNormalization( epsilon=1e-6, name="attention_norm" ) self.attention = tf.keras.layers.MultiHeadAttention( num_heads=self.heads, key_dim=self.channels // self.heads, output_shape=self.channels, name="mhsa", ) self.mlp_norm = tf.keras.layers.LayerNormalization( epsilon=1e-6, name="mlp_norm" ) self.mlp_expand = tf.keras.layers.Dense( self.channels * self.mlp_ratio, activation=tf.nn.relu6, name="mlp_expand", ) self.mlp_project = tf.keras.layers.Dense( self.channels, name="mlp_project" ) def build(self, input_shape): if input_shape[-1] is not None and int(input_shape[-1]) != self.channels: raise ValueError( f"channels={self.channels} does not match input channels={input_shape[-1]}" ) self.attention_scale = self.add_weight( name="attention_scale", shape=(self.channels,), initializer="zeros", trainable=True, ) self.mlp_scale = self.add_weight( name="mlp_scale", shape=(self.channels,), initializer="zeros", trainable=True, ) super().build(input_shape) def _window_attention(self, tokens, training): window = self.window_size shape = tf.shape(tokens) batch, height, width = shape[0], shape[1], shape[2] pad_height = tf.math.floormod(-height, window) pad_width = tf.math.floormod(-width, window) paddings = tf.stack( [ tf.constant([0, 0], dtype=tf.int32), tf.stack([tf.constant(0, tf.int32), pad_height]), tf.stack([tf.constant(0, tf.int32), pad_width]), tf.constant([0, 0], dtype=tf.int32), ] ) padded = tf.pad(tokens, paddings) padded_shape = tf.shape(padded) windows_h = padded_shape[1] // window windows_w = padded_shape[2] // window windows = tf.reshape( padded, [batch, windows_h, window, windows_w, window, self.channels], ) windows = tf.transpose(windows, [0, 1, 3, 2, 4, 5]) windows = tf.reshape(windows, [-1, window * window, self.channels]) windows = self.attention_norm(windows) attended = self.attention( windows, windows, training=training ) attended = tf.reshape( attended, [batch, windows_h, windows_w, window, window, self.channels], ) attended = tf.transpose(attended, [0, 1, 3, 2, 4, 5]) attended = tf.reshape( attended, [batch, padded_shape[1], padded_shape[2], self.channels], ) return attended[:, :height, :width, :] def _global_attention(self, tokens, training): shape = tf.shape(tokens) flattened = tf.reshape(tokens, [shape[0], shape[1] * shape[2], self.channels]) normalized = self.attention_norm(flattened) attended = self.attention( normalized, normalized, training=training ) return tf.reshape(attended, shape) def call(self, inputs, training=None): inputs = tf.cast(inputs, self.compute_dtype) position = self.position_conv(inputs) positioned = inputs + tf.cast(position, inputs.dtype) if self.window_size is None: attended = self._global_attention(positioned, training) else: attended = self._window_attention(positioned, training) attended = tf.cast(attended, inputs.dtype) x = inputs + tf.cast(self.attention_scale, inputs.dtype) * attended shape = tf.shape(x) tokens = tf.reshape(x, [shape[0], shape[1] * shape[2], self.channels]) mlp = self.mlp_norm(tokens) mlp = self.mlp_expand(mlp) mlp = self.mlp_project(mlp) tokens = tokens + tf.cast(self.mlp_scale, tokens.dtype) * tf.cast( mlp, tokens.dtype ) output = tf.reshape(tokens, shape) return tf.ensure_shape(output, inputs.shape) def get_config(self): config = super().get_config() config.update( { "channels": self.channels, "heads": self.heads, "mlp_ratio": self.mlp_ratio, "window_size": self.window_size, } ) return config def build_efficientnet_attention_conv( variant: str, num_classes: int = 1_000, ) -> tf.keras.Model: """Build EfficientNet with P3-windowed and P4/P5-global attention adapters.""" if variant not in VARIANTS: raise ValueError(f"Unknown EfficientNet scale {variant!r}; use {tuple(VARIANTS)}") scale = VARIANTS[variant] inputs = tf.keras.Input( shape=(scale.image_size, scale.image_size, 3), name="image" ) stem_filters = round_filters(32, scale.width) x = _conv_bn_relu6(inputs, stem_filters, 3, 2, "stem") repeats_by_stage = [round_repeats(spec[3], scale.depth) for spec in STAGE_SPECS] total_blocks = sum(repeats_by_stage) filters_in = stem_filters block_index = 0 for stage_index, (kernel, expansion, base_filters, _, first_stride) in enumerate( STAGE_SPECS ): filters_out = round_filters(base_filters, scale.width) repeats = repeats_by_stage[stage_index] for block in range(repeats): stride = first_stride if block == 0 else 1 drop_rate = 0.1 * block_index / max(total_blocks - 1, 1) x = _mbconv( x, filters_in=filters_in, filters_out=filters_out, kernel=kernel, stride=stride, expansion=expansion, se_ratio=0.25, drop_rate=drop_rate, name=f"stage{stage_index + 1}_block{block + 1}", ) filters_in = filters_out block_index += 1 if stage_index in ATTENTION_STAGE_INDICES: channels = filters_out window_size = P3_WINDOW_SIZE if stage_index == 2 else None x = ConvolutionalAttentionAdapter( channels=channels, heads=ATTENTION_HEADS, mlp_ratio=ATTENTION_MLP_RATIO, window_size=window_size, name=f"attnconv_stage{stage_index + 1}", )(x) head_filters = round_filters(1280, scale.width) x = _conv_bn_relu6(x, head_filters, 1, 1, "head") x = tf.keras.layers.GlobalAveragePooling2D(name="global_pool")(x) x = tf.keras.layers.Dropout(scale.dropout, name="classifier_dropout")(x) logits = tf.keras.layers.Dense(num_classes, dtype="float32", name="logits")(x) return tf.keras.Model( inputs, logits, name=f"efficientnet_{variant}_sparse_attnconv_relu6", ) def transfer_matching_backbone_weights( source_model: tf.keras.Model, target_model: tf.keras.Model, ) -> dict[str, object]: """Copy every exact name-and-shape match and report new target weights. Fails closed if any weighted source layer is missing or has a different weight layout, or if the target-only weighted layers are not exactly the three planned attention adapters. Call this before the parity audit so a transferred EfficientNet function is the actual audit baseline. """ source_layers = {layer.name: layer for layer in source_model.layers} target_layers = {layer.name: layer for layer in target_model.layers} copied_layers: list[str] = [] missing_layers: list[str] = [] shape_mismatches: dict[str, dict[str, list[tuple[int, ...]]]] = {} for name, source_layer in source_layers.items(): source_weights = source_layer.get_weights() if not source_weights: continue target_layer = target_layers.get(name) if target_layer is None: missing_layers.append(name) continue target_weights = target_layer.get_weights() source_shapes = [tuple(array.shape) for array in source_weights] target_shapes = [tuple(array.shape) for array in target_weights] if source_shapes != target_shapes: shape_mismatches[name] = { "source": source_shapes, "target": target_shapes, } continue target_layer.set_weights(source_weights) copied_layers.append(name) source_weighted_names = { layer.name for layer in source_model.layers if layer.get_weights() } target_only_weighted = sorted( layer.name for layer in target_model.layers if layer.get_weights() and layer.name not in source_weighted_names ) expected_target_only = sorted(EXPECTED_ATTENTION_LAYER_NAMES) target_only_set_matches_expected = target_only_weighted == expected_target_only report: dict[str, object] = { "copied_weighted_layer_count": len(copied_layers), "copied_weighted_layers": copied_layers, "missing_source_layers": missing_layers, "shape_mismatches": shape_mismatches, "target_only_weighted_layers": target_only_weighted, "expected_target_only_weighted_layers": expected_target_only, "target_only_set_matches_expected": target_only_set_matches_expected, "complete_source_transfer": ( not missing_layers and not shape_mismatches and target_only_set_matches_expected ), } if missing_layers or shape_mismatches or not target_only_set_matches_expected: raise ValueError(f"Incomplete EfficientNet weight transfer: {report}") return report def audit_initial_logits_parity( source_model: tf.keras.Model, target_model: tf.keras.Model, images: tf.Tensor, *, atol: float = 1e-6, rtol: float = 1e-6, ) -> dict[str, object]: """Measure inference-logit parity on a deterministic input batch. Use validation examples with ``training=False`` after weight transfer. The audit returns metrics instead of silently accepting a mismatch; the caller should record the report and require ``within_tolerance`` before training the attention-convolution arm. """ images = tf.convert_to_tensor(images) if images.shape.rank != 4: raise ValueError("images must be a rank-4 NHWC batch") if images.shape[0] == 0: raise ValueError("images must contain at least one example") reference_logits = tf.cast(source_model(images, training=False), tf.float64) target_logits = tf.cast(target_model(images, training=False), tf.float64) difference = tf.abs(reference_logits - target_logits) finite = tf.reduce_all(tf.math.is_finite(difference)) within_tolerance = tf.reduce_all( difference <= tf.cast(atol, tf.float64) + tf.cast(rtol, tf.float64) * tf.abs(reference_logits) ) return { "example_count": int(tf.shape(images)[0].numpy()), "logit_count": int(tf.size(difference).numpy()), "max_absolute_difference": float(tf.reduce_max(difference).numpy()), "mean_absolute_difference": float(tf.reduce_mean(difference).numpy()), "finite": bool(finite.numpy()), "within_tolerance": bool(tf.logical_and(finite, within_tolerance).numpy()), "atol": float(atol), "rtol": float(rtol), }