File size: 14,364 Bytes
587d427
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
"""Sparse multiscale attention adapters on an EfficientNet convolutional trunk.

This builder is intentionally kept separate from ``models.py``. It preserves
every MBConv stage and
adds three residual attention adapters: windowed attention at P3 and global
attention at P4 and P5.  Zero-initialized LayerScale gates make the inserted
blocks an identity at initialization, so a transferred EfficientNet checkpoint
starts with unchanged logits.

The strict training launcher supports the ``attention_conv`` architecture
name. A run still requires a committed source-pinned registration and queue
admission after the existing priority sequence.
"""

from __future__ import annotations

import tensorflow as tf

from imagenet_research.models import (
    STAGE_SPECS,
    VARIANTS,
    _conv_bn_relu6,
    _mbconv,
    round_filters,
    round_repeats,
)


ATTENTION_STAGE_INDICES = (2, 4, 6)
EXPECTED_ATTENTION_LAYER_NAMES = (
    "attnconv_stage3",
    "attnconv_stage5",
    "attnconv_stage7",
)
P3_WINDOW_SIZE = 7
ATTENTION_HEADS = 4
ATTENTION_MLP_RATIO = 2


@tf.keras.utils.register_keras_serializable(package="imagenet_research")
class ConvolutionalAttentionAdapter(tf.keras.layers.Layer):
    """Residual spatial attention with convolutional positional mixing.

    ``window_size=None`` uses global attention.  A positive window size
    partitions a feature map into non-overlapping windows, padding only the
    bottom and right edges when needed.  The depthwise positional convolution
    is applied before partitioning, so nearby windows still receive local
    convolutional context.

    The attention and MLP branches have independent per-channel LayerScale
    gates initialized to zero.  At the initial checkpoint, the adapter is
    exactly the identity; training can first open the gates and then update
    each branch.
    """

    def __init__(
        self,
        channels: int,
        heads: int = ATTENTION_HEADS,
        mlp_ratio: int = ATTENTION_MLP_RATIO,
        window_size: int | None = None,
        **kwargs,
    ):
        super().__init__(**kwargs)
        if channels < 1 or heads < 1 or channels % heads:
            raise ValueError("channels must be positive and divisible by heads")
        if mlp_ratio < 1:
            raise ValueError("mlp_ratio must be positive")
        if window_size is not None and window_size < 1:
            raise ValueError("window_size must be positive or None")
        self.channels = int(channels)
        self.heads = int(heads)
        self.mlp_ratio = int(mlp_ratio)
        self.window_size = None if window_size is None else int(window_size)
        self.position_conv = tf.keras.layers.DepthwiseConv2D(
            3,
            padding="same",
            depthwise_initializer="he_normal",
            name="position_conv",
        )
        self.attention_norm = tf.keras.layers.LayerNormalization(
            epsilon=1e-6, name="attention_norm"
        )
        self.attention = tf.keras.layers.MultiHeadAttention(
            num_heads=self.heads,
            key_dim=self.channels // self.heads,
            output_shape=self.channels,
            name="mhsa",
        )
        self.mlp_norm = tf.keras.layers.LayerNormalization(
            epsilon=1e-6, name="mlp_norm"
        )
        self.mlp_expand = tf.keras.layers.Dense(
            self.channels * self.mlp_ratio,
            activation=tf.nn.relu6,
            name="mlp_expand",
        )
        self.mlp_project = tf.keras.layers.Dense(
            self.channels, name="mlp_project"
        )

    def build(self, input_shape):
        if input_shape[-1] is not None and int(input_shape[-1]) != self.channels:
            raise ValueError(
                f"channels={self.channels} does not match input channels={input_shape[-1]}"
            )
        self.attention_scale = self.add_weight(
            name="attention_scale",
            shape=(self.channels,),
            initializer="zeros",
            trainable=True,
        )
        self.mlp_scale = self.add_weight(
            name="mlp_scale",
            shape=(self.channels,),
            initializer="zeros",
            trainable=True,
        )
        super().build(input_shape)

    def _window_attention(self, tokens, training):
        window = self.window_size
        shape = tf.shape(tokens)
        batch, height, width = shape[0], shape[1], shape[2]
        pad_height = tf.math.floormod(-height, window)
        pad_width = tf.math.floormod(-width, window)
        paddings = tf.stack(
            [
                tf.constant([0, 0], dtype=tf.int32),
                tf.stack([tf.constant(0, tf.int32), pad_height]),
                tf.stack([tf.constant(0, tf.int32), pad_width]),
                tf.constant([0, 0], dtype=tf.int32),
            ]
        )
        padded = tf.pad(tokens, paddings)
        padded_shape = tf.shape(padded)
        windows_h = padded_shape[1] // window
        windows_w = padded_shape[2] // window
        windows = tf.reshape(
            padded,
            [batch, windows_h, window, windows_w, window, self.channels],
        )
        windows = tf.transpose(windows, [0, 1, 3, 2, 4, 5])
        windows = tf.reshape(windows, [-1, window * window, self.channels])
        windows = self.attention_norm(windows)
        attended = self.attention(
            windows, windows, training=training
        )
        attended = tf.reshape(
            attended,
            [batch, windows_h, windows_w, window, window, self.channels],
        )
        attended = tf.transpose(attended, [0, 1, 3, 2, 4, 5])
        attended = tf.reshape(
            attended,
            [batch, padded_shape[1], padded_shape[2], self.channels],
        )
        return attended[:, :height, :width, :]

    def _global_attention(self, tokens, training):
        shape = tf.shape(tokens)
        flattened = tf.reshape(tokens, [shape[0], shape[1] * shape[2], self.channels])
        normalized = self.attention_norm(flattened)
        attended = self.attention(
            normalized, normalized, training=training
        )
        return tf.reshape(attended, shape)

    def call(self, inputs, training=None):
        inputs = tf.cast(inputs, self.compute_dtype)
        position = self.position_conv(inputs)
        positioned = inputs + tf.cast(position, inputs.dtype)
        if self.window_size is None:
            attended = self._global_attention(positioned, training)
        else:
            attended = self._window_attention(positioned, training)
        attended = tf.cast(attended, inputs.dtype)
        x = inputs + tf.cast(self.attention_scale, inputs.dtype) * attended

        shape = tf.shape(x)
        tokens = tf.reshape(x, [shape[0], shape[1] * shape[2], self.channels])
        mlp = self.mlp_norm(tokens)
        mlp = self.mlp_expand(mlp)
        mlp = self.mlp_project(mlp)
        tokens = tokens + tf.cast(self.mlp_scale, tokens.dtype) * tf.cast(
            mlp, tokens.dtype
        )
        output = tf.reshape(tokens, shape)
        return tf.ensure_shape(output, inputs.shape)

    def get_config(self):
        config = super().get_config()
        config.update(
            {
                "channels": self.channels,
                "heads": self.heads,
                "mlp_ratio": self.mlp_ratio,
                "window_size": self.window_size,
            }
        )
        return config


def build_efficientnet_attention_conv(
    variant: str,
    num_classes: int = 1_000,
) -> tf.keras.Model:
    """Build EfficientNet with P3-windowed and P4/P5-global attention adapters."""
    if variant not in VARIANTS:
        raise ValueError(f"Unknown EfficientNet scale {variant!r}; use {tuple(VARIANTS)}")
    scale = VARIANTS[variant]
    inputs = tf.keras.Input(
        shape=(scale.image_size, scale.image_size, 3), name="image"
    )
    stem_filters = round_filters(32, scale.width)
    x = _conv_bn_relu6(inputs, stem_filters, 3, 2, "stem")
    repeats_by_stage = [round_repeats(spec[3], scale.depth) for spec in STAGE_SPECS]
    total_blocks = sum(repeats_by_stage)
    filters_in = stem_filters
    block_index = 0

    for stage_index, (kernel, expansion, base_filters, _, first_stride) in enumerate(
        STAGE_SPECS
    ):
        filters_out = round_filters(base_filters, scale.width)
        repeats = repeats_by_stage[stage_index]
        for block in range(repeats):
            stride = first_stride if block == 0 else 1
            drop_rate = 0.1 * block_index / max(total_blocks - 1, 1)
            x = _mbconv(
                x,
                filters_in=filters_in,
                filters_out=filters_out,
                kernel=kernel,
                stride=stride,
                expansion=expansion,
                se_ratio=0.25,
                drop_rate=drop_rate,
                name=f"stage{stage_index + 1}_block{block + 1}",
            )
            filters_in = filters_out
            block_index += 1

        if stage_index in ATTENTION_STAGE_INDICES:
            channels = filters_out
            window_size = P3_WINDOW_SIZE if stage_index == 2 else None
            x = ConvolutionalAttentionAdapter(
                channels=channels,
                heads=ATTENTION_HEADS,
                mlp_ratio=ATTENTION_MLP_RATIO,
                window_size=window_size,
                name=f"attnconv_stage{stage_index + 1}",
            )(x)

    head_filters = round_filters(1280, scale.width)
    x = _conv_bn_relu6(x, head_filters, 1, 1, "head")
    x = tf.keras.layers.GlobalAveragePooling2D(name="global_pool")(x)
    x = tf.keras.layers.Dropout(scale.dropout, name="classifier_dropout")(x)
    logits = tf.keras.layers.Dense(num_classes, dtype="float32", name="logits")(x)
    return tf.keras.Model(
        inputs,
        logits,
        name=f"efficientnet_{variant}_sparse_attnconv_relu6",
    )


def transfer_matching_backbone_weights(
    source_model: tf.keras.Model,
    target_model: tf.keras.Model,
) -> dict[str, object]:
    """Copy every exact name-and-shape match and report new target weights.

    Fails closed if any weighted source layer is missing or has a different
    weight layout, or if the target-only weighted layers are not exactly the
    three planned attention adapters. Call this before the parity audit so a
    transferred EfficientNet function is the actual audit baseline.
    """
    source_layers = {layer.name: layer for layer in source_model.layers}
    target_layers = {layer.name: layer for layer in target_model.layers}
    copied_layers: list[str] = []
    missing_layers: list[str] = []
    shape_mismatches: dict[str, dict[str, list[tuple[int, ...]]]] = {}

    for name, source_layer in source_layers.items():
        source_weights = source_layer.get_weights()
        if not source_weights:
            continue
        target_layer = target_layers.get(name)
        if target_layer is None:
            missing_layers.append(name)
            continue
        target_weights = target_layer.get_weights()
        source_shapes = [tuple(array.shape) for array in source_weights]
        target_shapes = [tuple(array.shape) for array in target_weights]
        if source_shapes != target_shapes:
            shape_mismatches[name] = {
                "source": source_shapes,
                "target": target_shapes,
            }
            continue
        target_layer.set_weights(source_weights)
        copied_layers.append(name)

    source_weighted_names = {
        layer.name for layer in source_model.layers if layer.get_weights()
    }
    target_only_weighted = sorted(
        layer.name
        for layer in target_model.layers
        if layer.get_weights() and layer.name not in source_weighted_names
    )
    expected_target_only = sorted(EXPECTED_ATTENTION_LAYER_NAMES)
    target_only_set_matches_expected = target_only_weighted == expected_target_only
    report: dict[str, object] = {
        "copied_weighted_layer_count": len(copied_layers),
        "copied_weighted_layers": copied_layers,
        "missing_source_layers": missing_layers,
        "shape_mismatches": shape_mismatches,
        "target_only_weighted_layers": target_only_weighted,
        "expected_target_only_weighted_layers": expected_target_only,
        "target_only_set_matches_expected": target_only_set_matches_expected,
        "complete_source_transfer": (
            not missing_layers
            and not shape_mismatches
            and target_only_set_matches_expected
        ),
    }
    if missing_layers or shape_mismatches or not target_only_set_matches_expected:
        raise ValueError(f"Incomplete EfficientNet weight transfer: {report}")
    return report


def audit_initial_logits_parity(
    source_model: tf.keras.Model,
    target_model: tf.keras.Model,
    images: tf.Tensor,
    *,
    atol: float = 1e-6,
    rtol: float = 1e-6,
) -> dict[str, object]:
    """Measure inference-logit parity on a deterministic input batch.

    Use validation examples with ``training=False`` after weight transfer.
    The audit returns metrics instead of silently accepting a mismatch; the
    caller should record the report and require ``within_tolerance`` before
    training the attention-convolution arm.
    """
    images = tf.convert_to_tensor(images)
    if images.shape.rank != 4:
        raise ValueError("images must be a rank-4 NHWC batch")
    if images.shape[0] == 0:
        raise ValueError("images must contain at least one example")
    reference_logits = tf.cast(source_model(images, training=False), tf.float64)
    target_logits = tf.cast(target_model(images, training=False), tf.float64)
    difference = tf.abs(reference_logits - target_logits)
    finite = tf.reduce_all(tf.math.is_finite(difference))
    within_tolerance = tf.reduce_all(
        difference <= tf.cast(atol, tf.float64)
        + tf.cast(rtol, tf.float64) * tf.abs(reference_logits)
    )
    return {
        "example_count": int(tf.shape(images)[0].numpy()),
        "logit_count": int(tf.size(difference).numpy()),
        "max_absolute_difference": float(tf.reduce_max(difference).numpy()),
        "mean_absolute_difference": float(tf.reduce_mean(difference).numpy()),
        "finite": bool(finite.numpy()),
        "within_tolerance": bool(tf.logical_and(finite, within_tolerance).numpy()),
        "atol": float(atol),
        "rtol": float(rtol),
    }