File size: 7,402 Bytes
2abfed0
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
"""Per-layer fake-quant QAT model construction for adaptive B0 profiles.

W3/W4/W8 kernels use TensorFlow fake quantization with its straight-through
gradient. W16 means FP16-rounded weights with Float32 activations and
arithmetic; it does not claim an FP16 execution path.
"""

from __future__ import annotations

from collections import Counter
from typing import Mapping

import tensorflow as tf

from imagenet_research.mixed_precision_profiles import eligible_weight_layers
from imagenet_research.quantization import (
    FakeQuantConv2D,
    FakeQuantDense,
    FakeQuantDepthwiseConv2D,
)


SUPPORTED_BITS = frozenset((3, 4, 8, 16))


@tf.custom_gradient
def _fp16_weight_round_ste(kernel):
    rounded = tf.cast(tf.cast(kernel, tf.float16), kernel.dtype)

    def grad(upstream):
        return tf.cast(upstream, kernel.dtype)

    return rounded, grad


@tf.keras.utils.register_keras_serializable(package="imagenet_research")
class Fp16WeightConv2D(tf.keras.layers.Conv2D):
    """Round Conv2D weights to FP16 while retaining Float32 convolution math."""

    def convolution_op(self, inputs, kernel):
        return super().convolution_op(inputs, _fp16_weight_round_ste(kernel))


@tf.keras.utils.register_keras_serializable(package="imagenet_research")
class Fp16WeightDepthwiseConv2D(tf.keras.layers.DepthwiseConv2D):
    """Round depthwise weights to FP16 while retaining Float32 convolution math."""

    def call(self, inputs):
        kernel = tf.cast(self.kernel, tf.float32)
        input_channels = self._get_input_channel(inputs.shape)
        output_channels = input_channels * self.depth_multiplier
        kernel = tf.reshape(
            kernel, [self.kernel_size[0], self.kernel_size[1], output_channels]
        )
        kernel = _fp16_weight_round_ste(kernel)
        kernel = tf.reshape(kernel, tf.shape(self.kernel))
        kernel = tf.cast(kernel, self.compute_dtype)
        inputs = tf.cast(inputs, self.compute_dtype)
        outputs = tf.keras.ops.depthwise_conv(
            inputs,
            kernel,
            strides=self.strides,
            padding=self.padding,
            dilation_rate=self.dilation_rate,
            data_format=self.data_format,
        )
        if self.use_bias:
            if self.data_format == "channels_last":
                bias_shape = (1,) * (self.rank + 1) + (output_channels,)
            else:
                bias_shape = (1, output_channels) + (1,) * self.rank
            outputs = tf.keras.ops.add(
                outputs, tf.keras.ops.reshape(tf.cast(self.bias, self.compute_dtype), bias_shape)
            )
        if self.activation is not None:
            outputs = self.activation(outputs)
        return outputs


@tf.keras.utils.register_keras_serializable(package="imagenet_research")
class Fp16WeightDense(tf.keras.layers.Dense):
    """Round Dense weights to FP16 while retaining Float32 matmul math."""

    def call(self, inputs, training=None):
        kernel = tf.cast(_fp16_weight_round_ste(tf.cast(self.kernel, tf.float32)), self.compute_dtype)
        inputs = tf.cast(inputs, self.compute_dtype)
        outputs = tf.keras.ops.matmul(inputs, kernel)
        if self.bias is not None:
            outputs = tf.keras.ops.add(outputs, tf.cast(self.bias, self.compute_dtype))
        if self.activation is not None:
            outputs = self.activation(outputs)
        return outputs


def _clone_layer_for_bits(layer: tf.keras.layers.Layer, bits: int):
    config = layer.get_config()
    if bits == 16:
        if isinstance(layer, tf.keras.layers.DepthwiseConv2D):
            return Fp16WeightDepthwiseConv2D(**config)
        if isinstance(layer, tf.keras.layers.Conv2D):
            return Fp16WeightConv2D(**config)
        if isinstance(layer, tf.keras.layers.Dense):
            return Fp16WeightDense(**config)
    elif isinstance(layer, tf.keras.layers.DepthwiseConv2D):
        return FakeQuantDepthwiseConv2D(num_bits=bits, **config)
    elif isinstance(layer, tf.keras.layers.Conv2D):
        return FakeQuantConv2D(num_bits=bits, **config)
    elif isinstance(layer, tf.keras.layers.Dense):
        return FakeQuantDense(num_bits=bits, **config)
    return layer.__class__.from_config(config)


def build_adaptive_qat_model(
    source_model: tf.keras.Model, bits_by_layer: Mapping[str, int]
) -> tuple[tf.keras.Model, dict]:
    """Clone a model with one registered weight precision per eligible kernel."""
    layers = eligible_weight_layers(source_model)
    normalized = {name: int(bits) for name, bits in bits_by_layer.items()}
    if set(normalized) != set(layers):
        raise ValueError(
            "Adaptive QAT map must cover exactly the materialized weighted layers"
        )
    invalid = {name: bits for name, bits in normalized.items() if bits not in SUPPORTED_BITS}
    if invalid:
        raise ValueError(f"Unsupported adaptive QAT bitwidths: {invalid}")

    clone = tf.keras.models.clone_model(
        source_model,
        clone_function=lambda layer: _clone_layer_for_bits(layer, normalized[layer.name])
        if layer.name in normalized
        else layer.__class__.from_config(layer.get_config()),
        recursive=True,
    )
    for layer in source_model.layers:
        if layer.weights:
            clone.get_layer(layer.name).set_weights(layer.get_weights())

    source_rng = {}
    target_layers = {layer.name: layer for layer in clone.layers}
    for layer in source_model.layers:
        generator = getattr(layer, "seed_generator", None)
        if generator is None:
            continue
        target = getattr(target_layers.get(layer.name), "seed_generator", None)
        if target is None:
            raise RuntimeError(f"Adaptive QAT clone lacks RNG state for {layer.name}")
        target.state.assign(generator.state)
        source_rng[layer.name] = [int(item) for item in generator.state.numpy().tolist()]
        if target.state.numpy().tolist() != generator.state.numpy().tolist():
            raise RuntimeError(f"Adaptive QAT RNG-state copy failed for {layer.name}")
    if not source_rng:
        raise RuntimeError("Expected EfficientNet Dropout/stochastic-depth RNG state")

    parameter_counts: Counter[int] = Counter()
    layer_counts: Counter[int] = Counter()
    for name, bits in normalized.items():
        parameter_counts[bits] += int(layers[name].kernel.shape.num_elements())
        layer_counts[bits] += 1
    total = sum(parameter_counts.values())
    audit = {
        "layer_bits": dict(sorted(normalized.items())),
        "eligible_kernel_layers": len(normalized),
        "eligible_kernel_parameters": total,
        "layer_counts_by_bits": {str(k): layer_counts[k] for k in sorted(layer_counts)},
        "parameter_counts_by_bits": {
            str(k): parameter_counts[k] for k in sorted(parameter_counts)
        },
        "weighted_mean_bits_per_kernel_parameter": sum(
            bits * count for bits, count in parameter_counts.items()
        )
        / total,
        "fp16_semantics": "W16 kernels are rounded to IEEE FP16 with an identity straight-through gradient, then used by Float32 arithmetic.",
        "low_bit_semantics": "W3/W4/W8 kernels use per-output-channel TensorFlow fake quantization and Float32 arithmetic.",
        "dropout_rng": {
            "status": "copied_and_verified_before_fit",
            "layer_states": source_rng,
        },
    }
    return clone, audit