efficientnet-b0-sub4bit-mixed-precision / code /adaptive_mixed_precision_qat.py
meatloafer's picture
Upload verified model weights, LiteRT FlatBuffers, DP profiles, code, and model card
2abfed0 verified
Raw History Blame Contribute Delete
7.4 kB
"""Per-layer fake-quant QAT model construction for adaptive B0 profiles.
W3/W4/W8 kernels use TensorFlow fake quantization with its straight-through
gradient. W16 means FP16-rounded weights with Float32 activations and
arithmetic; it does not claim an FP16 execution path.
"""
from __future__ import annotations
from collections import Counter
from typing import Mapping
import tensorflow as tf
from imagenet_research.mixed_precision_profiles import eligible_weight_layers
from imagenet_research.quantization import (
FakeQuantConv2D,
FakeQuantDense,
FakeQuantDepthwiseConv2D,
)
SUPPORTED_BITS = frozenset((3, 4, 8, 16))
@tf.custom_gradient
def _fp16_weight_round_ste(kernel):
rounded = tf.cast(tf.cast(kernel, tf.float16), kernel.dtype)
def grad(upstream):
return tf.cast(upstream, kernel.dtype)
return rounded, grad
@tf.keras.utils.register_keras_serializable(package="imagenet_research")
class Fp16WeightConv2D(tf.keras.layers.Conv2D):
"""Round Conv2D weights to FP16 while retaining Float32 convolution math."""
def convolution_op(self, inputs, kernel):
return super().convolution_op(inputs, _fp16_weight_round_ste(kernel))
@tf.keras.utils.register_keras_serializable(package="imagenet_research")
class Fp16WeightDepthwiseConv2D(tf.keras.layers.DepthwiseConv2D):
"""Round depthwise weights to FP16 while retaining Float32 convolution math."""
def call(self, inputs):
kernel = tf.cast(self.kernel, tf.float32)
input_channels = self._get_input_channel(inputs.shape)
output_channels = input_channels * self.depth_multiplier
kernel = tf.reshape(
kernel, [self.kernel_size[0], self.kernel_size[1], output_channels]
)
kernel = _fp16_weight_round_ste(kernel)
kernel = tf.reshape(kernel, tf.shape(self.kernel))
kernel = tf.cast(kernel, self.compute_dtype)
inputs = tf.cast(inputs, self.compute_dtype)
outputs = tf.keras.ops.depthwise_conv(
inputs,
kernel,
strides=self.strides,
padding=self.padding,
dilation_rate=self.dilation_rate,
data_format=self.data_format,
)
if self.use_bias:
if self.data_format == "channels_last":
bias_shape = (1,) * (self.rank + 1) + (output_channels,)
else:
bias_shape = (1, output_channels) + (1,) * self.rank
outputs = tf.keras.ops.add(
outputs, tf.keras.ops.reshape(tf.cast(self.bias, self.compute_dtype), bias_shape)
)
if self.activation is not None:
outputs = self.activation(outputs)
return outputs
@tf.keras.utils.register_keras_serializable(package="imagenet_research")
class Fp16WeightDense(tf.keras.layers.Dense):
"""Round Dense weights to FP16 while retaining Float32 matmul math."""
def call(self, inputs, training=None):
kernel = tf.cast(_fp16_weight_round_ste(tf.cast(self.kernel, tf.float32)), self.compute_dtype)
inputs = tf.cast(inputs, self.compute_dtype)
outputs = tf.keras.ops.matmul(inputs, kernel)
if self.bias is not None:
outputs = tf.keras.ops.add(outputs, tf.cast(self.bias, self.compute_dtype))
if self.activation is not None:
outputs = self.activation(outputs)
return outputs
def _clone_layer_for_bits(layer: tf.keras.layers.Layer, bits: int):
config = layer.get_config()
if bits == 16:
if isinstance(layer, tf.keras.layers.DepthwiseConv2D):
return Fp16WeightDepthwiseConv2D(**config)
if isinstance(layer, tf.keras.layers.Conv2D):
return Fp16WeightConv2D(**config)
if isinstance(layer, tf.keras.layers.Dense):
return Fp16WeightDense(**config)
elif isinstance(layer, tf.keras.layers.DepthwiseConv2D):
return FakeQuantDepthwiseConv2D(num_bits=bits, **config)
elif isinstance(layer, tf.keras.layers.Conv2D):
return FakeQuantConv2D(num_bits=bits, **config)
elif isinstance(layer, tf.keras.layers.Dense):
return FakeQuantDense(num_bits=bits, **config)
return layer.__class__.from_config(config)
def build_adaptive_qat_model(
source_model: tf.keras.Model, bits_by_layer: Mapping[str, int]
) -> tuple[tf.keras.Model, dict]:
"""Clone a model with one registered weight precision per eligible kernel."""
layers = eligible_weight_layers(source_model)
normalized = {name: int(bits) for name, bits in bits_by_layer.items()}
if set(normalized) != set(layers):
raise ValueError(
"Adaptive QAT map must cover exactly the materialized weighted layers"
)
invalid = {name: bits for name, bits in normalized.items() if bits not in SUPPORTED_BITS}
if invalid:
raise ValueError(f"Unsupported adaptive QAT bitwidths: {invalid}")
clone = tf.keras.models.clone_model(
source_model,
clone_function=lambda layer: _clone_layer_for_bits(layer, normalized[layer.name])
if layer.name in normalized
else layer.__class__.from_config(layer.get_config()),
recursive=True,
)
for layer in source_model.layers:
if layer.weights:
clone.get_layer(layer.name).set_weights(layer.get_weights())
source_rng = {}
target_layers = {layer.name: layer for layer in clone.layers}
for layer in source_model.layers:
generator = getattr(layer, "seed_generator", None)
if generator is None:
continue
target = getattr(target_layers.get(layer.name), "seed_generator", None)
if target is None:
raise RuntimeError(f"Adaptive QAT clone lacks RNG state for {layer.name}")
target.state.assign(generator.state)
source_rng[layer.name] = [int(item) for item in generator.state.numpy().tolist()]
if target.state.numpy().tolist() != generator.state.numpy().tolist():
raise RuntimeError(f"Adaptive QAT RNG-state copy failed for {layer.name}")
if not source_rng:
raise RuntimeError("Expected EfficientNet Dropout/stochastic-depth RNG state")
parameter_counts: Counter[int] = Counter()
layer_counts: Counter[int] = Counter()
for name, bits in normalized.items():
parameter_counts[bits] += int(layers[name].kernel.shape.num_elements())
layer_counts[bits] += 1
total = sum(parameter_counts.values())
audit = {
"layer_bits": dict(sorted(normalized.items())),
"eligible_kernel_layers": len(normalized),
"eligible_kernel_parameters": total,
"layer_counts_by_bits": {str(k): layer_counts[k] for k in sorted(layer_counts)},
"parameter_counts_by_bits": {
str(k): parameter_counts[k] for k in sorted(parameter_counts)
},
"weighted_mean_bits_per_kernel_parameter": sum(
bits * count for bits, count in parameter_counts.items()
)
/ total,
"fp16_semantics": "W16 kernels are rounded to IEEE FP16 with an identity straight-through gradient, then used by Float32 arithmetic.",
"low_bit_semantics": "W3/W4/W8 kernels use per-output-channel TensorFlow fake quantization and Float32 arithmetic.",
"dropout_rng": {
"status": "copied_and_verified_before_fit",
"layer_states": source_rng,
},
}
return clone, audit