Update src/pipeline.py
Browse files- src/pipeline.py +19 -20
src/pipeline.py
CHANGED
|
@@ -48,25 +48,24 @@ class W8A16LinearLayer(nn.Module):
|
|
| 48 |
return w8_a16_forward(self.int8_weights, input, self.scales, self.bias)
|
| 49 |
|
| 50 |
def replace_linear_with_target_and_quantize(module, target_class, module_name_to_exclude):
|
| 51 |
-
|
| 52 |
-
|
| 53 |
-
|
| 54 |
-
|
| 55 |
-
|
| 56 |
-
|
| 57 |
-
|
| 58 |
-
|
| 59 |
-
|
| 60 |
-
|
| 61 |
-
|
| 62 |
-
|
| 63 |
-
|
| 64 |
-
|
| 65 |
-
|
| 66 |
-
|
| 67 |
-
|
| 68 |
-
|
| 69 |
-
replace_linear_with_target_and_quantize(child, target_class, module_name_to_exclude)
|
| 70 |
|
| 71 |
|
| 72 |
ckpt_id = "black-forest-labs/FLUX.1-schnell"
|
|
@@ -99,7 +98,7 @@ def load_pipeline() -> Pipeline:
|
|
| 99 |
pipeline.text_encoder.to(memory_format=torch.channels_last)
|
| 100 |
pipeline.transformer.to(memory_format=torch.channels_last)
|
| 101 |
replace_linear_with_target_and_quantize(pipeline.transformer, W8A16LinearLayer, [])
|
| 102 |
-
|
| 103 |
|
| 104 |
pipeline.vae.to(memory_format=torch.channels_last)
|
| 105 |
pipeline.vae = torch.compile(pipeline.vae)
|
|
|
|
| 48 |
return w8_a16_forward(self.int8_weights, input, self.scales, self.bias)
|
| 49 |
|
| 50 |
def replace_linear_with_target_and_quantize(module, target_class, module_name_to_exclude):
|
| 51 |
+
with open(output_file, "w") as f:
|
| 52 |
+
for name, child in module.named_children():
|
| 53 |
+
if isinstance(child, nn.Linear): # and 'linear' in name: #and not any([x == name for x in module_name_to_exclude]):
|
| 54 |
+
old_bias = child.bias
|
| 55 |
+
old_weight = child.weight
|
| 56 |
+
new_module = target_class(child.in_features, child.out_features, old_bias is not None, child.weight.dtype)
|
| 57 |
+
setattr(module, name, new_module)
|
| 58 |
+
getattr(module, name).quantize(old_weight)
|
| 59 |
+
if old_bias is not None:
|
| 60 |
+
getattr(module, name).bias = old_bias
|
| 61 |
+
|
| 62 |
+
# Print the replaced layer name and calculate the change in size
|
| 63 |
+
old_size = old_weight.numel() * old_weight.element_size()
|
| 64 |
+
new_size = new_module.int8_weights.numel() * new_module.int8_weights.element_size()
|
| 65 |
+
f.write(f"Replaced layer: {name}" + f" Size reduction: {old_size} bytes -> {new_size} bytes ({(old_size - new_size) / old_size * 100:.2f}% reduction)")
|
| 66 |
+
else:
|
| 67 |
+
# Recursively call the function for nested modules
|
| 68 |
+
replace_linear_with_target_and_quantize(child, target_class, module_name_to_exclude)
|
|
|
|
| 69 |
|
| 70 |
|
| 71 |
ckpt_id = "black-forest-labs/FLUX.1-schnell"
|
|
|
|
| 98 |
pipeline.text_encoder.to(memory_format=torch.channels_last)
|
| 99 |
pipeline.transformer.to(memory_format=torch.channels_last)
|
| 100 |
replace_linear_with_target_and_quantize(pipeline.transformer, W8A16LinearLayer, [])
|
| 101 |
+
exit()
|
| 102 |
|
| 103 |
pipeline.vae.to(memory_format=torch.channels_last)
|
| 104 |
pipeline.vae = torch.compile(pipeline.vae)
|