manbeast3b commited on
Commit
95be3ba
·
verified ·
1 Parent(s): 8dbe1ce

Update src/pipeline.py

Browse files
Files changed (1) hide show
  1. src/pipeline.py +19 -20
src/pipeline.py CHANGED
@@ -48,25 +48,24 @@ class W8A16LinearLayer(nn.Module):
48
  return w8_a16_forward(self.int8_weights, input, self.scales, self.bias)
49
 
50
  def replace_linear_with_target_and_quantize(module, target_class, module_name_to_exclude):
51
- for name, child in module.named_children():
52
- if isinstance(child, nn.Linear) and 'linear' in name: #and not any([x == name for x in module_name_to_exclude]):
53
- old_bias = child.bias
54
- old_weight = child.weight
55
- new_module = target_class(child.in_features, child.out_features, old_bias is not None, child.weight.dtype)
56
- setattr(module, name, new_module)
57
- getattr(module, name).quantize(old_weight)
58
- if old_bias is not None:
59
- getattr(module, name).bias = old_bias
60
-
61
- # # Print the replaced layer name and calculate the change in size
62
- # print(f"Replaced layer: {name}")
63
- # old_size = old_weight.numel() * old_weight.element_size()
64
- # new_size = new_module.int8_weights.numel() * new_module.int8_weights.element_size()
65
- # print(f"Size reduction: {old_size} bytes -> {new_size} bytes ({(old_size - new_size) / old_size * 100:.2f}% reduction)")
66
-
67
- else:
68
- # Recursively call the function for nested modules
69
- replace_linear_with_target_and_quantize(child, target_class, module_name_to_exclude)
70
 
71
 
72
  ckpt_id = "black-forest-labs/FLUX.1-schnell"
@@ -99,7 +98,7 @@ def load_pipeline() -> Pipeline:
99
  pipeline.text_encoder.to(memory_format=torch.channels_last)
100
  pipeline.transformer.to(memory_format=torch.channels_last)
101
  replace_linear_with_target_and_quantize(pipeline.transformer, W8A16LinearLayer, [])
102
- # exit()
103
 
104
  pipeline.vae.to(memory_format=torch.channels_last)
105
  pipeline.vae = torch.compile(pipeline.vae)
 
48
  return w8_a16_forward(self.int8_weights, input, self.scales, self.bias)
49
 
50
  def replace_linear_with_target_and_quantize(module, target_class, module_name_to_exclude):
51
+ with open(output_file, "w") as f:
52
+ for name, child in module.named_children():
53
+ if isinstance(child, nn.Linear): # and 'linear' in name: #and not any([x == name for x in module_name_to_exclude]):
54
+ old_bias = child.bias
55
+ old_weight = child.weight
56
+ new_module = target_class(child.in_features, child.out_features, old_bias is not None, child.weight.dtype)
57
+ setattr(module, name, new_module)
58
+ getattr(module, name).quantize(old_weight)
59
+ if old_bias is not None:
60
+ getattr(module, name).bias = old_bias
61
+
62
+ # Print the replaced layer name and calculate the change in size
63
+ old_size = old_weight.numel() * old_weight.element_size()
64
+ new_size = new_module.int8_weights.numel() * new_module.int8_weights.element_size()
65
+ f.write(f"Replaced layer: {name}" + f" Size reduction: {old_size} bytes -> {new_size} bytes ({(old_size - new_size) / old_size * 100:.2f}% reduction)")
66
+ else:
67
+ # Recursively call the function for nested modules
68
+ replace_linear_with_target_and_quantize(child, target_class, module_name_to_exclude)
 
69
 
70
 
71
  ckpt_id = "black-forest-labs/FLUX.1-schnell"
 
98
  pipeline.text_encoder.to(memory_format=torch.channels_last)
99
  pipeline.transformer.to(memory_format=torch.channels_last)
100
  replace_linear_with_target_and_quantize(pipeline.transformer, W8A16LinearLayer, [])
101
+ exit()
102
 
103
  pipeline.vae.to(memory_format=torch.channels_last)
104
  pipeline.vae = torch.compile(pipeline.vae)