Upload handler.py

handler.py

@@ -25,7 +25,7 @@ print("Device Capability:", torch.cuda.get_device_capability())
 
 IS_TURBO = False
 IS_4BIT = False
-IS_PARA = False
+IS_PARA = True
 IS_LVRAM = False
 IS_COMPILE = True
 IS_AUTOQ = False
@@ -158,14 +158,16 @@ def load_pipeline_opt(repo_id: str, dtype: torch.dtype) -> Any:
     elif IS_CC89: quantize_(transformer, float8_dynamic_activation_float8_weight(), device="cuda")
     else: quantize_(transformer, weight, device="cuda")
     transformer.to(memory_format=torch.channels_last)
-    transformer = torch.compile(transformer, mode="max-autotune", fullgraph=True)
+    #transformer = torch.compile(transformer, mode="max-autotune", fullgraph=True)
+    transformer = torch.compile(transformer, mode="reduce-overhead", fullgraph=False)
     vae = AutoencoderKL.from_pretrained(repo_id, subfolder="vae", torch_dtype=dtype)
     vae.fuse_qkv_projections()
     if IS_CC90: quantize_(vae, float8_dynamic_activation_float8_weight(granularity=PerRow()), device="cuda")
     elif IS_CC89: quantize_(vae, float8_dynamic_activation_float8_weight(), device="cuda")
     else: quantize_(vae, weight, device="cuda")
     vae.to(memory_format=torch.channels_last)
-    vae = torch.compile(vae, mode="max-autotune", fullgraph=True)
+    #vae = torch.compile(vae, mode="max-autotune", fullgraph=True)
+    vae = torch.compile(vae, mode="reduce-overhead", fullgraph=False)
     pipe = FluxPipeline.from_pretrained(repo_id, transformer=None, vae=None, torch_dtype=dtype, quantization_config=quantization_config)
     pipe.transformer = transformer
     pipe.vae = vae