fix: revert LoRA to startup merge to avoid OOM during generation
Browse files- __pycache__/app.cpython-310.pyc +0 -0
- app.py +19 -45
__pycache__/app.cpython-310.pyc
CHANGED
|
Binary files a/__pycache__/app.cpython-310.pyc and b/__pycache__/app.cpython-310.pyc differ
|
|
|
app.py
CHANGED
|
@@ -79,30 +79,9 @@ MANAGER = None
|
|
| 79 |
LOAD_ERROR: str | None = None
|
| 80 |
LOADED_IN: float | None = None
|
| 81 |
TURBO_AVAILABLE: bool = False
|
| 82 |
-
|
| 83 |
-
|
| 84 |
-
|
| 85 |
-
|
| 86 |
-
def _apply_turbo():
|
| 87 |
-
"""Merge the LoRA delta into the transformer weights (idempotent)."""
|
| 88 |
-
global _TURBO_merged
|
| 89 |
-
if _TURBO_merged or _TURBO_Deltas is None:
|
| 90 |
-
return
|
| 91 |
-
for mod, delta in _TURBO_Deltas:
|
| 92 |
-
mod.weight.data += delta.to(mod.weight.device, non_blocking=True)
|
| 93 |
-
_TURBO_merged = True
|
| 94 |
-
print("[gen] turbo LoRA merged", flush=True)
|
| 95 |
-
|
| 96 |
-
|
| 97 |
-
def _remove_turbo():
|
| 98 |
-
"""Restore the base weights by undoing the LoRA merge (idempotent)."""
|
| 99 |
-
global _TURBO_merged
|
| 100 |
-
if not _TURBO_merged or _TURBO_Deltas is None:
|
| 101 |
-
return
|
| 102 |
-
for mod, delta in _TURBO_Deltas:
|
| 103 |
-
mod.weight.data -= delta.to(mod.weight.device, non_blocking=True)
|
| 104 |
-
_TURBO_merged = False
|
| 105 |
-
print("[gen] turbo LoRA removed", flush=True)
|
| 106 |
|
| 107 |
def status() -> str:
|
| 108 |
if LOAD_ERROR:
|
|
@@ -167,8 +146,8 @@ def load_models() -> str | None:
|
|
| 167 |
_base = _k.split(".lora_B.")[0]
|
| 168 |
_lora_pairs.setdefault(_base, {})["B"] = _v
|
| 169 |
|
| 170 |
-
#
|
| 171 |
-
|
| 172 |
for _base_name, _pair in _lora_pairs.items():
|
| 173 |
if "A" not in _pair or "B" not in _pair:
|
| 174 |
continue
|
|
@@ -179,12 +158,13 @@ def load_models() -> str | None:
|
|
| 179 |
_obj = pipe.transformer
|
| 180 |
for _p in _parts:
|
| 181 |
_obj = _obj[int(_p)] if _p.isdigit() else getattr(_obj, _p)
|
| 182 |
-
|
|
|
|
| 183 |
|
| 184 |
del _lora_sd, _lora_pairs
|
| 185 |
import gc as _gc; _gc.collect()
|
| 186 |
TURBO_AVAILABLE = True
|
| 187 |
-
print(f"[gen] turbo LoRA
|
| 188 |
|
| 189 |
# Still startup, still free: an AoTI package carries no weights and opens its archive lazily inside the GPU
|
| 190 |
# worker. Off unless `H3_AOTI=1`.
|
|
@@ -341,23 +321,17 @@ def generate(prompt, image_path=None, last_image_path=None, canvas=DEFAULT_CANVA
|
|
| 341 |
|
| 342 |
progress(0.1, desc=f"Denoising {steps} steps at {width}x{height}, {num_frames} frames ...")
|
| 343 |
started = time.time()
|
| 344 |
-
|
| 345 |
-
|
| 346 |
-
|
| 347 |
-
|
| 348 |
-
|
| 349 |
-
|
| 350 |
-
|
| 351 |
-
|
| 352 |
-
|
| 353 |
-
|
| 354 |
-
|
| 355 |
-
steps,
|
| 356 |
-
seed,
|
| 357 |
-
)
|
| 358 |
-
finally:
|
| 359 |
-
if turbo and TURBO_AVAILABLE:
|
| 360 |
-
_remove_turbo()
|
| 361 |
generate_seconds = time.time() - started
|
| 362 |
|
| 363 |
directory = os.path.join(tempfile.gettempdir(), "h3-outputs")
|
|
|
|
| 79 |
LOAD_ERROR: str | None = None
|
| 80 |
LOADED_IN: float | None = None
|
| 81 |
TURBO_AVAILABLE: bool = False
|
| 82 |
+
|
| 83 |
+
|
| 84 |
+
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 85 |
|
| 86 |
def status() -> str:
|
| 87 |
if LOAD_ERROR:
|
|
|
|
| 146 |
_base = _k.split(".lora_B.")[0]
|
| 147 |
_lora_pairs.setdefault(_base, {})["B"] = _v
|
| 148 |
|
| 149 |
+
# Merge the LoRA delta into the transformer at startup (CPU, before ZeroGPU packing).
|
| 150 |
+
_merged = 0
|
| 151 |
for _base_name, _pair in _lora_pairs.items():
|
| 152 |
if "A" not in _pair or "B" not in _pair:
|
| 153 |
continue
|
|
|
|
| 158 |
_obj = pipe.transformer
|
| 159 |
for _p in _parts:
|
| 160 |
_obj = _obj[int(_p)] if _p.isdigit() else getattr(_obj, _p)
|
| 161 |
+
_obj.weight.data += _delta.to(_obj.weight.dtype)
|
| 162 |
+
_merged += 1
|
| 163 |
|
| 164 |
del _lora_sd, _lora_pairs
|
| 165 |
import gc as _gc; _gc.collect()
|
| 166 |
TURBO_AVAILABLE = True
|
| 167 |
+
print(f"[gen] turbo LoRA merged ({_merged} modules)", flush=True)
|
| 168 |
|
| 169 |
# Still startup, still free: an AoTI package carries no weights and opens its archive lazily inside the GPU
|
| 170 |
# worker. Off unless `H3_AOTI=1`.
|
|
|
|
| 321 |
|
| 322 |
progress(0.1, desc=f"Denoising {steps} steps at {width}x{height}, {num_frames} frames ...")
|
| 323 |
started = time.time()
|
| 324 |
+
frames, audio, sampling_rate = _generate(
|
| 325 |
+
prompt_embeds,
|
| 326 |
+
text_token_tags,
|
| 327 |
+
keyframe(image_path),
|
| 328 |
+
keyframe(last_image_path),
|
| 329 |
+
height,
|
| 330 |
+
width,
|
| 331 |
+
num_frames,
|
| 332 |
+
steps,
|
| 333 |
+
seed,
|
| 334 |
+
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 335 |
generate_seconds = time.time() - started
|
| 336 |
|
| 337 |
directory = os.path.join(tempfile.gettempdir(), "h3-outputs")
|