observantdistressed commited on
Commit
c43e4d2
·
verified ·
1 Parent(s): 74fdf62

fix: revert LoRA to startup merge to avoid OOM during generation

Browse files
Files changed (2) hide show
  1. __pycache__/app.cpython-310.pyc +0 -0
  2. app.py +19 -45
__pycache__/app.cpython-310.pyc CHANGED
Binary files a/__pycache__/app.cpython-310.pyc and b/__pycache__/app.cpython-310.pyc differ
 
app.py CHANGED
@@ -79,30 +79,9 @@ MANAGER = None
79
  LOAD_ERROR: str | None = None
80
  LOADED_IN: float | None = None
81
  TURBO_AVAILABLE: bool = False
82
- _TURBO_Deltas: list | None = None # list of (module, original_weight_copy) tuples
83
- _TURBO_merged = False
84
-
85
-
86
- def _apply_turbo():
87
- """Merge the LoRA delta into the transformer weights (idempotent)."""
88
- global _TURBO_merged
89
- if _TURBO_merged or _TURBO_Deltas is None:
90
- return
91
- for mod, delta in _TURBO_Deltas:
92
- mod.weight.data += delta.to(mod.weight.device, non_blocking=True)
93
- _TURBO_merged = True
94
- print("[gen] turbo LoRA merged", flush=True)
95
-
96
-
97
- def _remove_turbo():
98
- """Restore the base weights by undoing the LoRA merge (idempotent)."""
99
- global _TURBO_merged
100
- if not _TURBO_merged or _TURBO_Deltas is None:
101
- return
102
- for mod, delta in _TURBO_Deltas:
103
- mod.weight.data -= delta.to(mod.weight.device, non_blocking=True)
104
- _TURBO_merged = False
105
- print("[gen] turbo LoRA removed", flush=True)
106
 
107
  def status() -> str:
108
  if LOAD_ERROR:
@@ -167,8 +146,8 @@ def load_models() -> str | None:
167
  _base = _k.split(".lora_B.")[0]
168
  _lora_pairs.setdefault(_base, {})["B"] = _v
169
 
170
- # Compute and cache the deltas + snapshot original weights (CPU, no merge yet).
171
- _TURBO_Deltas = []
172
  for _base_name, _pair in _lora_pairs.items():
173
  if "A" not in _pair or "B" not in _pair:
174
  continue
@@ -179,12 +158,13 @@ def load_models() -> str | None:
179
  _obj = pipe.transformer
180
  for _p in _parts:
181
  _obj = _obj[int(_p)] if _p.isdigit() else getattr(_obj, _p)
182
- _TURBO_Deltas.append((_obj, _delta.to(_obj.weight.dtype)))
 
183
 
184
  del _lora_sd, _lora_pairs
185
  import gc as _gc; _gc.collect()
186
  TURBO_AVAILABLE = True
187
- print(f"[gen] turbo LoRA parsed ({len(_TURBO_Deltas)} modules, merge pending)", flush=True)
188
 
189
  # Still startup, still free: an AoTI package carries no weights and opens its archive lazily inside the GPU
190
  # worker. Off unless `H3_AOTI=1`.
@@ -341,23 +321,17 @@ def generate(prompt, image_path=None, last_image_path=None, canvas=DEFAULT_CANVA
341
 
342
  progress(0.1, desc=f"Denoising {steps} steps at {width}x{height}, {num_frames} frames ...")
343
  started = time.time()
344
- if turbo and TURBO_AVAILABLE:
345
- _apply_turbo()
346
- try:
347
- frames, audio, sampling_rate = _generate(
348
- prompt_embeds,
349
- text_token_tags,
350
- keyframe(image_path),
351
- keyframe(last_image_path),
352
- height,
353
- width,
354
- num_frames,
355
- steps,
356
- seed,
357
- )
358
- finally:
359
- if turbo and TURBO_AVAILABLE:
360
- _remove_turbo()
361
  generate_seconds = time.time() - started
362
 
363
  directory = os.path.join(tempfile.gettempdir(), "h3-outputs")
 
79
  LOAD_ERROR: str | None = None
80
  LOADED_IN: float | None = None
81
  TURBO_AVAILABLE: bool = False
82
+
83
+
84
+
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
85
 
86
  def status() -> str:
87
  if LOAD_ERROR:
 
146
  _base = _k.split(".lora_B.")[0]
147
  _lora_pairs.setdefault(_base, {})["B"] = _v
148
 
149
+ # Merge the LoRA delta into the transformer at startup (CPU, before ZeroGPU packing).
150
+ _merged = 0
151
  for _base_name, _pair in _lora_pairs.items():
152
  if "A" not in _pair or "B" not in _pair:
153
  continue
 
158
  _obj = pipe.transformer
159
  for _p in _parts:
160
  _obj = _obj[int(_p)] if _p.isdigit() else getattr(_obj, _p)
161
+ _obj.weight.data += _delta.to(_obj.weight.dtype)
162
+ _merged += 1
163
 
164
  del _lora_sd, _lora_pairs
165
  import gc as _gc; _gc.collect()
166
  TURBO_AVAILABLE = True
167
+ print(f"[gen] turbo LoRA merged ({_merged} modules)", flush=True)
168
 
169
  # Still startup, still free: an AoTI package carries no weights and opens its archive lazily inside the GPU
170
  # worker. Off unless `H3_AOTI=1`.
 
321
 
322
  progress(0.1, desc=f"Denoising {steps} steps at {width}x{height}, {num_frames} frames ...")
323
  started = time.time()
324
+ frames, audio, sampling_rate = _generate(
325
+ prompt_embeds,
326
+ text_token_tags,
327
+ keyframe(image_path),
328
+ keyframe(last_image_path),
329
+ height,
330
+ width,
331
+ num_frames,
332
+ steps,
333
+ seed,
334
+ )
 
 
 
 
 
 
335
  generate_seconds = time.time() - started
336
 
337
  directory = os.path.join(tempfile.gettempdir(), "h3-outputs")