observantdistressed commited on
Commit
7f0f4e1
·
verified ·
1 Parent(s): 297fb55

fix: deltas stay on CPU at startup (avoid 132GB packing overfill), lazy-move to CUDA on first merge + debug prints

Browse files
Files changed (2) hide show
  1. __pycache__/app.cpython-310.pyc +0 -0
  2. app.py +25 -20
__pycache__/app.cpython-310.pyc CHANGED
Binary files a/__pycache__/app.cpython-310.pyc and b/__pycache__/app.cpython-310.pyc differ
 
app.py CHANGED
@@ -84,18 +84,25 @@ _TURBO_merged = False
84
 
85
 
86
  def _apply_turbo():
87
- """Merge LoRA deltas into transformer weights. Deltas live on CUDA so this is zero-transfer in-place +=."""
88
- global _TURBO_merged
 
 
 
 
 
89
  if _TURBO_merged or _TURBO_Deltas is None:
90
  return
91
  _t0 = time.time()
92
- for _i, (mod, delta) in enumerate(_TURBO_Deltas):
93
- if _i == 0:
94
- print(f"[gen] _apply_turbo: mod.device={mod.weight.device}, delta.device={delta.device}, "
95
- f"mod.dtype={mod.weight.dtype}, delta.dtype={delta.dtype}, mod.shape={mod.weight.shape}, "
96
- f"delta.shape={delta.shape}", flush=True)
97
- elif _i == 1:
98
- print(f"[gen] _apply_turbo: delta[1] device={delta.device}, shape={delta.shape}", flush=True)
 
 
99
  mod.weight.data += delta
100
  _elapsed = time.time() - _t0
101
  _TURBO_merged = True
@@ -182,12 +189,10 @@ def load_models() -> str | None:
182
  _base = _k.split(".lora_B.")[0]
183
  _lora_pairs.setdefault(_base, {})["B"] = _v
184
 
185
- # Compute deltas and locate target modules. In pack mode ZeroGPU has already packed the
186
- # transformer (weights replaced by CPU placeholders), so _obj.weight.device == cpu.
187
- # We must force deltas to CUDA explicitly — inside @spaces.GPU the real weights are on cuda.
188
  _TURBO_Deltas = []
189
- _dev = "cuda" if torch.cuda.is_available() else "cpu"
190
- _sample_dev = None
191
  for _base_name, _pair in _lora_pairs.items():
192
  if "A" not in _pair or "B" not in _pair:
193
  continue
@@ -197,17 +202,17 @@ def load_models() -> str | None:
197
  _obj = pipe.transformer
198
  for _p in _parts:
199
  _obj = _obj[int(_p)] if _p.isdigit() else getattr(_obj, _p)
200
- _delta_cuda = _delta.to(_dev, dtype=_obj.weight.dtype)
201
- if _sample_dev is None:
202
- _sample_dev = (type(_obj).__name__, _obj.weight.device, _delta_cuda.device, _delta_cuda.shape, _delta_cuda.dtype)
203
- _TURBO_Deltas.append((_obj, _delta_cuda))
204
 
205
  del _lora_sd, _lora_pairs
206
  import gc as _gc; _gc.collect()
207
  TURBO_AVAILABLE = True
 
208
  print(f"[gen] turbo LoRA parsed ({len(_TURBO_Deltas)} modules, merge pending)", flush=True)
209
- print(f"[gen] sample delta: module={_sample_dev[0]}, mod.device={_sample_dev[1]}, "
210
- f"delta.device={_sample_dev[2]}, shape={_sample_dev[3]}, dtype={_sample_dev[4]}", flush=True)
 
 
211
 
212
  # Still startup, still free: an AoTI package carries no weights and opens its archive lazily inside the GPU
213
  # worker. Off unless `H3_AOTI=1`.
 
84
 
85
 
86
  def _apply_turbo():
87
+ """Merge LoRA deltas into transformer weights.
88
+
89
+ Deltas start on CPU so ZeroGPU doesn't try to pack them (they'd push the 132 GB packing
90
+ over the disk quota). On the first call we bulk-transfer them to CUDA and cache; every
91
+ subsequent call is pure in-place += with zero allocation.
92
+ """
93
+ global _TURBO_merged, _TURBO_Deltas
94
  if _TURBO_merged or _TURBO_Deltas is None:
95
  return
96
  _t0 = time.time()
97
+ # Lazy CUDA transfer on first call (one-time cost, then cached).
98
+ if _TURBO_Deltas and _TURBO_Deltas[0][1].device.type != "cuda":
99
+ print(f"[gen] _apply_turbo: moving {len(_TURBO_Deltas)} deltas to CUDA ...", flush=True)
100
+ for _i, (mod, delta) in enumerate(_TURBO_Deltas):
101
+ _TURBO_Deltas[_i] = (mod, delta.to("cuda", non_blocking=True))
102
+ print(f"[gen] _apply_turbo: deltas on CUDA after {time.time() - _t0:.2f}s", flush=True)
103
+ for mod, delta in _TURBO_Deltas:
104
+ if _TURBO_merged is False and delta.device.type != "cuda":
105
+ print(f"[gen] _apply_turbo BUG: delta still on {delta.device}", flush=True)
106
  mod.weight.data += delta
107
  _elapsed = time.time() - _t0
108
  _TURBO_merged = True
 
189
  _base = _k.split(".lora_B.")[0]
190
  _lora_pairs.setdefault(_base, {})["B"] = _v
191
 
192
+ # Compute deltas on CPU and locate target modules. Deltas intentionally stay on CPU so
193
+ # ZeroGPU's pack step (132 GB) does not include them — they'd push past the disk quota.
194
+ # They are moved to CUDA lazily on first _apply_turbo() call and cached there.
195
  _TURBO_Deltas = []
 
 
196
  for _base_name, _pair in _lora_pairs.items():
197
  if "A" not in _pair or "B" not in _pair:
198
  continue
 
202
  _obj = pipe.transformer
203
  for _p in _parts:
204
  _obj = _obj[int(_p)] if _p.isdigit() else getattr(_obj, _p)
205
+ _TURBO_Deltas.append((_obj, _delta.to(_obj.weight.dtype)))
 
 
 
206
 
207
  del _lora_sd, _lora_pairs
208
  import gc as _gc; _gc.collect()
209
  TURBO_AVAILABLE = True
210
+ _sample = _TURBO_Deltas[0] if _TURBO_Deltas else None
211
  print(f"[gen] turbo LoRA parsed ({len(_TURBO_Deltas)} modules, merge pending)", flush=True)
212
+ if _sample:
213
+ print(f"[gen] sample delta: module={type(_sample[0]).__name__}, "
214
+ f"mod.device={_sample[0].weight.device}, delta.device={_sample[1].device}, "
215
+ f"shape={_sample[1].shape}, dtype={_sample[1].dtype}", flush=True)
216
 
217
  # Still startup, still free: an AoTI package carries no weights and opens its archive lazily inside the GPU
218
  # worker. Off unless `H3_AOTI=1`.