fix: deltas stay on CPU at startup (avoid 132GB packing overfill), lazy-move to CUDA on first merge + debug prints
Browse files- __pycache__/app.cpython-310.pyc +0 -0
- app.py +25 -20
__pycache__/app.cpython-310.pyc
CHANGED
|
Binary files a/__pycache__/app.cpython-310.pyc and b/__pycache__/app.cpython-310.pyc differ
|
|
|
app.py
CHANGED
|
@@ -84,18 +84,25 @@ _TURBO_merged = False
|
|
| 84 |
|
| 85 |
|
| 86 |
def _apply_turbo():
|
| 87 |
-
"""Merge LoRA deltas into transformer weights.
|
| 88 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 89 |
if _TURBO_merged or _TURBO_Deltas is None:
|
| 90 |
return
|
| 91 |
_t0 = time.time()
|
| 92 |
-
|
| 93 |
-
|
| 94 |
-
|
| 95 |
-
|
| 96 |
-
|
| 97 |
-
|
| 98 |
-
|
|
|
|
|
|
|
| 99 |
mod.weight.data += delta
|
| 100 |
_elapsed = time.time() - _t0
|
| 101 |
_TURBO_merged = True
|
|
@@ -182,12 +189,10 @@ def load_models() -> str | None:
|
|
| 182 |
_base = _k.split(".lora_B.")[0]
|
| 183 |
_lora_pairs.setdefault(_base, {})["B"] = _v
|
| 184 |
|
| 185 |
-
# Compute deltas and locate target modules.
|
| 186 |
-
#
|
| 187 |
-
#
|
| 188 |
_TURBO_Deltas = []
|
| 189 |
-
_dev = "cuda" if torch.cuda.is_available() else "cpu"
|
| 190 |
-
_sample_dev = None
|
| 191 |
for _base_name, _pair in _lora_pairs.items():
|
| 192 |
if "A" not in _pair or "B" not in _pair:
|
| 193 |
continue
|
|
@@ -197,17 +202,17 @@ def load_models() -> str | None:
|
|
| 197 |
_obj = pipe.transformer
|
| 198 |
for _p in _parts:
|
| 199 |
_obj = _obj[int(_p)] if _p.isdigit() else getattr(_obj, _p)
|
| 200 |
-
|
| 201 |
-
if _sample_dev is None:
|
| 202 |
-
_sample_dev = (type(_obj).__name__, _obj.weight.device, _delta_cuda.device, _delta_cuda.shape, _delta_cuda.dtype)
|
| 203 |
-
_TURBO_Deltas.append((_obj, _delta_cuda))
|
| 204 |
|
| 205 |
del _lora_sd, _lora_pairs
|
| 206 |
import gc as _gc; _gc.collect()
|
| 207 |
TURBO_AVAILABLE = True
|
|
|
|
| 208 |
print(f"[gen] turbo LoRA parsed ({len(_TURBO_Deltas)} modules, merge pending)", flush=True)
|
| 209 |
-
|
| 210 |
-
|
|
|
|
|
|
|
| 211 |
|
| 212 |
# Still startup, still free: an AoTI package carries no weights and opens its archive lazily inside the GPU
|
| 213 |
# worker. Off unless `H3_AOTI=1`.
|
|
|
|
| 84 |
|
| 85 |
|
| 86 |
def _apply_turbo():
|
| 87 |
+
"""Merge LoRA deltas into transformer weights.
|
| 88 |
+
|
| 89 |
+
Deltas start on CPU so ZeroGPU doesn't try to pack them (they'd push the 132 GB packing
|
| 90 |
+
over the disk quota). On the first call we bulk-transfer them to CUDA and cache; every
|
| 91 |
+
subsequent call is pure in-place += with zero allocation.
|
| 92 |
+
"""
|
| 93 |
+
global _TURBO_merged, _TURBO_Deltas
|
| 94 |
if _TURBO_merged or _TURBO_Deltas is None:
|
| 95 |
return
|
| 96 |
_t0 = time.time()
|
| 97 |
+
# Lazy CUDA transfer on first call (one-time cost, then cached).
|
| 98 |
+
if _TURBO_Deltas and _TURBO_Deltas[0][1].device.type != "cuda":
|
| 99 |
+
print(f"[gen] _apply_turbo: moving {len(_TURBO_Deltas)} deltas to CUDA ...", flush=True)
|
| 100 |
+
for _i, (mod, delta) in enumerate(_TURBO_Deltas):
|
| 101 |
+
_TURBO_Deltas[_i] = (mod, delta.to("cuda", non_blocking=True))
|
| 102 |
+
print(f"[gen] _apply_turbo: deltas on CUDA after {time.time() - _t0:.2f}s", flush=True)
|
| 103 |
+
for mod, delta in _TURBO_Deltas:
|
| 104 |
+
if _TURBO_merged is False and delta.device.type != "cuda":
|
| 105 |
+
print(f"[gen] _apply_turbo BUG: delta still on {delta.device}", flush=True)
|
| 106 |
mod.weight.data += delta
|
| 107 |
_elapsed = time.time() - _t0
|
| 108 |
_TURBO_merged = True
|
|
|
|
| 189 |
_base = _k.split(".lora_B.")[0]
|
| 190 |
_lora_pairs.setdefault(_base, {})["B"] = _v
|
| 191 |
|
| 192 |
+
# Compute deltas on CPU and locate target modules. Deltas intentionally stay on CPU so
|
| 193 |
+
# ZeroGPU's pack step (132 GB) does not include them — they'd push past the disk quota.
|
| 194 |
+
# They are moved to CUDA lazily on first _apply_turbo() call and cached there.
|
| 195 |
_TURBO_Deltas = []
|
|
|
|
|
|
|
| 196 |
for _base_name, _pair in _lora_pairs.items():
|
| 197 |
if "A" not in _pair or "B" not in _pair:
|
| 198 |
continue
|
|
|
|
| 202 |
_obj = pipe.transformer
|
| 203 |
for _p in _parts:
|
| 204 |
_obj = _obj[int(_p)] if _p.isdigit() else getattr(_obj, _p)
|
| 205 |
+
_TURBO_Deltas.append((_obj, _delta.to(_obj.weight.dtype)))
|
|
|
|
|
|
|
|
|
|
| 206 |
|
| 207 |
del _lora_sd, _lora_pairs
|
| 208 |
import gc as _gc; _gc.collect()
|
| 209 |
TURBO_AVAILABLE = True
|
| 210 |
+
_sample = _TURBO_Deltas[0] if _TURBO_Deltas else None
|
| 211 |
print(f"[gen] turbo LoRA parsed ({len(_TURBO_Deltas)} modules, merge pending)", flush=True)
|
| 212 |
+
if _sample:
|
| 213 |
+
print(f"[gen] sample delta: module={type(_sample[0]).__name__}, "
|
| 214 |
+
f"mod.device={_sample[0].weight.device}, delta.device={_sample[1].device}, "
|
| 215 |
+
f"shape={_sample[1].shape}, dtype={_sample[1].dtype}", flush=True)
|
| 216 |
|
| 217 |
# Still startup, still free: an AoTI package carries no weights and opens its archive lazily inside the GPU
|
| 218 |
# worker. Off unless `H3_AOTI=1`.
|