Upload app.py
Browse filesFixing OOM Error
app.py
CHANGED
|
@@ -227,7 +227,7 @@ def dequantize_state_dict(state_dict):
|
|
| 227 |
return dequantized
|
| 228 |
|
| 229 |
# ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 230 |
-
# β
|
| 231 |
# ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 232 |
|
| 233 |
meta = json.load(open(meta_path))
|
|
@@ -239,23 +239,62 @@ device = "cuda" if torch.cuda.is_available() else "cpu"
|
|
| 239 |
dtype = torch.float16 if device == "cuda" else torch.float32
|
| 240 |
|
| 241 |
model = GPT(config)
|
|
|
|
| 242 |
|
| 243 |
-
# Load 8-bit quantized weights and dequantize
|
| 244 |
print("Loading 8-bit quantized weights...")
|
| 245 |
raw_state_dict = torch.load(weights_path, map_location="cpu", weights_only=True)
|
| 246 |
|
| 247 |
-
#
|
|
|
|
|
|
|
|
|
|
| 248 |
has_int8 = any(v.dtype == torch.int8 for v in raw_state_dict.values())
|
|
|
|
| 249 |
if has_int8:
|
| 250 |
-
print("Detected 8-bit quantized weights
|
| 251 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 252 |
else:
|
| 253 |
print("Weights are not int8, loading directly...")
|
| 254 |
-
|
|
|
|
|
|
|
|
|
|
| 255 |
|
| 256 |
-
model.
|
| 257 |
-
model = model.to(device=device, dtype=dtype).eval()
|
| 258 |
-
del raw_state_dict, state_dict # free memory
|
| 259 |
|
| 260 |
if device == "cuda":
|
| 261 |
hd = config.n_embd // config.n_head
|
|
|
|
| 227 |
return dequantized
|
| 228 |
|
| 229 |
# ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 230 |
+
# β 8-bit Dequantization Utilities & Model Loading β
|
| 231 |
# ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 232 |
|
| 233 |
meta = json.load(open(meta_path))
|
|
|
|
| 239 |
dtype = torch.float16 if device == "cuda" else torch.float32
|
| 240 |
|
| 241 |
model = GPT(config)
|
| 242 |
+
model = model.to(device=device, dtype=dtype)
|
| 243 |
|
|
|
|
| 244 |
print("Loading 8-bit quantized weights...")
|
| 245 |
raw_state_dict = torch.load(weights_path, map_location="cpu", weights_only=True)
|
| 246 |
|
| 247 |
+
# Delete large downloaded file references to free memory early if possible
|
| 248 |
+
import gc
|
| 249 |
+
gc.collect()
|
| 250 |
+
|
| 251 |
has_int8 = any(v.dtype == torch.int8 for v in raw_state_dict.values())
|
| 252 |
+
|
| 253 |
if has_int8:
|
| 254 |
+
print("Detected 8-bit quantized weights. Dequantizing directly into model to save RAM...")
|
| 255 |
+
|
| 256 |
+
# We will iterate through the model's named parameters and pull from raw_state_dict
|
| 257 |
+
# This prevents creating a second massive dict in memory.
|
| 258 |
+
model_state = model.state_dict()
|
| 259 |
+
|
| 260 |
+
for key in list(model_state.keys()):
|
| 261 |
+
if key in raw_state_dict:
|
| 262 |
+
tensor = raw_state_dict[key]
|
| 263 |
+
|
| 264 |
+
if tensor.dtype == torch.int8:
|
| 265 |
+
scale_key = key + '_scale'
|
| 266 |
+
if scale_key in raw_state_dict:
|
| 267 |
+
# Dequantize directly into the target dtype
|
| 268 |
+
dequantized = tensor.to(dtype) * raw_state_dict[scale_key].to(dtype)
|
| 269 |
+
model_state[key].copy_(dequantized)
|
| 270 |
+
|
| 271 |
+
# Delete the scale from raw dict to free memory as we go
|
| 272 |
+
del raw_state_dict[scale_key]
|
| 273 |
+
else:
|
| 274 |
+
print(f"β οΈ No scale found for {key}, casting directly")
|
| 275 |
+
model_state[key].copy_(tensor.to(dtype))
|
| 276 |
+
else:
|
| 277 |
+
model_state[key].copy_(tensor.to(dtype))
|
| 278 |
+
|
| 279 |
+
# Free the tensor from raw_state_dict as soon as we're done with it
|
| 280 |
+
del raw_state_dict[key]
|
| 281 |
+
|
| 282 |
+
# Ensure any remaining keys in raw_state_dict (like buffers not in model_state) are also copied
|
| 283 |
+
# though this shouldn't usually happen for standard SFT weights
|
| 284 |
+
for key in list(raw_state_dict.keys()):
|
| 285 |
+
if key not in model_state and not key.endswith('_scale'):
|
| 286 |
+
pass # Ignore extra keys
|
| 287 |
+
|
| 288 |
+
print("Dequantization complete.")
|
| 289 |
+
|
| 290 |
else:
|
| 291 |
print("Weights are not int8, loading directly...")
|
| 292 |
+
model.load_state_dict(raw_state_dict)
|
| 293 |
+
|
| 294 |
+
del raw_state_dict
|
| 295 |
+
gc.collect()
|
| 296 |
|
| 297 |
+
model.eval()
|
|
|
|
|
|
|
| 298 |
|
| 299 |
if device == "cuda":
|
| 300 |
hd = config.n_embd // config.n_head
|