PlusBIntelligenceLab commited on
Commit
80ee897
Β·
verified Β·
1 Parent(s): 70e8753

Upload app.py

Browse files

Fixing OOM Error

Files changed (1) hide show
  1. app.py +48 -9
app.py CHANGED
@@ -227,7 +227,7 @@ def dequantize_state_dict(state_dict):
227
  return dequantized
228
 
229
  # ╔══════════════════════════════════════════════════════════════════════════╗
230
- # β•‘ Load Model β•‘
231
  # β•šβ•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•
232
 
233
  meta = json.load(open(meta_path))
@@ -239,23 +239,62 @@ device = "cuda" if torch.cuda.is_available() else "cpu"
239
  dtype = torch.float16 if device == "cuda" else torch.float32
240
 
241
  model = GPT(config)
 
242
 
243
- # Load 8-bit quantized weights and dequantize
244
  print("Loading 8-bit quantized weights...")
245
  raw_state_dict = torch.load(weights_path, map_location="cpu", weights_only=True)
246
 
247
- # Check if any weights are int8 (quantized)
 
 
 
248
  has_int8 = any(v.dtype == torch.int8 for v in raw_state_dict.values())
 
249
  if has_int8:
250
- print("Detected 8-bit quantized weights, dequantizing...")
251
- state_dict = dequantize_state_dict(raw_state_dict)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
252
  else:
253
  print("Weights are not int8, loading directly...")
254
- state_dict = raw_state_dict
 
 
 
255
 
256
- model.load_state_dict(state_dict)
257
- model = model.to(device=device, dtype=dtype).eval()
258
- del raw_state_dict, state_dict # free memory
259
 
260
  if device == "cuda":
261
  hd = config.n_embd // config.n_head
 
227
  return dequantized
228
 
229
  # ╔══════════════════════════════════════════════════════════════════════════╗
230
+ # β•‘ 8-bit Dequantization Utilities & Model Loading β•‘
231
  # β•šβ•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•
232
 
233
  meta = json.load(open(meta_path))
 
239
  dtype = torch.float16 if device == "cuda" else torch.float32
240
 
241
  model = GPT(config)
242
+ model = model.to(device=device, dtype=dtype)
243
 
 
244
  print("Loading 8-bit quantized weights...")
245
  raw_state_dict = torch.load(weights_path, map_location="cpu", weights_only=True)
246
 
247
+ # Delete large downloaded file references to free memory early if possible
248
+ import gc
249
+ gc.collect()
250
+
251
  has_int8 = any(v.dtype == torch.int8 for v in raw_state_dict.values())
252
+
253
  if has_int8:
254
+ print("Detected 8-bit quantized weights. Dequantizing directly into model to save RAM...")
255
+
256
+ # We will iterate through the model's named parameters and pull from raw_state_dict
257
+ # This prevents creating a second massive dict in memory.
258
+ model_state = model.state_dict()
259
+
260
+ for key in list(model_state.keys()):
261
+ if key in raw_state_dict:
262
+ tensor = raw_state_dict[key]
263
+
264
+ if tensor.dtype == torch.int8:
265
+ scale_key = key + '_scale'
266
+ if scale_key in raw_state_dict:
267
+ # Dequantize directly into the target dtype
268
+ dequantized = tensor.to(dtype) * raw_state_dict[scale_key].to(dtype)
269
+ model_state[key].copy_(dequantized)
270
+
271
+ # Delete the scale from raw dict to free memory as we go
272
+ del raw_state_dict[scale_key]
273
+ else:
274
+ print(f"⚠️ No scale found for {key}, casting directly")
275
+ model_state[key].copy_(tensor.to(dtype))
276
+ else:
277
+ model_state[key].copy_(tensor.to(dtype))
278
+
279
+ # Free the tensor from raw_state_dict as soon as we're done with it
280
+ del raw_state_dict[key]
281
+
282
+ # Ensure any remaining keys in raw_state_dict (like buffers not in model_state) are also copied
283
+ # though this shouldn't usually happen for standard SFT weights
284
+ for key in list(raw_state_dict.keys()):
285
+ if key not in model_state and not key.endswith('_scale'):
286
+ pass # Ignore extra keys
287
+
288
+ print("Dequantization complete.")
289
+
290
  else:
291
  print("Weights are not int8, loading directly...")
292
+ model.load_state_dict(raw_state_dict)
293
+
294
+ del raw_state_dict
295
+ gc.collect()
296
 
297
+ model.eval()
 
 
298
 
299
  if device == "cuda":
300
  hd = config.n_embd // config.n_head