Spaces:
Sleeping
Sleeping
bf16 from_config build: avoid fp32 startup RAM peak (ZeroGPU OOM)
Browse files- echogemma/echogemma.py +4 -4
echogemma/echogemma.py
CHANGED
|
@@ -45,11 +45,11 @@ class EchoGemma(torch.nn.Module):
|
|
| 45 |
super().__init__()
|
| 46 |
self.tokenizer = AutoTokenizer.from_pretrained(self.BASE_MODEL_ID, use_fast=True)
|
| 47 |
|
| 48 |
-
# Build the architecture
|
| 49 |
-
#
|
| 50 |
-
#
|
| 51 |
config = AutoConfig.from_pretrained(self.BASE_MODEL_ID)
|
| 52 |
-
self.medgemma = AutoModelForCausalLM.from_config(config,
|
| 53 |
|
| 54 |
lora_target_modules = ["q_proj", "k_proj", "v_proj", "o_proj",
|
| 55 |
"gate_proj", "up_proj", "down_proj"]
|
|
|
|
| 45 |
super().__init__()
|
| 46 |
self.tokenizer = AutoTokenizer.from_pretrained(self.BASE_MODEL_ID, use_fast=True)
|
| 47 |
|
| 48 |
+
# Build the architecture directly in bf16 (8.5 GB resident) — no weight download,
|
| 49 |
+
# and no fp32 build peak (17 GB) which OOM-kills the ZeroGPU startup container.
|
| 50 |
+
# load_state_dict below casts each checkpoint tensor into the bf16 params.
|
| 51 |
config = AutoConfig.from_pretrained(self.BASE_MODEL_ID)
|
| 52 |
+
self.medgemma = AutoModelForCausalLM.from_config(config, dtype=torch.bfloat16)
|
| 53 |
|
| 54 |
lora_target_modules = ["q_proj", "k_proj", "v_proj", "o_proj",
|
| 55 |
"gate_proj", "up_proj", "down_proj"]
|