perchinkov commited on
Commit
6ab11ea
·
verified ·
1 Parent(s): 37cf9c1

bf16 from_config build: avoid fp32 startup RAM peak (ZeroGPU OOM)

Browse files
Files changed (1) hide show
  1. echogemma/echogemma.py +4 -4
echogemma/echogemma.py CHANGED
@@ -45,11 +45,11 @@ class EchoGemma(torch.nn.Module):
45
  super().__init__()
46
  self.tokenizer = AutoTokenizer.from_pretrained(self.BASE_MODEL_ID, use_fast=True)
47
 
48
- # Build the architecture from config with a random init — no weight download.
49
- # (from_pretrained on the mirror pulls ~8.6 GB of safetensors whose materialization
50
- # is pathologically slow on ZeroGPU startup and is fully overwritten below anyway.)
51
  config = AutoConfig.from_pretrained(self.BASE_MODEL_ID)
52
- self.medgemma = AutoModelForCausalLM.from_config(config, torch_dtype=torch.float)
53
 
54
  lora_target_modules = ["q_proj", "k_proj", "v_proj", "o_proj",
55
  "gate_proj", "up_proj", "down_proj"]
 
45
  super().__init__()
46
  self.tokenizer = AutoTokenizer.from_pretrained(self.BASE_MODEL_ID, use_fast=True)
47
 
48
+ # Build the architecture directly in bf16 (8.5 GB resident) — no weight download,
49
+ # and no fp32 build peak (17 GB) which OOM-kills the ZeroGPU startup container.
50
+ # load_state_dict below casts each checkpoint tensor into the bf16 params.
51
  config = AutoConfig.from_pretrained(self.BASE_MODEL_ID)
52
+ self.medgemma = AutoModelForCausalLM.from_config(config, dtype=torch.bfloat16)
53
 
54
  lora_target_modules = ["q_proj", "k_proj", "v_proj", "o_proj",
55
  "gate_proj", "up_proj", "down_proj"]