Fix: use PEFT torch_device=cpu (not device_map, which was silently ignored) to force CPU-side adapter load

app.py

@@ -49,16 +49,21 @@ SYSTEM_PROMPT = SYSTEM_PROMPT_TEMPLATE.format(
 
 tokenizer = AutoTokenizer.from_pretrained(BASE_MODEL)
 base_model = AutoModelForCausalLM.from_pretrained(BASE_MODEL, torch_dtype="auto")
-# device_map="cpu" is required here: PeftModel.from_pretrained's internal
-# safetensors load otherwise infers a CUDA target device from
-# torch.cuda.is_available() (which ZeroGPU's emulation layer reports as
-# True even at module-load time, before a real GPU is allocated), and that
-# specific direct-to-device safetensors load path is not covered by the
-# emulation, causing "RuntimeError: No CUDA GPUs are available". Loading
-# on CPU here and moving the whole composed model to CUDA afterward avoids
-# that code path entirely.
+# torch_device="cpu" is required here: PeftModel.from_pretrained forwards
+# unrecognized kwargs to load_adapter(), whose `torch_device` parameter
+# controls what device load_peft_weights()/safetensors loads the adapter
+# weights onto. Left unset, it calls infer_device(), which sees
+# torch.cuda.is_available() == True (ZeroGPU's emulation layer reports
+# this as True even at module-load time, before a real GPU is allocated)
+# and picks "cuda" -- a direct-to-device safetensors load that the
+# emulation layer does not cover, causing "RuntimeError: No CUDA GPUs are
+# available". Forcing torch_device="cpu" here and moving the whole
+# composed model to CUDA afterward avoids that code path entirely. Note
+# this is NOT the same kwarg as transformers' `device_map` (which is
+# unrelated to this PEFT-internal loading path and is silently ignored if
+# passed here instead).
 model = PeftModel.from_pretrained(
-    base_model, ADAPTER, subfolder=ADAPTER_SUBFOLDER, device_map="cpu"
+    base_model, ADAPTER, subfolder=ADAPTER_SUBFOLDER, torch_device="cpu"
 )
 model.to("cuda")
 model.eval()