Load 8B directly inside allocated ZeroGPU call

app.py

@@ -42,11 +42,7 @@ gte_model = None
 
 QWEN8_MODEL_PATH = Path("/models/qwen3-embedding-8b")
 qwen8_tokenizer = AutoTokenizer.from_pretrained(QWEN8_MODEL_PATH, padding_side="left")
-qwen8_model = AutoModel.from_pretrained(
-    QWEN8_MODEL_PATH,
-    dtype=torch.bfloat16,
-    trust_remote_code=True,
-).to("cuda").eval()
+qwen8_model = None
 
 
 def detailed_instruction(query: str) -> str:
@@ -293,10 +289,18 @@ def _run_family_bf16_control(family: str) -> dict:
     allocation_before = int(torch.cuda.memory_allocated())
     if family == "qwen3-embedding-8b":
         family_tokenizer = qwen8_tokenizer
-        family_model = qwen8_model
-        load_seconds = 0.0
-        loading_strategy = "root-module-cuda-emulation"
-        owns_model = False
+        load_started = time.perf_counter()
+        family_model = AutoModel.from_pretrained(
+            QWEN8_MODEL_PATH,
+            dtype=torch.bfloat16,
+            trust_remote_code=True,
+            low_cpu_mem_usage=True,
+            device_map={"": "cuda"},
+        ).eval()
+        torch.cuda.synchronize()
+        load_seconds = time.perf_counter() - load_started
+        loading_strategy = "in-call-direct-cuda-device-map"
+        owns_model = True
     else:
         raise RuntimeError("GTE 1.5B BF16 control is offline during the 8B phase")
     allocation_after_load = int(torch.cuda.memory_allocated())

requirements.txt

@@ -1,4 +1,5 @@
 transformers>=4.51,<5
+accelerate>=1,<2
 numpy>=2,<3
 spaces>=0.45
 torch>=2.8