Move GTE 1.5B CUDA placement to ZeroGPU startup

app.py

@@ -33,6 +33,17 @@ model = AutoModel.from_pretrained(
     trust_remote_code=True,
 ).to("cuda").eval()
 
+# ZeroGPU optimizes CUDA placements made during module startup.  The previous
+# larger-family path loaded this model inside the decorated call and exhausted
+# that reservation before producing an artifact.
+GTE_MODEL_PATH = Path("/models/gte-qwen2-1.5b")
+gte_tokenizer = AutoTokenizer.from_pretrained(GTE_MODEL_PATH, padding_side="left")
+gte_model = AutoModel.from_pretrained(
+    GTE_MODEL_PATH,
+    dtype=torch.bfloat16,
+    trust_remote_code=True,
+).to("cuda").eval()
+
 
 def detailed_instruction(query: str) -> str:
     return f"Instruct: {TASK}\nQuery:{query}"
@@ -272,18 +283,27 @@ def run_family_bf16_control(family: str) -> dict:
     query_texts = [detailed_instruction(item["query"]) for item in pairs]
     document_texts = [item["document"] for item in pairs]
 
-    gc.collect()
-    torch.cuda.empty_cache()
     torch.cuda.reset_peak_memory_stats()
     torch.cuda.synchronize()
     allocation_before = int(torch.cuda.memory_allocated())
-    load_started = time.perf_counter()
-    family_tokenizer = AutoTokenizer.from_pretrained(spec["path"], padding_side="left")
-    family_model = AutoModel.from_pretrained(
-        spec["path"], dtype=torch.bfloat16, trust_remote_code=True
-    ).to("cuda").eval()
-    torch.cuda.synchronize()
-    load_seconds = time.perf_counter() - load_started
+    if family == "gte-qwen2-1.5b":
+        family_tokenizer = gte_tokenizer
+        family_model = gte_model
+        load_seconds = 0.0
+        loading_strategy = "root-module-cuda-emulation"
+        owns_model = False
+    else:
+        load_started = time.perf_counter()
+        family_tokenizer = AutoTokenizer.from_pretrained(
+            spec["path"], padding_side="left"
+        )
+        family_model = AutoModel.from_pretrained(
+            spec["path"], dtype=torch.bfloat16, trust_remote_code=True
+        ).to("cuda").eval()
+        torch.cuda.synchronize()
+        load_seconds = time.perf_counter() - load_started
+        loading_strategy = "in-call-cuda-placement"
+        owns_model = True
     allocation_after_load = int(torch.cuda.memory_allocated())
 
     encode_started = time.perf_counter()
@@ -302,6 +322,7 @@ def run_family_bf16_control(family: str) -> dict:
         "model": spec["source"],
         "source_revision": spec["revision"],
         "load_seconds": load_seconds,
+        "loading_strategy": loading_strategy,
         "encode_seconds": encode_seconds,
         "texts_per_second": len(query_texts + document_texts) / encode_seconds,
         "torch_version": torch.__version__,
@@ -332,9 +353,10 @@ def run_family_bf16_control(family: str) -> dict:
     )
     (output_dir / "cuda-bf16.json").write_text(json.dumps(result, indent=2) + "\n")
 
-    del family_model, family_tokenizer
-    gc.collect()
-    torch.cuda.empty_cache()
+    if owns_model:
+        del family_model, family_tokenizer
+        gc.collect()
+        torch.cuda.empty_cache()
     return result