Lazy-load CUDA quantizer configuration inside GPU call

app.py

@@ -154,21 +154,27 @@ def compare_vectors(
     }
 
 
-QUANTIZERS = {
-    "bnb-int8": BitsAndBytesConfig(load_in_8bit=True),
-    "bnb-nf4": BitsAndBytesConfig(
-        load_in_4bit=True,
-        bnb_4bit_quant_type="nf4",
-        bnb_4bit_compute_dtype=torch.bfloat16,
-        bnb_4bit_use_double_quant=False,
-    ),
-}
-
-
-@spaces.GPU(duration=600)
+QUANTIZERS = ("bnb-int8", "bnb-nf4")
+
+
+def quantization_config(variant: str) -> BitsAndBytesConfig:
+    if variant == "bnb-int8":
+        return BitsAndBytesConfig(load_in_8bit=True)
+    if variant == "bnb-nf4":
+        return BitsAndBytesConfig(
+            load_in_4bit=True,
+            bnb_4bit_quant_type="nf4",
+            bnb_4bit_compute_dtype=torch.bfloat16,
+            bnb_4bit_use_double_quant=False,
+        )
+    raise ValueError(f"unsupported quantizer: {variant}")
+
+
+@spaces.GPU(duration=300)
 def run_quantized_control(variant: str) -> dict:
     if variant not in QUANTIZERS:
         raise ValueError(f"unsupported quantizer: {variant}")
+    config = quantization_config(variant)
     pairs = json.loads(INPUT_PATH.read_text())
     query_texts = [detailed_instruction(item["query"]) for item in pairs]
     document_texts = [item["document"] for item in pairs]
@@ -181,7 +187,7 @@ def run_quantized_control(variant: str) -> dict:
     load_started = time.perf_counter()
     quantized_model = AutoModel.from_pretrained(
         MODEL_PATH,
-        quantization_config=QUANTIZERS[variant],
+        quantization_config=config,
         device_map={"": 0},
         trust_remote_code=True,
     ).eval()
@@ -202,7 +208,7 @@ def run_quantized_control(variant: str) -> dict:
         "model": "Qwen/Qwen3-Embedding-0.6B",
         "source_revision": "97b0c614be4d77ee51c0cef4e5f07c00f9eb65b3",
         "quantizer": variant,
-        "quantization_config": QUANTIZERS[variant].to_dict(),
+        "quantization_config": config.to_dict(),
         "load_seconds": load_seconds,
         "encode_seconds": encode_seconds,
         "texts_per_second": len(query_texts + document_texts) / encode_seconds,