Add mounted 1.5B and 8B BF16 cross-runtime controls

README.md

@@ -20,3 +20,7 @@ BF16 vectors. CUDA results do not reproduce MLX/Metal performance.
 The second milestone adds CUDA-native bitsandbytes INT8 and NF4 controls.
 They are deliberately reported as separate quantizers and are not treated as
 equivalents of MLX Q, oQ, or oQe formats.
+
+Matched BF16 controls are also available for GTE-Qwen2-1.5B and
+Qwen3-Embedding-8B. Their CUDA vectors are compared against the corresponding
+saved MLX BF16 vectors from the private reproduction bucket.

app.py

@@ -38,8 +38,11 @@ def detailed_instruction(query: str) -> str:
     return f"Instruct: {TASK}\nQuery:{query}"
 
 
-def encode_with(active_model, text: str) -> np.ndarray:
-    batch = tokenizer(text, return_tensors="pt", truncation=True, max_length=32768)
+def encode_with(active_model, text: str, active_tokenizer=None) -> np.ndarray:
+    active_tokenizer = active_tokenizer or tokenizer
+    batch = active_tokenizer(
+        text, return_tensors="pt", truncation=True, max_length=32768
+    )
     batch = {key: value.to("cuda") for key, value in batch.items()}
     with torch.inference_mode():
         output = active_model(**batch)
@@ -246,6 +249,95 @@ def run_quantized_control(variant: str) -> dict:
     return result
 
 
+BF16_FAMILIES = {
+    "gte-qwen2-1.5b": {
+        "path": Path("/models/gte-qwen2-1.5b"),
+        "source": "Alibaba-NLP/gte-Qwen2-1.5B-instruct",
+        "revision": "a9af15a6372d7d6b25e9fb07c2ccb9e1fe645644",
+    },
+    "qwen3-embedding-8b": {
+        "path": Path("/models/qwen3-embedding-8b"),
+        "source": "Qwen/Qwen3-Embedding-8B",
+        "revision": "1d8ad4ca9b3dd8059ad90a75d4983776a23d44af",
+    },
+}
+
+
+@spaces.GPU(duration=300)
+def run_family_bf16_control(family: str) -> dict:
+    if family not in BF16_FAMILIES:
+        raise ValueError(f"unsupported family: {family}")
+    spec = BF16_FAMILIES[family]
+    pairs = json.loads(INPUT_PATH.read_text())
+    query_texts = [detailed_instruction(item["query"]) for item in pairs]
+    document_texts = [item["document"] for item in pairs]
+
+    gc.collect()
+    torch.cuda.empty_cache()
+    torch.cuda.reset_peak_memory_stats()
+    torch.cuda.synchronize()
+    allocation_before = int(torch.cuda.memory_allocated())
+    load_started = time.perf_counter()
+    family_tokenizer = AutoTokenizer.from_pretrained(spec["path"], padding_side="left")
+    family_model = AutoModel.from_pretrained(
+        spec["path"], dtype=torch.bfloat16, trust_remote_code=True
+    ).to("cuda").eval()
+    torch.cuda.synchronize()
+    load_seconds = time.perf_counter() - load_started
+    allocation_after_load = int(torch.cuda.memory_allocated())
+
+    encode_started = time.perf_counter()
+    queries = np.stack([
+        encode_with(family_model, text, family_tokenizer) for text in query_texts
+    ])
+    documents = np.stack([
+        encode_with(family_model, text, family_tokenizer) for text in document_texts
+    ])
+    torch.cuda.synchronize()
+    encode_seconds = time.perf_counter() - encode_started
+    metrics, scores, ranks = retrieval_metrics(queries, documents)
+    metrics.update({
+        "lane": "cuda-zerogpu-bf16",
+        "family": family,
+        "model": spec["source"],
+        "source_revision": spec["revision"],
+        "load_seconds": load_seconds,
+        "encode_seconds": encode_seconds,
+        "texts_per_second": len(query_texts + document_texts) / encode_seconds,
+        "torch_version": torch.__version__,
+        "cuda_device": torch.cuda.get_device_name(0),
+        "cuda_allocation_before_load": allocation_before,
+        "cuda_allocation_after_load": allocation_after_load,
+        "cuda_incremental_model_allocation": allocation_after_load - allocation_before,
+        "cuda_peak_bytes": int(torch.cuda.max_memory_allocated()),
+    })
+    local_reference = (
+        DATA_ROOT / "local-results/full-q4-q6-q8" / family / "quality/bf16.npz"
+    )
+    result = {
+        "metrics": metrics,
+        "mlx_bf16_comparison": compare_vectors(
+            queries, documents, scores, ranks, local_reference, "cuda_vs_mlx_bf16"
+        ),
+    }
+    output_dir = DATA_ROOT / "cloud-results" / family
+    output_dir.mkdir(parents=True, exist_ok=True)
+    np.savez_compressed(
+        output_dir / "cuda-bf16.npz",
+        queries=queries,
+        documents=documents,
+        scores=scores,
+        ranks=ranks,
+        metrics=np.array(json.dumps(metrics)),
+    )
+    (output_dir / "cuda-bf16.json").write_text(json.dumps(result, indent=2) + "\n")
+
+    del family_model, family_tokenizer
+    gc.collect()
+    torch.cuda.empty_cache()
+    return result
+
+
 with gr.Blocks() as demo:
     gr.Markdown(
         "# Embedding Quantization CUDA Control\n"
@@ -276,6 +368,21 @@ with gr.Blocks() as demo:
         concurrency_limit=1,
         api_name="run_quantized_control",
     )
+    gr.Markdown("## Larger-family BF16 cross-runtime controls")
+    family = gr.Dropdown(
+        choices=list(BF16_FAMILIES),
+        value="gte-qwen2-1.5b",
+        label="Model family",
+    )
+    family_button = gr.Button("Run bounded family BF16 control")
+    family_output = gr.JSON(label="Family result")
+    family_button.click(
+        fn=run_family_bf16_control,
+        inputs=family,
+        outputs=family_output,
+        concurrency_limit=1,
+        api_name="run_family_bf16_control",
+    )
 
 
 if __name__ == "__main__":