Spaces:
Running
Running
Update jina_fewshot.py
Browse files- jina_fewshot.py +71 -41
jina_fewshot.py
CHANGED
|
@@ -23,7 +23,6 @@ refs/ folder structure (3-10 images per class recommended):
|
|
| 23 |
import argparse
|
| 24 |
import csv
|
| 25 |
import json
|
| 26 |
-
import os
|
| 27 |
import time
|
| 28 |
from pathlib import Path
|
| 29 |
|
|
@@ -129,10 +128,34 @@ def parse_args():
|
|
| 129 |
return p.parse_args()
|
| 130 |
|
| 131 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 132 |
class JinaCLIPv2Encoder:
|
| 133 |
"""
|
| 134 |
ONNX Runtime based encoder for jina-clip-v2.
|
| 135 |
-
|
| 136 |
"""
|
| 137 |
|
| 138 |
def __init__(self, device="cuda"):
|
|
@@ -140,14 +163,10 @@ class JinaCLIPv2Encoder:
|
|
| 140 |
print("[*] Loading jina-clip-v2 (ONNX)...")
|
| 141 |
t0 = time.perf_counter()
|
| 142 |
|
| 143 |
-
# Download ONNX model
|
| 144 |
-
onnx_path =
|
| 145 |
-
repo_id="jinaai/jina-clip-v2",
|
| 146 |
-
filename="onnx/model.onnx",
|
| 147 |
-
)
|
| 148 |
-
print(f" ONNX model: {onnx_path}")
|
| 149 |
|
| 150 |
-
# Pick providers: prefer CUDA if available
|
| 151 |
available = ort.get_available_providers()
|
| 152 |
if "CUDAExecutionProvider" in available and device == "cuda":
|
| 153 |
providers = ["CUDAExecutionProvider", "CPUExecutionProvider"]
|
|
@@ -157,7 +176,7 @@ class JinaCLIPv2Encoder:
|
|
| 157 |
|
| 158 |
self.session = ort.InferenceSession(onnx_path, providers=providers)
|
| 159 |
|
| 160 |
-
# Load tokenizer and image processor
|
| 161 |
self.tokenizer = AutoTokenizer.from_pretrained(
|
| 162 |
"jinaai/jina-clip-v2", trust_remote_code=True
|
| 163 |
)
|
|
@@ -165,16 +184,27 @@ class JinaCLIPv2Encoder:
|
|
| 165 |
"jinaai/jina-clip-v2", trust_remote_code=True
|
| 166 |
)
|
| 167 |
|
| 168 |
-
#
|
| 169 |
self.input_names = [inp.name for inp in self.session.get_inputs()]
|
| 170 |
self.output_names = [out.name for out in self.session.get_outputs()]
|
| 171 |
print(f" ONNX inputs: {self.input_names}")
|
| 172 |
print(f" ONNX outputs: {self.output_names}")
|
| 173 |
|
| 174 |
-
#
|
| 175 |
-
self.
|
| 176 |
-
|
| 177 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 178 |
_dummy = Image.new("RGB", (224, 224), color=(255, 0, 0))
|
| 179 |
_test = self.encode_images([_dummy], dim=64)
|
| 180 |
_norm = float(np.linalg.norm(_test))
|
|
@@ -189,39 +219,40 @@ class JinaCLIPv2Encoder:
|
|
| 189 |
_tn = float(np.linalg.norm(_test_t))
|
| 190 |
print(f" [SANITY] dummy text embed norm={_tn:.4f}")
|
| 191 |
|
| 192 |
-
|
|
|
|
| 193 |
|
| 194 |
def _run_image(self, pixel_values: np.ndarray) -> np.ndarray:
|
| 195 |
-
"""Run ONNX
|
| 196 |
-
|
| 197 |
-
#
|
| 198 |
-
dummy_ids = np.zeros((
|
|
|
|
|
|
|
| 199 |
feeds = {}
|
| 200 |
-
|
| 201 |
-
|
| 202 |
-
|
| 203 |
-
|
| 204 |
-
|
| 205 |
-
|
| 206 |
-
feeds[name] = np.ones_like(dummy_ids, dtype=np.int64)
|
| 207 |
|
| 208 |
outputs = self.session.run(self.output_names, feeds)
|
| 209 |
-
# Output index 3 = normalized image embeddings (or 1 = unnormalized)
|
| 210 |
-
# Use the last output as image embeddings (normalized)
|
| 211 |
return outputs[_IMAGE_NORM_IDX]
|
| 212 |
|
| 213 |
def _run_text(self, input_ids: np.ndarray, attention_mask: np.ndarray) -> np.ndarray:
|
| 214 |
-
"""Run ONNX
|
| 215 |
-
|
| 216 |
-
|
|
|
|
|
|
|
| 217 |
feeds = {}
|
| 218 |
-
|
| 219 |
-
|
| 220 |
-
|
| 221 |
-
|
| 222 |
-
|
| 223 |
-
|
| 224 |
-
feeds[name] = attention_mask
|
| 225 |
|
| 226 |
outputs = self.session.run(self.output_names, feeds)
|
| 227 |
return outputs[_TEXT_NORM_IDX]
|
|
@@ -232,7 +263,6 @@ class JinaCLIPv2Encoder:
|
|
| 232 |
pixel_values = processed["pixel_values"].astype(np.float32)
|
| 233 |
|
| 234 |
embs = self._run_image(pixel_values)
|
| 235 |
-
# Truncate (matryoshka)
|
| 236 |
if dim and dim < embs.shape[-1]:
|
| 237 |
embs = embs[:, :dim]
|
| 238 |
embs = np.nan_to_num(embs, nan=0.0, posinf=0.0, neginf=0.0)
|
|
@@ -422,7 +452,7 @@ def main():
|
|
| 422 |
print(f"\n{'='*70}")
|
| 423 |
print("SUMMARY")
|
| 424 |
print(f"{'='*70}")
|
| 425 |
-
print(f" Model : jina-clip-v2 (ONNX Runtime)")
|
| 426 |
print(f" Embed dim : {args.dim}")
|
| 427 |
print(f" Text weight : {args.text_weight}")
|
| 428 |
print(f" Conf threshold : {args.conf_threshold}")
|
|
|
|
| 23 |
import argparse
|
| 24 |
import csv
|
| 25 |
import json
|
|
|
|
| 26 |
import time
|
| 27 |
from pathlib import Path
|
| 28 |
|
|
|
|
| 128 |
return p.parse_args()
|
| 129 |
|
| 130 |
|
| 131 |
+
def _download_onnx_model():
|
| 132 |
+
"""
|
| 133 |
+
Download the ONNX model from HF Hub.
|
| 134 |
+
Try fp32 (model.onnx + model.onnx_data) first.
|
| 135 |
+
Both files must be in the same directory for ONNX Runtime to find the
|
| 136 |
+
external data file.
|
| 137 |
+
"""
|
| 138 |
+
print(" Downloading ONNX model files from jinaai/jina-clip-v2...")
|
| 139 |
+
|
| 140 |
+
# Download both files β hf_hub_download puts them in the same snapshot dir
|
| 141 |
+
onnx_path = hf_hub_download(
|
| 142 |
+
repo_id="jinaai/jina-clip-v2",
|
| 143 |
+
filename="onnx/model.onnx",
|
| 144 |
+
)
|
| 145 |
+
# External weights file β MUST be downloaded to same directory
|
| 146 |
+
hf_hub_download(
|
| 147 |
+
repo_id="jinaai/jina-clip-v2",
|
| 148 |
+
filename="onnx/model.onnx_data",
|
| 149 |
+
)
|
| 150 |
+
print(f" Downloaded: {onnx_path}")
|
| 151 |
+
print(f" External data: model.onnx_data (same directory)")
|
| 152 |
+
return onnx_path
|
| 153 |
+
|
| 154 |
+
|
| 155 |
class JinaCLIPv2Encoder:
|
| 156 |
"""
|
| 157 |
ONNX Runtime based encoder for jina-clip-v2.
|
| 158 |
+
Completely bypasses PyTorch β no dtype/NaN issues.
|
| 159 |
"""
|
| 160 |
|
| 161 |
def __init__(self, device="cuda"):
|
|
|
|
| 163 |
print("[*] Loading jina-clip-v2 (ONNX)...")
|
| 164 |
t0 = time.perf_counter()
|
| 165 |
|
| 166 |
+
# Download ONNX model (fp32 with external data)
|
| 167 |
+
onnx_path = _download_onnx_model()
|
|
|
|
|
|
|
|
|
|
|
|
|
| 168 |
|
| 169 |
+
# Pick providers: prefer CUDA if available
|
| 170 |
available = ort.get_available_providers()
|
| 171 |
if "CUDAExecutionProvider" in available and device == "cuda":
|
| 172 |
providers = ["CUDAExecutionProvider", "CPUExecutionProvider"]
|
|
|
|
| 176 |
|
| 177 |
self.session = ort.InferenceSession(onnx_path, providers=providers)
|
| 178 |
|
| 179 |
+
# Load tokenizer and image processor
|
| 180 |
self.tokenizer = AutoTokenizer.from_pretrained(
|
| 181 |
"jinaai/jina-clip-v2", trust_remote_code=True
|
| 182 |
)
|
|
|
|
| 184 |
"jinaai/jina-clip-v2", trust_remote_code=True
|
| 185 |
)
|
| 186 |
|
| 187 |
+
# Inspect ONNX model I/O
|
| 188 |
self.input_names = [inp.name for inp in self.session.get_inputs()]
|
| 189 |
self.output_names = [out.name for out in self.session.get_outputs()]
|
| 190 |
print(f" ONNX inputs: {self.input_names}")
|
| 191 |
print(f" ONNX outputs: {self.output_names}")
|
| 192 |
|
| 193 |
+
# Build input name mapping
|
| 194 |
+
self._pixel_name = None
|
| 195 |
+
self._ids_name = None
|
| 196 |
+
self._mask_name = None
|
| 197 |
+
for name in self.input_names:
|
| 198 |
+
nl = name.lower()
|
| 199 |
+
if "pixel" in nl:
|
| 200 |
+
self._pixel_name = name
|
| 201 |
+
elif "input_id" in nl:
|
| 202 |
+
self._ids_name = name
|
| 203 |
+
elif "attention" in nl or "mask" in nl:
|
| 204 |
+
self._mask_name = name
|
| 205 |
+
print(f" Mapped: pixel={self._pixel_name}, ids={self._ids_name}, mask={self._mask_name}")
|
| 206 |
+
|
| 207 |
+
# Sanity checks
|
| 208 |
_dummy = Image.new("RGB", (224, 224), color=(255, 0, 0))
|
| 209 |
_test = self.encode_images([_dummy], dim=64)
|
| 210 |
_norm = float(np.linalg.norm(_test))
|
|
|
|
| 219 |
_tn = float(np.linalg.norm(_test_t))
|
| 220 |
print(f" [SANITY] dummy text embed norm={_tn:.4f}")
|
| 221 |
|
| 222 |
+
elapsed = time.perf_counter() - t0
|
| 223 |
+
print(f"[*] Loaded in {elapsed:.1f}s (ONNX, providers={providers})\n")
|
| 224 |
|
| 225 |
def _run_image(self, pixel_values: np.ndarray) -> np.ndarray:
|
| 226 |
+
"""Run ONNX for images only. Returns normalized image embeddings."""
|
| 227 |
+
bs = pixel_values.shape[0]
|
| 228 |
+
# Dummy text input (minimal tokens)
|
| 229 |
+
dummy_ids = np.zeros((bs, 1), dtype=np.int64)
|
| 230 |
+
dummy_mask = np.ones((bs, 1), dtype=np.int64)
|
| 231 |
+
|
| 232 |
feeds = {}
|
| 233 |
+
if self._pixel_name:
|
| 234 |
+
feeds[self._pixel_name] = pixel_values.astype(np.float32)
|
| 235 |
+
if self._ids_name:
|
| 236 |
+
feeds[self._ids_name] = dummy_ids
|
| 237 |
+
if self._mask_name:
|
| 238 |
+
feeds[self._mask_name] = dummy_mask
|
|
|
|
| 239 |
|
| 240 |
outputs = self.session.run(self.output_names, feeds)
|
|
|
|
|
|
|
| 241 |
return outputs[_IMAGE_NORM_IDX]
|
| 242 |
|
| 243 |
def _run_text(self, input_ids: np.ndarray, attention_mask: np.ndarray) -> np.ndarray:
|
| 244 |
+
"""Run ONNX for text only. Returns normalized text embeddings."""
|
| 245 |
+
bs = input_ids.shape[0]
|
| 246 |
+
# Dummy pixel values (1 pixel image β minimal memory)
|
| 247 |
+
dummy_pv = np.zeros((bs, 3, 224, 224), dtype=np.float32)
|
| 248 |
+
|
| 249 |
feeds = {}
|
| 250 |
+
if self._pixel_name:
|
| 251 |
+
feeds[self._pixel_name] = dummy_pv
|
| 252 |
+
if self._ids_name:
|
| 253 |
+
feeds[self._ids_name] = input_ids.astype(np.int64)
|
| 254 |
+
if self._mask_name:
|
| 255 |
+
feeds[self._mask_name] = attention_mask.astype(np.int64)
|
|
|
|
| 256 |
|
| 257 |
outputs = self.session.run(self.output_names, feeds)
|
| 258 |
return outputs[_TEXT_NORM_IDX]
|
|
|
|
| 263 |
pixel_values = processed["pixel_values"].astype(np.float32)
|
| 264 |
|
| 265 |
embs = self._run_image(pixel_values)
|
|
|
|
| 266 |
if dim and dim < embs.shape[-1]:
|
| 267 |
embs = embs[:, :dim]
|
| 268 |
embs = np.nan_to_num(embs, nan=0.0, posinf=0.0, neginf=0.0)
|
|
|
|
| 452 |
print(f"\n{'='*70}")
|
| 453 |
print("SUMMARY")
|
| 454 |
print(f"{'='*70}")
|
| 455 |
+
print(f" Model : jina-clip-v2 (ONNX Runtime, fp32)")
|
| 456 |
print(f" Embed dim : {args.dim}")
|
| 457 |
print(f" Text weight : {args.text_weight}")
|
| 458 |
print(f" Conf threshold : {args.conf_threshold}")
|