Napron commited on
Commit
bfacea2
Β·
verified Β·
1 Parent(s): 11c9018

Update jina_fewshot.py

Browse files
Files changed (1) hide show
  1. jina_fewshot.py +71 -41
jina_fewshot.py CHANGED
@@ -23,7 +23,6 @@ refs/ folder structure (3-10 images per class recommended):
23
  import argparse
24
  import csv
25
  import json
26
- import os
27
  import time
28
  from pathlib import Path
29
 
@@ -129,10 +128,34 @@ def parse_args():
129
  return p.parse_args()
130
 
131
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
132
  class JinaCLIPv2Encoder:
133
  """
134
  ONNX Runtime based encoder for jina-clip-v2.
135
- No PyTorch dependency for inference β€” avoids all dtype/NaN issues.
136
  """
137
 
138
  def __init__(self, device="cuda"):
@@ -140,14 +163,10 @@ class JinaCLIPv2Encoder:
140
  print("[*] Loading jina-clip-v2 (ONNX)...")
141
  t0 = time.perf_counter()
142
 
143
- # Download ONNX model from HF Hub
144
- onnx_path = hf_hub_download(
145
- repo_id="jinaai/jina-clip-v2",
146
- filename="onnx/model.onnx",
147
- )
148
- print(f" ONNX model: {onnx_path}")
149
 
150
- # Pick providers: prefer CUDA if available, fallback to CPU
151
  available = ort.get_available_providers()
152
  if "CUDAExecutionProvider" in available and device == "cuda":
153
  providers = ["CUDAExecutionProvider", "CPUExecutionProvider"]
@@ -157,7 +176,7 @@ class JinaCLIPv2Encoder:
157
 
158
  self.session = ort.InferenceSession(onnx_path, providers=providers)
159
 
160
- # Load tokenizer and image processor from HF
161
  self.tokenizer = AutoTokenizer.from_pretrained(
162
  "jinaai/jina-clip-v2", trust_remote_code=True
163
  )
@@ -165,16 +184,27 @@ class JinaCLIPv2Encoder:
165
  "jinaai/jina-clip-v2", trust_remote_code=True
166
  )
167
 
168
- # Figure out input/output names
169
  self.input_names = [inp.name for inp in self.session.get_inputs()]
170
  self.output_names = [out.name for out in self.session.get_outputs()]
171
  print(f" ONNX inputs: {self.input_names}")
172
  print(f" ONNX outputs: {self.output_names}")
173
 
174
- # Create a dummy token for image-only inference
175
- self._dummy_ids = self.tokenizer([""], return_tensors="np", padding=True)["input_ids"]
176
-
177
- # Sanity check
 
 
 
 
 
 
 
 
 
 
 
178
  _dummy = Image.new("RGB", (224, 224), color=(255, 0, 0))
179
  _test = self.encode_images([_dummy], dim=64)
180
  _norm = float(np.linalg.norm(_test))
@@ -189,39 +219,40 @@ class JinaCLIPv2Encoder:
189
  _tn = float(np.linalg.norm(_test_t))
190
  print(f" [SANITY] dummy text embed norm={_tn:.4f}")
191
 
192
- print(f"[*] Loaded in {time.perf_counter() - t0:.1f}s (ONNX, providers={providers})\n")
 
193
 
194
  def _run_image(self, pixel_values: np.ndarray) -> np.ndarray:
195
- """Run ONNX session for images. Returns normalized image embeddings."""
196
- # ONNX model expects both input_ids and pixel_values
197
- # Pass dummy text tokens when encoding images only
198
- dummy_ids = np.zeros((pixel_values.shape[0], 1), dtype=np.int64)
 
 
199
  feeds = {}
200
- for name in self.input_names:
201
- if "pixel" in name.lower():
202
- feeds[name] = pixel_values.astype(np.float32)
203
- elif "input_id" in name.lower():
204
- feeds[name] = dummy_ids
205
- elif "attention" in name.lower():
206
- feeds[name] = np.ones_like(dummy_ids, dtype=np.int64)
207
 
208
  outputs = self.session.run(self.output_names, feeds)
209
- # Output index 3 = normalized image embeddings (or 1 = unnormalized)
210
- # Use the last output as image embeddings (normalized)
211
  return outputs[_IMAGE_NORM_IDX]
212
 
213
  def _run_text(self, input_ids: np.ndarray, attention_mask: np.ndarray) -> np.ndarray:
214
- """Run ONNX session for text. Returns normalized text embeddings."""
215
- # Create dummy pixel values (1x3x224x224) β€” minimal overhead
216
- dummy_pv = np.zeros((input_ids.shape[0], 3, 224, 224), dtype=np.float32)
 
 
217
  feeds = {}
218
- for name in self.input_names:
219
- if "pixel" in name.lower():
220
- feeds[name] = dummy_pv
221
- elif "input_id" in name.lower():
222
- feeds[name] = input_ids
223
- elif "attention" in name.lower():
224
- feeds[name] = attention_mask
225
 
226
  outputs = self.session.run(self.output_names, feeds)
227
  return outputs[_TEXT_NORM_IDX]
@@ -232,7 +263,6 @@ class JinaCLIPv2Encoder:
232
  pixel_values = processed["pixel_values"].astype(np.float32)
233
 
234
  embs = self._run_image(pixel_values)
235
- # Truncate (matryoshka)
236
  if dim and dim < embs.shape[-1]:
237
  embs = embs[:, :dim]
238
  embs = np.nan_to_num(embs, nan=0.0, posinf=0.0, neginf=0.0)
@@ -422,7 +452,7 @@ def main():
422
  print(f"\n{'='*70}")
423
  print("SUMMARY")
424
  print(f"{'='*70}")
425
- print(f" Model : jina-clip-v2 (ONNX Runtime)")
426
  print(f" Embed dim : {args.dim}")
427
  print(f" Text weight : {args.text_weight}")
428
  print(f" Conf threshold : {args.conf_threshold}")
 
23
  import argparse
24
  import csv
25
  import json
 
26
  import time
27
  from pathlib import Path
28
 
 
128
  return p.parse_args()
129
 
130
 
131
+ def _download_onnx_model():
132
+ """
133
+ Download the ONNX model from HF Hub.
134
+ Try fp32 (model.onnx + model.onnx_data) first.
135
+ Both files must be in the same directory for ONNX Runtime to find the
136
+ external data file.
137
+ """
138
+ print(" Downloading ONNX model files from jinaai/jina-clip-v2...")
139
+
140
+ # Download both files β€” hf_hub_download puts them in the same snapshot dir
141
+ onnx_path = hf_hub_download(
142
+ repo_id="jinaai/jina-clip-v2",
143
+ filename="onnx/model.onnx",
144
+ )
145
+ # External weights file β€” MUST be downloaded to same directory
146
+ hf_hub_download(
147
+ repo_id="jinaai/jina-clip-v2",
148
+ filename="onnx/model.onnx_data",
149
+ )
150
+ print(f" Downloaded: {onnx_path}")
151
+ print(f" External data: model.onnx_data (same directory)")
152
+ return onnx_path
153
+
154
+
155
  class JinaCLIPv2Encoder:
156
  """
157
  ONNX Runtime based encoder for jina-clip-v2.
158
+ Completely bypasses PyTorch β€” no dtype/NaN issues.
159
  """
160
 
161
  def __init__(self, device="cuda"):
 
163
  print("[*] Loading jina-clip-v2 (ONNX)...")
164
  t0 = time.perf_counter()
165
 
166
+ # Download ONNX model (fp32 with external data)
167
+ onnx_path = _download_onnx_model()
 
 
 
 
168
 
169
+ # Pick providers: prefer CUDA if available
170
  available = ort.get_available_providers()
171
  if "CUDAExecutionProvider" in available and device == "cuda":
172
  providers = ["CUDAExecutionProvider", "CPUExecutionProvider"]
 
176
 
177
  self.session = ort.InferenceSession(onnx_path, providers=providers)
178
 
179
+ # Load tokenizer and image processor
180
  self.tokenizer = AutoTokenizer.from_pretrained(
181
  "jinaai/jina-clip-v2", trust_remote_code=True
182
  )
 
184
  "jinaai/jina-clip-v2", trust_remote_code=True
185
  )
186
 
187
+ # Inspect ONNX model I/O
188
  self.input_names = [inp.name for inp in self.session.get_inputs()]
189
  self.output_names = [out.name for out in self.session.get_outputs()]
190
  print(f" ONNX inputs: {self.input_names}")
191
  print(f" ONNX outputs: {self.output_names}")
192
 
193
+ # Build input name mapping
194
+ self._pixel_name = None
195
+ self._ids_name = None
196
+ self._mask_name = None
197
+ for name in self.input_names:
198
+ nl = name.lower()
199
+ if "pixel" in nl:
200
+ self._pixel_name = name
201
+ elif "input_id" in nl:
202
+ self._ids_name = name
203
+ elif "attention" in nl or "mask" in nl:
204
+ self._mask_name = name
205
+ print(f" Mapped: pixel={self._pixel_name}, ids={self._ids_name}, mask={self._mask_name}")
206
+
207
+ # Sanity checks
208
  _dummy = Image.new("RGB", (224, 224), color=(255, 0, 0))
209
  _test = self.encode_images([_dummy], dim=64)
210
  _norm = float(np.linalg.norm(_test))
 
219
  _tn = float(np.linalg.norm(_test_t))
220
  print(f" [SANITY] dummy text embed norm={_tn:.4f}")
221
 
222
+ elapsed = time.perf_counter() - t0
223
+ print(f"[*] Loaded in {elapsed:.1f}s (ONNX, providers={providers})\n")
224
 
225
  def _run_image(self, pixel_values: np.ndarray) -> np.ndarray:
226
+ """Run ONNX for images only. Returns normalized image embeddings."""
227
+ bs = pixel_values.shape[0]
228
+ # Dummy text input (minimal tokens)
229
+ dummy_ids = np.zeros((bs, 1), dtype=np.int64)
230
+ dummy_mask = np.ones((bs, 1), dtype=np.int64)
231
+
232
  feeds = {}
233
+ if self._pixel_name:
234
+ feeds[self._pixel_name] = pixel_values.astype(np.float32)
235
+ if self._ids_name:
236
+ feeds[self._ids_name] = dummy_ids
237
+ if self._mask_name:
238
+ feeds[self._mask_name] = dummy_mask
 
239
 
240
  outputs = self.session.run(self.output_names, feeds)
 
 
241
  return outputs[_IMAGE_NORM_IDX]
242
 
243
  def _run_text(self, input_ids: np.ndarray, attention_mask: np.ndarray) -> np.ndarray:
244
+ """Run ONNX for text only. Returns normalized text embeddings."""
245
+ bs = input_ids.shape[0]
246
+ # Dummy pixel values (1 pixel image β€” minimal memory)
247
+ dummy_pv = np.zeros((bs, 3, 224, 224), dtype=np.float32)
248
+
249
  feeds = {}
250
+ if self._pixel_name:
251
+ feeds[self._pixel_name] = dummy_pv
252
+ if self._ids_name:
253
+ feeds[self._ids_name] = input_ids.astype(np.int64)
254
+ if self._mask_name:
255
+ feeds[self._mask_name] = attention_mask.astype(np.int64)
 
256
 
257
  outputs = self.session.run(self.output_names, feeds)
258
  return outputs[_TEXT_NORM_IDX]
 
263
  pixel_values = processed["pixel_values"].astype(np.float32)
264
 
265
  embs = self._run_image(pixel_values)
 
266
  if dim and dim < embs.shape[-1]:
267
  embs = embs[:, :dim]
268
  embs = np.nan_to_num(embs, nan=0.0, posinf=0.0, neginf=0.0)
 
452
  print(f"\n{'='*70}")
453
  print("SUMMARY")
454
  print(f"{'='*70}")
455
+ print(f" Model : jina-clip-v2 (ONNX Runtime, fp32)")
456
  print(f" Embed dim : {args.dim}")
457
  print(f" Text weight : {args.text_weight}")
458
  print(f" Conf threshold : {args.conf_threshold}")