Napron commited on
Commit
11c9018
·
verified ·
1 Parent(s): 66890cd

Update jina_fewshot.py

Browse files
Files changed (1) hide show
  1. jina_fewshot.py +107 -96
jina_fewshot.py CHANGED
@@ -1,5 +1,6 @@
1
  """
2
- Few-shot object classification using jina-clip-v2 (EVA02-L, 304M).
 
3
  Combines IMAGE embeddings from reference photos + TEXT embeddings
4
  from class names. Dual threshold: confidence + gap between top-1 and top-2.
5
  Usage:
@@ -22,24 +23,22 @@ refs/ folder structure (3-10 images per class recommended):
22
  import argparse
23
  import csv
24
  import json
 
25
  import time
26
  from pathlib import Path
27
 
28
  import numpy as np
29
- import torch
30
- import torch.nn.functional as F
31
  from PIL import Image, ImageDraw, ImageFont
32
- from transformers import AutoModel, AutoImageProcessor
 
33
 
34
  IMAGE_EXTS = {".jpg", ".jpeg", ".png", ".bmp", ".webp", ".tiff"}
35
  TRUNCATE_DIM = 1024
36
 
37
-
38
- def _to_numpy(embs):
39
- """Convert to numpy; Jina may return tensor on some code paths."""
40
- if hasattr(embs, "cpu"):
41
- embs = embs.cpu().float().numpy()
42
- return np.asarray(embs, dtype=np.float64)
43
 
44
 
45
  def draw_label_on_image(img: Image.Image, label: str, confidence: float) -> Image.Image:
@@ -113,7 +112,7 @@ CLASS_PROMPTS = {
113
 
114
 
115
  def parse_args():
116
- p = argparse.ArgumentParser(description="Jina-CLIP-v2 few-shot classifier")
117
  p.add_argument("--refs", required=True, help="Reference images folder")
118
  p.add_argument("--input", required=True, help="Query crop images folder")
119
  p.add_argument("--output", default="jinaclip_results", help="Output folder")
@@ -132,99 +131,125 @@ def parse_args():
132
 
133
  class JinaCLIPv2Encoder:
134
  """
135
- Wraps jina-clip-v2 but BYPASSES encode_image to avoid bfloat16/NaN
136
- issues on T4. Calls vision_model + visual_projection directly with
137
- explicit float32 pixel values.
138
  """
139
 
140
  def __init__(self, device="cuda"):
141
  self.device = device
142
- print("[*] Loading jina-clip-v2...")
143
  t0 = time.perf_counter()
144
 
145
- _orig_linspace = torch.linspace
146
-
147
- def _safe_linspace(*args, **kwargs):
148
- kwargs.pop("device", None)
149
- return _orig_linspace(*args, **kwargs, device="cpu")
150
-
151
- torch.linspace = _safe_linspace
152
- try:
153
- self.model = AutoModel.from_pretrained(
154
- "jinaai/jina-clip-v2",
155
- trust_remote_code=True,
156
- low_cpu_mem_usage=False,
157
- revision="main",
158
- torch_dtype=torch.float32,
159
- device_map=None,
160
- )
161
- finally:
162
- torch.linspace = _orig_linspace
163
-
164
- # Force EVERYTHING to float32 — model, all submodules, all buffers
165
- self.model = self.model.to(device).eval().float()
166
-
167
- # Load the image preprocessor ourselves so we control dtype
168
- self.preprocess = AutoImageProcessor.from_pretrained(
169
  "jinaai/jina-clip-v2", trust_remote_code=True
170
  )
171
 
172
- # Sanity: encode a dummy image via our direct path
 
 
 
 
 
 
 
 
 
173
  _dummy = Image.new("RGB", (224, 224), color=(255, 0, 0))
174
- _test = self._encode_images_direct([_dummy], 64)
175
  _norm = float(np.linalg.norm(_test))
176
  _is_nan = bool(np.isnan(_norm))
177
  print(f" [SANITY] dummy image embed norm={_norm:.4f}, nan={_is_nan}")
178
  if _is_nan or _norm < 0.01:
179
- print(" [ERROR] Vision encoder still broken!")
180
  else:
181
- print(" [OK] Vision encoder producing valid embeddings via direct path")
182
 
183
- # Also test Jina's own encode_text (this works fine)
184
- _test_t = self.model.encode_text(["a red square"], truncate_dim=64)
185
  _tn = float(np.linalg.norm(_test_t))
186
  print(f" [SANITY] dummy text embed norm={_tn:.4f}")
187
 
188
- print(f"[*] Loaded in {time.perf_counter() - t0:.1f}s (device={device})\n")
189
-
190
- def _encode_images_direct(self, pil_images: list, dim: int) -> np.ndarray:
191
- """
192
- Bypass Jina's encode_image entirely.
193
- Preprocess → float32 pixel_values → vision_model → visual_projection → truncate → normalize.
194
- """
195
- rgb = [img.convert("RGB") for img in pil_images]
196
-
197
- # Preprocess with our own processor
198
- batch = self.preprocess(rgb, return_tensors="pt")
199
-
200
- # CRITICAL: force float32 before sending to GPU
201
- pixel_values = batch["pixel_values"].to(dtype=torch.float32, device=self.device)
202
-
203
- with torch.no_grad(), torch.amp.autocast("cuda", enabled=False):
204
- # Call vision model directly (EVA02)
205
- vision_out = self.model.vision_model(x=pixel_values)
206
- # Project
207
- image_features = self.model.visual_projection(vision_out)
208
- # Truncate (matryoshka)
209
- if dim and dim < image_features.shape[-1]:
210
- image_features = image_features[:, :dim]
211
- # Normalize
212
- image_features = F.normalize(image_features, p=2, dim=1)
213
-
214
- return image_features.cpu().float().numpy()
 
 
 
 
 
 
 
 
 
215
 
216
  def encode_images(self, images: list[Image.Image], dim: int = TRUNCATE_DIM) -> np.ndarray:
217
- embs = self._encode_images_direct(images, dim)
 
 
 
 
 
 
 
218
  embs = np.nan_to_num(embs, nan=0.0, posinf=0.0, neginf=0.0)
219
  norms = np.linalg.norm(embs, axis=-1, keepdims=True)
220
  norms = np.maximum(norms, 1e-12)
221
  return (embs / norms).astype(np.float32)
222
 
223
  def encode_texts(self, texts: list[str], dim: int = TRUNCATE_DIM) -> np.ndarray:
224
- # Jina's encode_text works fine — text encoder has no NaN issues
225
- with torch.no_grad():
226
- embs = self.model.encode_text(texts, truncate_dim=dim)
227
- embs = _to_numpy(embs)
 
 
 
 
 
228
  embs = np.nan_to_num(embs, nan=0.0, posinf=0.0, neginf=0.0)
229
  norms = np.linalg.norm(embs, axis=-1, keepdims=True)
230
  norms = np.maximum(norms, 1e-12)
@@ -246,8 +271,7 @@ def build_refs(encoder: JinaCLIPv2Encoder, refs_dir: Path,
246
  raise ValueError(f"No subfolders in {refs_dir}")
247
 
248
  labels, embeddings = [], []
249
- _device = getattr(encoder, "device", "?")
250
- print(f" Device: {_device} | Text weight: {text_weight:.1f} | Image weight: {1 - text_weight:.1f}\n")
251
 
252
  for d in class_dirs:
253
  name = d.name
@@ -255,16 +279,13 @@ def build_refs(encoder: JinaCLIPv2Encoder, refs_dir: Path,
255
  if not paths:
256
  continue
257
 
258
- # Image embeddings
259
  img_embs = encoder.encode_image_paths(paths, dim, batch_size)
260
  img_avg = np.nan_to_num(img_embs.mean(axis=0), nan=0.0, posinf=0.0, neginf=0.0)
261
 
262
- # Text embeddings
263
  prompts = CLASS_PROMPTS.get(name, [f"a {name}", f"a person holding a {name}"])
264
  text_embs = encoder.encode_texts(prompts, dim)
265
  text_avg = np.nan_to_num(text_embs.mean(axis=0), nan=0.0, posinf=0.0, neginf=0.0)
266
 
267
- # Combine
268
  combined = (1.0 - text_weight) * img_avg + text_weight * text_avg
269
  combined = np.nan_to_num(combined, nan=0.0, posinf=0.0, neginf=0.0)
270
  combined = combined / (np.linalg.norm(combined) + 1e-12)
@@ -333,21 +354,18 @@ def main():
333
 
334
  encoder = JinaCLIPv2Encoder("cuda")
335
 
336
- # Build references
337
  print("[*] Building references...")
338
  ref_labels, ref_embs = build_refs(
339
  encoder, Path(args.refs), args.dim, args.text_weight, args.batch_size
340
  )
341
  print(f"\n[*] {len(ref_labels)} classes: {ref_labels}\n")
342
 
343
- # Save refs if requested
344
  if args.save_refs:
345
  np.save(output_dir / "ref_embeddings.npy", ref_embs)
346
  with open(output_dir / "ref_labels.json", "w") as jf:
347
  json.dump(ref_labels, jf)
348
  print(f"[*] Saved refs to {output_dir}\n")
349
 
350
- # CSV
351
  csv_path = output_dir / "classifications.csv"
352
  f = open(csv_path, "w", newline="")
353
  w = csv.writer(f)
@@ -355,19 +373,16 @@ def main():
355
  "second_best", "second_conf", "status"] +
356
  [f"sim_{l}" for l in ref_labels] + ["time_ms"])
357
 
358
- # Stats
359
  times = []
360
  counts = {"unknown": 0}
361
  for l in ref_labels:
362
  counts[l] = 0
363
  accepted, rejected = 0, 0
364
 
365
- # Header
366
  hdr = " ".join(f"{l:>10}" for l in ref_labels)
367
  print(f"{'Image':<30} {'Result':<10} {'Conf':>6} {'Gap':>6} {hdr} {'Status'}")
368
  print("=" * (30 + 10 + 14 + len(hdr) + 40))
369
 
370
- # Classify
371
  for p in paths:
372
  t0 = time.perf_counter()
373
  img = Image.open(p)
@@ -393,13 +408,9 @@ def main():
393
  f"{sim_str} {result['status']}")
394
 
395
  w.writerow([
396
- p.name,
397
- result["prediction"],
398
- result["raw_prediction"],
399
- f"{result['confidence']:.4f}",
400
- f"{result['gap']:.4f}",
401
- result["second_best"],
402
- f"{result['second_conf']:.4f}",
403
  result["status"],
404
  ] + [f"{result['all_sims'][l]:.4f}" for l in ref_labels] +
405
  [f"{ms:.1f}"])
@@ -411,7 +422,7 @@ def main():
411
  print(f"\n{'='*70}")
412
  print("SUMMARY")
413
  print(f"{'='*70}")
414
- print(f" Model : jina-clip-v2 (EVA02-L, 304M, CLS pooling)")
415
  print(f" Embed dim : {args.dim}")
416
  print(f" Text weight : {args.text_weight}")
417
  print(f" Conf threshold : {args.conf_threshold}")
 
1
  """
2
+ Few-shot object classification using jina-clip-v2 via ONNX Runtime.
3
+ Bypasses all PyTorch custom code / dtype issues on HF Spaces (T4).
4
  Combines IMAGE embeddings from reference photos + TEXT embeddings
5
  from class names. Dual threshold: confidence + gap between top-1 and top-2.
6
  Usage:
 
23
  import argparse
24
  import csv
25
  import json
26
+ import os
27
  import time
28
  from pathlib import Path
29
 
30
  import numpy as np
31
+ import onnxruntime as ort
 
32
  from PIL import Image, ImageDraw, ImageFont
33
+ from huggingface_hub import hf_hub_download
34
+ from transformers import AutoImageProcessor, AutoTokenizer
35
 
36
  IMAGE_EXTS = {".jpg", ".jpeg", ".png", ".bmp", ".webp", ".tiff"}
37
  TRUNCATE_DIM = 1024
38
 
39
+ # ONNX model outputs: [text_unnorm, image_unnorm, text_norm, image_norm]
40
+ _TEXT_NORM_IDX = 2
41
+ _IMAGE_NORM_IDX = 3
 
 
 
42
 
43
 
44
  def draw_label_on_image(img: Image.Image, label: str, confidence: float) -> Image.Image:
 
112
 
113
 
114
  def parse_args():
115
+ p = argparse.ArgumentParser(description="Jina-CLIP-v2 few-shot classifier (ONNX)")
116
  p.add_argument("--refs", required=True, help="Reference images folder")
117
  p.add_argument("--input", required=True, help="Query crop images folder")
118
  p.add_argument("--output", default="jinaclip_results", help="Output folder")
 
131
 
132
  class JinaCLIPv2Encoder:
133
  """
134
+ ONNX Runtime based encoder for jina-clip-v2.
135
+ No PyTorch dependency for inference — avoids all dtype/NaN issues.
 
136
  """
137
 
138
  def __init__(self, device="cuda"):
139
  self.device = device
140
+ print("[*] Loading jina-clip-v2 (ONNX)...")
141
  t0 = time.perf_counter()
142
 
143
+ # Download ONNX model from HF Hub
144
+ onnx_path = hf_hub_download(
145
+ repo_id="jinaai/jina-clip-v2",
146
+ filename="onnx/model.onnx",
147
+ )
148
+ print(f" ONNX model: {onnx_path}")
149
+
150
+ # Pick providers: prefer CUDA if available, fallback to CPU
151
+ available = ort.get_available_providers()
152
+ if "CUDAExecutionProvider" in available and device == "cuda":
153
+ providers = ["CUDAExecutionProvider", "CPUExecutionProvider"]
154
+ else:
155
+ providers = ["CPUExecutionProvider"]
156
+ print(f" ONNX providers: {providers}")
157
+
158
+ self.session = ort.InferenceSession(onnx_path, providers=providers)
159
+
160
+ # Load tokenizer and image processor from HF
161
+ self.tokenizer = AutoTokenizer.from_pretrained(
162
+ "jinaai/jina-clip-v2", trust_remote_code=True
163
+ )
164
+ self.image_processor = AutoImageProcessor.from_pretrained(
 
 
165
  "jinaai/jina-clip-v2", trust_remote_code=True
166
  )
167
 
168
+ # Figure out input/output names
169
+ self.input_names = [inp.name for inp in self.session.get_inputs()]
170
+ self.output_names = [out.name for out in self.session.get_outputs()]
171
+ print(f" ONNX inputs: {self.input_names}")
172
+ print(f" ONNX outputs: {self.output_names}")
173
+
174
+ # Create a dummy token for image-only inference
175
+ self._dummy_ids = self.tokenizer([""], return_tensors="np", padding=True)["input_ids"]
176
+
177
+ # Sanity check
178
  _dummy = Image.new("RGB", (224, 224), color=(255, 0, 0))
179
+ _test = self.encode_images([_dummy], dim=64)
180
  _norm = float(np.linalg.norm(_test))
181
  _is_nan = bool(np.isnan(_norm))
182
  print(f" [SANITY] dummy image embed norm={_norm:.4f}, nan={_is_nan}")
183
  if _is_nan or _norm < 0.01:
184
+ print(" [ERROR] ONNX vision encoder broken!")
185
  else:
186
+ print(" [OK] ONNX vision encoder producing valid embeddings")
187
 
188
+ _test_t = self.encode_texts(["a red square"], dim=64)
 
189
  _tn = float(np.linalg.norm(_test_t))
190
  print(f" [SANITY] dummy text embed norm={_tn:.4f}")
191
 
192
+ print(f"[*] Loaded in {time.perf_counter() - t0:.1f}s (ONNX, providers={providers})\n")
193
+
194
+ def _run_image(self, pixel_values: np.ndarray) -> np.ndarray:
195
+ """Run ONNX session for images. Returns normalized image embeddings."""
196
+ # ONNX model expects both input_ids and pixel_values
197
+ # Pass dummy text tokens when encoding images only
198
+ dummy_ids = np.zeros((pixel_values.shape[0], 1), dtype=np.int64)
199
+ feeds = {}
200
+ for name in self.input_names:
201
+ if "pixel" in name.lower():
202
+ feeds[name] = pixel_values.astype(np.float32)
203
+ elif "input_id" in name.lower():
204
+ feeds[name] = dummy_ids
205
+ elif "attention" in name.lower():
206
+ feeds[name] = np.ones_like(dummy_ids, dtype=np.int64)
207
+
208
+ outputs = self.session.run(self.output_names, feeds)
209
+ # Output index 3 = normalized image embeddings (or 1 = unnormalized)
210
+ # Use the last output as image embeddings (normalized)
211
+ return outputs[_IMAGE_NORM_IDX]
212
+
213
+ def _run_text(self, input_ids: np.ndarray, attention_mask: np.ndarray) -> np.ndarray:
214
+ """Run ONNX session for text. Returns normalized text embeddings."""
215
+ # Create dummy pixel values (1x3x224x224) — minimal overhead
216
+ dummy_pv = np.zeros((input_ids.shape[0], 3, 224, 224), dtype=np.float32)
217
+ feeds = {}
218
+ for name in self.input_names:
219
+ if "pixel" in name.lower():
220
+ feeds[name] = dummy_pv
221
+ elif "input_id" in name.lower():
222
+ feeds[name] = input_ids
223
+ elif "attention" in name.lower():
224
+ feeds[name] = attention_mask
225
+
226
+ outputs = self.session.run(self.output_names, feeds)
227
+ return outputs[_TEXT_NORM_IDX]
228
 
229
  def encode_images(self, images: list[Image.Image], dim: int = TRUNCATE_DIM) -> np.ndarray:
230
+ rgb = [img.convert("RGB") for img in images]
231
+ processed = self.image_processor(rgb, return_tensors="np")
232
+ pixel_values = processed["pixel_values"].astype(np.float32)
233
+
234
+ embs = self._run_image(pixel_values)
235
+ # Truncate (matryoshka)
236
+ if dim and dim < embs.shape[-1]:
237
+ embs = embs[:, :dim]
238
  embs = np.nan_to_num(embs, nan=0.0, posinf=0.0, neginf=0.0)
239
  norms = np.linalg.norm(embs, axis=-1, keepdims=True)
240
  norms = np.maximum(norms, 1e-12)
241
  return (embs / norms).astype(np.float32)
242
 
243
  def encode_texts(self, texts: list[str], dim: int = TRUNCATE_DIM) -> np.ndarray:
244
+ tokens = self.tokenizer(
245
+ texts, return_tensors="np", padding=True, truncation=True, max_length=512
246
+ )
247
+ input_ids = tokens["input_ids"].astype(np.int64)
248
+ attention_mask = tokens["attention_mask"].astype(np.int64)
249
+
250
+ embs = self._run_text(input_ids, attention_mask)
251
+ if dim and dim < embs.shape[-1]:
252
+ embs = embs[:, :dim]
253
  embs = np.nan_to_num(embs, nan=0.0, posinf=0.0, neginf=0.0)
254
  norms = np.linalg.norm(embs, axis=-1, keepdims=True)
255
  norms = np.maximum(norms, 1e-12)
 
271
  raise ValueError(f"No subfolders in {refs_dir}")
272
 
273
  labels, embeddings = [], []
274
+ print(f" Text weight: {text_weight:.1f} | Image weight: {1 - text_weight:.1f}\n")
 
275
 
276
  for d in class_dirs:
277
  name = d.name
 
279
  if not paths:
280
  continue
281
 
 
282
  img_embs = encoder.encode_image_paths(paths, dim, batch_size)
283
  img_avg = np.nan_to_num(img_embs.mean(axis=0), nan=0.0, posinf=0.0, neginf=0.0)
284
 
 
285
  prompts = CLASS_PROMPTS.get(name, [f"a {name}", f"a person holding a {name}"])
286
  text_embs = encoder.encode_texts(prompts, dim)
287
  text_avg = np.nan_to_num(text_embs.mean(axis=0), nan=0.0, posinf=0.0, neginf=0.0)
288
 
 
289
  combined = (1.0 - text_weight) * img_avg + text_weight * text_avg
290
  combined = np.nan_to_num(combined, nan=0.0, posinf=0.0, neginf=0.0)
291
  combined = combined / (np.linalg.norm(combined) + 1e-12)
 
354
 
355
  encoder = JinaCLIPv2Encoder("cuda")
356
 
 
357
  print("[*] Building references...")
358
  ref_labels, ref_embs = build_refs(
359
  encoder, Path(args.refs), args.dim, args.text_weight, args.batch_size
360
  )
361
  print(f"\n[*] {len(ref_labels)} classes: {ref_labels}\n")
362
 
 
363
  if args.save_refs:
364
  np.save(output_dir / "ref_embeddings.npy", ref_embs)
365
  with open(output_dir / "ref_labels.json", "w") as jf:
366
  json.dump(ref_labels, jf)
367
  print(f"[*] Saved refs to {output_dir}\n")
368
 
 
369
  csv_path = output_dir / "classifications.csv"
370
  f = open(csv_path, "w", newline="")
371
  w = csv.writer(f)
 
373
  "second_best", "second_conf", "status"] +
374
  [f"sim_{l}" for l in ref_labels] + ["time_ms"])
375
 
 
376
  times = []
377
  counts = {"unknown": 0}
378
  for l in ref_labels:
379
  counts[l] = 0
380
  accepted, rejected = 0, 0
381
 
 
382
  hdr = " ".join(f"{l:>10}" for l in ref_labels)
383
  print(f"{'Image':<30} {'Result':<10} {'Conf':>6} {'Gap':>6} {hdr} {'Status'}")
384
  print("=" * (30 + 10 + 14 + len(hdr) + 40))
385
 
 
386
  for p in paths:
387
  t0 = time.perf_counter()
388
  img = Image.open(p)
 
408
  f"{sim_str} {result['status']}")
409
 
410
  w.writerow([
411
+ p.name, result["prediction"], result["raw_prediction"],
412
+ f"{result['confidence']:.4f}", f"{result['gap']:.4f}",
413
+ result["second_best"], f"{result['second_conf']:.4f}",
 
 
 
 
414
  result["status"],
415
  ] + [f"{result['all_sims'][l]:.4f}" for l in ref_labels] +
416
  [f"{ms:.1f}"])
 
422
  print(f"\n{'='*70}")
423
  print("SUMMARY")
424
  print(f"{'='*70}")
425
+ print(f" Model : jina-clip-v2 (ONNX Runtime)")
426
  print(f" Embed dim : {args.dim}")
427
  print(f" Text weight : {args.text_weight}")
428
  print(f" Conf threshold : {args.conf_threshold}")