Napron commited on
Commit
2c8f9da
·
verified ·
1 Parent(s): 0dd2530

Update jina_fewshot.py

Browse files
Files changed (1) hide show
  1. jina_fewshot.py +84 -16
jina_fewshot.py CHANGED
@@ -1,18 +1,15 @@
1
  """
2
  Few-shot object classification using jina-clip-v2 (EVA02-L, 304M).
3
-
4
  Combines IMAGE embeddings from reference photos + TEXT embeddings
5
  from class names. Dual threshold: confidence + gap between top-1 and top-2.
6
-
7
  Usage:
8
- python jina_clipv2_fewshot.py \
9
  --refs refs/ \
10
  --input crops/ \
11
  --output results/ \
12
  --text-weight 0.3 \
13
  --conf-threshold 0.75 \
14
  --gap-threshold 0.05
15
-
16
  refs/ folder structure (3-10 images per class recommended):
17
  refs/
18
  ├── cigarette/
@@ -64,7 +61,6 @@ def draw_label_on_image(img: Image.Image, label: str, confidence: float) -> Imag
64
  ddraw = ImageDraw.Draw(dummy)
65
  bbox = ddraw.textbbox((0, 0), text, font=font)
66
  tw, th = bbox[2] - bbox[0], bbox[3] - bbox[1]
67
- # Shrink font until text fits within image width (only when using truetype)
68
  if font_size is not None:
69
  while tw > max_text_w and font_size > 8:
70
  font_size = max(8, font_size - 2)
@@ -73,7 +69,6 @@ def draw_label_on_image(img: Image.Image, label: str, confidence: float) -> Imag
73
  tw, th = bbox[2] - bbox[0], bbox[3] - bbox[1]
74
  bar_height = th + 2 * margin
75
 
76
- # Canvas: bar on top (full width) + image below
77
  out = Image.new("RGB", (w, bar_height + h), color=(255, 255, 255))
78
  draw = ImageDraw.Draw(out)
79
  draw.rectangle([0, 0, w, bar_height], fill=(0, 0, 0))
@@ -83,6 +78,7 @@ def draw_label_on_image(img: Image.Image, label: str, confidence: float) -> Imag
83
  out.paste(img, (0, bar_height))
84
  return out
85
 
 
86
  CLASS_PROMPTS = {
87
  "knife": [
88
  "a knife",
@@ -138,10 +134,7 @@ class JinaCLIPv2Encoder:
138
  self.device = device
139
  print("[*] Loading jina-clip-v2...")
140
  t0 = time.perf_counter()
141
- # On HF Spaces, accelerate is pre-installed and transformers uses its meta-device
142
- # context during init, so set_default_device("cpu") is overridden. Jina's
143
- # eva_model.py:606 does torch.linspace(0, drop_path_rate, depth).item() and
144
- # crashes on meta tensors. Monkey-patch linspace to force device="cpu" for init.
145
  _orig_linspace = torch.linspace
146
 
147
  def _safe_linspace(*args, **kwargs):
@@ -155,8 +148,8 @@ class JinaCLIPv2Encoder:
155
  trust_remote_code=True,
156
  low_cpu_mem_usage=False,
157
  revision="main",
158
- torch_dtype=torch.float32, # ← force fp32, prevents NaN from bad dtype
159
- device_map=None, # ← prevent accelerate meta device init
160
  )
161
  finally:
162
  torch.linspace = _orig_linspace
@@ -165,10 +158,76 @@ class JinaCLIPv2Encoder:
165
  self.model = self.model.float()
166
  print(f"[*] Loaded in {time.perf_counter() - t0:.1f}s (device={device})\n")
167
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
168
  def encode_images(self, images: list[Image.Image], dim: int = TRUNCATE_DIM) -> np.ndarray:
169
  rgb = [img.convert("RGB") for img in images]
170
  with torch.no_grad():
171
  embs = self.model.encode_image(rgb, truncate_dim=dim)
 
 
 
 
 
 
 
 
 
 
 
172
  embs = _to_numpy(embs)
173
  embs = np.nan_to_num(embs, nan=0.0, posinf=0.0, neginf=0.0)
174
  norms = np.linalg.norm(embs, axis=-1, keepdims=True)
@@ -178,6 +237,17 @@ class JinaCLIPv2Encoder:
178
  def encode_texts(self, texts: list[str], dim: int = TRUNCATE_DIM) -> np.ndarray:
179
  with torch.no_grad():
180
  embs = self.model.encode_text(texts, truncate_dim=dim)
 
 
 
 
 
 
 
 
 
 
 
181
  embs = _to_numpy(embs)
182
  embs = np.nan_to_num(embs, nan=0.0, posinf=0.0, neginf=0.0)
183
  norms = np.linalg.norm(embs, axis=-1, keepdims=True)
@@ -203,7 +273,8 @@ def build_refs(encoder: JinaCLIPv2Encoder, refs_dir: Path,
203
  _device = getattr(encoder, "device", "?")
204
  print(f" Device: {_device} | Text weight: {text_weight:.1f} | Image weight: {1 - text_weight:.1f}\n")
205
  if _device == "cpu":
206
- print(" [WARNING] Jina is on CPU. Ref embeddings are often all zeros on CPU. Use a Space with GPU (e.g. T4) for D-FINE + Classify.\n")
 
207
 
208
  for d in class_dirs:
209
  name = d.name
@@ -251,7 +322,6 @@ def classify(query_emb: np.ndarray, ref_labels: list[str], ref_embs: np.ndarray,
251
  conf = float(sims[best_idx])
252
  gap = float(sims[best_idx] - sims[second_idx])
253
 
254
- # Dual threshold
255
  conf_ok = conf >= conf_threshold
256
  gap_ok = gap >= gap_threshold
257
 
@@ -343,7 +413,6 @@ def main():
343
  else:
344
  rejected += 1
345
 
346
- # Draw label on image and save to output folder
347
  annotated = draw_label_on_image(img, result["prediction"], result["confidence"])
348
  out_path = output_dir / p.name
349
  annotated.save(out_path)
@@ -367,7 +436,6 @@ def main():
367
 
368
  f.close()
369
 
370
- # Summary
371
  n = len(times)
372
  total = sum(times)
373
  print(f"\n{'='*70}")
 
1
  """
2
  Few-shot object classification using jina-clip-v2 (EVA02-L, 304M).
 
3
  Combines IMAGE embeddings from reference photos + TEXT embeddings
4
  from class names. Dual threshold: confidence + gap between top-1 and top-2.
 
5
  Usage:
6
+ python jina_fewshot.py \
7
  --refs refs/ \
8
  --input crops/ \
9
  --output results/ \
10
  --text-weight 0.3 \
11
  --conf-threshold 0.75 \
12
  --gap-threshold 0.05
 
13
  refs/ folder structure (3-10 images per class recommended):
14
  refs/
15
  ├── cigarette/
 
61
  ddraw = ImageDraw.Draw(dummy)
62
  bbox = ddraw.textbbox((0, 0), text, font=font)
63
  tw, th = bbox[2] - bbox[0], bbox[3] - bbox[1]
 
64
  if font_size is not None:
65
  while tw > max_text_w and font_size > 8:
66
  font_size = max(8, font_size - 2)
 
69
  tw, th = bbox[2] - bbox[0], bbox[3] - bbox[1]
70
  bar_height = th + 2 * margin
71
 
 
72
  out = Image.new("RGB", (w, bar_height + h), color=(255, 255, 255))
73
  draw = ImageDraw.Draw(out)
74
  draw.rectangle([0, 0, w, bar_height], fill=(0, 0, 0))
 
78
  out.paste(img, (0, bar_height))
79
  return out
80
 
81
+
82
  CLASS_PROMPTS = {
83
  "knife": [
84
  "a knife",
 
134
  self.device = device
135
  print("[*] Loading jina-clip-v2...")
136
  t0 = time.perf_counter()
137
+
 
 
 
138
  _orig_linspace = torch.linspace
139
 
140
  def _safe_linspace(*args, **kwargs):
 
148
  trust_remote_code=True,
149
  low_cpu_mem_usage=False,
150
  revision="main",
151
+ torch_dtype=torch.float32,
152
+ device_map=None,
153
  )
154
  finally:
155
  torch.linspace = _orig_linspace
 
158
  self.model = self.model.float()
159
  print(f"[*] Loaded in {time.perf_counter() - t0:.1f}s (device={device})\n")
160
 
161
+ # ===== DEBUG: check model state right after loading =====
162
+ print(f" [DEBUG] torch.version.cuda = {torch.version.cuda}")
163
+ print(f" [DEBUG] torch.cuda.available = {torch.cuda.is_available()}")
164
+ if torch.cuda.is_available():
165
+ print(f" [DEBUG] GPU = {torch.cuda.get_device_name()}")
166
+ # Check a few parameter norms to verify weights loaded
167
+ param_norms = {}
168
+ for name, p in self.model.named_parameters():
169
+ param_norms[name] = p.data.float().norm().item()
170
+ if len(param_norms) >= 5:
171
+ break
172
+ print(f" [DEBUG] first 5 param norms: {param_norms}")
173
+ all_zero_params = sum(1 for _, p in self.model.named_parameters() if p.data.float().norm().item() == 0.0)
174
+ total_params = sum(1 for _ in self.model.named_parameters())
175
+ print(f" [DEBUG] zero-norm params: {all_zero_params}/{total_params}")
176
+
177
+ # Dummy encode to check output
178
+ print(" [DEBUG] encoding dummy 224x224 red image...")
179
+ _dummy = Image.new("RGB", (224, 224), color=(255, 0, 0))
180
+ try:
181
+ _raw = self.model.encode_image([_dummy], truncate_dim=64)
182
+ if hasattr(_raw, 'cpu'):
183
+ _raw_np = _raw.cpu().float()
184
+ else:
185
+ _raw_np = torch.tensor(_raw).float()
186
+ print(f" [DEBUG] dummy image result: shape={list(_raw_np.shape)}, "
187
+ f"norm={_raw_np.norm().item():.6f}, "
188
+ f"has_nan={_raw_np.isnan().any().item()}, "
189
+ f"has_inf={_raw_np.isinf().any().item()}, "
190
+ f"all_zero={(_raw_np == 0).all().item()}, "
191
+ f"min={_raw_np.min().item():.6f}, max={_raw_np.max().item():.6f}")
192
+ except Exception as e:
193
+ print(f" [DEBUG] dummy encode FAILED: {e}")
194
+ import traceback
195
+ traceback.print_exc()
196
+
197
+ print(" [DEBUG] encoding dummy text 'a red square'...")
198
+ try:
199
+ _raw_t = self.model.encode_text(["a red square"], truncate_dim=64)
200
+ if hasattr(_raw_t, 'cpu'):
201
+ _raw_t_np = _raw_t.cpu().float()
202
+ else:
203
+ _raw_t_np = torch.tensor(_raw_t).float()
204
+ print(f" [DEBUG] dummy text result: shape={list(_raw_t_np.shape)}, "
205
+ f"norm={_raw_t_np.norm().item():.6f}, "
206
+ f"has_nan={_raw_t_np.isnan().any().item()}, "
207
+ f"has_inf={_raw_t_np.isinf().any().item()}, "
208
+ f"all_zero={(_raw_t_np == 0).all().item()}, "
209
+ f"min={_raw_t_np.min().item():.6f}, max={_raw_t_np.max().item():.6f}")
210
+ except Exception as e:
211
+ print(f" [DEBUG] dummy text encode FAILED: {e}")
212
+ import traceback
213
+ traceback.print_exc()
214
+ # ===== END DEBUG =====
215
+
216
  def encode_images(self, images: list[Image.Image], dim: int = TRUNCATE_DIM) -> np.ndarray:
217
  rgb = [img.convert("RGB") for img in images]
218
  with torch.no_grad():
219
  embs = self.model.encode_image(rgb, truncate_dim=dim)
220
+
221
+ # ===== DEBUG =====
222
+ if hasattr(embs, 'cpu'):
223
+ _t = embs.cpu().float()
224
+ print(f" [DEBUG encode_images] raw: shape={list(_t.shape)}, "
225
+ f"norm={_t.norm(dim=-1).tolist()}, "
226
+ f"has_nan={_t.isnan().any().item()}, "
227
+ f"all_zero={(_t == 0).all().item()}, "
228
+ f"min={_t.min().item():.6f}, max={_t.max().item():.6f}")
229
+ # ===== END DEBUG =====
230
+
231
  embs = _to_numpy(embs)
232
  embs = np.nan_to_num(embs, nan=0.0, posinf=0.0, neginf=0.0)
233
  norms = np.linalg.norm(embs, axis=-1, keepdims=True)
 
237
  def encode_texts(self, texts: list[str], dim: int = TRUNCATE_DIM) -> np.ndarray:
238
  with torch.no_grad():
239
  embs = self.model.encode_text(texts, truncate_dim=dim)
240
+
241
+ # ===== DEBUG =====
242
+ if hasattr(embs, 'cpu'):
243
+ _t = embs.cpu().float()
244
+ print(f" [DEBUG encode_texts] raw: shape={list(_t.shape)}, "
245
+ f"norm={_t.norm(dim=-1).tolist()}, "
246
+ f"has_nan={_t.isnan().any().item()}, "
247
+ f"all_zero={(_t == 0).all().item()}, "
248
+ f"min={_t.min().item():.6f}, max={_t.max().item():.6f}")
249
+ # ===== END DEBUG =====
250
+
251
  embs = _to_numpy(embs)
252
  embs = np.nan_to_num(embs, nan=0.0, posinf=0.0, neginf=0.0)
253
  norms = np.linalg.norm(embs, axis=-1, keepdims=True)
 
273
  _device = getattr(encoder, "device", "?")
274
  print(f" Device: {_device} | Text weight: {text_weight:.1f} | Image weight: {1 - text_weight:.1f}\n")
275
  if _device == "cpu":
276
+ print(" [WARNING] Jina is on CPU. Ref embeddings are often all zeros on CPU. "
277
+ "Use a Space with GPU (e.g. T4) for D-FINE + Classify.\n")
278
 
279
  for d in class_dirs:
280
  name = d.name
 
322
  conf = float(sims[best_idx])
323
  gap = float(sims[best_idx] - sims[second_idx])
324
 
 
325
  conf_ok = conf >= conf_threshold
326
  gap_ok = gap >= gap_threshold
327
 
 
413
  else:
414
  rejected += 1
415
 
 
416
  annotated = draw_label_on_image(img, result["prediction"], result["confidence"])
417
  out_path = output_dir / p.name
418
  annotated.save(out_path)
 
436
 
437
  f.close()
438
 
 
439
  n = len(times)
440
  total = sum(times)
441
  print(f"\n{'='*70}")