Napron commited on
Commit
cece4f2
·
verified ·
1 Parent(s): 2c8f9da

Update jina_fewshot.py

Browse files
Files changed (1) hide show
  1. jina_fewshot.py +6 -86
jina_fewshot.py CHANGED
@@ -153,81 +153,18 @@ class JinaCLIPv2Encoder:
153
  )
154
  finally:
155
  torch.linspace = _orig_linspace
156
- self.model = self.model.to(device).eval()
157
- if device == "cpu":
158
- self.model = self.model.float()
159
- print(f"[*] Loaded in {time.perf_counter() - t0:.1f}s (device={device})\n")
160
 
161
- # ===== DEBUG: check model state right after loading =====
162
- print(f" [DEBUG] torch.version.cuda = {torch.version.cuda}")
163
- print(f" [DEBUG] torch.cuda.available = {torch.cuda.is_available()}")
164
- if torch.cuda.is_available():
165
- print(f" [DEBUG] GPU = {torch.cuda.get_device_name()}")
166
- # Check a few parameter norms to verify weights loaded
167
- param_norms = {}
168
- for name, p in self.model.named_parameters():
169
- param_norms[name] = p.data.float().norm().item()
170
- if len(param_norms) >= 5:
171
- break
172
- print(f" [DEBUG] first 5 param norms: {param_norms}")
173
- all_zero_params = sum(1 for _, p in self.model.named_parameters() if p.data.float().norm().item() == 0.0)
174
- total_params = sum(1 for _ in self.model.named_parameters())
175
- print(f" [DEBUG] zero-norm params: {all_zero_params}/{total_params}")
176
-
177
- # Dummy encode to check output
178
- print(" [DEBUG] encoding dummy 224x224 red image...")
179
- _dummy = Image.new("RGB", (224, 224), color=(255, 0, 0))
180
- try:
181
- _raw = self.model.encode_image([_dummy], truncate_dim=64)
182
- if hasattr(_raw, 'cpu'):
183
- _raw_np = _raw.cpu().float()
184
- else:
185
- _raw_np = torch.tensor(_raw).float()
186
- print(f" [DEBUG] dummy image result: shape={list(_raw_np.shape)}, "
187
- f"norm={_raw_np.norm().item():.6f}, "
188
- f"has_nan={_raw_np.isnan().any().item()}, "
189
- f"has_inf={_raw_np.isinf().any().item()}, "
190
- f"all_zero={(_raw_np == 0).all().item()}, "
191
- f"min={_raw_np.min().item():.6f}, max={_raw_np.max().item():.6f}")
192
- except Exception as e:
193
- print(f" [DEBUG] dummy encode FAILED: {e}")
194
- import traceback
195
- traceback.print_exc()
196
-
197
- print(" [DEBUG] encoding dummy text 'a red square'...")
198
- try:
199
- _raw_t = self.model.encode_text(["a red square"], truncate_dim=64)
200
- if hasattr(_raw_t, 'cpu'):
201
- _raw_t_np = _raw_t.cpu().float()
202
- else:
203
- _raw_t_np = torch.tensor(_raw_t).float()
204
- print(f" [DEBUG] dummy text result: shape={list(_raw_t_np.shape)}, "
205
- f"norm={_raw_t_np.norm().item():.6f}, "
206
- f"has_nan={_raw_t_np.isnan().any().item()}, "
207
- f"has_inf={_raw_t_np.isinf().any().item()}, "
208
- f"all_zero={(_raw_t_np == 0).all().item()}, "
209
- f"min={_raw_t_np.min().item():.6f}, max={_raw_t_np.max().item():.6f}")
210
- except Exception as e:
211
- print(f" [DEBUG] dummy text encode FAILED: {e}")
212
- import traceback
213
- traceback.print_exc()
214
- # ===== END DEBUG =====
215
 
216
  def encode_images(self, images: list[Image.Image], dim: int = TRUNCATE_DIM) -> np.ndarray:
217
  rgb = [img.convert("RGB") for img in images]
218
  with torch.no_grad():
219
  embs = self.model.encode_image(rgb, truncate_dim=dim)
220
-
221
- # ===== DEBUG =====
222
- if hasattr(embs, 'cpu'):
223
- _t = embs.cpu().float()
224
- print(f" [DEBUG encode_images] raw: shape={list(_t.shape)}, "
225
- f"norm={_t.norm(dim=-1).tolist()}, "
226
- f"has_nan={_t.isnan().any().item()}, "
227
- f"all_zero={(_t == 0).all().item()}, "
228
- f"min={_t.min().item():.6f}, max={_t.max().item():.6f}")
229
- # ===== END DEBUG =====
230
-
231
  embs = _to_numpy(embs)
232
  embs = np.nan_to_num(embs, nan=0.0, posinf=0.0, neginf=0.0)
233
  norms = np.linalg.norm(embs, axis=-1, keepdims=True)
@@ -237,17 +174,6 @@ class JinaCLIPv2Encoder:
237
  def encode_texts(self, texts: list[str], dim: int = TRUNCATE_DIM) -> np.ndarray:
238
  with torch.no_grad():
239
  embs = self.model.encode_text(texts, truncate_dim=dim)
240
-
241
- # ===== DEBUG =====
242
- if hasattr(embs, 'cpu'):
243
- _t = embs.cpu().float()
244
- print(f" [DEBUG encode_texts] raw: shape={list(_t.shape)}, "
245
- f"norm={_t.norm(dim=-1).tolist()}, "
246
- f"has_nan={_t.isnan().any().item()}, "
247
- f"all_zero={(_t == 0).all().item()}, "
248
- f"min={_t.min().item():.6f}, max={_t.max().item():.6f}")
249
- # ===== END DEBUG =====
250
-
251
  embs = _to_numpy(embs)
252
  embs = np.nan_to_num(embs, nan=0.0, posinf=0.0, neginf=0.0)
253
  norms = np.linalg.norm(embs, axis=-1, keepdims=True)
@@ -272,9 +198,6 @@ def build_refs(encoder: JinaCLIPv2Encoder, refs_dir: Path,
272
  labels, embeddings = [], []
273
  _device = getattr(encoder, "device", "?")
274
  print(f" Device: {_device} | Text weight: {text_weight:.1f} | Image weight: {1 - text_weight:.1f}\n")
275
- if _device == "cpu":
276
- print(" [WARNING] Jina is on CPU. Ref embeddings are often all zeros on CPU. "
277
- "Use a Space with GPU (e.g. T4) for D-FINE + Classify.\n")
278
 
279
  for d in class_dirs:
280
  name = d.name
@@ -304,9 +227,6 @@ def build_refs(encoder: JinaCLIPv2Encoder, refs_dir: Path,
304
  sim = float(np.nan_to_num(np.dot(img_norm, text_norm), nan=0.0))
305
  print(f" {name:<14}: {len(paths)} imgs + {len(prompts)} prompts | "
306
  f"img-text sim: {sim:.4f}")
307
- if labels and np.allclose(np.stack(embeddings), 0.0):
308
- print("\n [WARNING] All ref embeddings are zero. Jina-CLIP often returns zeros on CPU. "
309
- "Use a Space with GPU (e.g. T4) for D-FINE + Classify to work correctly.")
310
 
311
  return labels, np.stack(embeddings)
312
 
 
153
  )
154
  finally:
155
  torch.linspace = _orig_linspace
 
 
 
 
156
 
157
+ # FIX: force entire model to fp32 — Jina's custom code internally casts
158
+ # the EVA02 vision encoder to fp16, which overflows to NaN on T4.
159
+ # .float() after .to(device) overrides that back to fp32.
160
+ self.model = self.model.to(device).eval().float()
161
+
162
+ print(f"[*] Loaded in {time.perf_counter() - t0:.1f}s (device={device})\n")
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
163
 
164
  def encode_images(self, images: list[Image.Image], dim: int = TRUNCATE_DIM) -> np.ndarray:
165
  rgb = [img.convert("RGB") for img in images]
166
  with torch.no_grad():
167
  embs = self.model.encode_image(rgb, truncate_dim=dim)
 
 
 
 
 
 
 
 
 
 
 
168
  embs = _to_numpy(embs)
169
  embs = np.nan_to_num(embs, nan=0.0, posinf=0.0, neginf=0.0)
170
  norms = np.linalg.norm(embs, axis=-1, keepdims=True)
 
174
  def encode_texts(self, texts: list[str], dim: int = TRUNCATE_DIM) -> np.ndarray:
175
  with torch.no_grad():
176
  embs = self.model.encode_text(texts, truncate_dim=dim)
 
 
 
 
 
 
 
 
 
 
 
177
  embs = _to_numpy(embs)
178
  embs = np.nan_to_num(embs, nan=0.0, posinf=0.0, neginf=0.0)
179
  norms = np.linalg.norm(embs, axis=-1, keepdims=True)
 
198
  labels, embeddings = [], []
199
  _device = getattr(encoder, "device", "?")
200
  print(f" Device: {_device} | Text weight: {text_weight:.1f} | Image weight: {1 - text_weight:.1f}\n")
 
 
 
201
 
202
  for d in class_dirs:
203
  name = d.name
 
227
  sim = float(np.nan_to_num(np.dot(img_norm, text_norm), nan=0.0))
228
  print(f" {name:<14}: {len(paths)} imgs + {len(prompts)} prompts | "
229
  f"img-text sim: {sim:.4f}")
 
 
 
230
 
231
  return labels, np.stack(embeddings)
232