Napron commited on
Commit
70eaed8
·
verified ·
1 Parent(s): d4af411

Update jina_fewshot.py

Browse files
Files changed (1) hide show
  1. jina_fewshot.py +62 -22
jina_fewshot.py CHANGED
@@ -23,6 +23,7 @@ import argparse
23
  import csv
24
  import json
25
  import time
 
26
  from pathlib import Path
27
 
28
  import numpy as np
@@ -34,6 +35,61 @@ IMAGE_EXTS = {".jpg", ".jpeg", ".png", ".bmp", ".webp", ".tiff"}
34
  TRUNCATE_DIM = 1024
35
 
36
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
37
  def _to_numpy(embs):
38
  """Convert to numpy; Jina may return tensor on some code paths."""
39
  if hasattr(embs, "cpu"):
@@ -41,16 +97,6 @@ def _to_numpy(embs):
41
  return np.asarray(embs, dtype=np.float64)
42
 
43
 
44
- def _force_fp32_recursive(module):
45
- """Force every submodule and parameter to fp32, and monkey-patch .half()
46
- so Jina's custom code can never cast back to fp16."""
47
- module.float()
48
- # Block any future .half() / .to(dtype=float16) calls
49
- module.half = lambda *a, **kw: module
50
- for child in module.children():
51
- _force_fp32_recursive(child)
52
-
53
-
54
  def draw_label_on_image(img: Image.Image, label: str, confidence: float) -> Image.Image:
55
  """Draw the label in a bar outside and on top of the image (full width). Returns new image."""
56
  img = img.convert("RGB")
@@ -164,17 +210,11 @@ class JinaCLIPv2Encoder:
164
  finally:
165
  torch.linspace = _orig_linspace
166
 
167
- self.model = self.model.to(device).eval()
168
-
169
- # NUCLEAR FIX: Jina's custom code internally casts the EVA02 vision
170
- # encoder to fp16 (via .half() or autocast). On T4 this overflows to
171
- # NaN. Force every submodule to fp32 AND monkey-patch .half() so it
172
- # can never be re-cast.
173
- _force_fp32_recursive(self.model)
174
 
175
- # Verify fix worked
176
  _dummy = Image.new("RGB", (224, 224), color=(255, 0, 0))
177
- with torch.no_grad(), torch.amp.autocast("cuda", enabled=False):
178
  _test = self.model.encode_image([_dummy], truncate_dim=64)
179
  if hasattr(_test, "cpu"):
180
  _test = _test.cpu().float()
@@ -182,7 +222,7 @@ class JinaCLIPv2Encoder:
182
  _is_nan = bool(np.isnan(_norm))
183
  print(f" [SANITY] dummy image embed norm={_norm:.4f}, nan={_is_nan}")
184
  if _is_nan or _norm < 0.01:
185
- print(" [ERROR] Image encoder still broken after fp32 fix!")
186
  else:
187
  print(" [OK] Image encoder producing valid embeddings")
188
 
@@ -190,7 +230,7 @@ class JinaCLIPv2Encoder:
190
 
191
  def encode_images(self, images: list[Image.Image], dim: int = TRUNCATE_DIM) -> np.ndarray:
192
  rgb = [img.convert("RGB") for img in images]
193
- with torch.no_grad(), torch.amp.autocast("cuda", enabled=False):
194
  embs = self.model.encode_image(rgb, truncate_dim=dim)
195
  embs = _to_numpy(embs)
196
  embs = np.nan_to_num(embs, nan=0.0, posinf=0.0, neginf=0.0)
@@ -199,7 +239,7 @@ class JinaCLIPv2Encoder:
199
  return (embs / norms).astype(np.float32)
200
 
201
  def encode_texts(self, texts: list[str], dim: int = TRUNCATE_DIM) -> np.ndarray:
202
- with torch.no_grad(), torch.amp.autocast("cuda", enabled=False):
203
  embs = self.model.encode_text(texts, truncate_dim=dim)
204
  embs = _to_numpy(embs)
205
  embs = np.nan_to_num(embs, nan=0.0, posinf=0.0, neginf=0.0)
 
23
  import csv
24
  import json
25
  import time
26
+ from contextlib import contextmanager
27
  from pathlib import Path
28
 
29
  import numpy as np
 
35
  TRUNCATE_DIM = 1024
36
 
37
 
38
+ # ---------------------------------------------------------------------------
39
+ # TENSOR-LEVEL fp16 blocker: Jina's custom encode_image does .half() or
40
+ # .to(dtype=float16) on TENSORS inside the forward pass. Module-level
41
+ # patches cannot catch this. We monkey-patch torch.Tensor.half and
42
+ # torch.Tensor.to while encode runs, blocking any fp16 conversion.
43
+ # ---------------------------------------------------------------------------
44
+ _BLOCK_FP16 = False
45
+
46
+ _orig_tensor_half = torch.Tensor.half
47
+ _orig_tensor_to = torch.Tensor.to
48
+
49
+
50
+ def _patched_half(self, *args, **kwargs):
51
+ if _BLOCK_FP16:
52
+ return self # no-op: stay in current dtype
53
+ return _orig_tensor_half(self, *args, **kwargs)
54
+
55
+
56
+ def _patched_to(self, *args, **kwargs):
57
+ if _BLOCK_FP16:
58
+ # Intercept .to(torch.float16) and .to(dtype=torch.float16)
59
+ new_args = list(args)
60
+ changed = False
61
+ for i, a in enumerate(new_args):
62
+ if a is torch.float16 or a is torch.bfloat16:
63
+ new_args[i] = torch.float32
64
+ changed = True
65
+ if "dtype" in kwargs and kwargs["dtype"] in (torch.float16, torch.bfloat16):
66
+ kwargs["dtype"] = torch.float32
67
+ changed = True
68
+ if changed:
69
+ return _orig_tensor_to(self, *new_args, **kwargs)
70
+ return _orig_tensor_to(self, *args, **kwargs)
71
+
72
+
73
+ # Install patches once at import time
74
+ torch.Tensor.half = _patched_half
75
+ torch.Tensor.to = _patched_to
76
+
77
+
78
+ @contextmanager
79
+ def block_fp16():
80
+ """Context manager: inside this block, all tensor .half() and .to(float16) become fp32."""
81
+ global _BLOCK_FP16
82
+ prev = _BLOCK_FP16
83
+ _BLOCK_FP16 = True
84
+ try:
85
+ yield
86
+ finally:
87
+ _BLOCK_FP16 = prev
88
+
89
+
90
+ # ---------------------------------------------------------------------------
91
+
92
+
93
  def _to_numpy(embs):
94
  """Convert to numpy; Jina may return tensor on some code paths."""
95
  if hasattr(embs, "cpu"):
 
97
  return np.asarray(embs, dtype=np.float64)
98
 
99
 
 
 
 
 
 
 
 
 
 
 
100
  def draw_label_on_image(img: Image.Image, label: str, confidence: float) -> Image.Image:
101
  """Draw the label in a bar outside and on top of the image (full width). Returns new image."""
102
  img = img.convert("RGB")
 
210
  finally:
211
  torch.linspace = _orig_linspace
212
 
213
+ self.model = self.model.to(device).eval().float()
 
 
 
 
 
 
214
 
215
+ # Sanity check with fp16 blocking
216
  _dummy = Image.new("RGB", (224, 224), color=(255, 0, 0))
217
+ with torch.no_grad(), block_fp16():
218
  _test = self.model.encode_image([_dummy], truncate_dim=64)
219
  if hasattr(_test, "cpu"):
220
  _test = _test.cpu().float()
 
222
  _is_nan = bool(np.isnan(_norm))
223
  print(f" [SANITY] dummy image embed norm={_norm:.4f}, nan={_is_nan}")
224
  if _is_nan or _norm < 0.01:
225
+ print(" [ERROR] Image encoder still broken!")
226
  else:
227
  print(" [OK] Image encoder producing valid embeddings")
228
 
 
230
 
231
  def encode_images(self, images: list[Image.Image], dim: int = TRUNCATE_DIM) -> np.ndarray:
232
  rgb = [img.convert("RGB") for img in images]
233
+ with torch.no_grad(), block_fp16():
234
  embs = self.model.encode_image(rgb, truncate_dim=dim)
235
  embs = _to_numpy(embs)
236
  embs = np.nan_to_num(embs, nan=0.0, posinf=0.0, neginf=0.0)
 
239
  return (embs / norms).astype(np.float32)
240
 
241
  def encode_texts(self, texts: list[str], dim: int = TRUNCATE_DIM) -> np.ndarray:
242
+ with torch.no_grad(), block_fp16():
243
  embs = self.model.encode_text(texts, truncate_dim=dim)
244
  embs = _to_numpy(embs)
245
  embs = np.nan_to_num(embs, nan=0.0, posinf=0.0, neginf=0.0)