Spaces:
Running
Running
Update jina_fewshot.py
Browse files- jina_fewshot.py +62 -22
jina_fewshot.py
CHANGED
|
@@ -23,6 +23,7 @@ import argparse
|
|
| 23 |
import csv
|
| 24 |
import json
|
| 25 |
import time
|
|
|
|
| 26 |
from pathlib import Path
|
| 27 |
|
| 28 |
import numpy as np
|
|
@@ -34,6 +35,61 @@ IMAGE_EXTS = {".jpg", ".jpeg", ".png", ".bmp", ".webp", ".tiff"}
|
|
| 34 |
TRUNCATE_DIM = 1024
|
| 35 |
|
| 36 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 37 |
def _to_numpy(embs):
|
| 38 |
"""Convert to numpy; Jina may return tensor on some code paths."""
|
| 39 |
if hasattr(embs, "cpu"):
|
|
@@ -41,16 +97,6 @@ def _to_numpy(embs):
|
|
| 41 |
return np.asarray(embs, dtype=np.float64)
|
| 42 |
|
| 43 |
|
| 44 |
-
def _force_fp32_recursive(module):
|
| 45 |
-
"""Force every submodule and parameter to fp32, and monkey-patch .half()
|
| 46 |
-
so Jina's custom code can never cast back to fp16."""
|
| 47 |
-
module.float()
|
| 48 |
-
# Block any future .half() / .to(dtype=float16) calls
|
| 49 |
-
module.half = lambda *a, **kw: module
|
| 50 |
-
for child in module.children():
|
| 51 |
-
_force_fp32_recursive(child)
|
| 52 |
-
|
| 53 |
-
|
| 54 |
def draw_label_on_image(img: Image.Image, label: str, confidence: float) -> Image.Image:
|
| 55 |
"""Draw the label in a bar outside and on top of the image (full width). Returns new image."""
|
| 56 |
img = img.convert("RGB")
|
|
@@ -164,17 +210,11 @@ class JinaCLIPv2Encoder:
|
|
| 164 |
finally:
|
| 165 |
torch.linspace = _orig_linspace
|
| 166 |
|
| 167 |
-
self.model = self.model.to(device).eval()
|
| 168 |
-
|
| 169 |
-
# NUCLEAR FIX: Jina's custom code internally casts the EVA02 vision
|
| 170 |
-
# encoder to fp16 (via .half() or autocast). On T4 this overflows to
|
| 171 |
-
# NaN. Force every submodule to fp32 AND monkey-patch .half() so it
|
| 172 |
-
# can never be re-cast.
|
| 173 |
-
_force_fp32_recursive(self.model)
|
| 174 |
|
| 175 |
-
#
|
| 176 |
_dummy = Image.new("RGB", (224, 224), color=(255, 0, 0))
|
| 177 |
-
with torch.no_grad(),
|
| 178 |
_test = self.model.encode_image([_dummy], truncate_dim=64)
|
| 179 |
if hasattr(_test, "cpu"):
|
| 180 |
_test = _test.cpu().float()
|
|
@@ -182,7 +222,7 @@ class JinaCLIPv2Encoder:
|
|
| 182 |
_is_nan = bool(np.isnan(_norm))
|
| 183 |
print(f" [SANITY] dummy image embed norm={_norm:.4f}, nan={_is_nan}")
|
| 184 |
if _is_nan or _norm < 0.01:
|
| 185 |
-
print(" [ERROR] Image encoder still broken
|
| 186 |
else:
|
| 187 |
print(" [OK] Image encoder producing valid embeddings")
|
| 188 |
|
|
@@ -190,7 +230,7 @@ class JinaCLIPv2Encoder:
|
|
| 190 |
|
| 191 |
def encode_images(self, images: list[Image.Image], dim: int = TRUNCATE_DIM) -> np.ndarray:
|
| 192 |
rgb = [img.convert("RGB") for img in images]
|
| 193 |
-
with torch.no_grad(),
|
| 194 |
embs = self.model.encode_image(rgb, truncate_dim=dim)
|
| 195 |
embs = _to_numpy(embs)
|
| 196 |
embs = np.nan_to_num(embs, nan=0.0, posinf=0.0, neginf=0.0)
|
|
@@ -199,7 +239,7 @@ class JinaCLIPv2Encoder:
|
|
| 199 |
return (embs / norms).astype(np.float32)
|
| 200 |
|
| 201 |
def encode_texts(self, texts: list[str], dim: int = TRUNCATE_DIM) -> np.ndarray:
|
| 202 |
-
with torch.no_grad(),
|
| 203 |
embs = self.model.encode_text(texts, truncate_dim=dim)
|
| 204 |
embs = _to_numpy(embs)
|
| 205 |
embs = np.nan_to_num(embs, nan=0.0, posinf=0.0, neginf=0.0)
|
|
|
|
| 23 |
import csv
|
| 24 |
import json
|
| 25 |
import time
|
| 26 |
+
from contextlib import contextmanager
|
| 27 |
from pathlib import Path
|
| 28 |
|
| 29 |
import numpy as np
|
|
|
|
| 35 |
TRUNCATE_DIM = 1024
|
| 36 |
|
| 37 |
|
| 38 |
+
# ---------------------------------------------------------------------------
|
| 39 |
+
# TENSOR-LEVEL fp16 blocker: Jina's custom encode_image does .half() or
|
| 40 |
+
# .to(dtype=float16) on TENSORS inside the forward pass. Module-level
|
| 41 |
+
# patches cannot catch this. We monkey-patch torch.Tensor.half and
|
| 42 |
+
# torch.Tensor.to while encode runs, blocking any fp16 conversion.
|
| 43 |
+
# ---------------------------------------------------------------------------
|
| 44 |
+
_BLOCK_FP16 = False
|
| 45 |
+
|
| 46 |
+
_orig_tensor_half = torch.Tensor.half
|
| 47 |
+
_orig_tensor_to = torch.Tensor.to
|
| 48 |
+
|
| 49 |
+
|
| 50 |
+
def _patched_half(self, *args, **kwargs):
|
| 51 |
+
if _BLOCK_FP16:
|
| 52 |
+
return self # no-op: stay in current dtype
|
| 53 |
+
return _orig_tensor_half(self, *args, **kwargs)
|
| 54 |
+
|
| 55 |
+
|
| 56 |
+
def _patched_to(self, *args, **kwargs):
|
| 57 |
+
if _BLOCK_FP16:
|
| 58 |
+
# Intercept .to(torch.float16) and .to(dtype=torch.float16)
|
| 59 |
+
new_args = list(args)
|
| 60 |
+
changed = False
|
| 61 |
+
for i, a in enumerate(new_args):
|
| 62 |
+
if a is torch.float16 or a is torch.bfloat16:
|
| 63 |
+
new_args[i] = torch.float32
|
| 64 |
+
changed = True
|
| 65 |
+
if "dtype" in kwargs and kwargs["dtype"] in (torch.float16, torch.bfloat16):
|
| 66 |
+
kwargs["dtype"] = torch.float32
|
| 67 |
+
changed = True
|
| 68 |
+
if changed:
|
| 69 |
+
return _orig_tensor_to(self, *new_args, **kwargs)
|
| 70 |
+
return _orig_tensor_to(self, *args, **kwargs)
|
| 71 |
+
|
| 72 |
+
|
| 73 |
+
# Install patches once at import time
|
| 74 |
+
torch.Tensor.half = _patched_half
|
| 75 |
+
torch.Tensor.to = _patched_to
|
| 76 |
+
|
| 77 |
+
|
| 78 |
+
@contextmanager
|
| 79 |
+
def block_fp16():
|
| 80 |
+
"""Context manager: inside this block, all tensor .half() and .to(float16) become fp32."""
|
| 81 |
+
global _BLOCK_FP16
|
| 82 |
+
prev = _BLOCK_FP16
|
| 83 |
+
_BLOCK_FP16 = True
|
| 84 |
+
try:
|
| 85 |
+
yield
|
| 86 |
+
finally:
|
| 87 |
+
_BLOCK_FP16 = prev
|
| 88 |
+
|
| 89 |
+
|
| 90 |
+
# ---------------------------------------------------------------------------
|
| 91 |
+
|
| 92 |
+
|
| 93 |
def _to_numpy(embs):
|
| 94 |
"""Convert to numpy; Jina may return tensor on some code paths."""
|
| 95 |
if hasattr(embs, "cpu"):
|
|
|
|
| 97 |
return np.asarray(embs, dtype=np.float64)
|
| 98 |
|
| 99 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 100 |
def draw_label_on_image(img: Image.Image, label: str, confidence: float) -> Image.Image:
|
| 101 |
"""Draw the label in a bar outside and on top of the image (full width). Returns new image."""
|
| 102 |
img = img.convert("RGB")
|
|
|
|
| 210 |
finally:
|
| 211 |
torch.linspace = _orig_linspace
|
| 212 |
|
| 213 |
+
self.model = self.model.to(device).eval().float()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 214 |
|
| 215 |
+
# Sanity check with fp16 blocking
|
| 216 |
_dummy = Image.new("RGB", (224, 224), color=(255, 0, 0))
|
| 217 |
+
with torch.no_grad(), block_fp16():
|
| 218 |
_test = self.model.encode_image([_dummy], truncate_dim=64)
|
| 219 |
if hasattr(_test, "cpu"):
|
| 220 |
_test = _test.cpu().float()
|
|
|
|
| 222 |
_is_nan = bool(np.isnan(_norm))
|
| 223 |
print(f" [SANITY] dummy image embed norm={_norm:.4f}, nan={_is_nan}")
|
| 224 |
if _is_nan or _norm < 0.01:
|
| 225 |
+
print(" [ERROR] Image encoder still broken!")
|
| 226 |
else:
|
| 227 |
print(" [OK] Image encoder producing valid embeddings")
|
| 228 |
|
|
|
|
| 230 |
|
| 231 |
def encode_images(self, images: list[Image.Image], dim: int = TRUNCATE_DIM) -> np.ndarray:
|
| 232 |
rgb = [img.convert("RGB") for img in images]
|
| 233 |
+
with torch.no_grad(), block_fp16():
|
| 234 |
embs = self.model.encode_image(rgb, truncate_dim=dim)
|
| 235 |
embs = _to_numpy(embs)
|
| 236 |
embs = np.nan_to_num(embs, nan=0.0, posinf=0.0, neginf=0.0)
|
|
|
|
| 239 |
return (embs / norms).astype(np.float32)
|
| 240 |
|
| 241 |
def encode_texts(self, texts: list[str], dim: int = TRUNCATE_DIM) -> np.ndarray:
|
| 242 |
+
with torch.no_grad(), block_fp16():
|
| 243 |
embs = self.model.encode_text(texts, truncate_dim=dim)
|
| 244 |
embs = _to_numpy(embs)
|
| 245 |
embs = np.nan_to_num(embs, nan=0.0, posinf=0.0, neginf=0.0)
|