Spaces:
Running
Running
Update jina_fewshot.py
Browse files- jina_fewshot.py +6 -86
jina_fewshot.py
CHANGED
|
@@ -153,81 +153,18 @@ class JinaCLIPv2Encoder:
|
|
| 153 |
)
|
| 154 |
finally:
|
| 155 |
torch.linspace = _orig_linspace
|
| 156 |
-
self.model = self.model.to(device).eval()
|
| 157 |
-
if device == "cpu":
|
| 158 |
-
self.model = self.model.float()
|
| 159 |
-
print(f"[*] Loaded in {time.perf_counter() - t0:.1f}s (device={device})\n")
|
| 160 |
|
| 161 |
-
#
|
| 162 |
-
|
| 163 |
-
|
| 164 |
-
|
| 165 |
-
|
| 166 |
-
|
| 167 |
-
param_norms = {}
|
| 168 |
-
for name, p in self.model.named_parameters():
|
| 169 |
-
param_norms[name] = p.data.float().norm().item()
|
| 170 |
-
if len(param_norms) >= 5:
|
| 171 |
-
break
|
| 172 |
-
print(f" [DEBUG] first 5 param norms: {param_norms}")
|
| 173 |
-
all_zero_params = sum(1 for _, p in self.model.named_parameters() if p.data.float().norm().item() == 0.0)
|
| 174 |
-
total_params = sum(1 for _ in self.model.named_parameters())
|
| 175 |
-
print(f" [DEBUG] zero-norm params: {all_zero_params}/{total_params}")
|
| 176 |
-
|
| 177 |
-
# Dummy encode to check output
|
| 178 |
-
print(" [DEBUG] encoding dummy 224x224 red image...")
|
| 179 |
-
_dummy = Image.new("RGB", (224, 224), color=(255, 0, 0))
|
| 180 |
-
try:
|
| 181 |
-
_raw = self.model.encode_image([_dummy], truncate_dim=64)
|
| 182 |
-
if hasattr(_raw, 'cpu'):
|
| 183 |
-
_raw_np = _raw.cpu().float()
|
| 184 |
-
else:
|
| 185 |
-
_raw_np = torch.tensor(_raw).float()
|
| 186 |
-
print(f" [DEBUG] dummy image result: shape={list(_raw_np.shape)}, "
|
| 187 |
-
f"norm={_raw_np.norm().item():.6f}, "
|
| 188 |
-
f"has_nan={_raw_np.isnan().any().item()}, "
|
| 189 |
-
f"has_inf={_raw_np.isinf().any().item()}, "
|
| 190 |
-
f"all_zero={(_raw_np == 0).all().item()}, "
|
| 191 |
-
f"min={_raw_np.min().item():.6f}, max={_raw_np.max().item():.6f}")
|
| 192 |
-
except Exception as e:
|
| 193 |
-
print(f" [DEBUG] dummy encode FAILED: {e}")
|
| 194 |
-
import traceback
|
| 195 |
-
traceback.print_exc()
|
| 196 |
-
|
| 197 |
-
print(" [DEBUG] encoding dummy text 'a red square'...")
|
| 198 |
-
try:
|
| 199 |
-
_raw_t = self.model.encode_text(["a red square"], truncate_dim=64)
|
| 200 |
-
if hasattr(_raw_t, 'cpu'):
|
| 201 |
-
_raw_t_np = _raw_t.cpu().float()
|
| 202 |
-
else:
|
| 203 |
-
_raw_t_np = torch.tensor(_raw_t).float()
|
| 204 |
-
print(f" [DEBUG] dummy text result: shape={list(_raw_t_np.shape)}, "
|
| 205 |
-
f"norm={_raw_t_np.norm().item():.6f}, "
|
| 206 |
-
f"has_nan={_raw_t_np.isnan().any().item()}, "
|
| 207 |
-
f"has_inf={_raw_t_np.isinf().any().item()}, "
|
| 208 |
-
f"all_zero={(_raw_t_np == 0).all().item()}, "
|
| 209 |
-
f"min={_raw_t_np.min().item():.6f}, max={_raw_t_np.max().item():.6f}")
|
| 210 |
-
except Exception as e:
|
| 211 |
-
print(f" [DEBUG] dummy text encode FAILED: {e}")
|
| 212 |
-
import traceback
|
| 213 |
-
traceback.print_exc()
|
| 214 |
-
# ===== END DEBUG =====
|
| 215 |
|
| 216 |
def encode_images(self, images: list[Image.Image], dim: int = TRUNCATE_DIM) -> np.ndarray:
|
| 217 |
rgb = [img.convert("RGB") for img in images]
|
| 218 |
with torch.no_grad():
|
| 219 |
embs = self.model.encode_image(rgb, truncate_dim=dim)
|
| 220 |
-
|
| 221 |
-
# ===== DEBUG =====
|
| 222 |
-
if hasattr(embs, 'cpu'):
|
| 223 |
-
_t = embs.cpu().float()
|
| 224 |
-
print(f" [DEBUG encode_images] raw: shape={list(_t.shape)}, "
|
| 225 |
-
f"norm={_t.norm(dim=-1).tolist()}, "
|
| 226 |
-
f"has_nan={_t.isnan().any().item()}, "
|
| 227 |
-
f"all_zero={(_t == 0).all().item()}, "
|
| 228 |
-
f"min={_t.min().item():.6f}, max={_t.max().item():.6f}")
|
| 229 |
-
# ===== END DEBUG =====
|
| 230 |
-
|
| 231 |
embs = _to_numpy(embs)
|
| 232 |
embs = np.nan_to_num(embs, nan=0.0, posinf=0.0, neginf=0.0)
|
| 233 |
norms = np.linalg.norm(embs, axis=-1, keepdims=True)
|
|
@@ -237,17 +174,6 @@ class JinaCLIPv2Encoder:
|
|
| 237 |
def encode_texts(self, texts: list[str], dim: int = TRUNCATE_DIM) -> np.ndarray:
|
| 238 |
with torch.no_grad():
|
| 239 |
embs = self.model.encode_text(texts, truncate_dim=dim)
|
| 240 |
-
|
| 241 |
-
# ===== DEBUG =====
|
| 242 |
-
if hasattr(embs, 'cpu'):
|
| 243 |
-
_t = embs.cpu().float()
|
| 244 |
-
print(f" [DEBUG encode_texts] raw: shape={list(_t.shape)}, "
|
| 245 |
-
f"norm={_t.norm(dim=-1).tolist()}, "
|
| 246 |
-
f"has_nan={_t.isnan().any().item()}, "
|
| 247 |
-
f"all_zero={(_t == 0).all().item()}, "
|
| 248 |
-
f"min={_t.min().item():.6f}, max={_t.max().item():.6f}")
|
| 249 |
-
# ===== END DEBUG =====
|
| 250 |
-
|
| 251 |
embs = _to_numpy(embs)
|
| 252 |
embs = np.nan_to_num(embs, nan=0.0, posinf=0.0, neginf=0.0)
|
| 253 |
norms = np.linalg.norm(embs, axis=-1, keepdims=True)
|
|
@@ -272,9 +198,6 @@ def build_refs(encoder: JinaCLIPv2Encoder, refs_dir: Path,
|
|
| 272 |
labels, embeddings = [], []
|
| 273 |
_device = getattr(encoder, "device", "?")
|
| 274 |
print(f" Device: {_device} | Text weight: {text_weight:.1f} | Image weight: {1 - text_weight:.1f}\n")
|
| 275 |
-
if _device == "cpu":
|
| 276 |
-
print(" [WARNING] Jina is on CPU. Ref embeddings are often all zeros on CPU. "
|
| 277 |
-
"Use a Space with GPU (e.g. T4) for D-FINE + Classify.\n")
|
| 278 |
|
| 279 |
for d in class_dirs:
|
| 280 |
name = d.name
|
|
@@ -304,9 +227,6 @@ def build_refs(encoder: JinaCLIPv2Encoder, refs_dir: Path,
|
|
| 304 |
sim = float(np.nan_to_num(np.dot(img_norm, text_norm), nan=0.0))
|
| 305 |
print(f" {name:<14}: {len(paths)} imgs + {len(prompts)} prompts | "
|
| 306 |
f"img-text sim: {sim:.4f}")
|
| 307 |
-
if labels and np.allclose(np.stack(embeddings), 0.0):
|
| 308 |
-
print("\n [WARNING] All ref embeddings are zero. Jina-CLIP often returns zeros on CPU. "
|
| 309 |
-
"Use a Space with GPU (e.g. T4) for D-FINE + Classify to work correctly.")
|
| 310 |
|
| 311 |
return labels, np.stack(embeddings)
|
| 312 |
|
|
|
|
| 153 |
)
|
| 154 |
finally:
|
| 155 |
torch.linspace = _orig_linspace
|
|
|
|
|
|
|
|
|
|
|
|
|
| 156 |
|
| 157 |
+
# FIX: force entire model to fp32 — Jina's custom code internally casts
|
| 158 |
+
# the EVA02 vision encoder to fp16, which overflows to NaN on T4.
|
| 159 |
+
# .float() after .to(device) overrides that back to fp32.
|
| 160 |
+
self.model = self.model.to(device).eval().float()
|
| 161 |
+
|
| 162 |
+
print(f"[*] Loaded in {time.perf_counter() - t0:.1f}s (device={device})\n")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 163 |
|
| 164 |
def encode_images(self, images: list[Image.Image], dim: int = TRUNCATE_DIM) -> np.ndarray:
|
| 165 |
rgb = [img.convert("RGB") for img in images]
|
| 166 |
with torch.no_grad():
|
| 167 |
embs = self.model.encode_image(rgb, truncate_dim=dim)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 168 |
embs = _to_numpy(embs)
|
| 169 |
embs = np.nan_to_num(embs, nan=0.0, posinf=0.0, neginf=0.0)
|
| 170 |
norms = np.linalg.norm(embs, axis=-1, keepdims=True)
|
|
|
|
| 174 |
def encode_texts(self, texts: list[str], dim: int = TRUNCATE_DIM) -> np.ndarray:
|
| 175 |
with torch.no_grad():
|
| 176 |
embs = self.model.encode_text(texts, truncate_dim=dim)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 177 |
embs = _to_numpy(embs)
|
| 178 |
embs = np.nan_to_num(embs, nan=0.0, posinf=0.0, neginf=0.0)
|
| 179 |
norms = np.linalg.norm(embs, axis=-1, keepdims=True)
|
|
|
|
| 198 |
labels, embeddings = [], []
|
| 199 |
_device = getattr(encoder, "device", "?")
|
| 200 |
print(f" Device: {_device} | Text weight: {text_weight:.1f} | Image weight: {1 - text_weight:.1f}\n")
|
|
|
|
|
|
|
|
|
|
| 201 |
|
| 202 |
for d in class_dirs:
|
| 203 |
name = d.name
|
|
|
|
| 227 |
sim = float(np.nan_to_num(np.dot(img_norm, text_norm), nan=0.0))
|
| 228 |
print(f" {name:<14}: {len(paths)} imgs + {len(prompts)} prompts | "
|
| 229 |
f"img-text sim: {sim:.4f}")
|
|
|
|
|
|
|
|
|
|
| 230 |
|
| 231 |
return labels, np.stack(embeddings)
|
| 232 |
|