Spaces:
Runtime error
Runtime error
Commit Β·
cff04af
1
Parent(s): 80eac8a
Enforce image token in tiny mode
Browse files
app.py
CHANGED
|
@@ -309,6 +309,9 @@ def score_binary_logits(logits: torch.Tensor, tokenizer) -> tuple[float, str]:
|
|
| 309 |
|
| 310 |
|
| 311 |
def build_prompt(processor_obj, user_text: str) -> str:
|
|
|
|
|
|
|
|
|
|
| 312 |
return user_text
|
| 313 |
|
| 314 |
|
|
@@ -324,6 +327,16 @@ def infer_image(image: Image.Image):
|
|
| 324 |
inputs = processor_obj(text=prompt_text, images=prepared_image, return_tensors="pt", padding=True)
|
| 325 |
inputs = {name: tensor.to(device) for name, tensor in inputs.items()}
|
| 326 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 327 |
# Keep pixel dtype stable across CPU/GPU to avoid backend kernel errors.
|
| 328 |
if "pixel_values" in inputs:
|
| 329 |
inputs["pixel_values"] = inputs["pixel_values"].to(
|
|
|
|
| 309 |
|
| 310 |
|
| 311 |
def build_prompt(processor_obj, user_text: str) -> str:
|
| 312 |
+
if MODEL_MODE == "tiny":
|
| 313 |
+
# Tiny LLaVA forward expects an explicit image placeholder token.
|
| 314 |
+
return f"<image>\n{user_text}"
|
| 315 |
return user_text
|
| 316 |
|
| 317 |
|
|
|
|
| 327 |
inputs = processor_obj(text=prompt_text, images=prepared_image, return_tensors="pt", padding=True)
|
| 328 |
inputs = {name: tensor.to(device) for name, tensor in inputs.items()}
|
| 329 |
|
| 330 |
+
if MODEL_MODE == "tiny":
|
| 331 |
+
image_token_id = getattr(model_obj.config, "image_token_index", None)
|
| 332 |
+
if image_token_id is not None and "input_ids" in inputs:
|
| 333 |
+
image_token_count = int((inputs["input_ids"] == image_token_id).sum().item())
|
| 334 |
+
if image_token_count == 0:
|
| 335 |
+
# Defensive recovery if tokenizer template/path strips the image placeholder.
|
| 336 |
+
fallback_prompt = f"<image>\n{TINY_PROMPT}"
|
| 337 |
+
inputs = processor_obj(text=fallback_prompt, images=prepared_image, return_tensors="pt", padding=True)
|
| 338 |
+
inputs = {name: tensor.to(device) for name, tensor in inputs.items()}
|
| 339 |
+
|
| 340 |
# Keep pixel dtype stable across CPU/GPU to avoid backend kernel errors.
|
| 341 |
if "pixel_values" in inputs:
|
| 342 |
inputs["pixel_values"] = inputs["pixel_values"].to(
|