NeelakshSaxena commited on
Commit
cff04af
Β·
1 Parent(s): 80eac8a

Enforce image token in tiny mode

Browse files
Files changed (1) hide show
  1. app.py +13 -0
app.py CHANGED
@@ -309,6 +309,9 @@ def score_binary_logits(logits: torch.Tensor, tokenizer) -> tuple[float, str]:
309
 
310
 
311
  def build_prompt(processor_obj, user_text: str) -> str:
 
 
 
312
  return user_text
313
 
314
 
@@ -324,6 +327,16 @@ def infer_image(image: Image.Image):
324
  inputs = processor_obj(text=prompt_text, images=prepared_image, return_tensors="pt", padding=True)
325
  inputs = {name: tensor.to(device) for name, tensor in inputs.items()}
326
 
 
 
 
 
 
 
 
 
 
 
327
  # Keep pixel dtype stable across CPU/GPU to avoid backend kernel errors.
328
  if "pixel_values" in inputs:
329
  inputs["pixel_values"] = inputs["pixel_values"].to(
 
309
 
310
 
311
  def build_prompt(processor_obj, user_text: str) -> str:
312
+ if MODEL_MODE == "tiny":
313
+ # Tiny LLaVA forward expects an explicit image placeholder token.
314
+ return f"<image>\n{user_text}"
315
  return user_text
316
 
317
 
 
327
  inputs = processor_obj(text=prompt_text, images=prepared_image, return_tensors="pt", padding=True)
328
  inputs = {name: tensor.to(device) for name, tensor in inputs.items()}
329
 
330
+ if MODEL_MODE == "tiny":
331
+ image_token_id = getattr(model_obj.config, "image_token_index", None)
332
+ if image_token_id is not None and "input_ids" in inputs:
333
+ image_token_count = int((inputs["input_ids"] == image_token_id).sum().item())
334
+ if image_token_count == 0:
335
+ # Defensive recovery if tokenizer template/path strips the image placeholder.
336
+ fallback_prompt = f"<image>\n{TINY_PROMPT}"
337
+ inputs = processor_obj(text=fallback_prompt, images=prepared_image, return_tensors="pt", padding=True)
338
+ inputs = {name: tensor.to(device) for name, tensor in inputs.items()}
339
+
340
  # Keep pixel dtype stable across CPU/GPU to avoid backend kernel errors.
341
  if "pixel_values" in inputs:
342
  inputs["pixel_values"] = inputs["pixel_values"].to(