T1_VLM / classify.py
Arnav Tripathi
Upload 15 files
367e4ef verified
Raw History Blame Contribute Delete
5.57 kB
# classify.py
from typing import List, Tuple, Dict, Any
from PIL import Image
import numpy as np
import colorsys
import json
from utils.prompt import build_few_shot_messages
from model.qwen_vl import QwenVLWrapper
from config import EXAMPLE_IMAGES, LABELS
from utils.image import load_image, save_pil_temp
import warnings
# Load example images once helper
def load_example_data():
ex_data = []
for path, label in EXAMPLE_IMAGES:
img = load_image(str(path))
ex_data.append((img, label))
return ex_data
def _parse_text_output(text: str, labels=LABELS) -> Dict[str, Any]:
"""Parse model output and extract multiple labels with confidence scores."""
results = {
"labels": [],
"confidence": [],
"raw_output": text
}
if not text:
return results
# Pre-process text to avoid matching negations (e.g. "not Green")
# This is a simple heuristic: split by comma/newline, check if segment has "not "
# But since we updated examples to simple lists, risk is lower.
# Still, good to be safe. We'll check for positive presence.
text_lower = text.lower()
found_labels = set()
# Define keywords for each label to robustly find them
# Key: Label name from config input
# Value: List of unambiguous strings that indicate this label
keyword_map = {
"Clear/Transparent": ["clear", "transparent"],
"Light Blue": ["light blue", "blue"], # 'blue' might match 'dark blue', but for this task likely 'Light Blue'
"Green": ["green"],
"Brown/Amber": ["brown", "amber"],
"Mixed/Other colors": ["mixed", "other", "multicolor", "multi-color"]
}
# Helper to check if keyword is present as a whole word (or close to it)
# and not preceded by 'not '
def is_present(kw, txt):
idx = txt.find(kw)
if idx == -1: return False
# Check for negation "not " before it (within ~5 chars)
start_lookback = max(0, idx - 5)
preceding = txt[start_lookback:idx]
if "not" in preceding:
return False
return True
for lbl, keywords in keyword_map.items():
# Check if the exact label name is present
if is_present(lbl.lower(), text_lower):
found_labels.add(lbl)
continue
# Check keywords
for kw in keywords:
if is_present(kw, text_lower):
found_labels.add(lbl)
break
# If we found labels, assign confidence
if found_labels:
for lbl in found_labels:
results["labels"].append(lbl)
results["confidence"].append(0.8)
else:
# If model output something but we found nothing, maybe it's "Unknown" or just missed
pass
# We leave results empty, enabling fallback in caller if no labels found
return results
def dominant_color_bucket(pil_img: Image.Image, sample_pixels: int = 1500) -> str:
img = pil_img.convert("RGB").resize((160,160))
arr = np.array(img).reshape(-1,3)
if arr.shape[0] > sample_pixels:
idx = np.random.choice(arr.shape[0], sample_pixels, replace=False)
arr = arr[idx]
hs, ss, vs = [], [], []
for (r,g,b) in arr:
r_n, g_n, b_n = r/255.0, g/255.0, b/255.0
h,s,v = colorsys.rgb_to_hsv(r_n,g_n,b_n)
hs.append(h*360.0); ss.append(s); vs.append(v)
h_med = float(np.median(hs)); s_med = float(np.median(ss)); v_med = float(np.median(vs))
if s_med < 0.12 and v_med > 0.7:
return "Clear/Transparent"
if 180 <= h_med <= 260:
return "Light Blue"
if 70 <= h_med <= 160:
return "Green"
if (10 <= h_med <= 55) and (s_med > 0.15) and (v_med < 0.9):
return "Brown/Amber"
return "Mixed/Other colors"
class FewShotClassifier:
def __init__(self, model_name="vikhyatk/moondream2", use_hf_space=False):
if use_hf_space:
# Use remote Hugging Face Space
from model.hf_space_wrapper import HFSpaceWrapper
self.model = HFSpaceWrapper(space_url=model_name)
else:
# Use local model
use_quant = "qwen" in model_name.lower()
self.model = QwenVLWrapper(model_name=model_name, use_quantization=use_quant)
self.example_data = load_example_data()
def predict(self, query_image: Image.Image, use_hsv_fallback=True, max_new_tokens=30) -> Dict[str, Any]:
"""
Predict multiple suitable labels for an image.
Returns JSON with labels, confidence scores, and raw model output.
"""
messages = build_few_shot_messages(self.example_data, query_image, LABELS)
try:
raw = self.model.predict_from_messages(messages, max_new_tokens=max_new_tokens)
except Exception as e:
warnings.warn(f"Model call failed: {e}")
raw = ""
parsed = _parse_text_output(raw)
# If no labels found and HSV fallback enabled, add HSV prediction
if not parsed["labels"] and use_hsv_fallback:
try:
hsv_label = dominant_color_bucket(query_image)
parsed["labels"] = [hsv_label]
parsed["confidence"] = [0.7]
parsed["method"] = "hsv_fallback"
except Exception:
parsed["labels"] = ["Unknown"]
parsed["confidence"] = [0.5]
parsed["method"] = "fallback"
else:
parsed["method"] = "model"
return parsed