# classify.py from typing import List, Tuple, Dict, Any from PIL import Image import numpy as np import colorsys import json from utils.prompt import build_few_shot_messages from model.qwen_vl import QwenVLWrapper from config import EXAMPLE_IMAGES, LABELS from utils.image import load_image, save_pil_temp import warnings # Load example images once helper def load_example_data(): ex_data = [] for path, label in EXAMPLE_IMAGES: img = load_image(str(path)) ex_data.append((img, label)) return ex_data def _parse_text_output(text: str, labels=LABELS) -> Dict[str, Any]: """Parse model output and extract multiple labels with confidence scores.""" results = { "labels": [], "confidence": [], "raw_output": text } if not text: return results # Pre-process text to avoid matching negations (e.g. "not Green") # This is a simple heuristic: split by comma/newline, check if segment has "not " # But since we updated examples to simple lists, risk is lower. # Still, good to be safe. We'll check for positive presence. text_lower = text.lower() found_labels = set() # Define keywords for each label to robustly find them # Key: Label name from config input # Value: List of unambiguous strings that indicate this label keyword_map = { "Clear/Transparent": ["clear", "transparent"], "Light Blue": ["light blue", "blue"], # 'blue' might match 'dark blue', but for this task likely 'Light Blue' "Green": ["green"], "Brown/Amber": ["brown", "amber"], "Mixed/Other colors": ["mixed", "other", "multicolor", "multi-color"] } # Helper to check if keyword is present as a whole word (or close to it) # and not preceded by 'not ' def is_present(kw, txt): idx = txt.find(kw) if idx == -1: return False # Check for negation "not " before it (within ~5 chars) start_lookback = max(0, idx - 5) preceding = txt[start_lookback:idx] if "not" in preceding: return False return True for lbl, keywords in keyword_map.items(): # Check if the exact label name is present if is_present(lbl.lower(), text_lower): found_labels.add(lbl) continue # Check keywords for kw in keywords: if is_present(kw, text_lower): found_labels.add(lbl) break # If we found labels, assign confidence if found_labels: for lbl in found_labels: results["labels"].append(lbl) results["confidence"].append(0.8) else: # If model output something but we found nothing, maybe it's "Unknown" or just missed pass # We leave results empty, enabling fallback in caller if no labels found return results def dominant_color_bucket(pil_img: Image.Image, sample_pixels: int = 1500) -> str: img = pil_img.convert("RGB").resize((160,160)) arr = np.array(img).reshape(-1,3) if arr.shape[0] > sample_pixels: idx = np.random.choice(arr.shape[0], sample_pixels, replace=False) arr = arr[idx] hs, ss, vs = [], [], [] for (r,g,b) in arr: r_n, g_n, b_n = r/255.0, g/255.0, b/255.0 h,s,v = colorsys.rgb_to_hsv(r_n,g_n,b_n) hs.append(h*360.0); ss.append(s); vs.append(v) h_med = float(np.median(hs)); s_med = float(np.median(ss)); v_med = float(np.median(vs)) if s_med < 0.12 and v_med > 0.7: return "Clear/Transparent" if 180 <= h_med <= 260: return "Light Blue" if 70 <= h_med <= 160: return "Green" if (10 <= h_med <= 55) and (s_med > 0.15) and (v_med < 0.9): return "Brown/Amber" return "Mixed/Other colors" class FewShotClassifier: def __init__(self, model_name="vikhyatk/moondream2", use_hf_space=False): if use_hf_space: # Use remote Hugging Face Space from model.hf_space_wrapper import HFSpaceWrapper self.model = HFSpaceWrapper(space_url=model_name) else: # Use local model use_quant = "qwen" in model_name.lower() self.model = QwenVLWrapper(model_name=model_name, use_quantization=use_quant) self.example_data = load_example_data() def predict(self, query_image: Image.Image, use_hsv_fallback=True, max_new_tokens=30) -> Dict[str, Any]: """ Predict multiple suitable labels for an image. Returns JSON with labels, confidence scores, and raw model output. """ messages = build_few_shot_messages(self.example_data, query_image, LABELS) try: raw = self.model.predict_from_messages(messages, max_new_tokens=max_new_tokens) except Exception as e: warnings.warn(f"Model call failed: {e}") raw = "" parsed = _parse_text_output(raw) # If no labels found and HSV fallback enabled, add HSV prediction if not parsed["labels"] and use_hsv_fallback: try: hsv_label = dominant_color_bucket(query_image) parsed["labels"] = [hsv_label] parsed["confidence"] = [0.7] parsed["method"] = "hsv_fallback" except Exception: parsed["labels"] = ["Unknown"] parsed["confidence"] = [0.5] parsed["method"] = "fallback" else: parsed["method"] = "model" return parsed