Upload folder using huggingface_hub

.gitattributes

@@ -33,3 +33,7 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+examples/car_fruit_stand.png filter=lfs diff=lfs merge=lfs -text
+examples/pikachu_mario.png filter=lfs diff=lfs merge=lfs -text
+examples/sign_bianca_buda.png filter=lfs diff=lfs merge=lfs -text
+examples/throne_dystopia.png filter=lfs diff=lfs merge=lfs -text

README.md

@@ -1,13 +1,35 @@
 ---
 title: Structured Defect Grounding
-emoji: 🐨
-colorFrom: blue
-colorTo: blue
+emoji: 🔍
+colorFrom: red
+colorTo: yellow
 sdk: gradio
 sdk_version: 6.19.0
-python_version: '3.12'
 app_file: app.py
+short_description: Ground localized defects in text-to-image generations
+python_version: "3.12"
+startup_duration_timeout: 1h
 pinned: false
 ---
 
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+# Structured Defect Grounding (SDG)
+
+Interactive demo of the **SDG Detector** ([`P1n3/sdg-detector-grpo`](https://huggingface.co/P1n3/sdg-detector-grpo)),
+a Qwen3-VL-4B vision-language model trained with SFT + GRPO for the paper
+[*Where, What, Why, and Importance: Structured Defect Grounding for
+Text-to-Image Feedback*](https://huggingface.co/papers/2606.06113).
+
+Given a generated image (and optionally the caption it was generated from), the
+detector predicts a **structured set of localized defects**, each grounded as a
+tuple of *(where, what, why, importance)*:
+
+- **where** — a bounding box localizing the problematic region,
+- **what** — the defect type: 🔴 *artifact* (visual defect) or 🔵 *misalignment*
+  (image doesn't match the caption),
+- **why** — a natural-language explanation of the issue,
+- **importance** — a 1–100 severity score.
+
+The demo overlays the predicted boxes on the image and renders a structured
+feedback table plus the detector's raw reasoning.
+
+Model license: `cc-by-nc-4.0` (research / non-commercial use only).

app.py

@@ -0,0 +1,311 @@
+import os
+os.environ.setdefault("PYTORCH_CUDA_ALLOC_CONF", "expandable_segments:True")
+
+import spaces  # noqa: E402  must precede torch / transformers
+
+import json
+import re
+import html
+import tempfile
+
+import torch
+import gradio as gr
+from PIL import Image, ImageDraw, ImageFont
+from transformers import AutoProcessor, AutoModelForImageTextToText
+
+MODEL_ID = "P1n3/sdg-detector-grpo"
+
+# ---------------------------------------------------------------------------
+# Model (loaded once at module scope, moved to CUDA eagerly for ZeroGPU)
+# ---------------------------------------------------------------------------
+processor = AutoProcessor.from_pretrained(MODEL_ID)
+model = AutoModelForImageTextToText.from_pretrained(
+    MODEL_ID,
+    dtype=torch.bfloat16,
+    attn_implementation="sdpa",
+).to("cuda")
+model.eval()
+
+# ---------------------------------------------------------------------------
+# Prompt (the SDG detector's structured-grounding question template)
+# ---------------------------------------------------------------------------
+QUESTION_TEMPLATE = """You are an AI image quality evaluator. You will be given **one image** to analyze.
+
+### Definitions
+
+**Misalignment**: Areas where the image content does NOT match the text caption, including:
+- Missing objects: Objects mentioned in caption but not present in image
+- Extra objects: Objects present in image but not mentioned in caption
+- Wrong attributes: Incorrect color, size, material, count, or other properties
+- Wrong spatial relationships: Incorrect positions, orientations, or arrangements
+
+**Artifact**: Visual defects in the generated image, including:
+- Distorted anatomy: Malformed hands, extra/missing limbs, wrong number of fingers
+- Duplicated/missing parts: Repeated or absent body parts, objects
+- Warped geometry: Perspective errors, impossible shapes
+- Texture issues: Melted, smeared, or overly smooth textures
+- Unnatural edges: Jagged, broken, or blurry boundaries
+- Garbled text: Unreadable or malformed text/letters
+- Lighting inconsistencies: Wrong shadows, reflections, or light sources
+
+Text Caption: {caption}
+
+**Goal**: Produce a detailed analysis of the image quality and output bounding boxes for all detected issues.
+
+### Strict Output Rules
+Output **TWO blocks in this exact order**:
+1) `<think>` - Your detailed analysis
+2) `<answer>` - JSON list of bounding boxes
+
+### Answer Format (for <answer>)
+Return a JSON list:
+[
+    {{"box_2d": [x0, y0, x1, y1], "label": "misalignment"|"artifact", "description": "brief description of the issue", "importance": 1-100}}
+]
+
+Bounding box coordinates are in normalized 0-1000 space: [x0, y0, x1, y1].
+The "importance" is an integer from 1 (minor) to 100 (severe) rating how much the defect hurts image quality.
+If there are no issues, output an empty list.
+
+Now analyze the image and produce your output:
+"""
+
+ARTIFACT_COLOR = (239, 68, 68)       # red
+MISALIGNMENT_COLOR = (37, 99, 235)   # blue
+
+
+def _load_font(size):
+    for path in (
+        "/usr/share/fonts/truetype/dejavu/DejaVuSans-Bold.ttf",
+        "/usr/share/fonts/truetype/dejavu/DejaVuSans.ttf",
+    ):
+        try:
+            return ImageFont.truetype(path, size)
+        except Exception:
+            continue
+    return ImageFont.load_default()
+
+
+def parse_answer(response: str):
+    """Extract the structured defect list from the model's <answer> block."""
+    m = re.search(r"<answer>\s*(.*?)\s*</answer>", response, re.DOTALL)
+    payload = m.group(1) if m else response
+    start, end = payload.find("["), payload.rfind("]")
+    if start == -1 or end == -1 or end <= start:
+        return []
+    try:
+        data = json.loads(payload[start:end + 1])
+    except Exception:
+        return []
+    if not isinstance(data, list):
+        return []
+
+    defects = []
+    for item in data:
+        if not isinstance(item, dict):
+            continue
+        box = item.get("box_2d")
+        if not (isinstance(box, list) and len(box) == 4):
+            continue
+        try:
+            box = [float(v) for v in box]
+        except Exception:
+            continue
+        label = str(item.get("label", "")).lower()
+        if label not in ("artifact", "misalignment"):
+            label = "artifact"
+        desc = item.get("description") or item.get("desc") or ""
+        imp = item.get("importance")
+        try:
+            imp = int(imp)
+        except Exception:
+            imp = None
+        defects.append({
+            "box_2d": box,
+            "label": label,
+            "description": str(desc),
+            "importance": imp,
+        })
+    return defects
+
+
+def extract_think(response: str) -> str:
+    m = re.search(r"<think>\s*(.*?)\s*</think>", response, re.DOTALL)
+    return m.group(1).strip() if m else ""
+
+
+def draw_defects(image: Image.Image, defects):
+    """Overlay defect bounding boxes (coords normalized 0-1000, Qwen order)."""
+    image = image.convert("RGB").copy()
+    w, h = image.size
+    draw = ImageDraw.Draw(image)
+    line_w = max(3, round(min(w, h) / 250))
+    font = _load_font(max(16, round(min(w, h) / 45)))
+
+    for idx, d in enumerate(defects, start=1):
+        x0, y0, x1, y1 = d["box_2d"]
+        px0, py0 = x0 / 1000 * w, y0 / 1000 * h
+        px1, py1 = x1 / 1000 * w, y1 / 1000 * h
+        if px1 < px0:
+            px0, px1 = px1, px0
+        if py1 < py0:
+            py0, py1 = py1, py0
+        color = ARTIFACT_COLOR if d["label"] == "artifact" else MISALIGNMENT_COLOR
+        draw.rectangle([px0, py0, px1, py1], outline=color, width=line_w)
+
+        imp = d["importance"]
+        tag = f"{idx}" + (f" \u00b7 {imp}" if imp is not None else "")
+        bbox = draw.textbbox((0, 0), tag, font=font)
+        tw, th = bbox[2] - bbox[0], bbox[3] - bbox[1]
+        ly = max(0, py0 - th - 4)
+        draw.rectangle([px0, ly, px0 + tw + 8, ly + th + 4], fill=color)
+        draw.text((px0 + 4, ly + 2), tag, fill=(255, 255, 255), font=font)
+
+    return image
+
+
+def defects_to_markdown(defects):
+    if not defects:
+        return ("### ✅ No defects detected\n\n"
+                "The SDG detector did not ground any localized artifacts or "
+                "caption misalignments in this image.")
+    n_art = sum(1 for d in defects if d["label"] == "artifact")
+    n_mis = sum(1 for d in defects if d["label"] == "misalignment")
+    lines = [
+        f"### Found {len(defects)} defect(s) — "
+        f"🔴 {n_art} artifact, 🔵 {n_mis} misalignment\n",
+        "| # | Type | Importance | Where / What / Why |",
+        "|---|------|-----------|--------------------|",
+    ]
+    for idx, d in enumerate(defects, start=1):
+        emoji = "🔴" if d["label"] == "artifact" else "🔵"
+        imp = d["importance"] if d["importance"] is not None else "—"
+        box = ", ".join(str(int(v)) for v in d["box_2d"])
+        desc = html.escape(d["description"]).replace("\n", " ").replace("|", "\\|")
+        lines.append(
+            f"| {idx} | {emoji} {d['label']} | {imp} | "
+            f"{desc} <br><sub>box (0–1000): [{box}]</sub> |"
+        )
+    return "\n".join(lines)
+
+
+@spaces.GPU(duration=90)
+def detect(image, caption, max_new_tokens=1024):
+    """Detect and ground structured defects in a text-to-image generation.
+
+    Args:
+        image: the generated image to inspect for defects.
+        caption: the text prompt the image was generated from (used to spot
+            caption/image misalignments). Optional.
+        max_new_tokens: generation budget for the detector's reasoning + answer.
+
+    Returns:
+        The image annotated with defect boxes, a structured defect table, and
+        the detector's raw reasoning.
+    """
+    if image is None:
+        raise gr.Error("Please provide an image to analyze.")
+    caption = (caption or "").strip()
+
+    messages = [
+        {
+            "role": "user",
+            "content": [
+                {"type": "image", "image": image},
+                {"type": "text", "text": QUESTION_TEMPLATE.format(caption=caption)},
+            ],
+        }
+    ]
+    inputs = processor.apply_chat_template(
+        messages,
+        tokenize=True,
+        add_generation_prompt=True,
+        return_dict=True,
+        return_tensors="pt",
+    ).to(model.device)
+
+    with torch.inference_mode():
+        generated = model.generate(
+            **inputs,
+            max_new_tokens=int(max_new_tokens),
+            do_sample=False,
+        )
+    trimmed = generated[:, inputs["input_ids"].shape[1]:]
+    response = processor.batch_decode(
+        trimmed, skip_special_tokens=True, clean_up_tokenization_spaces=False
+    )[0]
+
+    defects = parse_answer(response)
+    annotated = draw_defects(image, defects)
+    table = defects_to_markdown(defects)
+    think = extract_think(response)
+    reasoning = think if think else response.strip()
+    return annotated, table, reasoning
+
+
+CSS = """
+#col-container { max-width: 1200px; margin: 0 auto; }
+.dark .gradio-container { color: var(--body-text-color); }
+"""
+
+INTRO = """# 🔍 Structured Defect Grounding (SDG)
+
+Detect **localized defects** in text-to-image generations and get instance-level
+feedback — *where* the defect is, *what* type it is (🔴 artifact / 🔵 misalignment),
+*why* it's wrong, and *how important* it is (1–100).
+
+Powered by the **SDG Detector** ([`P1n3/sdg-detector-grpo`](https://huggingface.co/P1n3/sdg-detector-grpo)),
+a Qwen3-VL-4B model trained with SFT + GRPO from the paper
+[*Where, What, Why, and Importance: Structured Defect Grounding for Text-to-Image Feedback*](https://huggingface.co/papers/2606.06113).
+
+Upload a generated image and (optionally) the caption it was generated from,
+then press **Analyze**.
+"""
+
+with gr.Blocks(theme=gr.themes.Citrus(), css=CSS, title="Structured Defect Grounding") as demo:
+    with gr.Column(elem_id="col-container"):
+        gr.Markdown(INTRO)
+        with gr.Row():
+            with gr.Column(scale=1):
+                image_in = gr.Image(type="pil", label="Generated image", height=380)
+                caption_in = gr.Textbox(
+                    label="Caption / prompt (optional)",
+                    placeholder="The text prompt the image was generated from…",
+                    lines=2,
+                )
+                run = gr.Button("Analyze", variant="primary")
+                with gr.Accordion("Advanced settings", open=False):
+                    max_tokens = gr.Slider(
+                        256, 2048, value=1024, step=64,
+                        label="Max new tokens",
+                        info="Generation budget for the detector's reasoning + answer.",
+                    )
+            with gr.Column(scale=1):
+                image_out = gr.Image(type="pil", label="Detected defects", height=380)
+                table_out = gr.Markdown(label="Structured feedback")
+        with gr.Accordion("Detector reasoning (<think>)", open=False):
+            reasoning_out = gr.Textbox(label="Raw reasoning", lines=8, show_copy_button=True)
+
+        gr.Examples(
+            examples=[
+                ["examples/throne_dystopia.png", "a jung male sitting down on a throne in a dystopian world, digital art, epic"],
+                ["examples/sign_bianca_buda.png", "A sign that says Bianca Buda"],
+                ["examples/pikachu_mario.png", "Pikachu Ninja turtle Mewtwo super Mario"],
+                ["examples/car_fruit_stand.png", "a car driving through a fruit stand, movie action scene, fruits flying everywhere"],
+            ],
+            inputs=[image_in, caption_in],
+            outputs=[image_out, table_out, reasoning_out],
+            fn=detect,
+            cache_examples=True,
+            cache_mode="lazy",
+        )
+
+    run.click(
+        detect,
+        inputs=[image_in, caption_in, max_tokens],
+        outputs=[image_out, table_out, reasoning_out],
+        api_name="detect",
+    )
+
+if __name__ == "__main__":
+    demo.launch(mcp_server=True)

requirements.txt

@@ -0,0 +1,4 @@
+transformers>=5.10.0
+accelerate
+torchvision
+pillow