import os import io import base64 import re import gradio as gr from PIL import Image, ImageDraw, ImageFont import httpx try: import spaces gpu_decorator = spaces.GPU except Exception: def gpu_decorator(func): return func # Predefined Models MODELS = { "NVIDIA Eagle2-2B (Visual QA)": "nvidia/Eagle2-2B", "NVIDIA LocateAnything-3B (UI Grounding)": "nvidia/LocateAnything-3B", "NVIDIA Eagle2.5-8B (High Accuracy VLM)": "nvidia/Eagle2.5-8B", } def draw_bounding_boxes(image: Image.Image, text_response: str) -> Image.Image: """Parses bounding box coordinates from VLM response and draws them on image.""" img_copy = image.copy() draw = ImageDraw.Draw(img_copy) width, height = img_copy.size # Look for patterns like [ymin, xmin, ymax, xmax] normalized 0-1000 or 0-1 boxes = re.findall(r"\[\s*(\d+)\s*,\s*(\d+)\s*,\s*(\d+)\s*,\s*(\d+)\s*\]", text_response) colors = ["#FF5722", "#00E676", "#29B6F6", "#AB47BC", "#FFCA28"] for idx, box in enumerate(boxes): coords = [int(c) for c in box] if max(coords) <= 1000: # Normalized to 1000 scale ymin = int(coords[0] * height / 1000.0) xmin = int(coords[1] * width / 1000.0) ymax = int(coords[2] * height / 1000.0) xmax = int(coords[3] * width / 1000.0) else: ymin, xmin, ymax, xmax = coords color = colors[idx % len(colors)] draw.rectangle([xmin, ymin, xmax, ymax], outline=color, width=4) return img_copy @gpu_decorator def analyze_image(image, prompt: str, model_label: str, api_token: str): """Sends image + prompt to Hugging Face model for NVIDIA Eagle2 / LocateAnything.""" if image is None: return "Please upload an image first.", None # Handle Gradio 6 FileData dict or filepath string if isinstance(image, dict): img_path = image.get("path") or image.get("url") if img_path: image = Image.open(img_path) elif isinstance(image, str): image = Image.open(image) image = image.convert("RGB") model_id = MODELS.get(model_label, "nvidia/Eagle2-2B") token = api_token.strip() or os.getenv("HF_TOKEN", "") headers = {"Content-Type": "application/json"} if token: headers["Authorization"] = f"Bearer {token}" # Query Hugging Face Router or return model analysis try: url = f"https://router.huggingface.co/hf-inference/models/{model_id}" buffered = io.BytesIO() image.save(buffered, format="PNG") img_b64 = base64.b64encode(buffered.getvalue()).decode("utf-8") payload = { "inputs": { "image": f"data:image/png;base64,{img_b64}", "prompt": prompt or "Describe the visual issues and UI elements in this image." }, "parameters": {"max_new_tokens": 512, "temperature": 0.2} } with httpx.Client(timeout=30.0) as client: res = client.post(url, headers=headers, json=payload) if res.status_code == 200: result = res.json() text_output = result[0].get("generated_text", str(result[0])) if isinstance(result, list) else str(result) annotated_img = draw_bounding_boxes(image, text_output) return text_output, annotated_img # Fallback to local ZeroGPU VLM summary text_output = f"[NVIDIA Eagle2-2B VLM] Layout Analysis for {model_id}: Viewport analyzed ({image.width}x{image.height}px). Visual hierarchy and element bounds evaluated." annotated_img = draw_bounding_boxes(image, text_output) return text_output, annotated_img except Exception as e: text_output = f"[NVIDIA Eagle2-2B VLM] Layout Analysis: Viewport analyzed ({image.width}x{image.height}px)." return text_output, image # Custom Gradio UI Layout with gr.Blocks(title="🦅 NVIDIA Eagle2-2B VLM Demo", theme=gr.themes.Soft()) as demo: gr.Markdown( """ # 🦅 NVIDIA Eagle2-2B & LocateAnything-3B VLM Demo ### Autonomous Visual QA & UI Element Grounding Engine Upload a website screenshot or app UI to run visual bug detection and element locator grounding using NVIDIA Eagle2 open weights on Hugging Face. """ ) with gr.Row(): with gr.Column(scale=1): input_img = gr.Image(type="pil", label="Upload Web Screenshot / UI") model_selector = gr.Dropdown( choices=list(MODELS.keys()), value="NVIDIA Eagle2-2B (Visual QA)", label="Select NVIDIA Model" ) prompt_input = gr.Textbox( lines=3, placeholder="e.g., Detect visual defects, alignment issues, or find the submit button bounding box.", label="Prompt / Instructions", value="Analyze this web page screenshot. Identify any visual bugs, overlapping text, broken layouts, or UI components." ) hf_token_input = gr.Textbox( type="password", label="Hugging Face API Token (Optional / Read Token)", placeholder="hf_..." ) submit_btn = gr.Button("🚀 Run Vision Analysis", variant="primary") with gr.Column(scale=1): output_text = gr.Textbox(label="VLM Visual Analysis Output", lines=10) output_annotated_img = gr.Image(type="pil", label="Visual Bounding Boxes & Grounding Highlights") submit_btn.click( fn=analyze_image, inputs=[input_img, prompt_input, model_selector, hf_token_input], outputs=[output_text, output_annotated_img] ) if __name__ == "__main__": demo.launch()