Spaces:
Running on Zero
Running on Zero
| """EVAN-8B: visually-grounded multimodal reasoning demo. | |
| EVAN (EVidence-Ablated Negatives) trains a VLM with RL to actually rely on the | |
| image instead of language priors. This Space runs the released EVAN-8B | |
| checkpoint (Qwen3-VL-8B-Instruct + EVAN/DAPO RL on ViRL39K) on an | |
| image + question pair, streaming the <think> reasoning trace and the final | |
| \boxed{} answer. | |
| """ | |
| import os | |
| os.environ.setdefault("PYTORCH_CUDA_ALLOC_CONF", "expandable_segments:True") | |
| import re | |
| import threading | |
| import spaces # noqa: E402 — must precede torch | |
| import torch | |
| from PIL import Image | |
| import gradio as gr | |
| from transformers import ( | |
| AutoProcessor, | |
| Qwen3VLForConditionalGeneration, | |
| TextIteratorStreamer, | |
| ) | |
| MODEL_ID = "DawnLiu/EVAN-8B" | |
| # The instruction suffix used during EVAN training / evaluation (model card). | |
| FORMAT_INSTRUCTION = ( | |
| "You first think through the reasoning process as an internal monologue, " | |
| "enclosed within <think> </think> tags. Then, provide your final answer " | |
| "enclosed within \\boxed{}." | |
| ) | |
| THINK_PATTERN = re.compile( | |
| r"<think>(.*?)</think>", re.DOTALL | re.IGNORECASE | |
| ) | |
| def _extract_boxed(text: str, start: int = 0): | |
| """Return (content, span) of the first \\boxed{...} at/after `start`. | |
| Handles nested braces (e.g. \\boxed{\\dfrac{7}{10}}), mirroring the | |
| authors' own `unbox()` helper in the EVAN training repo. | |
| """ | |
| i = text.find("\\boxed{", start) | |
| if i < 0: | |
| return None | |
| depth = 1 | |
| j = i + len("\\boxed{") | |
| while j < len(text): | |
| if text[j] == "{": | |
| depth += 1 | |
| elif text[j] == "}": | |
| depth -= 1 | |
| if depth == 0: | |
| return text[i + len("\\boxed{"): j], (i, j + 1) | |
| j += 1 | |
| # Unterminated (streaming) box: return what we have so far. | |
| return text[i + len("\\boxed{"):], (i, len(text)) | |
| # --------------------------------------------------------------------------- | |
| # Model load — module scope, eager .to("cuda") per ZeroGPU rules | |
| # --------------------------------------------------------------------------- | |
| processor = AutoProcessor.from_pretrained(MODEL_ID) | |
| model = Qwen3VLForConditionalGeneration.from_pretrained( | |
| MODEL_ID, | |
| dtype=torch.bfloat16, | |
| attn_implementation="sdpa", | |
| ).to("cuda") | |
| model.eval() | |
| # Chat-control tokens (<|im_end|> etc.) — stripped from the visible output. | |
| # NOTE: <think>/</think> are *added tokens* in this checkpoint, so the streamer | |
| # must decode with skip_special_tokens=False or the boundary disappears. | |
| CONTROL_TOKEN_PATTERN = re.compile(r"<\|[a-zA-Z_]+\|>") | |
| def _split_reasoning(text: str): | |
| """Split a generation into (reasoning trace, final answer). | |
| The trace is everything up to the final \\boxed{} (the internal | |
| monologue); the answer is the \\boxed{} content. If the model emits | |
| ... tags, the trace is the tagged block. | |
| """ | |
| if not text: | |
| return "", "" | |
| text = CONTROL_TOKEN_PATTERN.sub("", text).strip() | |
| if not text: | |
| return "", "" | |
| found = _extract_boxed(text) | |
| if not found: | |
| # No final answer yet — the whole text is (still-growing) reasoning. | |
| return text, "" | |
| answer, (box_start, box_end) = found | |
| answer = answer.strip() | |
| match = THINK_PATTERN.search(text) | |
| if match: | |
| # Keep any text before the opening tag, the tagged monologue itself, | |
| # and anything written after the closing tag but before the box. | |
| parts = [ | |
| text[: match.start()].strip(), | |
| (match.group(1) or "").strip(), | |
| text[match.end(): box_start].strip(), | |
| ] | |
| reasoning = "\n\n".join(p for p in parts if p) | |
| else: | |
| reasoning = text[:box_start].strip() | |
| return reasoning, answer | |
| def run_evan( | |
| image: Image.Image, | |
| question: str, | |
| max_new_tokens: int = 2048, | |
| temperature: float = 1.0, | |
| top_p: float = 0.7, | |
| use_format_instruction: bool = True, | |
| ): | |
| """Answer a question about an image with EVAN-8B visually-grounded reasoning. | |
| Streams the model's <think> reasoning trace, then returns the final | |
| \\boxed{} answer. The model was RL-trained (DAPO + evidence-ablated | |
| negatives) to ground its answers in the actual image rather than | |
| language priors. | |
| Args: | |
| image: The input image to reason about. | |
| question: The question to answer from the image. | |
| max_new_tokens: Cap on generated tokens (reasoning chains can be long). | |
| temperature: Sampling temperature (1.0 is the authors' eval default). | |
| top_p: Nucleus sampling threshold (0.7 is the authors' eval default). | |
| use_format_instruction: Append the training-time <think>/\\boxed{} | |
| format instruction to the question. | |
| """ | |
| if image is None: | |
| yield ("⚠️ Please upload an image first.", "") | |
| return | |
| if not question or not str(question).strip(): | |
| yield ("⚠️ Please enter a question.", "") | |
| return | |
| image = image.convert("RGB") | |
| prompt = question.strip() | |
| if use_format_instruction: | |
| prompt = f"{prompt}\n\n{FORMAT_INSTRUCTION}" | |
| messages = [ | |
| { | |
| "role": "user", | |
| "content": [ | |
| {"type": "image", "image": image}, | |
| {"type": "text", "text": prompt}, | |
| ], | |
| } | |
| ] | |
| inputs = processor.apply_chat_template( | |
| messages, | |
| tokenize=True, | |
| add_generation_prompt=True, | |
| return_dict=True, | |
| return_tensors="pt", | |
| ).to("cuda") | |
| streamer = _make_streamer(max_new_tokens) | |
| def _run(): | |
| try: | |
| with torch.inference_mode(): | |
| model.generate( | |
| **inputs, | |
| streamer=streamer, | |
| max_new_tokens=int(max_new_tokens), | |
| do_sample=float(temperature) > 0, | |
| temperature=max(float(temperature), 1e-5), | |
| top_p=float(top_p), | |
| ) | |
| except Exception as e: # keep the UI responsive on failure | |
| print("generation error:", repr(e)) | |
| streamer.end() | |
| thread = threading.Thread(target=_run, daemon=True) | |
| thread.start() | |
| buffer = "" | |
| try: | |
| for new_text in streamer: | |
| buffer += new_text | |
| reasoning, answer = _split_reasoning(buffer) | |
| yield (reasoning, answer) | |
| finally: | |
| thread.join(timeout=2.0) | |
| reasoning, answer = _split_reasoning(buffer) | |
| yield (reasoning, answer) | |
| def _make_streamer(max_new_tokens): | |
| """Build a TextIteratorStreamer bound to the loaded tokenizer. | |
| skip_special_tokens=False keeps .dtd/`` (added tokens in this | |
| checkpoint) so the reasoning boundary is visible; chat-control tokens | |
| (<|im_end|>, ...) are scrubbed in _split_reasoning. | |
| """ | |
| return TextIteratorStreamer( | |
| processor.tokenizer, | |
| skip_prompt=True, | |
| skip_special_tokens=False, | |
| timeout=max(60.0, float(max_new_tokens) * 0.5), | |
| ) | |
| # --------------------------------------------------------------------------- | |
| # Gradio UI | |
| # --------------------------------------------------------------------------- | |
| CSS = """ | |
| #col-container { max-width: 1100px; margin: 0 auto; } | |
| """ | |
| with gr.Blocks() as demo: | |
| gr.Markdown( | |
| "# 🖼️ EVAN-8B: Grounding Multimodal Reasoning with Evidence-Ablated Negatives\n" | |
| "EVAN penalizes answers the model produces *without* seeing the visual " | |
| "evidence, so the trained model actually **looks at the image** instead " | |
| "of leaning on language priors. Upload an image, ask a question, and " | |
| "watch the reasoning trace + final answer.\n\n" | |
| "Model: [`DawnLiu/EVAN-8B`](https://huggingface.co/DawnLiu/EVAN-8B) " | |
| "(Qwen3-VL-8B-Instruct + DAPO with EVAN on " | |
| "[ViRL39K](https://huggingface.co/datasets/TIGER-Lab/ViRL39K)) · " | |
| "Paper: [arXiv:2610.39237](https://arxiv.org/abs/2610.39237) · " | |
| "Code: [GitHub](https://github.com/DawnLIU35/EVAN)" | |
| ) | |
| with gr.Row(): | |
| with gr.Column(scale=1): | |
| image_input = gr.Image( | |
| label="Input image", type="pil", height=300 | |
| ) | |
| question_input = gr.Textbox( | |
| label="Question", | |
| placeholder="e.g. How many red objects are in this image?", | |
| lines=2, | |
| ) | |
| run_btn = gr.Button("Run EVAN-8B", variant="primary") | |
| with gr.Accordion("Advanced settings", open=False): | |
| max_tokens_slider = gr.Slider( | |
| 256, 4096, value=2048, step=256, | |
| label="Max new tokens", | |
| ) | |
| temp_slider = gr.Slider( | |
| 0.0, 1.5, value=1.0, step=0.1, label="Temperature" | |
| ) | |
| top_p_slider = gr.Slider( | |
| 0.1, 1.0, value=0.7, step=0.05, label="Top-p" | |
| ) | |
| format_toggle = gr.Checkbox( | |
| value=True, | |
| label="Append training-time format instruction " | |
| "(<think> + \\boxed{})", | |
| ) | |
| with gr.Column(scale=1): | |
| answer_output = gr.Textbox( | |
| label="Final answer", lines=2, interactive=False | |
| ) | |
| reasoning_output = gr.Textbox( | |
| label="Reasoning trace", lines=16, interactive=False | |
| ) | |
| run_btn.click( | |
| run_evan, | |
| inputs=[ | |
| image_input, | |
| question_input, | |
| max_tokens_slider, | |
| temp_slider, | |
| top_p_slider, | |
| format_toggle, | |
| ], | |
| outputs=[reasoning_output, answer_output], | |
| api_name="run_evan", | |
| ) | |
| gr.Examples( | |
| examples=[ | |
| ["examples/clocks.jpg", | |
| "The clocks show two things Lily did Tuesday evening. " | |
| "Which did Lily do earlier?\nChoices:\n(A) ate dinner\n" | |
| "(B) watched a movie"], | |
| ["examples/pie_chart.png", | |
| "What is the ratio of the Red section to the Aqua section " | |
| "in the pie chart?"], | |
| ["examples/marbles.jpg", | |
| "What is the ratio of black marbles to pink marbles?"], | |
| ["examples/diamonds.jpg", | |
| "Is the number of diamonds even or odd?\n" | |
| "Choices:\n(A) odd\n(B) even"], | |
| ["examples/spinning_wheel.png", | |
| "As shown in the figure, when the spinning wheel stops, the " | |
| "probability that the pointer points to the shaded area is ___. " | |
| "Express your answer as a fraction."], | |
| ], | |
| inputs=[image_input, question_input], | |
| outputs=[reasoning_output, answer_output], | |
| fn=run_evan, | |
| cache_examples=True, | |
| cache_mode="lazy", | |
| ) | |
| gr.Markdown( | |
| "---\n" | |
| "**How it works:** during training, EVAN masks the patches of the image " | |
| "a question depends on (by sIIG gradient magnitude), lets the policy " | |
| "answer without them, and penalizes plausible wrong answers under the " | |
| "clean image — suppressing shortcuts that plain RLVR leaves intact. " | |
| "Example images are from the " | |
| "[ViRL39K](https://huggingface.co/datasets/TIGER-Lab/ViRL39K) training " | |
| "set (MIT license)." | |
| ) | |
| demo.launch(mcp_server=True, css=CSS, theme=gr.themes.Citrus()) | |