"""EVAN-8B: visually-grounded multimodal reasoning demo. EVAN (EVidence-Ablated Negatives) trains a VLM with RL to actually rely on the image instead of language priors. This Space runs the released EVAN-8B checkpoint (Qwen3-VL-8B-Instruct + EVAN/DAPO RL on ViRL39K) on an image + question pair, streaming the reasoning trace and the final \boxed{} answer. """ import os os.environ.setdefault("PYTORCH_CUDA_ALLOC_CONF", "expandable_segments:True") import re import threading import spaces # noqa: E402 — must precede torch import torch from PIL import Image import gradio as gr from transformers import ( AutoProcessor, Qwen3VLForConditionalGeneration, TextIteratorStreamer, ) MODEL_ID = "DawnLiu/EVAN-8B" # The instruction suffix used during EVAN training / evaluation (model card). FORMAT_INSTRUCTION = ( "You first think through the reasoning process as an internal monologue, " "enclosed within tags. Then, provide your final answer " "enclosed within \\boxed{}." ) THINK_PATTERN = re.compile( r"(.*?)", re.DOTALL | re.IGNORECASE ) def _extract_boxed(text: str, start: int = 0): """Return (content, span) of the first \\boxed{...} at/after `start`. Handles nested braces (e.g. \\boxed{\\dfrac{7}{10}}), mirroring the authors' own `unbox()` helper in the EVAN training repo. """ i = text.find("\\boxed{", start) if i < 0: return None depth = 1 j = i + len("\\boxed{") while j < len(text): if text[j] == "{": depth += 1 elif text[j] == "}": depth -= 1 if depth == 0: return text[i + len("\\boxed{"): j], (i, j + 1) j += 1 # Unterminated (streaming) box: return what we have so far. return text[i + len("\\boxed{"):], (i, len(text)) # --------------------------------------------------------------------------- # Model load — module scope, eager .to("cuda") per ZeroGPU rules # --------------------------------------------------------------------------- processor = AutoProcessor.from_pretrained(MODEL_ID) model = Qwen3VLForConditionalGeneration.from_pretrained( MODEL_ID, dtype=torch.bfloat16, attn_implementation="sdpa", ).to("cuda") model.eval() # Chat-control tokens (<|im_end|> etc.) — stripped from the visible output. # NOTE: / are *added tokens* in this checkpoint, so the streamer # must decode with skip_special_tokens=False or the boundary disappears. CONTROL_TOKEN_PATTERN = re.compile(r"<\|[a-zA-Z_]+\|>") def _split_reasoning(text: str): """Split a generation into (reasoning trace, final answer). The trace is everything up to the final \\boxed{} (the internal monologue); the answer is the \\boxed{} content. If the model emits ... tags, the trace is the tagged block. """ if not text: return "", "" text = CONTROL_TOKEN_PATTERN.sub("", text).strip() if not text: return "", "" found = _extract_boxed(text) if not found: # No final answer yet — the whole text is (still-growing) reasoning. return text, "" answer, (box_start, box_end) = found answer = answer.strip() match = THINK_PATTERN.search(text) if match: # Keep any text before the opening tag, the tagged monologue itself, # and anything written after the closing tag but before the box. parts = [ text[: match.start()].strip(), (match.group(1) or "").strip(), text[match.end(): box_start].strip(), ] reasoning = "\n\n".join(p for p in parts if p) else: reasoning = text[:box_start].strip() return reasoning, answer @spaces.GPU(duration=120) def run_evan( image: Image.Image, question: str, max_new_tokens: int = 2048, temperature: float = 1.0, top_p: float = 0.7, use_format_instruction: bool = True, ): """Answer a question about an image with EVAN-8B visually-grounded reasoning. Streams the model's reasoning trace, then returns the final \\boxed{} answer. The model was RL-trained (DAPO + evidence-ablated negatives) to ground its answers in the actual image rather than language priors. Args: image: The input image to reason about. question: The question to answer from the image. max_new_tokens: Cap on generated tokens (reasoning chains can be long). temperature: Sampling temperature (1.0 is the authors' eval default). top_p: Nucleus sampling threshold (0.7 is the authors' eval default). use_format_instruction: Append the training-time /\\boxed{} format instruction to the question. """ if image is None: yield ("⚠️ Please upload an image first.", "") return if not question or not str(question).strip(): yield ("⚠️ Please enter a question.", "") return image = image.convert("RGB") prompt = question.strip() if use_format_instruction: prompt = f"{prompt}\n\n{FORMAT_INSTRUCTION}" messages = [ { "role": "user", "content": [ {"type": "image", "image": image}, {"type": "text", "text": prompt}, ], } ] inputs = processor.apply_chat_template( messages, tokenize=True, add_generation_prompt=True, return_dict=True, return_tensors="pt", ).to("cuda") streamer = _make_streamer(max_new_tokens) def _run(): try: with torch.inference_mode(): model.generate( **inputs, streamer=streamer, max_new_tokens=int(max_new_tokens), do_sample=float(temperature) > 0, temperature=max(float(temperature), 1e-5), top_p=float(top_p), ) except Exception as e: # keep the UI responsive on failure print("generation error:", repr(e)) streamer.end() thread = threading.Thread(target=_run, daemon=True) thread.start() buffer = "" try: for new_text in streamer: buffer += new_text reasoning, answer = _split_reasoning(buffer) yield (reasoning, answer) finally: thread.join(timeout=2.0) reasoning, answer = _split_reasoning(buffer) yield (reasoning, answer) def _make_streamer(max_new_tokens): """Build a TextIteratorStreamer bound to the loaded tokenizer. skip_special_tokens=False keeps .dtd/`` (added tokens in this checkpoint) so the reasoning boundary is visible; chat-control tokens (<|im_end|>, ...) are scrubbed in _split_reasoning. """ return TextIteratorStreamer( processor.tokenizer, skip_prompt=True, skip_special_tokens=False, timeout=max(60.0, float(max_new_tokens) * 0.5), ) # --------------------------------------------------------------------------- # Gradio UI # --------------------------------------------------------------------------- CSS = """ #col-container { max-width: 1100px; margin: 0 auto; } """ with gr.Blocks() as demo: gr.Markdown( "# 🖼️ EVAN-8B: Grounding Multimodal Reasoning with Evidence-Ablated Negatives\n" "EVAN penalizes answers the model produces *without* seeing the visual " "evidence, so the trained model actually **looks at the image** instead " "of leaning on language priors. Upload an image, ask a question, and " "watch the reasoning trace + final answer.\n\n" "Model: [`DawnLiu/EVAN-8B`](https://huggingface.co/DawnLiu/EVAN-8B) " "(Qwen3-VL-8B-Instruct + DAPO with EVAN on " "[ViRL39K](https://huggingface.co/datasets/TIGER-Lab/ViRL39K)) · " "Paper: [arXiv:2610.39237](https://arxiv.org/abs/2610.39237) · " "Code: [GitHub](https://github.com/DawnLIU35/EVAN)" ) with gr.Row(): with gr.Column(scale=1): image_input = gr.Image( label="Input image", type="pil", height=300 ) question_input = gr.Textbox( label="Question", placeholder="e.g. How many red objects are in this image?", lines=2, ) run_btn = gr.Button("Run EVAN-8B", variant="primary") with gr.Accordion("Advanced settings", open=False): max_tokens_slider = gr.Slider( 256, 4096, value=2048, step=256, label="Max new tokens", ) temp_slider = gr.Slider( 0.0, 1.5, value=1.0, step=0.1, label="Temperature" ) top_p_slider = gr.Slider( 0.1, 1.0, value=0.7, step=0.05, label="Top-p" ) format_toggle = gr.Checkbox( value=True, label="Append training-time format instruction " "( + \\boxed{})", ) with gr.Column(scale=1): answer_output = gr.Textbox( label="Final answer", lines=2, interactive=False ) reasoning_output = gr.Textbox( label="Reasoning trace", lines=16, interactive=False ) run_btn.click( run_evan, inputs=[ image_input, question_input, max_tokens_slider, temp_slider, top_p_slider, format_toggle, ], outputs=[reasoning_output, answer_output], api_name="run_evan", ) gr.Examples( examples=[ ["examples/clocks.jpg", "The clocks show two things Lily did Tuesday evening. " "Which did Lily do earlier?\nChoices:\n(A) ate dinner\n" "(B) watched a movie"], ["examples/pie_chart.png", "What is the ratio of the Red section to the Aqua section " "in the pie chart?"], ["examples/marbles.jpg", "What is the ratio of black marbles to pink marbles?"], ["examples/diamonds.jpg", "Is the number of diamonds even or odd?\n" "Choices:\n(A) odd\n(B) even"], ["examples/spinning_wheel.png", "As shown in the figure, when the spinning wheel stops, the " "probability that the pointer points to the shaded area is ___. " "Express your answer as a fraction."], ], inputs=[image_input, question_input], outputs=[reasoning_output, answer_output], fn=run_evan, cache_examples=True, cache_mode="lazy", ) gr.Markdown( "---\n" "**How it works:** during training, EVAN masks the patches of the image " "a question depends on (by sIIG gradient magnitude), lets the policy " "answer without them, and penalizes plausible wrong answers under the " "clean image — suppressing shortcuts that plain RLVR leaves intact. " "Example images are from the " "[ViRL39K](https://huggingface.co/datasets/TIGER-Lab/ViRL39K) training " "set (MIT license)." ) demo.launch(mcp_server=True, css=CSS, theme=gr.themes.Citrus())