import spaces # must be first: before torch / transformers import os import random import re import time import gradio as gr import torch from transformers import AutoProcessor, AutoModelForImageTextToText MODEL_ID = os.environ.get("MODEL_ID", "aurateam/AURA_2") # Authors' single-shot system prompt for AURA (adapted from the duplex # deployment's aura_system_prompt in the vLLM-Omni reference; this demo is # non-duplex, so the reply policy is stated for one clip instead of a stream). DEFAULT_SYSTEM_PROMPT = ( "You are watching a video clip. Answer the user's question about what you " "see accurately and concisely. Ground objects and regions in the frame " "when asked." ) # AURA_v2 control tokens (aurateam checkpoints, vocab 248320), per the reference # deployment's stage-1 config: generation stops on silent + im_end. SILENT = "<|silent|>" RESPOND = "<|respond|>" STOP_TOKEN_IDS = [248070, 248046] # <|silent|>, <|im_end|> THINK_RE = re.compile(r".*?", re.DOTALL | re.IGNORECASE) THINK_OPEN_RE = re.compile(r".*$", re.DOTALL | re.IGNORECASE) CHATML_RE = re.compile(r"<\|im_end\|>|<\|im_start\|>|<\|endoftext\|>") NEXT_TURN_RE = re.compile(r"<\|im_start\|>.*", re.DOTALL) # Grounding: <|box_start|>(x1,y1),(x2,y2)<|box_end|>, coords are 0-1000 on the frame. BOX_SPAN_RE = re.compile(r"<\|box_start\|>(.*?)<\|box_end\|>", re.DOTALL) BOX_TAIL_RE = re.compile(r"<\|box(?:_start(?:\|.*)?)?$") NUM_RE = re.compile(r"-?\d+(?:\.\d+)?") def _render_boxes(text: str) -> str: """Turn each grounding span into a readable coordinate line. The reference client keeps the box tokens and draws them on the frame (`_strip_box_tokens_for_speech` removes them only before TTS); since this demo is text-only we render the same numbers as a labelled line instead of leaving bare tokens in the markdown. """ lines = [] def repl(m): nums = NUM_RE.findall(m.group(1) or "") if len(nums) != 4: return "" vals = [max(0.0, min(1000.0, float(n))) for n in nums] x1, y1, x2, y2 = vals if x2 < x1: x1, x2 = x2, x1 if y2 < y1: y1, y2 = y2, y1 lines.append((x1, y1, x2, y2)) return "" text = BOX_SPAN_RE.sub(repl, text) text = text.replace("<|box_end|>", "") text = BOX_TAIL_RE.sub("", text) if lines: rendered = " ".join(f"`({a:.0f},{b:.0f},{c:.0f},{d:.0f})`" for a, b, c, d in lines) text = text.rstrip() + f"\n\n**Grounding boxes** (x1,y1,x2,y2 on a 0–1000 scale): {rendered}" return text def clean_response(text: str) -> str: """Mirror the reference's _strip_assistant_text: drop think wrappers, ChatML specials, control markers (silent / respond) and any hallucinated follow-up turn before display, then render grounding boxes readably.""" text = NEXT_TURN_RE.sub("", text or "") text = THINK_RE.sub("", text) text = THINK_OPEN_RE.sub("", text) text = text.replace("", "") text = CHATML_RE.sub("", text) # Object-ref tags name the thing being pointed at; the words inside stay. text = text.replace("<|object_ref_start|>", "").replace("<|object_ref_end|>", "") text = text.strip() if text.startswith(RESPOND): text = text[len(RESPOND):].lstrip() text = text.replace(SILENT, "") text = _render_boxes(text) return text.strip() processor = AutoProcessor.from_pretrained(MODEL_ID) model = AutoModelForImageTextToText.from_pretrained( MODEL_ID, dtype=torch.bfloat16, attn_implementation="sdpa", ) model.to("cuda") model.eval() # Measured worst case on this Space (zero-a10g, large): 19.5 s wall for a # 300 s / 720p clip with a 1024-token budget; typical clips run 1.5-8 s. @spaces.GPU(duration=25) def answer_video( video, question, system_prompt=DEFAULT_SYSTEM_PROMPT, max_new_tokens=256, temperature=0.5, seed=42, progress=gr.Progress(track_tqdm=True), ): """Watch a video clip and answer a question about what happens in it. Args: video: uploaded video clip (mp4/avi/mkv/webm). question: what to ask about the clip. system_prompt: instruction shown to the model. max_new_tokens: response length budget. temperature: sampling temperature (authors' default 0.5). seed: random seed for reproducibility (-1 = random). Returns: The model's grounded answer as markdown text. """ if video is None: raise gr.Error("Please upload a video clip first.") question = (question or "").strip() or "Describe this video." if seed is None or seed < 0: seed = random.randint(0, 2**31 - 1) torch.manual_seed(seed) messages = [ {"role": "system", "content": [{"type": "text", "text": system_prompt}]}, { "role": "user", "content": [ {"type": "video", "video": video}, {"type": "text", "text": question}, ], }, ] prompt = processor.apply_chat_template( messages, tokenize=False, add_generation_prompt=True, enable_thinking=False ) # cap_pixels_per_frame=True adopts the reference (qwen-vl-utils) per-frame # pixel budget the authors' runtime uses; HF's default is uncapped, which # over-spends vision tokens on short clips. inputs = processor( text=[prompt], videos=[video], videos_kwargs={"cap_pixels_per_frame": True}, padding=True, return_tensors="pt", ).to("cuda") t0 = time.perf_counter() with torch.inference_mode(): out = model.generate( **inputs, max_new_tokens=int(max_new_tokens), do_sample=float(temperature) > 0, temperature=float(temperature), top_p=1.0, repetition_penalty=1.0, eos_token_id=STOP_TOKEN_IDS, ) gen_only = out[:, inputs["input_ids"].shape[1]:] raw = processor.tokenizer.batch_decode(gen_only, skip_special_tokens=False)[0] answer = clean_response(raw) elapsed = time.perf_counter() - t0 footer = f"\n\n_{len(gen_only[0])} tokens · {elapsed:.1f}s · seed {seed}_" return (answer or "(no spoken content)") + footer DESCRIPTION = """ # AURA 2 — streaming video–language model **AURA 2** ([aurateam/AURA_2](https://huggingface.co/aurateam/AURA_2)) is a 9.4B streaming video-language model that watches video and answers grounded questions about what it sees. This demo runs the *single-shot* path of the reference deployment: upload a clip, ask about it, and the model answers (the full [vLLM-Omni deployment](https://github.com/aurateam2026/Aura2.0) adds duplex speech and tool use on top of this same thinker). Leave the question empty for a general description. """ CSS = """ #col-container { max-width: 1100px; margin: 0 auto; } .dark .gradio-container { color: var(--body-text-color); } """ with gr.Blocks(title="AURA 2 · video understanding") as demo: gr.Markdown(DESCRIPTION) with gr.Row(): with gr.Column(): video_in = gr.Video(label="Video clip", sources=["upload"]) question = gr.Textbox( label="Question", placeholder="Ask about the clip — e.g. 'What is happening?'", lines=2, ) with gr.Accordion("Advanced options", open=False): system_prompt = gr.Textbox( label="System prompt", value=DEFAULT_SYSTEM_PROMPT, lines=4 ) max_new_tokens = gr.Slider(64, 1024, value=256, step=64, label="Max new tokens") temperature = gr.Slider(0.0, 1.5, value=0.5, step=0.05, label="Temperature") seed = gr.Number(value=42, label="Seed (-1 = random)", precision=0) submit = gr.Button("Answer", variant="primary", scale=1) with gr.Column(): answer_out = gr.Markdown(label="Answer") submit.click( fn=answer_video, inputs=[video_in, question, system_prompt, max_new_tokens, temperature, seed], outputs=[answer_out], api_name="answer_video", ) gr.Examples( examples=[ ["sample_demo_1.mp4", "Describe what the baby is doing."], ["sample_demo_1.mp4", "What book is the baby reading? Point to it."], ["sample_demo_3.mp4", "What product is advertised and what is the safety message?"], ["Big_Buck_Bunny_720_10s_10MB.mp4", "Describe the scene in this clip."], ], inputs=[video_in, question], fn=answer_video, outputs=[answer_out], cache_examples=True, cache_mode="lazy", label="Example clips (from the reference deployment's test videos)", ) gr.Markdown( "Example clips: [raushan-testing-hf/videos-test](https://huggingface.co/datasets/raushan-testing-hf/videos-test) " "(Apache-2.0) — the same clips used by the AURA reference implementation. " "Model: [aurateam/AURA_2](https://huggingface.co/aurateam/AURA_2) · " "[project page](https://aurateam2026.github.io/Aura-2.0/) · " "[code](https://github.com/aurateam2026/Aura2.0)." ) demo.launch(mcp_server=True, theme=gr.themes.Citrus(), css=CSS)