import spaces # must be first: before torch / transformers
import os
import random
import re
import time
import gradio as gr
import torch
from transformers import AutoProcessor, AutoModelForImageTextToText
MODEL_ID = os.environ.get("MODEL_ID", "aurateam/AURA_2")
# Authors' single-shot system prompt for AURA (adapted from the duplex
# deployment's aura_system_prompt in the vLLM-Omni reference; this demo is
# non-duplex, so the reply policy is stated for one clip instead of a stream).
DEFAULT_SYSTEM_PROMPT = (
"You are watching a video clip. Answer the user's question about what you "
"see accurately and concisely. Ground objects and regions in the frame "
"when asked."
)
# AURA_v2 control tokens (aurateam checkpoints, vocab 248320), per the reference
# deployment's stage-1 config: generation stops on silent + im_end.
SILENT = "<|silent|>"
RESPOND = "<|respond|>"
STOP_TOKEN_IDS = [248070, 248046] # <|silent|>, <|im_end|>
THINK_RE = re.compile(r".*?", re.DOTALL | re.IGNORECASE)
THINK_OPEN_RE = re.compile(r".*$", re.DOTALL | re.IGNORECASE)
CHATML_RE = re.compile(r"<\|im_end\|>|<\|im_start\|>|<\|endoftext\|>")
NEXT_TURN_RE = re.compile(r"<\|im_start\|>.*", re.DOTALL)
# Grounding: <|box_start|>(x1,y1),(x2,y2)<|box_end|>, coords are 0-1000 on the frame.
BOX_SPAN_RE = re.compile(r"<\|box_start\|>(.*?)<\|box_end\|>", re.DOTALL)
BOX_TAIL_RE = re.compile(r"<\|box(?:_start(?:\|.*)?)?$")
NUM_RE = re.compile(r"-?\d+(?:\.\d+)?")
def _render_boxes(text: str) -> str:
"""Turn each grounding span into a readable coordinate line.
The reference client keeps the box tokens and draws them on the frame
(`_strip_box_tokens_for_speech` removes them only before TTS); since this
demo is text-only we render the same numbers as a labelled line instead of
leaving bare tokens in the markdown.
"""
lines = []
def repl(m):
nums = NUM_RE.findall(m.group(1) or "")
if len(nums) != 4:
return ""
vals = [max(0.0, min(1000.0, float(n))) for n in nums]
x1, y1, x2, y2 = vals
if x2 < x1:
x1, x2 = x2, x1
if y2 < y1:
y1, y2 = y2, y1
lines.append((x1, y1, x2, y2))
return ""
text = BOX_SPAN_RE.sub(repl, text)
text = text.replace("<|box_end|>", "")
text = BOX_TAIL_RE.sub("", text)
if lines:
rendered = " ".join(f"`({a:.0f},{b:.0f},{c:.0f},{d:.0f})`" for a, b, c, d in lines)
text = text.rstrip() + f"\n\n**Grounding boxes** (x1,y1,x2,y2 on a 0–1000 scale): {rendered}"
return text
def clean_response(text: str) -> str:
"""Mirror the reference's _strip_assistant_text: drop think wrappers,
ChatML specials, control markers (silent / respond) and any hallucinated
follow-up turn before display, then render grounding boxes readably."""
text = NEXT_TURN_RE.sub("", text or "")
text = THINK_RE.sub("", text)
text = THINK_OPEN_RE.sub("", text)
text = text.replace("", "")
text = CHATML_RE.sub("", text)
# Object-ref tags name the thing being pointed at; the words inside stay.
text = text.replace("<|object_ref_start|>", "").replace("<|object_ref_end|>", "")
text = text.strip()
if text.startswith(RESPOND):
text = text[len(RESPOND):].lstrip()
text = text.replace(SILENT, "")
text = _render_boxes(text)
return text.strip()
processor = AutoProcessor.from_pretrained(MODEL_ID)
model = AutoModelForImageTextToText.from_pretrained(
MODEL_ID,
dtype=torch.bfloat16,
attn_implementation="sdpa",
)
model.to("cuda")
model.eval()
# Measured worst case on this Space (zero-a10g, large): 19.5 s wall for a
# 300 s / 720p clip with a 1024-token budget; typical clips run 1.5-8 s.
@spaces.GPU(duration=25)
def answer_video(
video,
question,
system_prompt=DEFAULT_SYSTEM_PROMPT,
max_new_tokens=256,
temperature=0.5,
seed=42,
progress=gr.Progress(track_tqdm=True),
):
"""Watch a video clip and answer a question about what happens in it.
Args:
video: uploaded video clip (mp4/avi/mkv/webm).
question: what to ask about the clip.
system_prompt: instruction shown to the model.
max_new_tokens: response length budget.
temperature: sampling temperature (authors' default 0.5).
seed: random seed for reproducibility (-1 = random).
Returns:
The model's grounded answer as markdown text.
"""
if video is None:
raise gr.Error("Please upload a video clip first.")
question = (question or "").strip() or "Describe this video."
if seed is None or seed < 0:
seed = random.randint(0, 2**31 - 1)
torch.manual_seed(seed)
messages = [
{"role": "system", "content": [{"type": "text", "text": system_prompt}]},
{
"role": "user",
"content": [
{"type": "video", "video": video},
{"type": "text", "text": question},
],
},
]
prompt = processor.apply_chat_template(
messages, tokenize=False, add_generation_prompt=True, enable_thinking=False
)
# cap_pixels_per_frame=True adopts the reference (qwen-vl-utils) per-frame
# pixel budget the authors' runtime uses; HF's default is uncapped, which
# over-spends vision tokens on short clips.
inputs = processor(
text=[prompt],
videos=[video],
videos_kwargs={"cap_pixels_per_frame": True},
padding=True,
return_tensors="pt",
).to("cuda")
t0 = time.perf_counter()
with torch.inference_mode():
out = model.generate(
**inputs,
max_new_tokens=int(max_new_tokens),
do_sample=float(temperature) > 0,
temperature=float(temperature),
top_p=1.0,
repetition_penalty=1.0,
eos_token_id=STOP_TOKEN_IDS,
)
gen_only = out[:, inputs["input_ids"].shape[1]:]
raw = processor.tokenizer.batch_decode(gen_only, skip_special_tokens=False)[0]
answer = clean_response(raw)
elapsed = time.perf_counter() - t0
footer = f"\n\n_{len(gen_only[0])} tokens · {elapsed:.1f}s · seed {seed}_"
return (answer or "(no spoken content)") + footer
DESCRIPTION = """
# AURA 2 — streaming video–language model
**AURA 2** ([aurateam/AURA_2](https://huggingface.co/aurateam/AURA_2)) is a 9.4B streaming
video-language model that watches video and answers grounded questions about what it sees.
This demo runs the *single-shot* path of the reference deployment: upload a clip, ask about it,
and the model answers (the full
[vLLM-Omni deployment](https://github.com/aurateam2026/Aura2.0) adds duplex speech and tool use
on top of this same thinker). Leave the question empty for a general description.
"""
CSS = """
#col-container { max-width: 1100px; margin: 0 auto; }
.dark .gradio-container { color: var(--body-text-color); }
"""
with gr.Blocks(title="AURA 2 · video understanding") as demo:
gr.Markdown(DESCRIPTION)
with gr.Row():
with gr.Column():
video_in = gr.Video(label="Video clip", sources=["upload"])
question = gr.Textbox(
label="Question",
placeholder="Ask about the clip — e.g. 'What is happening?'",
lines=2,
)
with gr.Accordion("Advanced options", open=False):
system_prompt = gr.Textbox(
label="System prompt", value=DEFAULT_SYSTEM_PROMPT, lines=4
)
max_new_tokens = gr.Slider(64, 1024, value=256, step=64, label="Max new tokens")
temperature = gr.Slider(0.0, 1.5, value=0.5, step=0.05, label="Temperature")
seed = gr.Number(value=42, label="Seed (-1 = random)", precision=0)
submit = gr.Button("Answer", variant="primary", scale=1)
with gr.Column():
answer_out = gr.Markdown(label="Answer")
submit.click(
fn=answer_video,
inputs=[video_in, question, system_prompt, max_new_tokens, temperature, seed],
outputs=[answer_out],
api_name="answer_video",
)
gr.Examples(
examples=[
["sample_demo_1.mp4", "Describe what the baby is doing."],
["sample_demo_1.mp4", "What book is the baby reading? Point to it."],
["sample_demo_3.mp4", "What product is advertised and what is the safety message?"],
["Big_Buck_Bunny_720_10s_10MB.mp4", "Describe the scene in this clip."],
],
inputs=[video_in, question],
fn=answer_video,
outputs=[answer_out],
cache_examples=True,
cache_mode="lazy",
label="Example clips (from the reference deployment's test videos)",
)
gr.Markdown(
"Example clips: [raushan-testing-hf/videos-test](https://huggingface.co/datasets/raushan-testing-hf/videos-test) "
"(Apache-2.0) — the same clips used by the AURA reference implementation. "
"Model: [aurateam/AURA_2](https://huggingface.co/aurateam/AURA_2) · "
"[project page](https://aurateam2026.github.io/Aura-2.0/) · "
"[code](https://github.com/aurateam2026/Aura2.0)."
)
demo.launch(mcp_server=True, theme=gr.themes.Citrus(), css=CSS)