Spaces:
Running on Zero
Running on Zero
Download app.py from hugging-apps/aura-2: direct link, hf CLI and curl.
- Browser
- Download file 9.29 kB
-
https://huggingface.co/spaces/hugging-apps/aura-2/resolve/main/app.py
- Command line
-
hf download hf://spaces/hugging-apps/aura-2/app.py
-
curl -L -o app.py https://huggingface.co/spaces/hugging-apps/aura-2/resolve/main/app.py
9.29 kB
| import spaces # must be first: before torch / transformers | |
| import os | |
| import random | |
| import re | |
| import time | |
| import gradio as gr | |
| import torch | |
| from transformers import AutoProcessor, AutoModelForImageTextToText | |
| MODEL_ID = os.environ.get("MODEL_ID", "aurateam/AURA_2") | |
| # Authors' single-shot system prompt for AURA (adapted from the duplex | |
| # deployment's aura_system_prompt in the vLLM-Omni reference; this demo is | |
| # non-duplex, so the reply policy is stated for one clip instead of a stream). | |
| DEFAULT_SYSTEM_PROMPT = ( | |
| "You are watching a video clip. Answer the user's question about what you " | |
| "see accurately and concisely. Ground objects and regions in the frame " | |
| "when asked." | |
| ) | |
| # AURA_v2 control tokens (aurateam checkpoints, vocab 248320), per the reference | |
| # deployment's stage-1 config: generation stops on silent + im_end. | |
| SILENT = "<|silent|>" | |
| RESPOND = "<|respond|>" | |
| STOP_TOKEN_IDS = [248070, 248046] # <|silent|>, <|im_end|> | |
| THINK_RE = re.compile(r"<think>.*?</think>", re.DOTALL | re.IGNORECASE) | |
| THINK_OPEN_RE = re.compile(r"<think>.*$", re.DOTALL | re.IGNORECASE) | |
| CHATML_RE = re.compile(r"<\|im_end\|>|<\|im_start\|>|<\|endoftext\|>") | |
| NEXT_TURN_RE = re.compile(r"<\|im_start\|>.*", re.DOTALL) | |
| # Grounding: <|box_start|>(x1,y1),(x2,y2)<|box_end|>, coords are 0-1000 on the frame. | |
| BOX_SPAN_RE = re.compile(r"<\|box_start\|>(.*?)<\|box_end\|>", re.DOTALL) | |
| BOX_TAIL_RE = re.compile(r"<\|box(?:_start(?:\|.*)?)?$") | |
| NUM_RE = re.compile(r"-?\d+(?:\.\d+)?") | |
| def _render_boxes(text: str) -> str: | |
| """Turn each grounding span into a readable coordinate line. | |
| The reference client keeps the box tokens and draws them on the frame | |
| (`_strip_box_tokens_for_speech` removes them only before TTS); since this | |
| demo is text-only we render the same numbers as a labelled line instead of | |
| leaving bare tokens in the markdown. | |
| """ | |
| lines = [] | |
| def repl(m): | |
| nums = NUM_RE.findall(m.group(1) or "") | |
| if len(nums) != 4: | |
| return "" | |
| vals = [max(0.0, min(1000.0, float(n))) for n in nums] | |
| x1, y1, x2, y2 = vals | |
| if x2 < x1: | |
| x1, x2 = x2, x1 | |
| if y2 < y1: | |
| y1, y2 = y2, y1 | |
| lines.append((x1, y1, x2, y2)) | |
| return "" | |
| text = BOX_SPAN_RE.sub(repl, text) | |
| text = text.replace("<|box_end|>", "") | |
| text = BOX_TAIL_RE.sub("", text) | |
| if lines: | |
| rendered = " ".join(f"`({a:.0f},{b:.0f},{c:.0f},{d:.0f})`" for a, b, c, d in lines) | |
| text = text.rstrip() + f"\n\n**Grounding boxes** (x1,y1,x2,y2 on a 0–1000 scale): {rendered}" | |
| return text | |
| def clean_response(text: str) -> str: | |
| """Mirror the reference's _strip_assistant_text: drop think wrappers, | |
| ChatML specials, control markers (silent / respond) and any hallucinated | |
| follow-up turn before display, then render grounding boxes readably.""" | |
| text = NEXT_TURN_RE.sub("", text or "") | |
| text = THINK_RE.sub("", text) | |
| text = THINK_OPEN_RE.sub("", text) | |
| text = text.replace("</think>", "") | |
| text = CHATML_RE.sub("", text) | |
| # Object-ref tags name the thing being pointed at; the words inside stay. | |
| text = text.replace("<|object_ref_start|>", "").replace("<|object_ref_end|>", "") | |
| text = text.strip() | |
| if text.startswith(RESPOND): | |
| text = text[len(RESPOND):].lstrip() | |
| text = text.replace(SILENT, "") | |
| text = _render_boxes(text) | |
| return text.strip() | |
| processor = AutoProcessor.from_pretrained(MODEL_ID) | |
| model = AutoModelForImageTextToText.from_pretrained( | |
| MODEL_ID, | |
| dtype=torch.bfloat16, | |
| attn_implementation="sdpa", | |
| ) | |
| model.to("cuda") | |
| model.eval() | |
| # Measured worst case on this Space (zero-a10g, large): 19.5 s wall for a | |
| # 300 s / 720p clip with a 1024-token budget; typical clips run 1.5-8 s. | |
| def answer_video( | |
| video, | |
| question, | |
| system_prompt=DEFAULT_SYSTEM_PROMPT, | |
| max_new_tokens=256, | |
| temperature=0.5, | |
| seed=42, | |
| progress=gr.Progress(track_tqdm=True), | |
| ): | |
| """Watch a video clip and answer a question about what happens in it. | |
| Args: | |
| video: uploaded video clip (mp4/avi/mkv/webm). | |
| question: what to ask about the clip. | |
| system_prompt: instruction shown to the model. | |
| max_new_tokens: response length budget. | |
| temperature: sampling temperature (authors' default 0.5). | |
| seed: random seed for reproducibility (-1 = random). | |
| Returns: | |
| The model's grounded answer as markdown text. | |
| """ | |
| if video is None: | |
| raise gr.Error("Please upload a video clip first.") | |
| question = (question or "").strip() or "Describe this video." | |
| if seed is None or seed < 0: | |
| seed = random.randint(0, 2**31 - 1) | |
| torch.manual_seed(seed) | |
| messages = [ | |
| {"role": "system", "content": [{"type": "text", "text": system_prompt}]}, | |
| { | |
| "role": "user", | |
| "content": [ | |
| {"type": "video", "video": video}, | |
| {"type": "text", "text": question}, | |
| ], | |
| }, | |
| ] | |
| prompt = processor.apply_chat_template( | |
| messages, tokenize=False, add_generation_prompt=True, enable_thinking=False | |
| ) | |
| # cap_pixels_per_frame=True adopts the reference (qwen-vl-utils) per-frame | |
| # pixel budget the authors' runtime uses; HF's default is uncapped, which | |
| # over-spends vision tokens on short clips. | |
| inputs = processor( | |
| text=[prompt], | |
| videos=[video], | |
| videos_kwargs={"cap_pixels_per_frame": True}, | |
| padding=True, | |
| return_tensors="pt", | |
| ).to("cuda") | |
| t0 = time.perf_counter() | |
| with torch.inference_mode(): | |
| out = model.generate( | |
| **inputs, | |
| max_new_tokens=int(max_new_tokens), | |
| do_sample=float(temperature) > 0, | |
| temperature=float(temperature), | |
| top_p=1.0, | |
| repetition_penalty=1.0, | |
| eos_token_id=STOP_TOKEN_IDS, | |
| ) | |
| gen_only = out[:, inputs["input_ids"].shape[1]:] | |
| raw = processor.tokenizer.batch_decode(gen_only, skip_special_tokens=False)[0] | |
| answer = clean_response(raw) | |
| elapsed = time.perf_counter() - t0 | |
| footer = f"\n\n_{len(gen_only[0])} tokens · {elapsed:.1f}s · seed {seed}_" | |
| return (answer or "(no spoken content)") + footer | |
| DESCRIPTION = """ | |
| # AURA 2 — streaming video–language model | |
| **AURA 2** ([aurateam/AURA_2](https://huggingface.co/aurateam/AURA_2)) is a 9.4B streaming | |
| video-language model that watches video and answers grounded questions about what it sees. | |
| This demo runs the *single-shot* path of the reference deployment: upload a clip, ask about it, | |
| and the model answers (the full | |
| [vLLM-Omni deployment](https://github.com/aurateam2026/Aura2.0) adds duplex speech and tool use | |
| on top of this same thinker). Leave the question empty for a general description. | |
| """ | |
| CSS = """ | |
| #col-container { max-width: 1100px; margin: 0 auto; } | |
| .dark .gradio-container { color: var(--body-text-color); } | |
| """ | |
| with gr.Blocks(title="AURA 2 · video understanding") as demo: | |
| gr.Markdown(DESCRIPTION) | |
| with gr.Row(): | |
| with gr.Column(): | |
| video_in = gr.Video(label="Video clip", sources=["upload"]) | |
| question = gr.Textbox( | |
| label="Question", | |
| placeholder="Ask about the clip — e.g. 'What is happening?'", | |
| lines=2, | |
| ) | |
| with gr.Accordion("Advanced options", open=False): | |
| system_prompt = gr.Textbox( | |
| label="System prompt", value=DEFAULT_SYSTEM_PROMPT, lines=4 | |
| ) | |
| max_new_tokens = gr.Slider(64, 1024, value=256, step=64, label="Max new tokens") | |
| temperature = gr.Slider(0.0, 1.5, value=0.5, step=0.05, label="Temperature") | |
| seed = gr.Number(value=42, label="Seed (-1 = random)", precision=0) | |
| submit = gr.Button("Answer", variant="primary", scale=1) | |
| with gr.Column(): | |
| answer_out = gr.Markdown(label="Answer") | |
| submit.click( | |
| fn=answer_video, | |
| inputs=[video_in, question, system_prompt, max_new_tokens, temperature, seed], | |
| outputs=[answer_out], | |
| api_name="answer_video", | |
| ) | |
| gr.Examples( | |
| examples=[ | |
| ["sample_demo_1.mp4", "Describe what the baby is doing."], | |
| ["sample_demo_1.mp4", "What book is the baby reading? Point to it."], | |
| ["sample_demo_3.mp4", "What product is advertised and what is the safety message?"], | |
| ["Big_Buck_Bunny_720_10s_10MB.mp4", "Describe the scene in this clip."], | |
| ], | |
| inputs=[video_in, question], | |
| fn=answer_video, | |
| outputs=[answer_out], | |
| cache_examples=True, | |
| cache_mode="lazy", | |
| label="Example clips (from the reference deployment's test videos)", | |
| ) | |
| gr.Markdown( | |
| "Example clips: [raushan-testing-hf/videos-test](https://huggingface.co/datasets/raushan-testing-hf/videos-test) " | |
| "(Apache-2.0) — the same clips used by the AURA reference implementation. " | |
| "Model: [aurateam/AURA_2](https://huggingface.co/aurateam/AURA_2) · " | |
| "[project page](https://aurateam2026.github.io/Aura-2.0/) · " | |
| "[code](https://github.com/aurateam2026/Aura2.0)." | |
| ) | |
| demo.launch(mcp_server=True, theme=gr.themes.Citrus(), css=CSS) | |