import sys
from pathlib import Path
project_root = Path(__file__).parent.parent
sys.path.append(str(project_root))
import torch
import time
import gc
from transformers import AutoProcessor
from thinkstream.model import MODEL_CLS, DEFAULT_VIDEO_FLEX_WINDOW_SIZE
from thinkstream.model.inference import (
StreamingWindowInferenceEngine,
streaming_video_chat,
think_budget_sample,
)
from thinkstream.data.stream_data_processor import (
SYSTEM_PROMPT,
QWEN_TEMPLATE_WO_SYSTEM,
)
# ================= 配置参数 =================
MODEL_ID = "/your/model/ckpt"
VIDEO_PATH = "/path/to/a/video.mp4"
NUM_SECONDS = 90
FPS = 2.0
MAX_NEW_TOKENS = 30
MAX_THINK_TOKENS = 20
NUM_GENERATIONS = 1
MODEL_TYPE = "qwen2.5vl"
MIN_PIXELS = 100352
MAX_PIXELS = 150528
def split_think_and_response(text: str):
"""将 ... 部分与后面内容分开,返回 (think_parts, response)。"""
if not text or "" not in text:
return ([], text.strip() if text else "")
parts = text.split("")
# 最后一段是 之后的「回复」;前面每段要再剥掉 得到「思考」
think_parts = []
for i, block in enumerate(parts[:-1]):
block = block.strip()
if block.startswith(""):
block = block[len(""):].strip() # 去掉开头的
if block:
think_parts.append(block)
response = (parts[-1] or "").strip()
if response.endswith("<|im_end|>"):
response = response[:-len("<|im_end|>")]
return (think_parts, response)
def clear_gpu():
gc.collect()
torch.cuda.empty_cache()
def run_video_streaming_benchmark():
print(f"\n{'='*20} Testing: Real Video Streaming (2 FPS) {'='*20}")
clear_gpu()
# 1. 加载模型与处理器
processor = AutoProcessor.from_pretrained(MODEL_ID)
vp = processor.video_processor
vp.min_pixels = MIN_PIXELS
vp.max_pixels = MAX_PIXELS
vp.size["shortest_edge"] = MIN_PIXELS
vp.size["longest_edge"] = MAX_PIXELS
model = MODEL_CLS[MODEL_TYPE].from_pretrained(
MODEL_ID,
torch_dtype=torch.bfloat16,
attn_implementation="flash_attention_2",
device_map="cuda:0",
)
model.config.text_config._attn_implementation = "flash_attention_2_infer"
model.eval()
video_token_id = processor.tokenizer.convert_tokens_to_ids(["<|video_pad|>"])[0]
video_flex_window_size = getattr(
model.config, "video_flex_window_size", DEFAULT_VIDEO_FLEX_WINDOW_SIZE,
)
if MODEL_TYPE == "qwen3vl":
text_config = model.config.text_config
else:
text_config = model.config
engine = StreamingWindowInferenceEngine(
model,
batch_size=NUM_GENERATIONS,
max_len=16384,
num_hidden_layers=text_config.num_hidden_layers,
num_key_value_heads=text_config.num_key_value_heads,
head_dim=text_config.hidden_size // text_config.num_attention_heads,
vocab_size=text_config.vocab_size,
pad_token_id=model.generation_config.pad_token_id,
eos_token_ids=model.generation_config.eos_token_id,
video_token_id=video_token_id,
video_flex_window_size=video_flex_window_size,
)
# 2. 思考长度限制(与 GRPO rollout 一致)
think_end_token_id = processor.tokenizer.convert_tokens_to_ids("")
sample_kwargs = {
"think_end_token_id": think_end_token_id,
"max_think_tokens": MAX_THINK_TOKENS,
}
# 3. 使用 streaming_video_chat 统一推理
queries = [
{
"content": "Question/Instruction here",
"timestamp": 0.0, # Question time here.
},
]
total_token_count = 0
start_time = time.time()
for result in streaming_video_chat(
engine=engine,
processor=processor,
video_path=VIDEO_PATH,
queries=queries,
video_start=0.0,
video_end=float(NUM_SECONDS),
frames_per_chunk=int(FPS),
max_chunks=NUM_SECONDS,
min_pixels=MIN_PIXELS,
max_pixels=MAX_PIXELS,
max_new_tokens=MAX_NEW_TOKENS,
num_generations=NUM_GENERATIONS,
system_prompt=SYSTEM_PROMPT,
chat_template_wo_system=QWEN_TEMPLATE_WO_SYSTEM,
sample=think_budget_sample,
sample_kwargs=sample_kwargs,
model_type=MODEL_TYPE,
break_on_answer=False,
):
idx = result["chunk_idx"]
gen_tokens = result["generated_tokens"]
decoded = processor.batch_decode([gen_tokens[0]])[0]
think_parts, response = split_think_and_response(decoded)
print(f"\n[Chunk {idx}] has_query={result['has_query']}")
if think_parts:
for i, think in enumerate(think_parts, 1):
print(f" [think {i}] {think}")
if response:
print(f" [response] {response}")
if not think_parts and not response:
print(f" (empty) {decoded!r}")
total_token_count += max(t.shape[0] for t in gen_tokens) * NUM_GENERATIONS
torch.cuda.synchronize()
end_time = time.time()
# 4. 指标计算
latency = end_time - start_time
print(f"\nBenchmark Results:")
print(f"Total Time: {latency:.2f}s for {NUM_SECONDS}s video")
print(f"Total new: {total_token_count}")
print(f"Speed: {total_token_count / latency} tokens/s (batch_size={NUM_GENERATIONS})")
if __name__ == "__main__":
run_video_streaming_benchmark()