| { |
| "format": "iclr_video.locked_evaluation_protocol.v1", |
| "protocol_id": "recent4_full_v1", |
| "status": "template", |
| "applies_to": "all baseline and candidate checkpoints", |
| "inference": { |
| "causal_prefix_only": true, |
| "sampling_fps": 1.0, |
| "context_policy": "recent_n", |
| "recent_frames": 4, |
| "timestamp_rule": "match the SimpleStream release: decode a four-second half-open causal prefix window [max(0,t-4),t) at nominal 1 FPS and retain exactly four independently encoded image frames; only prefixes shorter than two raw frames repeat frame zero", |
| "allow_future_frames": false, |
| "allow_global_frames": false, |
| "allow_extra_frame_inspection": false, |
| "allow_persistent_visual_or_text_memory": false, |
| "visual_input_mode": "four independent image frames using qwen-vl-utils fetch_video (image_patch_size=14), never Qwen video tokens", |
| "processor_contract": { |
| "preprocessor_config_sha256": "", |
| "image_processor_type": "Qwen2VLImageProcessorFast", |
| "patch_size": 16, |
| "merge_size": 2, |
| "temporal_patch_size": 2, |
| "shortest_edge": 65536, |
| "longest_edge": 16777216, |
| "reference_visual_tokens_per_frame": 576, |
| "reason": "Bind the processor used by your evaluation before inference." |
| }, |
| "prompt_source": "StreamingBench SimpleStream release MCQ template; OVO backward/realtime SimpleStream release one-letter template; OVO forward frozen official task wrapper; no system message", |
| "decode": { |
| "do_sample": false, |
| "temperature": 0.0 |
| } |
| }, |
| "benchmarks": { |
| "streamingbench": { |
| "scope": "Real-Time full", |
| "export": "data/streamingbench/queries.jsonl", |
| "export_sha256": "", |
| "expected_queries": 2495, |
| "expected_groups": { |
| "real_time_visual": 2495 |
| }, |
| "primary_metric": "official Real-Time accuracy" |
| }, |
| "ovo": { |
| "scope": "all 12 tasks: Real-Time, Backward, and Forward", |
| "export": "data/ovo/queries.jsonl", |
| "export_sha256": "", |
| "expected_queries": 3035, |
| "expected_tasks": { |
| "ACR": 109, |
| "ASI": 148, |
| "ATR": 116, |
| "CRR": 240, |
| "EPM": 297, |
| "FPD": 101, |
| "HLD": 186, |
| "OCR": 149, |
| "OJR": 184, |
| "REC": 698, |
| "SSR": 629, |
| "STU": 178 |
| }, |
| "primary_metric": "official Total Avg", |
| "secondary_metrics": [ |
| "micro Overall", |
| "Real-Time Avg", |
| "Backward Avg", |
| "Forward Avg", |
| "RT/BT Avg" |
| ] |
| } |
| } |
| } |
|
|
