Card: import prompt constants from primo_prompts / frame helper from primo_video_utils

README.md

@@ -96,89 +96,31 @@ The prompt template is equally load-bearing: the answer extractor is a regex ove
 
 This example is transcribed from `src/eval/eval_interleave.py`, the harness that produced the published numbers.
 
+The prompts and the frame extraction are **imported, not pasted**. Both live in the repo cloned during setup: `src/primo_prompts.py` holds every prompt constant and `src/primo_video_utils.py` holds the frame helpers. They are the same objects the eval harness uses, so an example that imports them cannot drift out of sync with the checkpoint. `setup.sh` puts `src/` on `PYTHONPATH`; from elsewhere, add it yourself:
+
+```python
+import sys
+sys.path.insert(0, "/path/to/PRIMO-R1/src")
+```
+
 ```python
-import cv2
 import torch
-from PIL import Image
 from transformers import AutoProcessor, AutoTokenizer
 from vllm import LLM, SamplingParams
 from qwen_vl_utils import process_vision_info
 
+# The single source of truth for the prompt format and the anchor frames.
+from primo_prompts import SYSTEM_PROMPT, build_question
+from primo_video_utils import extract_frames_on_demand
+
 MODEL_PATH = "models/PRIMO-R1-7B"   # or "LeonOverload/PRIMO-R1-7B"
 video_path = "path/to/your/episode.mp4"
 question = "What is the completion percentage of the task in the video?"
 problem_type = "regression"
 
-SYSTEM_PROMPT = (
-    "A conversation between User and Assistant. The Assistant is an expert AI specializing in embodied procedure and event reasoning based on visual input (video or images). "
-    "The assistant must strictly follow a specific thought process and output format. "
-    "The reasoning process is enclosed within <think> </think> tags, and the final answer is within <answer> </answer> tags. "
-    "The <think> block must contain three ordered subsections: <planning>, <observation>, and <reasoning>. "
-    "The <answer> block must contain only the final output required by the question type and no other commentary."
-)
-
-QUESTION_TEMPLATE = (
-    "QUESTION:\n{Question}\n\n"
-    "QUESTION TYPE:\n{question_type}\n\n"
-    "Analyze the provided visual data and reason about the ongoing task.\n\n"
-    "Please think about this question as if you were a human pondering deeply. "
-    "Provide your detailed reasoning between the <think> and </think> tags, following the subsections <planning>, <observation>, and <reasoning>. "
-    "Then give your final answer between the <answer> and </answer> tags.\n\n"
-    "Below is the required template:\n\n"
-    "<think>\n"
-    "<planning>\n"
-    "Identify the high-level goal of the agent, what is the initial state? What does successful completion look like?\n"
-    "Break down the high-level goal into a logical sequence of canonical steps. This serves as your mental plan for interpreting the task.\n"
-    "Use this plan to interpret actions, map observed behaviors to steps, assess progress, detect anomalies, and predict what happens next.\n"
-    "</planning>\n"
-    "<observation>\n"
-    "View the video as a temporal sequence of actions contributing to the procedure.\n"
-    "Objectively describe what is occurring in the current moment, noting evidence of progress or state changes.\n"
-    "Identify fine-grained actions and explain how they move the task forward.\n"
-    "List relevant objects, tools, and environmental context, emphasizing functional states and transformations.\n"
-    "Note cues—repetition, transitions, or completion indicators—that situate the action in the procedural script.\n"
-    "</observation>\n"
-    "<reasoning>\n"
-    "Think through the question as a human would, Engage in an internal dialogue using expressions such as 'let me think', 'wait', 'hmm', 'oh, I see', 'let's break it down', etc.\n"
-    "Connect observations to the procedural plan to determine which step is being executed, progress, correctness, or anomalies.\n"
-    "Reflect on assumptions, verify interpretations, and, if appropriate, predict the agent's next likely action.\n"
-    "Synthesize understanding of what the agent is doing, how it fits into the broader task, and whether the process seems successful.\n"
-    "You are encouraged to include self-reflection or verification in your reasoning process.\n"
-    "</reasoning>\n"
-    "</think>\n"
-    "<answer>\n"
-    "[Final answer here — strictly follow the `{question_type}` output format and include no extra commentary.]\n"
-    "</answer>"
-)
-
-TYPE_TEMPLATE = {
-    "multiple choice": " Please provide only the single option letter (e.g., A, B, C, D, etc.) within the <answer> </answer> tags.",
-    "numerical": " Please provide the numerical value (e.g., 42 or 3.14) within the <answer> </answer> tags.",
-    "OCR": " Please transcribe text from the image/video clearly and provide your text answer within the <answer> </answer> tags.",
-    "free-form": " Please provide your text answer within the <answer> </answer> tags.",
-    "regression": " Please provide the numerical value (e.g., 42 or 3.14) within the <answer> </answer> tags.",
-    "boolean": " Please provide only 'Yes' or 'No' as your answer within the <answer> </answer> tags.",
-}
-
-
-def extract_first_and_last_frame(path):
-    """Initial state = first frame, current state = last decodable frame."""
-    cap = cv2.VideoCapture(path)
-    ok, first = cap.read()
-    if not ok:
-        raise RuntimeError(f"cannot read {path}")
-    total = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))
-    cap.set(cv2.CAP_PROP_POS_FRAMES, max(0, total - 1))
-    ok, last = cap.read()
-    if not ok:                                    # some containers mis-report the count
-        cap.set(cv2.CAP_PROP_POS_FRAMES, max(0, total - 2))
-        ok, last = cap.read()
-    cap.release()
-    to_pil = lambda f: Image.fromarray(cv2.cvtColor(f, cv2.COLOR_BGR2RGB))
-    return to_pil(first), to_pil(last if ok else first)
-
-
-init_img, current_img = extract_first_and_last_frame(video_path)
+# (initial state, current state) as PIL images. LRU-cached, so calling this
+# again for the same video is free.
+init_img, current_img = extract_frames_on_demand(video_path)
 
 messages = [
     {"role": "system", "content": [{"type": "text", "text": SYSTEM_PROMPT}]},
@@ -188,12 +130,8 @@ messages = [
             {"type": "image", "image": init_img},                      # 1. initial state
             {"type": "video", "video": video_path, "nframes": 22},     # 2. the clip
             {"type": "image", "image": current_img},                   # 3. current state
-            {
-                "type": "text",
-                "text": QUESTION_TEMPLATE.format(
-                    Question=question, question_type=problem_type
-                ) + TYPE_TEMPLATE[problem_type],
-            },
+            # QUESTION_TEMPLATE.format(...) + TYPE_TEMPLATE[problem_type]
+            {"type": "text", "text": build_question(question, problem_type)},
         ],
     },
 ]
@@ -250,11 +188,25 @@ For `regression` and `numerical` questions the answer is a **progress percentage
 
 ### Question types
 
-`regression` and `numerical` (progress estimation), `multiple choice`, `boolean` (failure detection), `free-form`, `OCR`. Pass the type both in `QUESTION_TEMPLATE` and via `TYPE_TEMPLATE`; the model was trained with both present.
+`regression` and `numerical` (progress estimation), `multiple choice`, `boolean` (failure detection), `free-form`, `OCR`. `build_question` puts the type in both places — inside `QUESTION_TEMPLATE` and as the `TYPE_TEMPLATE` hint appended after it — because the model was trained with both present.
+
+### The prompt constants
+
+If you need to inspect or extend them rather than just call `build_question`:
+
+```python
+from primo_prompts import SYSTEM_PROMPT, QUESTION_TEMPLATE, TYPE_TEMPLATE
+
+print(QUESTION_TEMPLATE.format(Question="...", question_type="regression"))
+print(sorted(TYPE_TEMPLATE))
+# ['OCR', 'boolean', 'free-form', 'multiple choice', 'numerical', 'regression']
+```
+
+`primo_prompts.py` also carries the baseline and ablation prompts under `*_BASELINE` / `*_PARSER_ONLY` / `*_QUESTION_ONLY` names. Those are for reproducing the paper's comparison rows, **not** for this checkpoint — it was trained on `QUESTION_TEMPLATE` and expects it.
 
 ### A note on frame counts
 
-The published results were produced with the per-video frame cap at 22, so the launcher's `--nframes 32` effectively sampled 22. `nframes=22` above reproduces that. Training capped videos at 16 frames; eval samples more at higher resolution.
+The published results were produced with the per-video frame cap at 22, so the launcher's `--nframes 32` effectively sampled 22. `nframes=22` above reproduces that; `primo_video_utils.choose_nframes` is what applies the cap in the harness (`MAX_NFRAMES`, overridable via `INTERLEAVE_MAX_NFRAMES`). Training capped videos at 16 frames; eval samples more at higher resolution.
 
 ## Performance