Fix skipped videos without a runtime limit; calibrate video pricing
#2
by pepijn223 HF Staff - opened
- README.md +15 -11
- frontend/src/main.tsx +12 -10
- human_video.py +28 -11
- model_catalog.py +24 -5
- provider.py +5 -4
- studio.py +14 -12
- test_human_video.py +84 -24
- worker.py +13 -1
README.md
CHANGED
|
@@ -121,17 +121,21 @@ a first-person human demonstration, following LeRobot's `--human_video` annotati
|
|
| 121 |
Both media models run on fal through HF Inference Providers with the signed-in user's
|
| 122 |
token. Selecting videos also selects Subtasks, since there is one video per subtask.
|
| 123 |
|
| 124 |
-
The estimate
|
| 125 |
-
|
| 126 |
-
|
| 127 |
-
|
| 128 |
-
|
| 129 |
-
|
| 130 |
-
|
| 131 |
-
|
| 132 |
-
|
| 133 |
-
|
| 134 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 135 |
|
| 136 |
The output holds `annotation_studio/human_videos/episode_XXXXXX/segment_XXX{.mp4,
|
| 137 |
_first_frame.png,_robot_frame.png}` (named by source episode) and `index.json`, which lists
|
|
|
|
| 121 |
Both media models run on fal through HF Inference Providers with the signed-in user's
|
| 122 |
token. Selecting videos also selects Subtasks, since there is one video per subtask.
|
| 123 |
|
| 124 |
+
The estimate uses the selected episodes' total seconds and count, calibrated on the first
|
| 125 |
+
video run (3 ReBot episodes, 159.7 s, 33 subtasks): **1 subtask per episode + 1 per 5.3 s**,
|
| 126 |
+
up to 1.5× that in the upper bound, with the per-episode cap applied. Each video is priced at
|
| 127 |
+
fal's list prices: the edit at $0.03 per output megapixel (1024×1024) and the 5 s 480p video
|
| 128 |
+
at $0.05/s, about $0.28 in total, plus about 1,750 input and 550 output VLM tokens.
|
| 129 |
+
**Parallel generations** (1–32, default 8) limits simultaneous media requests across the
|
| 130 |
+
whole job. The review warns when the expected videos at that parallelism would outlast the
|
| 131 |
+
job timeout (about 3 min per video).
|
| 132 |
+
|
| 133 |
+
Videos start only after an episode's annotations pass validation. Each request holds a
|
| 134 |
+
conservative reservation (`model_catalog.HUMAN_VIDEO`) from the spending target while in
|
| 135 |
+
flight and is reconciled to its list price when it returns. A failed video is recorded and
|
| 136 |
+
the others continue. Videos stop at the spending target, or before the runtime limit (the job
|
| 137 |
+
timeout when none is set), without paying for an edit whose video could not finish; the
|
| 138 |
+
episode's annotations are kept. The launch checks that both fal mappings are live.
|
| 139 |
|
| 140 |
The output holds `annotation_studio/human_videos/episode_XXXXXX/segment_XXX{.mp4,
|
| 141 |
_first_frame.png,_robot_frame.png}` (named by source episode) and `index.json`, which lists
|
frontend/src/main.tsx
CHANGED
|
@@ -94,13 +94,15 @@ type HumanVideoProfile = {
|
|
| 94 |
video_model: string;
|
| 95 |
resolution: string;
|
| 96 |
duration_s: number;
|
| 97 |
-
|
| 98 |
-
|
|
|
|
| 99 |
checked_at: string;
|
| 100 |
pricing_url: string;
|
| 101 |
segment_choices: number[];
|
| 102 |
parallel_choices: number[];
|
| 103 |
seconds_per_subtask: number;
|
|
|
|
| 104 |
minutes_per_video: number;
|
| 105 |
};
|
| 106 |
type ModelCatalog = {
|
|
@@ -1129,14 +1131,14 @@ function App() {
|
|
| 1129 |
{modelCatalog.human_video.resolution} via{" "}
|
| 1130 |
{modelCatalog.human_video.provider_name} on Inference
|
| 1131 |
Providers · about{" "}
|
| 1132 |
-
{money(
|
| 1133 |
-
|
| 1134 |
-
|
| 1135 |
-
|
| 1136 |
-
|
| 1137 |
-
|
| 1138 |
-
{modelCatalog.human_video.seconds_per_subtask}
|
| 1139 |
-
episode.
|
| 1140 |
{options.features.includes(HUMAN_VIDEOS) &&
|
| 1141 |
" One video per subtask, so Subtasks are included."}{" "}
|
| 1142 |
<a
|
|
|
|
| 94 |
video_model: string;
|
| 95 |
resolution: string;
|
| 96 |
duration_s: number;
|
| 97 |
+
segment_usd: number;
|
| 98 |
+
edit_usd_per_megapixel: number;
|
| 99 |
+
video_usd_per_second: number;
|
| 100 |
checked_at: string;
|
| 101 |
pricing_url: string;
|
| 102 |
segment_choices: number[];
|
| 103 |
parallel_choices: number[];
|
| 104 |
seconds_per_subtask: number;
|
| 105 |
+
subtasks_per_episode: number;
|
| 106 |
minutes_per_video: number;
|
| 107 |
};
|
| 108 |
type ModelCatalog = {
|
|
|
|
| 1131 |
{modelCatalog.human_video.resolution} via{" "}
|
| 1132 |
{modelCatalog.human_video.provider_name} on Inference
|
| 1133 |
Providers · about{" "}
|
| 1134 |
+
{money(modelCatalog.human_video.segment_usd)} per video
|
| 1135 |
+
(edit ${modelCatalog.human_video.edit_usd_per_megapixel}
|
| 1136 |
+
/MP + video $
|
| 1137 |
+
{modelCatalog.human_video.video_usd_per_second}/s), billed
|
| 1138 |
+
to your account. The estimate counts{" "}
|
| 1139 |
+
{modelCatalog.human_video.subtasks_per_episode} + one
|
| 1140 |
+
subtask per {modelCatalog.human_video.seconds_per_subtask}{" "}
|
| 1141 |
+
s of each episode.
|
| 1142 |
{options.features.includes(HUMAN_VIDEOS) &&
|
| 1143 |
" One video per subtask, so Subtasks are included."}{" "}
|
| 1144 |
<a
|
human_video.py
CHANGED
|
@@ -52,20 +52,32 @@ def subtask_spans(record, staging):
|
|
| 52 |
|
| 53 |
|
| 54 |
class FalMedia:
|
| 55 |
-
"""Qwen-Image-Edit + MiniMax-H3 on fal through the HF router, charged to ``budget``.
|
| 56 |
|
| 57 |
-
|
| 58 |
-
|
| 59 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 60 |
self.poll_interval = poll_interval
|
| 61 |
# Job-wide limit on simultaneous media requests, shared by every episode thread.
|
| 62 |
self.concurrency = max(1, min(int(concurrency or self.profile["concurrency"]), 32))
|
| 63 |
self.semaphore = threading.BoundedSemaphore(self.concurrency)
|
| 64 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 65 |
def edit_image(self, image, prompt):
|
| 66 |
from huggingface_hub import InferenceClient
|
| 67 |
|
| 68 |
-
self.
|
| 69 |
with self.semaphore:
|
| 70 |
reservation = self.budget.reserve_media(self.profile["edit_usd"])
|
| 71 |
client = InferenceClient(provider=self.profile["provider"], api_key=self.token)
|
|
@@ -75,7 +87,8 @@ class FalMedia:
|
|
| 75 |
model=self.profile["edit_model"],
|
| 76 |
guidance_scale=self.profile["edit_guidance_scale"],
|
| 77 |
)
|
| 78 |
-
|
|
|
|
| 79 |
# fal returns a square image; keep the camera's size and aspect ratio.
|
| 80 |
return edited.convert("RGB").resize(image.size, Image.LANCZOS)
|
| 81 |
|
|
@@ -83,8 +96,7 @@ class FalMedia:
|
|
| 83 |
from huggingface_hub.inference._providers.fal_ai import FalAIImageToVideoTask
|
| 84 |
from huggingface_hub.utils import get_session, hf_raise_for_status
|
| 85 |
|
| 86 |
-
|
| 87 |
-
raise DeadlineExceededError("Not enough runtime left for another video.")
|
| 88 |
# InferenceClient has no image-text-to-video method yet; fal's image-to-video helper speaks
|
| 89 |
# the same queue protocol once its task matches the model's mapping.
|
| 90 |
helper = FalAIImageToVideoTask()
|
|
@@ -119,7 +131,8 @@ class FalMedia:
|
|
| 119 |
while status != "COMPLETED":
|
| 120 |
if status in ("FAILED", "ERROR", "CANCELLED"):
|
| 121 |
raise RuntimeError(f"The video provider reported {status}.")
|
| 122 |
-
self.
|
|
|
|
| 123 |
time.sleep(self.poll_interval)
|
| 124 |
polled = session.get(f"{base}{path}/status{query}", headers=request.headers, timeout=30)
|
| 125 |
hf_raise_for_status(polled)
|
|
@@ -130,7 +143,9 @@ class FalMedia:
|
|
| 130 |
# The result is a public CDN link; the token is never sent there.
|
| 131 |
video = session.get(url, timeout=120)
|
| 132 |
hf_raise_for_status(video)
|
| 133 |
-
self.budget.record_media(
|
|
|
|
|
|
|
| 134 |
return video.content
|
| 135 |
|
| 136 |
|
|
@@ -199,6 +214,7 @@ def generate_human_videos(record, staging, vlm, frames, full_frames, media, out_
|
|
| 199 |
entry["status"] = "skipped"
|
| 200 |
return entry
|
| 201 |
try:
|
|
|
|
| 202 |
if not isinstance(plan, dict) or not plan.get("edit_prompt"):
|
| 203 |
raise ValueError("The model returned no human-video plan.")
|
| 204 |
robot = full_frames.frames_at(record, [_nearest(ts, span["start"])])[0].convert("RGB")
|
|
@@ -230,7 +246,8 @@ def generate_human_videos(record, staging, vlm, frames, full_frames, media, out_
|
|
| 230 |
stop.set()
|
| 231 |
entry["status"] = "time_stopped" if isinstance(exc, DeadlineExceededError) else "budget_stopped"
|
| 232 |
except Exception as exc:
|
| 233 |
-
|
|
|
|
| 234 |
return entry
|
| 235 |
|
| 236 |
with ThreadPoolExecutor(max_workers=min(len(spans), getattr(media, "concurrency", 1))) as pool:
|
|
|
|
| 52 |
|
| 53 |
|
| 54 |
class FalMedia:
|
| 55 |
+
"""Qwen-Image-Edit + MiniMax-H3 on fal through the HF router, charged to ``budget``.
|
| 56 |
|
| 57 |
+
``deadline`` is the epoch by which the job must finish (None = no limit).
|
| 58 |
+
"""
|
| 59 |
+
|
| 60 |
+
def __init__(self, token, budget, deadline=None, profile=None, concurrency=None, poll_interval=2.0):
|
| 61 |
+
self.token, self.budget, self.deadline = token, budget, deadline
|
| 62 |
+
# Pinned manifests may predate newer price fields; fill them from the current catalog.
|
| 63 |
+
self.profile = {**human_video_profile(), **(profile or {})}
|
| 64 |
self.poll_interval = poll_interval
|
| 65 |
# Job-wide limit on simultaneous media requests, shared by every episode thread.
|
| 66 |
self.concurrency = max(1, min(int(concurrency or self.profile["concurrency"]), 32))
|
| 67 |
self.semaphore = threading.BoundedSemaphore(self.concurrency)
|
| 68 |
|
| 69 |
+
def time_left(self):
|
| 70 |
+
return self.deadline - time.time() if self.deadline else float("inf")
|
| 71 |
+
|
| 72 |
+
def check_time(self):
|
| 73 |
+
"""Raise before paying for a segment that could not finish before the deadline."""
|
| 74 |
+
if self.time_left() < VIDEO_SECONDS_NEEDED:
|
| 75 |
+
raise DeadlineExceededError("Not enough runtime left for another video.")
|
| 76 |
+
|
| 77 |
def edit_image(self, image, prompt):
|
| 78 |
from huggingface_hub import InferenceClient
|
| 79 |
|
| 80 |
+
self.check_time()
|
| 81 |
with self.semaphore:
|
| 82 |
reservation = self.budget.reserve_media(self.profile["edit_usd"])
|
| 83 |
client = InferenceClient(provider=self.profile["provider"], api_key=self.token)
|
|
|
|
| 87 |
model=self.profile["edit_model"],
|
| 88 |
guidance_scale=self.profile["edit_guidance_scale"],
|
| 89 |
)
|
| 90 |
+
megapixels = edited.width * edited.height / 1e6
|
| 91 |
+
self.budget.record_media(reservation, megapixels * self.profile["edit_usd_per_megapixel"])
|
| 92 |
# fal returns a square image; keep the camera's size and aspect ratio.
|
| 93 |
return edited.convert("RGB").resize(image.size, Image.LANCZOS)
|
| 94 |
|
|
|
|
| 96 |
from huggingface_hub.inference._providers.fal_ai import FalAIImageToVideoTask
|
| 97 |
from huggingface_hub.utils import get_session, hf_raise_for_status
|
| 98 |
|
| 99 |
+
self.check_time()
|
|
|
|
| 100 |
# InferenceClient has no image-text-to-video method yet; fal's image-to-video helper speaks
|
| 101 |
# the same queue protocol once its task matches the model's mapping.
|
| 102 |
helper = FalAIImageToVideoTask()
|
|
|
|
| 131 |
while status != "COMPLETED":
|
| 132 |
if status in ("FAILED", "ERROR", "CANCELLED"):
|
| 133 |
raise RuntimeError(f"The video provider reported {status}.")
|
| 134 |
+
if self.time_left() <= 5:
|
| 135 |
+
raise DeadlineExceededError("Runtime limit reached while a video was generating.")
|
| 136 |
time.sleep(self.poll_interval)
|
| 137 |
polled = session.get(f"{base}{path}/status{query}", headers=request.headers, timeout=30)
|
| 138 |
hf_raise_for_status(polled)
|
|
|
|
| 143 |
# The result is a public CDN link; the token is never sent there.
|
| 144 |
video = session.get(url, timeout=120)
|
| 145 |
hf_raise_for_status(video)
|
| 146 |
+
self.budget.record_media(
|
| 147 |
+
reservation, self.profile["video_usd_per_second"] * self.profile["duration_s"]
|
| 148 |
+
)
|
| 149 |
return video.content
|
| 150 |
|
| 151 |
|
|
|
|
| 214 |
entry["status"] = "skipped"
|
| 215 |
return entry
|
| 216 |
try:
|
| 217 |
+
media.check_time()
|
| 218 |
if not isinstance(plan, dict) or not plan.get("edit_prompt"):
|
| 219 |
raise ValueError("The model returned no human-video plan.")
|
| 220 |
robot = full_frames.frames_at(record, [_nearest(ts, span["start"])])[0].convert("RGB")
|
|
|
|
| 246 |
stop.set()
|
| 247 |
entry["status"] = "time_stopped" if isinstance(exc, DeadlineExceededError) else "budget_stopped"
|
| 248 |
except Exception as exc:
|
| 249 |
+
# A short reason for index.json; provider errors carry no credentials.
|
| 250 |
+
entry.update(status="failed", error_type=type(exc).__name__, error=str(exc)[:300])
|
| 251 |
return entry
|
| 252 |
|
| 253 |
with ThreadPoolExecutor(max_workers=min(len(spans), getattr(media, "concurrency", 1))) as pool:
|
model_catalog.py
CHANGED
|
@@ -219,15 +219,23 @@ HUMAN_VIDEO = {
|
|
| 219 |
"video_task": "image-text-to-video",
|
| 220 |
"resolution": "480P",
|
| 221 |
"duration_s": 5,
|
| 222 |
-
#
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 223 |
"edit_usd": 0.04,
|
| 224 |
-
# $0.05 per second at 480p; the clip is 5 s, reserved as 6 s.
|
| 225 |
"video_usd": 0.30,
|
| 226 |
# Default simultaneous media requests per job (user-selectable up to 32).
|
| 227 |
"concurrency": 8,
|
| 228 |
-
# Estimation only
|
| 229 |
-
#
|
| 230 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 231 |
"minutes_per_video": 3,
|
| 232 |
"checked_at": "2026-09-30",
|
| 233 |
"pricing_url": "https://fal.ai/models/minimax/h3/image-to-video",
|
|
@@ -248,6 +256,16 @@ def human_video_profile():
|
|
| 248 |
|
| 249 |
|
| 250 |
def human_video_segment_cost(profile=None):
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 251 |
profile = profile or HUMAN_VIDEO
|
| 252 |
return profile["edit_usd"] + profile["video_usd"]
|
| 253 |
|
|
@@ -261,5 +279,6 @@ def public_models():
|
|
| 261 |
**human_video_profile(),
|
| 262 |
"segment_choices": list(HUMAN_VIDEO_SEGMENT_CHOICES),
|
| 263 |
"parallel_choices": list(HUMAN_VIDEO_PARALLEL_CHOICES),
|
|
|
|
| 264 |
},
|
| 265 |
}
|
|
|
|
| 219 |
"video_task": "image-text-to-video",
|
| 220 |
"resolution": "480P",
|
| 221 |
"duration_s": 5,
|
| 222 |
+
# fal list prices. The edit is billed per output megapixel (fal returns 1024x1024), the video
|
| 223 |
+
# per second of output.
|
| 224 |
+
"edit_usd_per_megapixel": 0.03,
|
| 225 |
+
"edit_megapixels": 1.05,
|
| 226 |
+
"video_usd_per_second": 0.05,
|
| 227 |
+
# Reserved before each request and reconciled to the price above on success.
|
| 228 |
"edit_usd": 0.04,
|
|
|
|
| 229 |
"video_usd": 0.30,
|
| 230 |
# Default simultaneous media requests per job (user-selectable up to 32).
|
| 231 |
"concurrency": 8,
|
| 232 |
+
# Estimation only, calibrated on the first Studio video run (3 ReBot episodes, 159.7 s,
|
| 233 |
+
# 33 subtasks): subtasks ≈ 1 per episode + 1 per 5.3 s. The two VLM calls per video used
|
| 234 |
+
# about 1,750 input and 550 output tokens together. A video takes up to ~3 min on fal.
|
| 235 |
+
"subtasks_per_episode": 1,
|
| 236 |
+
"seconds_per_subtask": 5.3,
|
| 237 |
+
"vlm_input_tokens": 1750,
|
| 238 |
+
"vlm_output_tokens": 550,
|
| 239 |
"minutes_per_video": 3,
|
| 240 |
"checked_at": "2026-09-30",
|
| 241 |
"pricing_url": "https://fal.ai/models/minimax/h3/image-to-video",
|
|
|
|
| 256 |
|
| 257 |
|
| 258 |
def human_video_segment_cost(profile=None):
|
| 259 |
+
"""Expected fal charge for one video (edit + animation) at list prices."""
|
| 260 |
+
profile = profile or HUMAN_VIDEO
|
| 261 |
+
return (
|
| 262 |
+
profile["edit_usd_per_megapixel"] * profile["edit_megapixels"]
|
| 263 |
+
+ profile["video_usd_per_second"] * profile["duration_s"]
|
| 264 |
+
)
|
| 265 |
+
|
| 266 |
+
|
| 267 |
+
def human_video_segment_reservation(profile=None):
|
| 268 |
+
"""Conservative amount held from the spending target while one video is in flight."""
|
| 269 |
profile = profile or HUMAN_VIDEO
|
| 270 |
return profile["edit_usd"] + profile["video_usd"]
|
| 271 |
|
|
|
|
| 279 |
**human_video_profile(),
|
| 280 |
"segment_choices": list(HUMAN_VIDEO_SEGMENT_CHOICES),
|
| 281 |
"parallel_choices": list(HUMAN_VIDEO_PARALLEL_CHOICES),
|
| 282 |
+
"segment_usd": human_video_segment_cost(),
|
| 283 |
},
|
| 284 |
}
|
provider.py
CHANGED
|
@@ -65,11 +65,12 @@ class Budget:
|
|
| 65 |
self.media_attempts += 1
|
| 66 |
return amount
|
| 67 |
|
| 68 |
-
def record_media(self,
|
| 69 |
-
"""A media request returned its output
|
| 70 |
with self.lock:
|
| 71 |
-
self.
|
| 72 |
-
self.
|
|
|
|
| 73 |
self.media_completed += 1
|
| 74 |
|
| 75 |
def reconcile(self, reservation, usage):
|
|
|
|
| 65 |
self.media_attempts += 1
|
| 66 |
return amount
|
| 67 |
|
| 68 |
+
def record_media(self, reservation, cost):
|
| 69 |
+
"""A media request returned its output: replace its reservation with its list price."""
|
| 70 |
with self.lock:
|
| 71 |
+
self.reserved += cost - reservation
|
| 72 |
+
self.measured += cost
|
| 73 |
+
self.media_usd += cost
|
| 74 |
self.media_completed += 1
|
| 75 |
|
| 76 |
def reconcile(self, reservation, usage):
|
studio.py
CHANGED
|
@@ -20,6 +20,7 @@ from model_catalog import (
|
|
| 20 |
HUMAN_VIDEO_SEGMENT_CHOICES,
|
| 21 |
human_video_profile,
|
| 22 |
human_video_segment_cost,
|
|
|
|
| 23 |
model_profile,
|
| 24 |
)
|
| 25 |
from prompt_config import TEMPLATES, validate_prompts
|
|
@@ -215,20 +216,21 @@ def estimate(
|
|
| 215 |
videos = videos_max = media = media_max = video_minutes = 0
|
| 216 |
if HUMAN_VIDEOS in features:
|
| 217 |
video_profile = human_video_profile()
|
| 218 |
-
# One video per subtask
|
| 219 |
-
#
|
| 220 |
cap = human_video_segments or math.inf
|
| 221 |
-
per = video_profile["seconds_per_subtask"]
|
| 222 |
fps = selection["info"]["fps"]
|
| 223 |
-
|
| 224 |
-
videos = sum(min(cap,
|
| 225 |
-
videos_max = sum(min(cap,
|
| 226 |
media = videos * human_video_segment_cost()
|
| 227 |
media_max = videos_max * human_video_segment_cost()
|
| 228 |
video_minutes = math.ceil(videos / human_video_parallel) * video_profile["minutes_per_video"]
|
| 229 |
-
# Two VLM calls per video: plan
|
| 230 |
-
|
| 231 |
-
|
|
|
|
| 232 |
calls += videos * 2
|
| 233 |
if "plan_render" in prompts:
|
| 234 |
plan_calls = max(n, seconds / 5) + (n * 3 if "Interjections & speech" in features else 0)
|
|
@@ -252,12 +254,12 @@ def estimate(
|
|
| 252 |
cpu_max = math.ceil(hours * 60 + 1) * cpu_rate
|
| 253 |
# Range includes retries, grounding, variable segment counts and recovery.
|
| 254 |
return {
|
| 255 |
-
"low": inference * 0.6 + media * 0.
|
| 256 |
"high": inference * 2.5 + media_max + cpu_max,
|
| 257 |
"inference": inference,
|
| 258 |
"media": media,
|
| 259 |
"media_max": media_max,
|
| 260 |
-
"videos":
|
| 261 |
"videos_max": videos_max,
|
| 262 |
"video_minutes": video_minutes,
|
| 263 |
"video_parallel": human_video_parallel if videos else 0,
|
|
@@ -507,7 +509,7 @@ def launch(
|
|
| 507 |
if human_video["profile"] != human_video_profile():
|
| 508 |
raise ValueError("Video pricing changed. Calculate a fresh estimate.")
|
| 509 |
# One video must fit next to one conservative text reservation.
|
| 510 |
-
minimum_reservation +=
|
| 511 |
if budget - plan["costs"]["cpu_max"] < minimum_reservation:
|
| 512 |
raise ValueError(
|
| 513 |
f"This model needs at least ${minimum_reservation + plan['costs']['cpu_max']:.2f} for one conservative request reservation. Increase the target or choose a lower-cost model."
|
|
|
|
| 20 |
HUMAN_VIDEO_SEGMENT_CHOICES,
|
| 21 |
human_video_profile,
|
| 22 |
human_video_segment_cost,
|
| 23 |
+
human_video_segment_reservation,
|
| 24 |
model_profile,
|
| 25 |
)
|
| 26 |
from prompt_config import TEMPLATES, validate_prompts
|
|
|
|
| 216 |
videos = videos_max = media = media_max = video_minutes = 0
|
| 217 |
if HUMAN_VIDEOS in features:
|
| 218 |
video_profile = human_video_profile()
|
| 219 |
+
# One video per subtask. Subtasks grow with each episode's length (calibrated rate in
|
| 220 |
+
# model_catalog.HUMAN_VIDEO); the upper bound allows 1.5x that; the per-episode cap applies.
|
| 221 |
cap = human_video_segments or math.inf
|
| 222 |
+
base, per = video_profile["subtasks_per_episode"], video_profile["seconds_per_subtask"]
|
| 223 |
fps = selection["info"]["fps"]
|
| 224 |
+
expected = [base + r["length"] / fps / per for r in selection["episodes"]]
|
| 225 |
+
videos = sum(min(cap, e) for e in expected)
|
| 226 |
+
videos_max = sum(min(cap, math.ceil(1.5 * e)) for e in expected)
|
| 227 |
media = videos * human_video_segment_cost()
|
| 228 |
media_max = videos_max * human_video_segment_cost()
|
| 229 |
video_minutes = math.ceil(videos / human_video_parallel) * video_profile["minutes_per_video"]
|
| 230 |
+
# Two VLM calls per video: the plan and edit prompt (6 thumbnails), then the video prompt
|
| 231 |
+
# (edited frame + 6 thumbnails). The plan call runs for every capped subtask.
|
| 232 |
+
inputs += videos * video_profile["vlm_input_tokens"]
|
| 233 |
+
outputs += videos * video_profile["vlm_output_tokens"]
|
| 234 |
calls += videos * 2
|
| 235 |
if "plan_render" in prompts:
|
| 236 |
plan_calls = max(n, seconds / 5) + (n * 3 if "Interjections & speech" in features else 0)
|
|
|
|
| 254 |
cpu_max = math.ceil(hours * 60 + 1) * cpu_rate
|
| 255 |
# Range includes retries, grounding, variable segment counts and recovery.
|
| 256 |
return {
|
| 257 |
+
"low": inference * 0.6 + media * 0.8 + cpu_max,
|
| 258 |
"high": inference * 2.5 + media_max + cpu_max,
|
| 259 |
"inference": inference,
|
| 260 |
"media": media,
|
| 261 |
"media_max": media_max,
|
| 262 |
+
"videos": round(videos),
|
| 263 |
"videos_max": videos_max,
|
| 264 |
"video_minutes": video_minutes,
|
| 265 |
"video_parallel": human_video_parallel if videos else 0,
|
|
|
|
| 509 |
if human_video["profile"] != human_video_profile():
|
| 510 |
raise ValueError("Video pricing changed. Calculate a fresh estimate.")
|
| 511 |
# One video must fit next to one conservative text reservation.
|
| 512 |
+
minimum_reservation += human_video_segment_reservation()
|
| 513 |
if budget - plan["costs"]["cpu_max"] < minimum_reservation:
|
| 514 |
raise ValueError(
|
| 515 |
f"This model needs at least ${minimum_reservation + plan['costs']['cpu_max']:.2f} for one conservative request reservation. Increase the target or choose a lower-cost model."
|
test_human_video.py
CHANGED
|
@@ -13,7 +13,11 @@ from PIL import Image
|
|
| 13 |
sys.path.insert(0, str(Path(__file__).parent))
|
| 14 |
import human_video # noqa: E402
|
| 15 |
import studio # noqa: E402
|
| 16 |
-
from model_catalog import
|
|
|
|
|
|
|
|
|
|
|
|
|
| 17 |
from provider import Budget, BudgetExceededError, DeadlineExceededError # noqa: E402
|
| 18 |
|
| 19 |
from lerobot.annotations.steerable_pipeline.reader import EpisodeRecord
|
|
@@ -64,29 +68,41 @@ def episodes(*seconds, fps=30):
|
|
| 64 |
}
|
| 65 |
|
| 66 |
|
| 67 |
-
def
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 68 |
base = studio.estimate(selection(), ["Subtasks"], 0.1, 4, 0.0005)
|
| 69 |
assert base["media"] == 0 and base["videos"] == 0
|
| 70 |
-
# 10 s and 20 s episodes:
|
| 71 |
every = video_estimate(selection())
|
| 72 |
-
|
| 73 |
-
assert every["
|
| 74 |
-
assert every["
|
|
|
|
| 75 |
assert every["high"] - base["high"] > every["media_max"]
|
| 76 |
-
assert every["calls"] == base["calls"] +
|
| 77 |
-
#
|
| 78 |
assert video_estimate(episodes(40, 40))["media"] > every["media"]
|
| 79 |
-
assert video_estimate(episodes(10, 20, 10, 20))["
|
| 80 |
-
assert video_estimate(episodes(
|
|
|
|
| 81 |
# A per-episode cap bounds both the expected and maximum count.
|
| 82 |
capped = video_estimate(selection(), segments=3)
|
| 83 |
-
assert capped["videos"] ==
|
| 84 |
|
| 85 |
|
| 86 |
def test_parallel_generations_shorten_the_video_time_estimate():
|
| 87 |
-
long = episodes(*[60] * 20) #
|
| 88 |
slow, fast = video_estimate(long, parallel=1), video_estimate(long, parallel=32)
|
| 89 |
-
assert slow["video_minutes"] ==
|
| 90 |
assert slow["media"] == fast["media"]
|
| 91 |
with pytest.raises(ValueError, match="parallel"):
|
| 92 |
video_estimate(selection(), parallel=64)
|
|
@@ -179,9 +195,11 @@ def test_launch_refuses_offline_fal_models_or_stale_video_prices():
|
|
| 179 |
def test_media_budget_reserves_before_request_and_stops_at_target():
|
| 180 |
budget = Budget(0.5, 0.15, 0.5)
|
| 181 |
reservation = budget.reserve_media(0.3)
|
| 182 |
-
budget.record_media(reservation)
|
| 183 |
-
|
| 184 |
-
assert budget.summary()["
|
|
|
|
|
|
|
| 185 |
with pytest.raises(BudgetExceededError):
|
| 186 |
budget.reserve_media(0.3)
|
| 187 |
assert budget.summary()["media_requests"] == 1
|
|
@@ -190,11 +208,15 @@ def test_media_budget_reserves_before_request_and_stops_at_target():
|
|
| 190 |
class FakeMedia:
|
| 191 |
concurrency = 4
|
| 192 |
|
| 193 |
-
def __init__(self, fail_first=False, budget_after=None):
|
| 194 |
self.profile = human_video_profile()
|
| 195 |
-
self.fail_first, self.budget_after = fail_first, budget_after
|
| 196 |
self.edits, self.videos = [], []
|
| 197 |
|
|
|
|
|
|
|
|
|
|
|
|
|
| 198 |
def edit_image(self, image, prompt):
|
| 199 |
self.edits.append((image.size, prompt))
|
| 200 |
return image.transpose(Image.FLIP_LEFT_RIGHT)
|
|
@@ -313,9 +335,9 @@ def test_parallel_setting_bounds_simultaneous_media_requests():
|
|
| 313 |
import threading
|
| 314 |
|
| 315 |
budget = Budget(5, 0.15, 0.5)
|
| 316 |
-
assert human_video.FalMedia("t", budget
|
| 317 |
-
assert human_video.FalMedia("t", budget,
|
| 318 |
-
media = human_video.FalMedia("t", budget,
|
| 319 |
active, peak, lock = [0], [0], threading.Lock()
|
| 320 |
|
| 321 |
def request():
|
|
@@ -415,7 +437,7 @@ def test_video_polls_router_charges_quoted_price_and_never_sends_token_to_cdn(fa
|
|
| 415 |
session = FakeSession(["IN_PROGRESS", "COMPLETED"])
|
| 416 |
use(session)
|
| 417 |
budget = Budget(5, 0.15, 0.5)
|
| 418 |
-
media = human_video.FalMedia("hf_token", budget,
|
| 419 |
assert media.image_to_video(Image.new("RGB", (64, 48)), "a prompt") == b"mp4-bytes"
|
| 420 |
assert seen["task"] == "image-text-to-video" and seen["model"] == "MiniMaxAI/MiniMax-H3"
|
| 421 |
assert seen["parameters"] == {"prompt": "a prompt", "resolution": "480P", "duration": 5}
|
|
@@ -423,7 +445,8 @@ def test_video_polls_router_charges_quoted_price_and_never_sends_token_to_cdn(fa
|
|
| 423 |
assert [c[1] for c in session.calls if c[0] == "get"][:2] == [status_url, status_url]
|
| 424 |
cdn = [c for c in session.calls if c[1].startswith("https://cdn.fal.media")]
|
| 425 |
assert cdn and cdn[0][2] is None
|
| 426 |
-
assert budget.summary()["media_usd"] == pytest.approx(
|
|
|
|
| 427 |
|
| 428 |
|
| 429 |
def test_video_is_not_started_without_runtime_for_it(fal):
|
|
@@ -431,7 +454,7 @@ def test_video_is_not_started_without_runtime_for_it(fal):
|
|
| 431 |
session = FakeSession([])
|
| 432 |
use(session)
|
| 433 |
budget = Budget(5, 0.15, 0.5)
|
| 434 |
-
media = human_video.FalMedia("hf_token", budget,
|
| 435 |
with pytest.raises(DeadlineExceededError):
|
| 436 |
media.image_to_video(Image.new("RGB", (64, 48)), "a prompt")
|
| 437 |
assert session.calls == [] and budget.summary()["media_requests"] == 0
|
|
@@ -483,3 +506,40 @@ def test_media_endpoint_serves_only_generated_files(monkeypatch):
|
|
| 483 |
assert "secret-token" not in json.dumps(listed)
|
| 484 |
anonymous = TestClient(server.app).get("/api/jobs/job1/media?path=" + listed[0]["video"].split("=")[1])
|
| 485 |
assert anonymous.status_code == 401
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 13 |
sys.path.insert(0, str(Path(__file__).parent))
|
| 14 |
import human_video # noqa: E402
|
| 15 |
import studio # noqa: E402
|
| 16 |
+
from model_catalog import ( # noqa: E402
|
| 17 |
+
human_video_profile,
|
| 18 |
+
human_video_segment_cost,
|
| 19 |
+
human_video_segment_reservation,
|
| 20 |
+
)
|
| 21 |
from provider import Budget, BudgetExceededError, DeadlineExceededError # noqa: E402
|
| 22 |
|
| 23 |
from lerobot.annotations.steerable_pipeline.reader import EpisodeRecord
|
|
|
|
| 68 |
}
|
| 69 |
|
| 70 |
|
| 71 |
+
def test_video_estimate_matches_the_calibration_run():
|
| 72 |
+
# First Studio video run: 3 ReBot episodes (68.6 s, 46.8 s, 44.4 s) had 33 subtasks.
|
| 73 |
+
run = video_estimate(episodes(68.6, 46.8, 44.4))
|
| 74 |
+
assert run["videos"] == 33
|
| 75 |
+
# fal list prices: $0.03/MP for a 1024x1024 edit + $0.05/s for a 5 s 480p video.
|
| 76 |
+
assert human_video_segment_cost() == pytest.approx(0.03 * 1.05 + 0.05 * 5)
|
| 77 |
+
assert run["media"] == pytest.approx(run["videos"] * human_video_segment_cost(), rel=0.03)
|
| 78 |
+
assert human_video_segment_reservation() > human_video_segment_cost()
|
| 79 |
+
|
| 80 |
+
|
| 81 |
+
def test_video_estimate_scales_with_total_seconds_and_episode_count():
|
| 82 |
base = studio.estimate(selection(), ["Subtasks"], 0.1, 4, 0.0005)
|
| 83 |
assert base["media"] == 0 and base["videos"] == 0
|
| 84 |
+
# 10 s and 20 s episodes: 1 + s / 5.3 subtasks each -> 7.66 videos, up to 5 + 8.
|
| 85 |
every = video_estimate(selection())
|
| 86 |
+
expected = (1 + 10 / 5.3) + (1 + 20 / 5.3)
|
| 87 |
+
assert every["videos"] == 8 and every["videos_max"] == 13
|
| 88 |
+
assert every["media"] == pytest.approx(expected * human_video_segment_cost())
|
| 89 |
+
assert every["media_max"] == pytest.approx(13 * human_video_segment_cost())
|
| 90 |
assert every["high"] - base["high"] > every["media_max"]
|
| 91 |
+
assert every["calls"] == pytest.approx(base["calls"] + expected * 2, abs=1)
|
| 92 |
+
# More seconds and more episodes both add videos; every episode gets at least one.
|
| 93 |
assert video_estimate(episodes(40, 40))["media"] > every["media"]
|
| 94 |
+
assert video_estimate(episodes(10, 20, 10, 20))["media"] == pytest.approx(2 * every["media"])
|
| 95 |
+
assert video_estimate(episodes(15, 15, 15))["videos"] > video_estimate(episodes(45))["videos"]
|
| 96 |
+
assert video_estimate(episodes(1, 2))["videos"] == 3
|
| 97 |
# A per-episode cap bounds both the expected and maximum count.
|
| 98 |
capped = video_estimate(selection(), segments=3)
|
| 99 |
+
assert capped["videos"] == 6 and capped["videos_max"] == 6
|
| 100 |
|
| 101 |
|
| 102 |
def test_parallel_generations_shorten_the_video_time_estimate():
|
| 103 |
+
long = episodes(*[60] * 20) # 20 * (1 + 60 / 5.3) = 246.4 videos expected
|
| 104 |
slow, fast = video_estimate(long, parallel=1), video_estimate(long, parallel=32)
|
| 105 |
+
assert slow["video_minutes"] == 247 * 3 and fast["video_minutes"] == 8 * 3
|
| 106 |
assert slow["media"] == fast["media"]
|
| 107 |
with pytest.raises(ValueError, match="parallel"):
|
| 108 |
video_estimate(selection(), parallel=64)
|
|
|
|
| 195 |
def test_media_budget_reserves_before_request_and_stops_at_target():
|
| 196 |
budget = Budget(0.5, 0.15, 0.5)
|
| 197 |
reservation = budget.reserve_media(0.3)
|
| 198 |
+
budget.record_media(reservation, 0.25)
|
| 199 |
+
# Success replaces the reservation with the list price.
|
| 200 |
+
assert budget.summary()["media_usd"] == pytest.approx(0.25)
|
| 201 |
+
assert budget.summary()["measured_inference_usd"] == pytest.approx(0.25)
|
| 202 |
+
assert budget.reserved == pytest.approx(0.25)
|
| 203 |
with pytest.raises(BudgetExceededError):
|
| 204 |
budget.reserve_media(0.3)
|
| 205 |
assert budget.summary()["media_requests"] == 1
|
|
|
|
| 208 |
class FakeMedia:
|
| 209 |
concurrency = 4
|
| 210 |
|
| 211 |
+
def __init__(self, fail_first=False, budget_after=None, out_of_time=False):
|
| 212 |
self.profile = human_video_profile()
|
| 213 |
+
self.fail_first, self.budget_after, self.out_of_time = fail_first, budget_after, out_of_time
|
| 214 |
self.edits, self.videos = [], []
|
| 215 |
|
| 216 |
+
def check_time(self):
|
| 217 |
+
if self.out_of_time:
|
| 218 |
+
raise DeadlineExceededError("Not enough runtime left for another video.")
|
| 219 |
+
|
| 220 |
def edit_image(self, image, prompt):
|
| 221 |
self.edits.append((image.size, prompt))
|
| 222 |
return image.transpose(Image.FLIP_LEFT_RIGHT)
|
|
|
|
| 335 |
import threading
|
| 336 |
|
| 337 |
budget = Budget(5, 0.15, 0.5)
|
| 338 |
+
assert human_video.FalMedia("t", budget).concurrency == human_video_profile()["concurrency"]
|
| 339 |
+
assert human_video.FalMedia("t", budget, concurrency=64).concurrency == 32
|
| 340 |
+
media = human_video.FalMedia("t", budget, concurrency=3)
|
| 341 |
active, peak, lock = [0], [0], threading.Lock()
|
| 342 |
|
| 343 |
def request():
|
|
|
|
| 437 |
session = FakeSession(["IN_PROGRESS", "COMPLETED"])
|
| 438 |
use(session)
|
| 439 |
budget = Budget(5, 0.15, 0.5)
|
| 440 |
+
media = human_video.FalMedia("hf_token", budget, None, poll_interval=0)
|
| 441 |
assert media.image_to_video(Image.new("RGB", (64, 48)), "a prompt") == b"mp4-bytes"
|
| 442 |
assert seen["task"] == "image-text-to-video" and seen["model"] == "MiniMaxAI/MiniMax-H3"
|
| 443 |
assert seen["parameters"] == {"prompt": "a prompt", "resolution": "480P", "duration": 5}
|
|
|
|
| 445 |
assert [c[1] for c in session.calls if c[0] == "get"][:2] == [status_url, status_url]
|
| 446 |
cdn = [c for c in session.calls if c[1].startswith("https://cdn.fal.media")]
|
| 447 |
assert cdn and cdn[0][2] is None
|
| 448 |
+
assert budget.summary()["media_usd"] == pytest.approx(0.05 * 5)
|
| 449 |
+
assert budget.reserved == pytest.approx(0.05 * 5)
|
| 450 |
|
| 451 |
|
| 452 |
def test_video_is_not_started_without_runtime_for_it(fal):
|
|
|
|
| 454 |
session = FakeSession([])
|
| 455 |
use(session)
|
| 456 |
budget = Budget(5, 0.15, 0.5)
|
| 457 |
+
media = human_video.FalMedia("hf_token", budget, time.time() + 60, poll_interval=0)
|
| 458 |
with pytest.raises(DeadlineExceededError):
|
| 459 |
media.image_to_video(Image.new("RGB", (64, 48)), "a prompt")
|
| 460 |
assert session.calls == [] and budget.summary()["media_requests"] == 0
|
|
|
|
| 506 |
assert "secret-token" not in json.dumps(listed)
|
| 507 |
anonymous = TestClient(server.app).get("/api/jobs/job1/media?path=" + listed[0]["video"].split("=")[1])
|
| 508 |
assert anonymous.status_code == 401
|
| 509 |
+
|
| 510 |
+
|
| 511 |
+
def test_jobs_without_a_runtime_limit_still_generate_videos():
|
| 512 |
+
"""Regression: the first run skipped every video because no runtime limit was set."""
|
| 513 |
+
from worker import video_deadline
|
| 514 |
+
|
| 515 |
+
manifest = {"costs": {"hours": 4}}
|
| 516 |
+
deadline = video_deadline(None, manifest, now=1000.0)
|
| 517 |
+
assert deadline == 1000.0 + 4 * 3600 - 900
|
| 518 |
+
assert video_deadline(5000.0, manifest) == 5000.0
|
| 519 |
+
media = human_video.FalMedia("t", Budget(5, 0.15, 0.5), video_deadline(None, manifest))
|
| 520 |
+
media.check_time() # hours of runtime left: no DeadlineExceededError
|
| 521 |
+
assert human_video.FalMedia("t", Budget(5, 0.15, 0.5), None).time_left() == float("inf")
|
| 522 |
+
|
| 523 |
+
|
| 524 |
+
def test_no_edit_is_paid_for_when_the_video_could_not_finish(tmp_path, episode):
|
| 525 |
+
media = FakeMedia(out_of_time=True)
|
| 526 |
+
result = annotate(tmp_path, episode, media, segments=0)
|
| 527 |
+
assert result["status"] == "completed"
|
| 528 |
+
assert media.edits == [] and media.videos == []
|
| 529 |
+
assert set(result["human_videos"]) <= {"time_stopped", "skipped"}
|
| 530 |
+
|
| 531 |
+
|
| 532 |
+
def test_older_pinned_profiles_get_current_price_fields():
|
| 533 |
+
old = {
|
| 534 |
+
k: v
|
| 535 |
+
for k, v in human_video_profile().items()
|
| 536 |
+
if k not in ("edit_usd_per_megapixel", "video_usd_per_second")
|
| 537 |
+
}
|
| 538 |
+
media = human_video.FalMedia("t", Budget(5, 0.15, 0.5), None, old)
|
| 539 |
+
assert media.profile["edit_usd_per_megapixel"] == 0.03 and media.profile["video_usd_per_second"] == 0.05
|
| 540 |
+
|
| 541 |
+
|
| 542 |
+
def test_failed_segments_record_a_short_reason(tmp_path, episode):
|
| 543 |
+
result = annotate(tmp_path, episode, FakeMedia(fail_first=True))
|
| 544 |
+
failed = next(e for e in result["human_video_entries"] if e["status"] == "failed")
|
| 545 |
+
assert failed["error_type"] == "RuntimeError" and failed["error"] == "provider error"
|
worker.py
CHANGED
|
@@ -301,6 +301,18 @@ def _annotate_episode(record, row, info, root, manifest, provider, staging_dir,
|
|
| 301 |
HUMAN_VIDEO_DIR = "annotation_studio/human_videos"
|
| 302 |
|
| 303 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 304 |
def human_video_operations(entries, index):
|
| 305 |
"""Upload an episode's media and the updated index (source episode IDs until materialized)."""
|
| 306 |
operations = []
|
|
@@ -422,7 +434,7 @@ def run(manifest, work=Path("/workspace/run")):
|
|
| 422 |
FalMedia(
|
| 423 |
token,
|
| 424 |
budget,
|
| 425 |
-
|
| 426 |
manifest["human_video"]["profile"],
|
| 427 |
concurrency=manifest["human_video"].get("parallel"),
|
| 428 |
)
|
|
|
|
| 301 |
HUMAN_VIDEO_DIR = "annotation_studio/human_videos"
|
| 302 |
|
| 303 |
|
| 304 |
+
def video_deadline(deadline, manifest, now=None):
|
| 305 |
+
"""When videos must stop: the runtime limit if one was set, else the job timeout.
|
| 306 |
+
|
| 307 |
+
Without a runtime limit the job still stops at its timeout (counted from job start, before
|
| 308 |
+
setup), so keep a 15 minute margin for setup and the final commits.
|
| 309 |
+
"""
|
| 310 |
+
if deadline:
|
| 311 |
+
return deadline
|
| 312 |
+
now = time.time() if now is None else now
|
| 313 |
+
return now + manifest["costs"]["hours"] * 3600 - 900
|
| 314 |
+
|
| 315 |
+
|
| 316 |
def human_video_operations(entries, index):
|
| 317 |
"""Upload an episode's media and the updated index (source episode IDs until materialized)."""
|
| 318 |
operations = []
|
|
|
|
| 434 |
FalMedia(
|
| 435 |
token,
|
| 436 |
budget,
|
| 437 |
+
video_deadline(deadline, manifest),
|
| 438 |
manifest["human_video"]["profile"],
|
| 439 |
concurrency=manifest["human_video"].get("parallel"),
|
| 440 |
)
|