robot-agent-gallery / scripts /validate_gallery.py
Alan0928's picture
Serve standalone gallery with versioned Hugging Face Dataset media
1f42c47 verified
Raw History Blame Contribute Delete
56 kB
#!/usr/bin/env python3
"""Validate the complete public gallery with the Python standard library."""
from __future__ import annotations
import argparse
from collections import Counter
from concurrent.futures import ThreadPoolExecutor
import csv
from fractions import Fraction
import hashlib
import json
import math
from pathlib import Path, PurePosixPath
import re
import shutil
import subprocess
import sys
DEFAULT_ROOT = Path(__file__).resolve().parents[1]
EXCLUDED = {".git", "node_modules", "_site", "artifacts", ".venv", "__pycache__", ".gallery-cache", ".cache", ".pytest_cache"}
# Space content guard for the complete local archive (~4.1 GB), including RoboCasa.
# Historical Pages/Release byte counters below remain part of the frozen export audit.
MAX_SITE_BYTES = 6_000_000_000
MAX_FILE_BYTES = 2 * 1024**3
EXPECTED_SUITES = {"libero_spatial": 95, "libero_goal": 79, "libero_object": 89, "libero_10": 59}
EXPECTED_RUNS = {
"libero": ("astra_libero_v5_calibrated_parallel10_r4",
"df08694520ff7ea3e9bfee061a29223e46c4db76bc54a07e637cbfcbe93f42f8"),
"robotwin": ("robotwin_astra_firstpass_20260917",
"8f0b9bfc28b7e2dccc7fe7a056114f1b7d2fdbf36e4bcbf72771931c5a9c6c38"),
"robocasa": ("robocasa365_astra_firstpass_20260918",
"3a2a7f6939e18b5ee3fb6d6bac10d8f2f8704cd422d91efc8e56a038cf28c12c"),
"robodojo": ("robodojo42_astra_firstpass_20260919",
"fe327cacc7615e868096a025c1287a8ad070408851bac9104cf132c1349f6fd4"),
"robotwin_nvidia10": ("robotwin_goal_spec_full_v1",
"5044e7c5eaed3389b689c0534c0863966d5ee1b3fbafc9e4990a0c85960a5f88"),
}
# Exact standard task matrix from the frozen RoboDojo manifest. Native names are
# case-sensitive; gallery IDs are lowercased and the suite ID uses underscores.
ROBODOJO_TASKS = {
"generalization": ("stack_bowls", "push_T", "pack_objects_into_box", "fold_clothes", "hang_mugs",
"sweep_blocks", "pour_liquid_into_cup", "make_toast", "arrange_largest_number",
"sort_nesting_dolls_by_size", "store_laptop_and_headphones", "stack_blocks"),
"memory": ("cover_blocks", "match_and_pick_from_conveyor", "swap_blocks", "swap_T", "press_by_number",
"imitate_sorting_sequence"),
"precision": ("fasten_screws", "plug_in_charger", "insert_tubes", "pour_balls_into_vase", "play_Xylophone",
"deposit_coin", "insert_key", "build_tower"),
"long-horizon": ("fill_pen_holder", "classify_objects", "put_bottles_into_dustbin", "play_tic_tac_toe",
"fill_egg_holder", "organize_table", "make_kong", "play_stacking_toy"),
"open": ("align_blocks", "general_pickup", "solve_equation", "stack_blocks_by_language",
"classify_objects_by_language", "pick_from_conveyor_by_image", "store_tools_in_toolbox",
"pour_by_language"),
}
ROBODOJO_SUITES = {"robodojo_" + group.replace("-", "_"): names for group, names in ROBODOJO_TASKS.items()}
ROBODOJO_HORIZONS = dict(zip(
(name for names in ROBODOJO_TASKS.values() for name in names),
(800, 600, 1300, 500, 800, 1000, 400, 1400, 1050, 1050, 800, 550,
800, 700, 700, 400, 700, 1600, 1900, 400, 500, 600, 500, 300, 300, 1050,
1100, 1100, 700, 1100, 700, 1000, 600, 1200, 200, 200, 300, 400, 1100, 700, 900, 800),
))
ROBODOJO_PROVENANCE_KEYS = {"run", "manifestSha256", "auditKind", "auditSha256", "policyBoundaryAuditSha256"}
ROBODOJO_EPISODE_KEYS = {"id", "index", "status", "seed", "initStateId", "steps", "maxSteps", "toolCalls",
"wallSeconds", "durationSeconds", "video", "poster", "width", "height", "frames",
"nativeScore", "terminationReason", "layoutId"}
ROBODOJO_SELECTION_KEYS = {"episode", "sourceEpisodeKey", "seed", "selectedAttempt", "resultSha256",
"sourceVideoSha256", "instructionSha256", "nativeSuccess", "nativeScore",
"status", "steps", "excludedAttempts", "nativeStateSha256"}
PRIVATE_KEYS = {
"auth", "authorization", "authentication", "credentials", "credential", "apikey",
"accesstoken", "refreshtoken", "idtoken", "bearertoken", "secret", "password",
"thread", "threads", "threadid", "conversation", "conversations", "conversationid",
"codexhome", "workspace", "workspacedir", "rawlogs", "rawevents", "reasoning",
"reasoningcontent", "chainofthought", "transcript", "stdout", "stderr", "prompt",
"systemprompt", "messages", "oauth", "authorizationheader", "eventlog", "codexevents",
}
class ValidationError(Exception):
"""An exported asset does not satisfy the publication contract."""
def require(condition, message):
if not condition:
raise ValidationError(message)
def read_json(path):
try:
return json.loads(path.read_text(encoding="utf-8"))
except (OSError, ValueError) as exc:
raise ValidationError(f"Cannot read JSON {path.name}: {exc}") from exc
def integer(value, label, minimum=0):
require(type(value) is int and value >= minimum, f"{label} must be an integer >= {minimum}")
return value
def public_metadata(value, location="data"):
"""Check public JSON without printing potentially private values in errors."""
if isinstance(value, dict):
for key, child in value.items():
normalized = re.sub(r"[^a-z0-9]", "", key.lower())
require(normalized not in PRIVATE_KEYS, f"Private metadata key at {location}")
public_metadata(child, f"{location}.{key}")
elif isinstance(value, list):
for index, child in enumerate(value):
public_metadata(child, f"{location}[{index}]")
elif isinstance(value, str):
require(not re.search(r"(?:^|[\s\"'=])/(?:home|playpen|tmp|Users|root|mnt|private)/", value),
f"Machine-local path at {location}")
require(not value.startswith(("/", "file:", "\\\\")) and not re.match(r"^[A-Za-z]:[\\/]", value),
f"Absolute local path at {location}")
require(not re.search(r"\bsk-[A-Za-z0-9_-]{20,}\b", value), f"Possible secret at {location}")
require(not any(marker in value.lower() for marker in ("auth.json", "codex_home", "codex-events.jsonl")),
f"Private artifact reference at {location}")
elif type(value) is float:
require(math.isfinite(value), f"Non-finite metadata number at {location}")
def local_asset(root, relative, label, *, allow_missing=False):
require(isinstance(relative, str) and relative, f"Missing {label} path")
path = PurePosixPath(relative)
require(not path.is_absolute() and ".." not in path.parts and "\\" not in relative
and ":" not in relative and "?" not in relative and "#" not in relative,
f"{label} must be a plain repository-relative path")
require(path.parts[0] in {"media", "assets"}, f"{label} must be a public media asset")
candidate = root.joinpath(*path.parts)
require(allow_missing or candidate.is_file(), f"Missing {label}: {relative}")
require(not candidate.is_symlink() and candidate.resolve().is_relative_to(root.resolve()),
f"Unsafe {label}: {relative}")
return candidate
def sha256(path):
digest = hashlib.sha256()
with path.open("rb") as stream:
for block in iter(lambda: stream.read(1024 * 1024), b""):
digest.update(block)
return digest.hexdigest()
def check_tree(root, external_media=()):
total, site_total, files = 0, 0, 0
for path in root.rglob("*"):
relative = path.relative_to(root)
if any(part in EXCLUDED for part in relative.parts):
continue
require(not path.is_symlink(), f"Symlinks are not allowed: {relative.as_posix()}")
if path.is_file():
size = path.stat().st_size
external = relative.as_posix() in external_media
require(size < (2 * 1024**3 if external else MAX_FILE_BYTES),
f"File exceeds its hosting limit: {relative.as_posix()}")
total += size
if not external:
site_total += size
files += 1
require(total < MAX_SITE_BYTES, f"Space archive exceeds the configured size limit: {total} bytes")
return {"files": files, "bytes": total, "siteBytes": site_total}
def probe_video(item):
path, expected = item
label = str(path).rsplit("/", 1)[-1]
completed = subprocess.run(
["ffprobe", "-v", "error", "-count_frames", "-select_streams", "v:0", "-show_entries",
"stream=codec_name,pix_fmt,width,height,nb_read_frames,avg_frame_rate", "-of", "json", str(path)],
check=False, capture_output=True, text=True, timeout=120,
)
require(completed.returncode == 0 and not completed.stderr.strip(), f"Video decode failed: {label}")
try:
streams = json.loads(completed.stdout)["streams"]
require(len(streams) == 1, f"Expected one video stream: {label}")
stream = streams[0]
require(stream["codec_name"] == "h264", f"Expected H.264: {label}")
require(stream["pix_fmt"] == "yuv420p", f"Expected yuv420p: {label}")
require(stream["width"] == expected["width"] and stream["height"] == expected["height"],
f"Video dimensions differ from metadata: {label}")
frames = int(stream["nb_read_frames"])
require(frames == expected["frames"], f"Video frame count differs from source mapping: {label}")
numerator, denominator = map(int, stream["avg_frame_rate"].split("/"))
require(denominator > 0 and abs(numerator / denominator - expected["fps"]) < 1e-8,
f"Video frame rate differs from metadata: {label}")
return frames
except (KeyError, ValueError, TypeError) as exc:
raise ValidationError(f"Incomplete video metadata: {label}") from exc
def counts(episodes, tasks=None):
statuses = Counter(episode["status"] for episode in episodes)
result = {
"episodes": len(episodes), "successes": statuses["success"],
"failures": statuses["failure"] + statuses["timeout"], "timeouts": statuses["timeout"],
"successRate": statuses["success"] / len(episodes) if episodes else 0,
}
if tasks is not None:
result["tasks"] = tasks
return result
def check_summary(actual, expected, label, keys=None):
require(isinstance(actual, dict), f"Missing {label}")
for key in keys or expected:
require(actual.get(key) == expected[key], f"Wrong {label}.{key}: expected {expected[key]}")
def check_digest(value, label):
require(isinstance(value, str) and re.fullmatch(r"[0-9a-f]{64}", value), f"Invalid {label} fingerprint")
def public_fields(value, allowed, label):
require(isinstance(value, dict) and set(value) <= allowed, f"Unexpected public {label} fields")
def validate_robodojo_task(task):
public_fields(task, {"id", "name", "instruction", "instructionSha256", "suite", "suiteName", "episodes",
"successes", "failures", "successRate", "nativeTaskName", "taskGroup"}, "RoboDojo task")
native_name = task.get("nativeTaskName")
group = task.get("taskGroup")
require(group in ROBODOJO_TASKS and native_name in ROBODOJO_TASKS[group]
and task["id"] == "robodojo_" + native_name.lower()
and task["suite"] == "robodojo_" + group.replace("-", "_"),
"RoboDojo native task/group identity mismatch")
digest = hashlib.sha256(task["instruction"].encode("utf-8")).hexdigest()
require(task.get("instructionSha256") == digest, "RoboDojo native instruction fingerprint mismatch")
def validate_robodojo_episode(episode, task):
public_fields(episode, ROBODOJO_EPISODE_KEYS, "RoboDojo episode")
require(episode["seed"] == 0 and episode["maxSteps"] == ROBODOJO_HORIZONS[task["nativeTaskName"]],
"Incorrect RoboDojo seed or native horizon")
require(type(episode.get("layoutId")) is int and episode["layoutId"] == 0, "Incorrect RoboDojo native layout")
score = episode.get("nativeScore")
require(type(score) in (int, float) and math.isfinite(score) and 0 <= score <= 1,
"Invalid RoboDojo native score")
reason = episode.get("terminationReason")
require(reason in {"step_budget", "agent_finished", "robodojo_success", "environment_done"},
"Invalid RoboDojo termination reason")
require((reason == "robodojo_success") == (episode["status"] == "success"),
"RoboDojo termination/outcome mismatch")
if reason == "step_budget":
require(episode["steps"] == episode["maxSteps"], "RoboDojo exhausted horizon mismatch")
def validate_training_demos(root, gallery, *, required=False):
"""Validate separate training references without changing evaluation counts."""
catalog_path = root / "data/task-demos.json"
if not catalog_path.exists():
require(not required, "Training demonstration catalog is required")
return None, [], set()
catalog = read_json(catalog_path)
public_metadata(catalog, "data/task-demos.json")
require(catalog.get("schemaVersion") == 1, "Unsupported training demo schema")
records = catalog.get("tasks")
expected = {task["id"]: (benchmark["id"], task)
for benchmark in gallery["benchmarks"] for task in benchmark["tasks"]}
# Older evaluation-only RoboDojo imports may have no demonstration group.
# Once imported, every task needs an available or unavailable source record.
if isinstance(records, dict) and not any(task_id in records for task_id, (benchmark_id, _) in expected.items()
if benchmark_id == "robodojo"):
expected = {task_id: value for task_id, value in expected.items() if value[0] != "robodojo"}
require(isinstance(records, dict) and set(records) == set(expected),
"Training demo catalog must account for every gallery task exactly once")
coverage = {benchmark["id"]: {"tasks": len(benchmark["tasks"]), "available": 0, "unavailable": 0}
for benchmark in gallery["benchmarks"]
if any(value[0] == benchmark["id"] for value in expected.values())}
videos, media_paths = [], set()
for task_id, record in records.items():
benchmark_id, task = expected[task_id]
require(record.get("taskId") == task_id and record.get("benchmark") == benchmark_id,
f"Training demo task identity mismatch: {task_id}")
status = record.get("status")
require(status in {"available", "unavailable"}, f"Invalid training demo status: {task_id}")
coverage[benchmark_id][status] += 1
source = record.get("source", {})
require(isinstance(source, dict) and isinstance(source.get("dataset"), str) and source["dataset"].strip()
and isinstance(source.get("url"), str) and source["url"].startswith("https://"),
f"Missing training dataset attribution: {task_id}")
if status == "unavailable":
require(isinstance(record.get("reason"), str) and record["reason"].strip(),
f"Missing unavailable-demo explanation: {task_id}")
require(not any(key in record for key in ("video", "poster", "remoteVideo", "frames")),
f"Unavailable training demo must not claim media: {task_id}")
continue
relative_source = source.get("relativePath")
require(isinstance(relative_source, str) and relative_source and
not PurePosixPath(relative_source).is_absolute() and ".." not in PurePosixPath(relative_source).parts,
f"Missing relative training source path: {task_id}")
require(source.get("episode") is not None, f"Missing training episode selection: {task_id}")
if benchmark_id in {"robotwin", "robocasa", "robodojo"}:
require(isinstance(source.get("taskName"), str) and
f"{benchmark_id}_{source['taskName'].lower()}" == task_id,
f"Training source task differs from gallery task: {task_id}")
if benchmark_id == "robodojo":
require(source["taskName"] == task.get("nativeTaskName"),
f"Training source native task differs from gallery task: {task_id}")
for key in ("width", "height", "frames"):
integer(record.get(key), f"{task_id} demo {key}", 1)
fps = record.get("fps")
require(type(fps) in (int, float) and math.isfinite(fps) and fps > 0,
f"Invalid training demo frame rate: {task_id}")
duration = record.get("durationSeconds")
require(type(duration) in (int, float) and math.isfinite(duration)
and abs(duration - record["frames"] / fps) < .001,
f"Training demo duration/frame mismatch: {task_id}")
require(isinstance(record.get("cameras"), list) and record["cameras"] and
all(isinstance(value, str) and value.strip() for value in record["cameras"]),
f"Missing training camera labels: {task_id}")
for key, suffix in (("video", ".mp4"), ("poster", ".jpg")):
relative = f"media/demos/{benchmark_id}/{task_id}{suffix}"
require(record.get(key) == relative and relative not in media_paths,
f"Incorrect training demo asset mapping: {task_id}")
path = local_asset(root, relative, f"training {key}")
check_digest(record.get(f"{key}Sha256"), f"training {key}")
require(sha256(path) == record[f"{key}Sha256"], f"Training media digest mismatch: {relative}")
if key == "video" or f"{key}Bytes" in record:
require(path.stat().st_size == record.get(f"{key}Bytes"),
f"Training media byte count mismatch: {relative}")
media_paths.add(relative)
videos.append((root / record["video"], {key: record[key] for key in ("width", "height", "frames", "fps")}))
require(catalog.get("coverage") == coverage, "Training demo coverage summary differs from catalog")
return coverage, videos, media_paths
def validate_blog_media(root):
"""Allow only explicit, complete essay clip references into the media tree.
The separate blog validator checks experiment claims and clip fingerprints.
This inventory check also supports references to existing baseline media.
"""
path = root / "data/libero-blog-media.json"
if not path.exists():
return set()
catalog = read_json(path)
public_metadata(catalog, "data/libero-blog-media.json")
require(isinstance(catalog, dict) and catalog.get("schemaVersion") == 1,
"Unsupported blog media schema")
require(catalog.get("complete") is True, "The blog media export is not complete")
clips = catalog.get("clips")
require(isinstance(clips, dict) and clips, "Blog media must declare its clips")
paths = set()
for identifier, clip in clips.items():
require(isinstance(clip, dict) and clip.get("id") == identifier,
"Invalid blog clip identity")
for key, suffix in (("video", ".mp4"), ("poster", ".jpg")):
relative = clip.get(key)
asset = local_asset(root, relative, f"blog {key}")
require(PurePosixPath(relative).parts[0] == "media" and asset.suffix == suffix,
f"Blog {key} must reference a {suffix} file under media")
paths.add(relative)
return paths
def validate_media_inventory(root, media_paths, external_media=()):
actual_media = {path.relative_to(root).as_posix() for path in (root / "media").rglob("*") if path.is_file()}
require(media_paths - set(external_media) <= actual_media <= media_paths,
"The media directory contains missing or unlisted files")
def validate_gallery(root, *, check_interface=True, require_robocasa=False, release_assets=None,
require_demos=False, require_robodojo=False):
required_files = ("index.html", "app.js", "styles.css", ".nojekyll", "METHODOLOGY.md") if check_interface else ()
for filename in required_files:
require((root / filename).is_file(), f"Required public file is missing: {filename}")
expected_data_files = {"gallery.json", "episodes.csv", "export-report.json"}
if (root / "data/task-demos.json").exists():
expected_data_files.add("task-demos.json")
for name in ("libero-blog-media.json", "libero-alignment.json", "libero-alignment-episodes.csv",
"libero-human-review.json", "robotwin-alignment-summary.json"):
if (root / "data" / name).exists():
expected_data_files.add(name)
require({path.relative_to(root / "data").as_posix() for path in (root / "data").rglob("*") if path.is_file()}
== expected_data_files,
"The public data directory contains missing or unexpected files")
gallery = read_json(root / "data/gallery.json")
report = read_json(root / "data/export-report.json")
for path in (root / "data").rglob("*.json"):
public_metadata(read_json(path), path.relative_to(root).as_posix())
require(gallery.get("schemaVersion") == 1 and report.get("schemaVersion") == 1, "Unsupported schema version")
require(gallery.get("model") == "gpt-6-astra", "Unexpected model")
require(report.get("complete") is True, "The media export is not complete")
benchmarks = gallery["benchmarks"]
benchmark_ids = {item["id"] for item in benchmarks}
has_robocasa = "robocasa" in benchmark_ids
has_robodojo = "robodojo" in benchmark_ids
has_robotwin_nvidia10 = "robotwin_nvidia10" in benchmark_ids
require(release_assets is None or has_robocasa, "Release validation requires RoboCasa365")
expected_benchmarks = ({"libero", "robotwin"} | ({"robocasa"} if has_robocasa else set())
| ({"robodojo"} if has_robodojo else set())
| ({"robotwin_nvidia10"} if has_robotwin_nvidia10 else set()))
require(benchmark_ids == expected_benchmarks and len(benchmarks) == len(expected_benchmarks),
"Expected LIBERO and RoboTwin, with complete optional benchmark collections")
require(not require_robocasa or has_robocasa, "The complete RoboCasa365 collection is required")
require(not require_robodojo or has_robodojo, "The complete RoboDojo42 collection is required")
expected_episodes, expected_tasks_total = (815, 455) if has_robocasa else (450, 90)
if has_robodojo:
expected_episodes += 42
expected_tasks_total += 42
if has_robotwin_nvidia10:
expected_episodes += 479
expected_tasks_total += 50
require(gallery.get("evaluationDate") == ("2026-09-22" if has_robotwin_nvidia10 else "2026-09-19" if has_robodojo else "2026-09-18" if has_robocasa else "2026-09-17"),
"Unexpected evaluation date")
if has_robocasa or has_robodojo or has_robotwin_nvidia10:
dates = ["2026-09-17"] + (["2026-09-18"] if has_robocasa else []) + (["2026-09-19"] if has_robodojo else []) + (["2026-09-22"] if has_robotwin_nvidia10 else [])
require(gallery.get("evaluationDates") == dates, "Incorrect evaluation dates")
require(report.get("expectedEpisodes") == expected_episodes and report.get("verifiedEpisodes") == expected_episodes,
f"The export must contain {expected_episodes} verified episodes")
episodes, task_ids, media_paths, summaries = {}, set(), set(), {}
benchmark_by_id = {item["id"]: item for item in benchmarks}
external_media = set()
if has_robocasa:
from release_media import RELEASE_HOSTING, remote_video_url
require(report.get("videoHosting") == {"robocasa": RELEASE_HOSTING}
and benchmark_by_id["robocasa"].get("videoHosting") == RELEASE_HOSTING,
"Incorrect RoboCasa video hosting declaration")
external_media = {episode["video"] for task in benchmark_by_id["robocasa"]["tasks"] for episode in task["episodes"]}
tree = check_tree(root, external_media)
for benchmark in benchmarks:
benchmark_id = benchmark["id"]
is_libero = benchmark_id == "libero"
is_robocasa = benchmark_id == "robocasa"
is_robodojo = benchmark_id == "robodojo"
is_robotwin_nvidia10 = benchmark_id == "robotwin_nvidia10"
if is_robodojo:
public_fields(benchmark, {"id", "name", "evaluationDate", "summary", "suites", "protocol", "provenance", "tasks"},
"RoboDojo benchmark")
public_fields(benchmark["protocol"], {"label", "description", "cameras", "videoNote", "stepsLabel", "episodesPerTask", "trainingDemoNote"},
"RoboDojo protocol")
public_fields(benchmark["provenance"], ROBODOJO_PROVENANCE_KEYS, "RoboDojo provenance")
public_fields(benchmark["summary"], set(counts([], 0)), "RoboDojo summary")
require(is_robocasa or "videoHosting" not in benchmark, "Unexpected external hosting for a historical benchmark")
if has_robocasa or has_robodojo or has_robotwin_nvidia10:
require(benchmark.get("evaluationDate") == ("2026-09-22" if is_robotwin_nvidia10 else "2026-09-19" if is_robodojo else "2026-09-18" if is_robocasa else "2026-09-17"),
f"Incorrect evaluation date: {benchmark_id}")
expected_tasks, per_task = {"libero": (40, 10), "robotwin": (50, 1), "robocasa": (365, 1), "robodojo": (42, 1),
"robotwin_nvidia10": (50, 10)}[benchmark_id]
tasks = benchmark["tasks"]
require(len(tasks) == expected_tasks, f"Wrong task count for {benchmark_id}")
require(benchmark["protocol"]["episodesPerTask"] == per_task, f"Wrong sampling for {benchmark_id}")
check_digest(benchmark["provenance"]["manifestSha256"], f"{benchmark_id} manifest")
require((benchmark["provenance"]["run"], benchmark["provenance"]["manifestSha256"]) == EXPECTED_RUNS[benchmark_id],
f"Unexpected frozen source run for {benchmark_id}")
expected_suites = set(EXPECTED_SUITES) if is_libero else ({"robocasa_atomic", "robocasa_composite"} if is_robocasa else {"robotwin_nvidia10"} if is_robotwin_nvidia10 else {"robotwin"})
if is_robodojo:
expected_suites = set(ROBODOJO_SUITES)
suites = benchmark["suites"]
require(len(suites) == len(expected_suites) and {item["id"] for item in suites} == expected_suites,
f"Wrong suites for {benchmark_id}")
all_episodes = []
for task in tasks:
task_id = task["id"]
require(isinstance(task_id, str) and re.fullmatch(r"[a-z0-9_]+", task_id), "Invalid task ID")
require(task_id not in task_ids, f"Duplicate task ID: {task_id}")
task_ids.add(task_id)
require(task["suite"] in expected_suites, f"Wrong suite for {task_id}")
require(isinstance(task["instruction"], str) and (is_robodojo or task["instruction"].strip()), f"Missing instruction: {task_id}")
if is_robodojo:
validate_robodojo_task(task)
task_episodes = task["episodes"]
if is_robotwin_nvidia10:
require(1 <= len(task_episodes) <= per_task, f"Wrong episode count for {task_id}")
require({item["index"] for item in task_episodes}.issubset(set(range(per_task))), f"Wrong rollout indices: {task_id}")
else:
require(len(task_episodes) == per_task, f"Wrong episode count for {task_id}")
require({item["index"] for item in task_episodes} == set(range(per_task)), f"Wrong rollout indices: {task_id}")
if is_libero:
require({item["initStateId"] for item in task_episodes} == set(range(10)), f"Wrong initial states: {task_id}")
for episode in task_episodes:
episode_id = episode["id"]
require(is_robocasa or "remoteVideo" not in episode, "Unexpected remote video for a historical benchmark")
require(isinstance(episode_id, str) and re.fullmatch(r"[a-z0-9_]+", episode_id), "Invalid episode ID")
require(episode_id not in episodes, f"Duplicate episode ID: {episode_id}")
require(is_libero or task_id.startswith(f"{benchmark_id}_"), f"Unexpected {benchmark_id} task identifier")
expected_prefix = task_id if is_libero or is_robocasa or is_robodojo else task_id.removeprefix("robotwin_")
if is_robotwin_nvidia10:
expected_prefix = task_id
require(episode_id == f"{expected_prefix}_r{episode['index']:02d}", f"Episode/task identity mismatch: {episode_id}")
require(episode["status"] in {"success", "failure", "timeout"}, f"Unscored episode: {episode_id}")
if is_robotwin_nvidia10:
require(isinstance(episode.get("instruction"), str) and episode["instruction"].strip(),
f"Missing episode instruction: {episode_id}")
for key in ("index", "seed", "steps", "maxSteps", "toolCalls", "width", "height", "frames"):
integer(episode[key], f"{episode_id}.{key}", 1 if key in {"width", "height", "frames", "maxSteps"} else 0)
require(episode["steps"] <= episode["maxSteps"] and episode["toolCalls"] <= (1500 if is_robocasa or is_robodojo else 750),
f"Recorded controls exceed the protocol: {episode_id}")
require(episode["steps"] > 0, f"Episode has no recorded action: {episode_id}")
fps = 25 if is_robodojo else 20 if is_libero or is_robocasa else 10
if is_libero:
require(episode["maxSteps"] == 500 and episode["seed"] == episode["index"] == episode["initStateId"],
f"Incorrect LIBERO episode protocol: {episode_id}")
require(episode["frames"] == episode["steps"] + 11, f"LIBERO warmup/frame mismatch: {episode_id}")
elif is_robotwin_nvidia10:
require(episode["initStateId"] is None and 0 <= episode["index"] < 10,
f"Incorrect RoboTwin 10-rollout protocol: {episode_id}")
require(episode["seed"] >= 100000,
f"Incorrect RoboTwin 10-rollout seed: {episode_id}")
require(episode["frames"] == episode["steps"] + 1, f"{benchmark_id} action/frame mismatch: {episode_id}")
require(episode["video"] == f"media/robotwin_nvidia10/{episode_id}.mp4"
and episode["poster"] == f"media/robotwin_nvidia10/{episode_id}.jpg",
f"Incorrect RoboTwin 10-rollout local media paths: {episode_id}")
else:
require(episode["index"] == 0 and episode["initStateId"] is None,
f"Incorrect {benchmark_id} episode protocol: {episode_id}")
require(episode["frames"] == episode["steps"] + 1, f"{benchmark_id} action/frame mismatch: {episode_id}")
if is_robodojo:
validate_robodojo_episode(episode, task)
require((episode["width"], episode["height"]) == (1920, 480),
f"Incorrect RoboDojo native camera layout: {episode_id}")
require(episode["video"] == f"media/robodojo/{episode_id}.mp4"
and episode["poster"] == f"media/robodojo/{episode_id}.jpg",
f"Incorrect RoboDojo local media paths: {episode_id}")
if is_robocasa:
require(episode["seed"] >= 100000 and episode["maxSteps"] <= 7200,
f"Incorrect RoboCasa365 seed or step budget: {episode_id}")
require(episode["width"] == 3 * episode["height"], f"Incorrect three-camera layout: {episode_id}")
require(episode.get("remoteVideo") == remote_video_url(episode_id),
f"Incorrect release video URL: {episode_id}")
require(episode["video"] == f"media/robocasa/{episode_id}.mp4"
and episode["poster"] == f"media/robocasa/{episode_id}.jpg",
f"Incorrect RoboCasa local media paths: {episode_id}")
for key in ("wallSeconds", "durationSeconds"):
require(type(episode[key]) in (int, float) and math.isfinite(episode[key]) and episode[key] >= 0,
f"Invalid duration: {episode_id}.{key}")
require(abs(episode["durationSeconds"] - episode["frames"] / fps) < 1e-6,
f"Playback duration/frame mismatch: {episode_id}")
for key, suffix in (("video", ".mp4"), ("poster", ".jpg")):
relative = episode[key]
require(relative not in media_paths, f"Duplicate media reference: {relative}")
media_paths.add(relative)
allow_missing = is_robocasa and key == "video" and release_assets is not None
require(local_asset(root, relative, key, allow_missing=allow_missing).suffix == suffix,
f"Unexpected media format: {relative}")
episodes[episode_id] = {"episode": episode, "benchmark": benchmark_id, "task": task, "fps": fps}
all_episodes.append(episode)
check_summary(task, counts(task_episodes), task_id, ("successes", "failures", "successRate"))
if is_libero:
require({task["id"] for task in tasks} == {f"{suite}_t{index:02d}" for suite in EXPECTED_SUITES for index in range(10)},
"LIBERO must contain tasks 0–9 in each official suite")
expected = counts(all_episodes, len(tasks))
fixed = {"tasks": 40, "episodes": 400, "successes": 322, "failures": 78, "timeouts": 0, "successRate": .805} if is_libero else {
"tasks": 50, "episodes": 50, "successes": 36, "failures": 14, "timeouts": 1, "successRate": .72}
if not is_robocasa and not is_robotwin_nvidia10:
if is_robodojo:
fixed = {"tasks": 42, "episodes": 42, "successes": 6, "failures": 36, "timeouts": 0, "successRate": 6 / 42}
require(expected == fixed, f"Wrong native outcome aggregate for {benchmark_id}")
check_summary(benchmark["summary"], expected, f"{benchmark_id} summary")
summaries[benchmark_id] = expected
for suite in suites:
suite_tasks = [task for task in tasks if task["suite"] == suite["id"]]
selected = [episode for task in suite_tasks for episode in task["episodes"]]
expected = counts(selected, len(suite_tasks))
if is_libero:
require(expected["tasks"] == 10 and expected["episodes"] == 100 and expected["successes"] == EXPECTED_SUITES[suite["id"]],
f"Wrong LIBERO suite outcome: {suite['id']}")
elif is_robocasa:
require(expected["tasks"] == expected["episodes"] == (65 if suite["id"] == "robocasa_atomic" else 300),
f"Wrong RoboCasa365 task group: {suite['id']}")
elif is_robodojo:
public_fields(suite, {"id", "name", *counts([], 0)}, "RoboDojo suite")
require({task["id"] for task in suite_tasks} ==
{"robodojo_" + name.lower() for name in ROBODOJO_SUITES[suite["id"]]},
f"Wrong RoboDojo standard task matrix: {suite['id']}")
check_summary(suite, expected, f"{suite['id']} summary")
require(len(episodes) == expected_episodes and len(task_ids) == expected_tasks_total,
f"Expected {expected_episodes} unique episodes across {expected_tasks_total} tasks")
runs = report["runs"]
require(len(runs) == len(expected_benchmarks) and {run["benchmark"] for run in runs} == expected_benchmarks,
"Wrong provenance runs")
for run in runs:
benchmark_id = run["benchmark"]
is_robocasa = benchmark_id == "robocasa"
is_robodojo = benchmark_id == "robodojo"
is_robotwin_nvidia10 = benchmark_id == "robotwin_nvidia10"
expected_ids = {key for key, item in episodes.items() if item["benchmark"] == benchmark_id}
provenance = benchmark_by_id[benchmark_id]["provenance"]
require(run["run"] == provenance["run"] and run["manifestSha256"] == provenance["manifestSha256"],
f"Provenance mismatch: {benchmark_id}")
sources = {}
if is_robodojo:
public_fields(run, ROBODOJO_PROVENANCE_KEYS | {"benchmark", "selectedEpisodes", "excludedAttemptCount", "attemptSelection"},
"RoboDojo run")
require(run.get("auditKind") == provenance.get("auditKind") == "robodojo_firstpass",
"RoboDojo requires the complete first-pass audits")
for key in ("auditSha256", "policyBoundaryAuditSha256"):
check_digest(run.get(key), f"RoboDojo {key}")
require(run[key] == provenance.get(key), f"RoboDojo {key} mismatch")
if is_robocasa:
require(run.get("auditKind") == provenance.get("auditKind") == "robocasa_collection",
"RoboCasa requires a complete collection audit")
for key in ("auditSha256", "planSha256"):
check_digest(run.get(key), f"RoboCasa {key}")
require(run[key] == provenance.get(key), f"RoboCasa {key} mismatch")
require(run.get("sources") == provenance.get("sources") and isinstance(run.get("sources"), list),
"RoboCasa source registry mismatch")
for source in run["sources"]:
name = source.get("run")
require(isinstance(name, str) and re.fullmatch(r"[A-Za-z0-9_.-]+", name) and name not in sources,
"Invalid or duplicate RoboCasa source run")
check_digest(source.get("manifestSha256"), "RoboCasa source manifest")
integer(source.get("seedBase"), "RoboCasa source seed", 100000)
sources[name] = source
require(run["run"] in sources and sources[run["run"]]["manifestSha256"] == run["manifestSha256"]
and sources[run["run"]]["seedBase"] == 100000, "RoboCasa primary source mismatch")
selections = run["attemptSelection"]
require(len(selections) == len(expected_ids) and {item["episode"] for item in selections} == expected_ids,
f"Incomplete source selection mapping: {benchmark_id}")
require(run["selectedEpisodes"] == len(expected_ids), f"Incorrect selected episode count: {benchmark_id}")
excluded_count = 0
for selection in selections:
episode = episodes[selection["episode"]]["episode"]
check_digest(selection["resultSha256"], "selected result")
selected_attempt = selection["selectedAttempt"]
if is_robotwin_nvidia10:
require(re.fullmatch(r"attempt_\d+_nvidia_goal_spec_full_v1", selected_attempt),
"Invalid selected attempt identifier")
else:
require(re.fullmatch(r"attempt_\d+", selected_attempt), "Invalid selected attempt identifier")
if is_robodojo:
public_fields(selection, ROBODOJO_SELECTION_KEYS, "RoboDojo selection")
task = episodes[selection["episode"]]["task"]
require(selection.get("sourceEpisodeKey") == task["nativeTaskName"] + "_r00"
and selected_attempt == "attempt_001" and selection["excludedAttempts"] == [],
"RoboDojo first-pass source identity mismatch")
require(selection.get("seed") == episode["seed"]
and selection.get("status") == episode["status"]
and selection.get("steps") == episode["steps"], "RoboDojo selected outcome mismatch")
require(type(selection.get("nativeSuccess")) is bool
and selection["nativeSuccess"] == (episode["status"] == "success"),
"RoboDojo native success mismatch")
require(type(selection.get("nativeScore")) in (int, float)
and selection["nativeScore"] == episode["nativeScore"], "RoboDojo native score mismatch")
require(selection.get("instructionSha256") == task["instructionSha256"],
"RoboDojo selected native instruction fingerprint mismatch")
check_digest(selection.get("sourceVideoSha256"), "RoboDojo selected source video")
check_digest(selection.get("nativeStateSha256"), "RoboDojo selected native state")
if is_robocasa:
source = sources.get(selection.get("run"))
require(source is not None and source["manifestSha256"] == selection.get("manifestSha256")
and source["seedBase"] == selection.get("seed") == episode["seed"],
"RoboCasa selected source or seed mismatch")
original_key = selection.get("sourceEpisodeKey")
require(isinstance(original_key, str) and re.fullmatch(r"[A-Za-z][A-Za-z0-9_]*_r00", original_key)
and selection["episode"] == f"robocasa_{original_key.lower()}", "RoboCasa source episode mismatch")
require(selection.get("status") == episode["status"] and selection.get("steps") == episode["steps"],
"RoboCasa selected outcome mismatch")
require(type(selection.get("nativeScore")) is int
and selection["nativeScore"] == int(episode["status"] == "success"),
"RoboCasa native score mismatch")
check_digest(selection.get("sourceVideoSha256"), "RoboCasa selected source video")
if is_robotwin_nvidia10:
require(selection.get("sourceEpisodeKey") == episode.get("sourceEpisodeKey")
and selection.get("status") == episode["status"]
and selection["excludedAttempts"] == [],
"RoboTwin 10-rollout source identity mismatch")
check_digest(selection.get("sourceVideoSha256"), "RoboTwin 10-rollout selected source video")
excluded_attempts = set()
for excluded in selection["excludedAttempts"]:
require(excluded["status"] in {"error", "interrupted"}, "A policy outcome was excluded by a retry")
require(re.fullmatch(r"attempt_\d+", excluded["attempt"]), "Invalid excluded attempt identifier")
identity = (excluded.get("run"), excluded["attempt"]) if is_robocasa else excluded["attempt"]
require(identity not in excluded_attempts, "Duplicate excluded attempt")
excluded_attempts.add(identity)
if not is_robocasa or excluded.get("run") == selection["run"]:
require(int(excluded["attempt"].split("_")[-1]) < int(selected_attempt.split("_")[-1]),
"The selected result is not the latest attempt")
if is_robocasa:
source = sources.get(excluded.get("run"))
require(source is not None and excluded.get("manifestSha256") == source["manifestSha256"]
and excluded.get("seed") == source["seedBase"] <= selection["seed"],
"RoboCasa excluded source or seed mismatch")
require(excluded.get("sourceEpisodeKey") == selection["sourceEpisodeKey"],
"RoboCasa excluded task mismatch")
require(excluded.get("countsAsValidEpisode") is False and excluded.get("countedAsValidSuccess") is False,
"RoboCasa infrastructure attempts must not count as policy episodes")
require(excluded["status"] == "error" and excluded.get("reason") in {
"simulation_error", "codex_error", "video_recording_error"}, "Unsupported RoboCasa excluded outcome")
integer(excluded.get("steps"), "RoboCasa excluded steps")
integer(excluded.get("visualObservations"), "RoboCasa excluded RGB observations")
if excluded["reason"] == "simulation_error":
require(excluded["steps"] == excluded["visualObservations"] == 0
and excluded.get("nativeSuccess") is True, "Unproved initial-scene exclusion")
else:
require(excluded.get("nativeSuccess") is False and excluded["steps"] > 0
and excluded["visualObservations"] > 0, "Unproved retained API failure")
require(excluded["seed"] == selection["seed"], "An API failure cannot advance the seed")
check_digest(excluded.get("videoSha256"), "RoboCasa excluded video")
check_digest(excluded["resultSha256"], "excluded result")
excluded_count += 1
if not is_robotwin_nvidia10:
require(excluded_count == run["excludedAttemptCount"],
f"Wrong infrastructure/interruption retry count: {benchmark_id}")
if not is_robocasa and not is_robotwin_nvidia10:
require(excluded_count == {"libero": 4, "robotwin": 6, "robodojo": 0}[benchmark_id],
f"Unexpected historical retry count: {benchmark_id}")
if is_robotwin_nvidia10:
require(run["excludedAttemptCount"] == 21, "Unexpected RoboTwin 10-rollout excluded episode count")
media = report["media"]
require(len(media) == expected_episodes and {item["episode"] for item in media} == set(episodes), "Incomplete media provenance mapping")
videos, video_bytes, poster_bytes = [], 0, 0
if release_assets is not None:
require(set(release_assets) == {f"{key}.mp4" for key, item in episodes.items() if item["benchmark"] == "robocasa"},
"Release must contain exactly the 365 selected RoboCasa videos")
for record in media:
item = episodes[record["episode"]]
episode = item["episode"]
if item["benchmark"] == "robodojo":
public_fields(record, {"episode", "benchmark", "sourceSha256", "sourceBytes", "video", "videoSha256", "videoBytes",
"poster", "posterSha256", "posterBytes", "posterFrame", "codec", "pixelFormat", "width",
"height", "fps", "frames", "durationSeconds", "fastStart", "verified"}, "RoboDojo media")
require(record["benchmark"] == item["benchmark"], "Wrong media benchmark mapping")
for key in ("video", "poster", "width", "height", "frames"):
require(record[key] == episode[key], f"Media/source mapping mismatch: {record['episode']}.{key}")
require(record["codec"] == "h264" and record["pixelFormat"] == "yuv420p"
and record["fastStart"] is True and record["verified"] is True, "Unverified or incompatible video export")
require(Fraction(record["fps"]) == item["fps"], "Video frame-rate mapping differs from the source protocol")
require(abs(record["durationSeconds"] - episode["durationSeconds"]) < 1e-3, "Video duration mapping differs from source")
require(0 <= integer(record["posterFrame"], "poster frame") < episode["frames"], "Poster references a missing source frame")
check_digest(record["sourceSha256"], "source video")
if item["benchmark"] in {"robocasa", "robodojo", "robotwin_nvidia10"}:
selection = next(selection for run in runs if run["benchmark"] == item["benchmark"]
for selection in run["attemptSelection"] if selection["episode"] == record["episode"])
label = {"robocasa": "RoboCasa", "robodojo": "RoboDojo", "robotwin_nvidia10": "RoboTwin 10-rollout"}[item["benchmark"]]
require(record["sourceSha256"] == selection["sourceVideoSha256"], f"{label} audit/source video mismatch")
integer(record["sourceBytes"], "source video size", 1)
for key in ("video", "poster"):
check_digest(record[f"{key}Sha256"], f"{key} export")
is_release_video = item["benchmark"] == "robocasa" and key == "video" and release_assets is not None
path = local_asset(root, record[key], key, allow_missing=is_release_video)
if is_release_video:
asset = release_assets[f"{record['episode']}.mp4"]
require(asset.get("state") == "uploaded" and asset.get("size") == record["videoBytes"]
and asset.get("digest") == f"sha256:{record['videoSha256']}"
and asset.get("browser_download_url") == episode["remoteVideo"],
f"Release asset fingerprint mismatch: {record['episode']}")
if path.exists():
require(path.is_file() and path.stat().st_size == record["videoBytes"]
and sha256(path) == record["videoSha256"],
f"Local release video fingerprint mismatch: {record['episode']}")
else:
require(path.stat().st_size == record[f"{key}Bytes"], f"Media byte count mismatch: {record[key]}")
require(sha256(path) == record[f"{key}Sha256"], f"Media digest mismatch: {record[key]}")
video_bytes += record["videoBytes"]
poster_bytes += record["posterBytes"]
video_source = episode["remoteVideo"] if item["benchmark"] == "robocasa" and release_assets is not None else root / record["video"]
videos.append((video_source, {"width": record["width"], "height": record["height"],
"frames": record["frames"], "fps": item["fps"]}))
require(report["videoBytes"] == video_bytes and report["posterBytes"] == poster_bytes
and report["publishedMediaBytes"] == video_bytes + poster_bytes, "Incorrect published media size total")
require(report["sourceBytes"] == sum(record["sourceBytes"] for record in media), "Incorrect source media size total")
if has_robocasa:
external_bytes = sum(record["videoBytes"] for record in media if record["benchmark"] == "robocasa")
pages_bytes = video_bytes + poster_bytes - external_bytes
require(report.get("externalVideoBytes") == external_bytes and report.get("pagesMediaBytes") == pages_bytes,
"Incorrect release/Pages media size totals")
require(pages_bytes < 1_500_000_000, "Pages media exceeds 1.5 GB")
require(report["validation"]["decodedFrameCount"] == sum(record["frames"] for record in media),
"Incorrect mapped frame total")
demo_coverage, demo_videos, demo_paths = validate_training_demos(root, gallery, required=require_demos)
require(not media_paths.intersection(demo_paths), "Training and evaluation media must have distinct assets")
demo_bytes = sum((root / relative).stat().st_size for relative in demo_paths)
if has_robocasa:
require(pages_bytes + demo_bytes < 1_500_000_000, "Pages media including training demos exceeds 1.5 GB")
media_paths.update(demo_paths)
videos.extend(demo_videos)
blog_paths = validate_blog_media(root)
additional_blog_paths = blog_paths - media_paths
blog_bytes = sum((root / relative).stat().st_size for relative in additional_blog_paths)
media_paths.update(blog_paths)
if has_robocasa:
require(pages_bytes + demo_bytes + blog_bytes < 1_500_000_000,
"Pages media including training demos and blog clips exceeds 1.5 GB")
validate_media_inventory(root, media_paths, external_media if release_assets is not None else ())
with (root / "data/episodes.csv").open(newline="", encoding="utf-8") as stream:
csv_rows = list(csv.DictReader(stream))
require(len(csv_rows) == expected_episodes and {row["episode"] for row in csv_rows} == set(episodes), "CSV episode mapping is incomplete")
for row in csv_rows:
public_metadata(row, "episodes.csv")
item = episodes[row["episode"]]
expected_row = {"benchmark": item["benchmark"], "suite": item["task"]["suite"], "task": item["task"]["id"],
"instruction": item["episode"].get("instruction", item["task"]["instruction"]), "episode": row["episode"],
**{key: item["episode"][key] for key in ("status", "seed", "initStateId", "steps", "maxSteps",
"toolCalls", "wallSeconds", "durationSeconds", "frames", "video", "poster")},
"rolloutIndex": item["episode"]["index"]}
if has_robocasa:
expected_row["remoteVideo"] = item["episode"].get("remoteVideo")
require(row == {key: "" if value is None else str(value) for key, value in expected_row.items()},
f"CSV/JSON mismatch: {row['episode']}")
return {"benchmarks": summaries, "episodes": len(episodes), "tasks": len(task_ids),
"training_demos": demo_coverage,
"training_demo_bytes": demo_bytes,
"blog_media_files": len(additional_blog_paths), "blog_media_bytes": blog_bytes,
"media_files": len(media_paths), "media_bytes": video_bytes + poster_bytes + demo_bytes + blog_bytes,
"repository_files": tree["files"], "repository_bytes": tree["bytes"], "pages_bytes": tree["siteBytes"]}, videos
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--root", type=Path, default=DEFAULT_ROOT, help="Gallery repository root")
parser.add_argument("--videos", action="store_true", help="Decode and validate every video with ffprobe")
parser.add_argument("--require-robocasa", action="store_true", help="Require all 365 RoboCasa episodes in addition to LIBERO and RoboTwin")
parser.add_argument("--require-robodojo", action="store_true", help="Require all 42 standard RoboDojo episodes")
parser.add_argument("--require-demos", action="store_true", help="Require the training catalog; RoboDojo may remain wholly unimported")
parser.add_argument("--release-media", action="store_true", help="Verify RoboCasa videos through the public Release asset SHA256/size registry")
parser.add_argument("--workers", type=int, default=4, help="Concurrent ffprobe processes (default: 4)")
args = parser.parse_args()
try:
require(args.root.is_dir(), "Gallery root is missing")
require(1 <= args.workers <= 32, "Workers must be between 1 and 32")
release_assets = None
if args.release_media:
from release_media import fetch_release_assets
release_assets = fetch_release_assets()
result, videos = validate_gallery(args.root.resolve(), require_robocasa=args.require_robocasa,
release_assets=release_assets, require_demos=args.require_demos,
require_robodojo=args.require_robodojo)
if args.videos:
require(shutil.which("ffprobe") is not None, "Install ffprobe to use --videos")
with ThreadPoolExecutor(max_workers=args.workers) as pool:
frames = sum(pool.map(probe_video, videos))
result.update(videos_decoded=len(videos), frames_decoded=frames)
result["passed"] = True
print(json.dumps(result, indent=2, sort_keys=True))
except (ValidationError, OSError, KeyError, TypeError, ValueError, subprocess.TimeoutExpired) as exc:
print(f"Gallery validation failed: {exc}", file=sys.stderr)
return 1
return 0
if __name__ == "__main__":
raise SystemExit(main())