Spaces:
Running
Running
File size: 55,991 Bytes
1f42c47 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 | #!/usr/bin/env python3
"""Validate the complete public gallery with the Python standard library."""
from __future__ import annotations
import argparse
from collections import Counter
from concurrent.futures import ThreadPoolExecutor
import csv
from fractions import Fraction
import hashlib
import json
import math
from pathlib import Path, PurePosixPath
import re
import shutil
import subprocess
import sys
DEFAULT_ROOT = Path(__file__).resolve().parents[1]
EXCLUDED = {".git", "node_modules", "_site", "artifacts", ".venv", "__pycache__", ".gallery-cache", ".cache", ".pytest_cache"}
# Space content guard for the complete local archive (~4.1 GB), including RoboCasa.
# Historical Pages/Release byte counters below remain part of the frozen export audit.
MAX_SITE_BYTES = 6_000_000_000
MAX_FILE_BYTES = 2 * 1024**3
EXPECTED_SUITES = {"libero_spatial": 95, "libero_goal": 79, "libero_object": 89, "libero_10": 59}
EXPECTED_RUNS = {
"libero": ("astra_libero_v5_calibrated_parallel10_r4",
"df08694520ff7ea3e9bfee061a29223e46c4db76bc54a07e637cbfcbe93f42f8"),
"robotwin": ("robotwin_astra_firstpass_20260917",
"8f0b9bfc28b7e2dccc7fe7a056114f1b7d2fdbf36e4bcbf72771931c5a9c6c38"),
"robocasa": ("robocasa365_astra_firstpass_20260918",
"3a2a7f6939e18b5ee3fb6d6bac10d8f2f8704cd422d91efc8e56a038cf28c12c"),
"robodojo": ("robodojo42_astra_firstpass_20260919",
"fe327cacc7615e868096a025c1287a8ad070408851bac9104cf132c1349f6fd4"),
"robotwin_nvidia10": ("robotwin_goal_spec_full_v1",
"5044e7c5eaed3389b689c0534c0863966d5ee1b3fbafc9e4990a0c85960a5f88"),
}
# Exact standard task matrix from the frozen RoboDojo manifest. Native names are
# case-sensitive; gallery IDs are lowercased and the suite ID uses underscores.
ROBODOJO_TASKS = {
"generalization": ("stack_bowls", "push_T", "pack_objects_into_box", "fold_clothes", "hang_mugs",
"sweep_blocks", "pour_liquid_into_cup", "make_toast", "arrange_largest_number",
"sort_nesting_dolls_by_size", "store_laptop_and_headphones", "stack_blocks"),
"memory": ("cover_blocks", "match_and_pick_from_conveyor", "swap_blocks", "swap_T", "press_by_number",
"imitate_sorting_sequence"),
"precision": ("fasten_screws", "plug_in_charger", "insert_tubes", "pour_balls_into_vase", "play_Xylophone",
"deposit_coin", "insert_key", "build_tower"),
"long-horizon": ("fill_pen_holder", "classify_objects", "put_bottles_into_dustbin", "play_tic_tac_toe",
"fill_egg_holder", "organize_table", "make_kong", "play_stacking_toy"),
"open": ("align_blocks", "general_pickup", "solve_equation", "stack_blocks_by_language",
"classify_objects_by_language", "pick_from_conveyor_by_image", "store_tools_in_toolbox",
"pour_by_language"),
}
ROBODOJO_SUITES = {"robodojo_" + group.replace("-", "_"): names for group, names in ROBODOJO_TASKS.items()}
ROBODOJO_HORIZONS = dict(zip(
(name for names in ROBODOJO_TASKS.values() for name in names),
(800, 600, 1300, 500, 800, 1000, 400, 1400, 1050, 1050, 800, 550,
800, 700, 700, 400, 700, 1600, 1900, 400, 500, 600, 500, 300, 300, 1050,
1100, 1100, 700, 1100, 700, 1000, 600, 1200, 200, 200, 300, 400, 1100, 700, 900, 800),
))
ROBODOJO_PROVENANCE_KEYS = {"run", "manifestSha256", "auditKind", "auditSha256", "policyBoundaryAuditSha256"}
ROBODOJO_EPISODE_KEYS = {"id", "index", "status", "seed", "initStateId", "steps", "maxSteps", "toolCalls",
"wallSeconds", "durationSeconds", "video", "poster", "width", "height", "frames",
"nativeScore", "terminationReason", "layoutId"}
ROBODOJO_SELECTION_KEYS = {"episode", "sourceEpisodeKey", "seed", "selectedAttempt", "resultSha256",
"sourceVideoSha256", "instructionSha256", "nativeSuccess", "nativeScore",
"status", "steps", "excludedAttempts", "nativeStateSha256"}
PRIVATE_KEYS = {
"auth", "authorization", "authentication", "credentials", "credential", "apikey",
"accesstoken", "refreshtoken", "idtoken", "bearertoken", "secret", "password",
"thread", "threads", "threadid", "conversation", "conversations", "conversationid",
"codexhome", "workspace", "workspacedir", "rawlogs", "rawevents", "reasoning",
"reasoningcontent", "chainofthought", "transcript", "stdout", "stderr", "prompt",
"systemprompt", "messages", "oauth", "authorizationheader", "eventlog", "codexevents",
}
class ValidationError(Exception):
"""An exported asset does not satisfy the publication contract."""
def require(condition, message):
if not condition:
raise ValidationError(message)
def read_json(path):
try:
return json.loads(path.read_text(encoding="utf-8"))
except (OSError, ValueError) as exc:
raise ValidationError(f"Cannot read JSON {path.name}: {exc}") from exc
def integer(value, label, minimum=0):
require(type(value) is int and value >= minimum, f"{label} must be an integer >= {minimum}")
return value
def public_metadata(value, location="data"):
"""Check public JSON without printing potentially private values in errors."""
if isinstance(value, dict):
for key, child in value.items():
normalized = re.sub(r"[^a-z0-9]", "", key.lower())
require(normalized not in PRIVATE_KEYS, f"Private metadata key at {location}")
public_metadata(child, f"{location}.{key}")
elif isinstance(value, list):
for index, child in enumerate(value):
public_metadata(child, f"{location}[{index}]")
elif isinstance(value, str):
require(not re.search(r"(?:^|[\s\"'=])/(?:home|playpen|tmp|Users|root|mnt|private)/", value),
f"Machine-local path at {location}")
require(not value.startswith(("/", "file:", "\\\\")) and not re.match(r"^[A-Za-z]:[\\/]", value),
f"Absolute local path at {location}")
require(not re.search(r"\bsk-[A-Za-z0-9_-]{20,}\b", value), f"Possible secret at {location}")
require(not any(marker in value.lower() for marker in ("auth.json", "codex_home", "codex-events.jsonl")),
f"Private artifact reference at {location}")
elif type(value) is float:
require(math.isfinite(value), f"Non-finite metadata number at {location}")
def local_asset(root, relative, label, *, allow_missing=False):
require(isinstance(relative, str) and relative, f"Missing {label} path")
path = PurePosixPath(relative)
require(not path.is_absolute() and ".." not in path.parts and "\\" not in relative
and ":" not in relative and "?" not in relative and "#" not in relative,
f"{label} must be a plain repository-relative path")
require(path.parts[0] in {"media", "assets"}, f"{label} must be a public media asset")
candidate = root.joinpath(*path.parts)
require(allow_missing or candidate.is_file(), f"Missing {label}: {relative}")
require(not candidate.is_symlink() and candidate.resolve().is_relative_to(root.resolve()),
f"Unsafe {label}: {relative}")
return candidate
def sha256(path):
digest = hashlib.sha256()
with path.open("rb") as stream:
for block in iter(lambda: stream.read(1024 * 1024), b""):
digest.update(block)
return digest.hexdigest()
def check_tree(root, external_media=()):
total, site_total, files = 0, 0, 0
for path in root.rglob("*"):
relative = path.relative_to(root)
if any(part in EXCLUDED for part in relative.parts):
continue
require(not path.is_symlink(), f"Symlinks are not allowed: {relative.as_posix()}")
if path.is_file():
size = path.stat().st_size
external = relative.as_posix() in external_media
require(size < (2 * 1024**3 if external else MAX_FILE_BYTES),
f"File exceeds its hosting limit: {relative.as_posix()}")
total += size
if not external:
site_total += size
files += 1
require(total < MAX_SITE_BYTES, f"Space archive exceeds the configured size limit: {total} bytes")
return {"files": files, "bytes": total, "siteBytes": site_total}
def probe_video(item):
path, expected = item
label = str(path).rsplit("/", 1)[-1]
completed = subprocess.run(
["ffprobe", "-v", "error", "-count_frames", "-select_streams", "v:0", "-show_entries",
"stream=codec_name,pix_fmt,width,height,nb_read_frames,avg_frame_rate", "-of", "json", str(path)],
check=False, capture_output=True, text=True, timeout=120,
)
require(completed.returncode == 0 and not completed.stderr.strip(), f"Video decode failed: {label}")
try:
streams = json.loads(completed.stdout)["streams"]
require(len(streams) == 1, f"Expected one video stream: {label}")
stream = streams[0]
require(stream["codec_name"] == "h264", f"Expected H.264: {label}")
require(stream["pix_fmt"] == "yuv420p", f"Expected yuv420p: {label}")
require(stream["width"] == expected["width"] and stream["height"] == expected["height"],
f"Video dimensions differ from metadata: {label}")
frames = int(stream["nb_read_frames"])
require(frames == expected["frames"], f"Video frame count differs from source mapping: {label}")
numerator, denominator = map(int, stream["avg_frame_rate"].split("/"))
require(denominator > 0 and abs(numerator / denominator - expected["fps"]) < 1e-8,
f"Video frame rate differs from metadata: {label}")
return frames
except (KeyError, ValueError, TypeError) as exc:
raise ValidationError(f"Incomplete video metadata: {label}") from exc
def counts(episodes, tasks=None):
statuses = Counter(episode["status"] for episode in episodes)
result = {
"episodes": len(episodes), "successes": statuses["success"],
"failures": statuses["failure"] + statuses["timeout"], "timeouts": statuses["timeout"],
"successRate": statuses["success"] / len(episodes) if episodes else 0,
}
if tasks is not None:
result["tasks"] = tasks
return result
def check_summary(actual, expected, label, keys=None):
require(isinstance(actual, dict), f"Missing {label}")
for key in keys or expected:
require(actual.get(key) == expected[key], f"Wrong {label}.{key}: expected {expected[key]}")
def check_digest(value, label):
require(isinstance(value, str) and re.fullmatch(r"[0-9a-f]{64}", value), f"Invalid {label} fingerprint")
def public_fields(value, allowed, label):
require(isinstance(value, dict) and set(value) <= allowed, f"Unexpected public {label} fields")
def validate_robodojo_task(task):
public_fields(task, {"id", "name", "instruction", "instructionSha256", "suite", "suiteName", "episodes",
"successes", "failures", "successRate", "nativeTaskName", "taskGroup"}, "RoboDojo task")
native_name = task.get("nativeTaskName")
group = task.get("taskGroup")
require(group in ROBODOJO_TASKS and native_name in ROBODOJO_TASKS[group]
and task["id"] == "robodojo_" + native_name.lower()
and task["suite"] == "robodojo_" + group.replace("-", "_"),
"RoboDojo native task/group identity mismatch")
digest = hashlib.sha256(task["instruction"].encode("utf-8")).hexdigest()
require(task.get("instructionSha256") == digest, "RoboDojo native instruction fingerprint mismatch")
def validate_robodojo_episode(episode, task):
public_fields(episode, ROBODOJO_EPISODE_KEYS, "RoboDojo episode")
require(episode["seed"] == 0 and episode["maxSteps"] == ROBODOJO_HORIZONS[task["nativeTaskName"]],
"Incorrect RoboDojo seed or native horizon")
require(type(episode.get("layoutId")) is int and episode["layoutId"] == 0, "Incorrect RoboDojo native layout")
score = episode.get("nativeScore")
require(type(score) in (int, float) and math.isfinite(score) and 0 <= score <= 1,
"Invalid RoboDojo native score")
reason = episode.get("terminationReason")
require(reason in {"step_budget", "agent_finished", "robodojo_success", "environment_done"},
"Invalid RoboDojo termination reason")
require((reason == "robodojo_success") == (episode["status"] == "success"),
"RoboDojo termination/outcome mismatch")
if reason == "step_budget":
require(episode["steps"] == episode["maxSteps"], "RoboDojo exhausted horizon mismatch")
def validate_training_demos(root, gallery, *, required=False):
"""Validate separate training references without changing evaluation counts."""
catalog_path = root / "data/task-demos.json"
if not catalog_path.exists():
require(not required, "Training demonstration catalog is required")
return None, [], set()
catalog = read_json(catalog_path)
public_metadata(catalog, "data/task-demos.json")
require(catalog.get("schemaVersion") == 1, "Unsupported training demo schema")
records = catalog.get("tasks")
expected = {task["id"]: (benchmark["id"], task)
for benchmark in gallery["benchmarks"] for task in benchmark["tasks"]}
# Older evaluation-only RoboDojo imports may have no demonstration group.
# Once imported, every task needs an available or unavailable source record.
if isinstance(records, dict) and not any(task_id in records for task_id, (benchmark_id, _) in expected.items()
if benchmark_id == "robodojo"):
expected = {task_id: value for task_id, value in expected.items() if value[0] != "robodojo"}
require(isinstance(records, dict) and set(records) == set(expected),
"Training demo catalog must account for every gallery task exactly once")
coverage = {benchmark["id"]: {"tasks": len(benchmark["tasks"]), "available": 0, "unavailable": 0}
for benchmark in gallery["benchmarks"]
if any(value[0] == benchmark["id"] for value in expected.values())}
videos, media_paths = [], set()
for task_id, record in records.items():
benchmark_id, task = expected[task_id]
require(record.get("taskId") == task_id and record.get("benchmark") == benchmark_id,
f"Training demo task identity mismatch: {task_id}")
status = record.get("status")
require(status in {"available", "unavailable"}, f"Invalid training demo status: {task_id}")
coverage[benchmark_id][status] += 1
source = record.get("source", {})
require(isinstance(source, dict) and isinstance(source.get("dataset"), str) and source["dataset"].strip()
and isinstance(source.get("url"), str) and source["url"].startswith("https://"),
f"Missing training dataset attribution: {task_id}")
if status == "unavailable":
require(isinstance(record.get("reason"), str) and record["reason"].strip(),
f"Missing unavailable-demo explanation: {task_id}")
require(not any(key in record for key in ("video", "poster", "remoteVideo", "frames")),
f"Unavailable training demo must not claim media: {task_id}")
continue
relative_source = source.get("relativePath")
require(isinstance(relative_source, str) and relative_source and
not PurePosixPath(relative_source).is_absolute() and ".." not in PurePosixPath(relative_source).parts,
f"Missing relative training source path: {task_id}")
require(source.get("episode") is not None, f"Missing training episode selection: {task_id}")
if benchmark_id in {"robotwin", "robocasa", "robodojo"}:
require(isinstance(source.get("taskName"), str) and
f"{benchmark_id}_{source['taskName'].lower()}" == task_id,
f"Training source task differs from gallery task: {task_id}")
if benchmark_id == "robodojo":
require(source["taskName"] == task.get("nativeTaskName"),
f"Training source native task differs from gallery task: {task_id}")
for key in ("width", "height", "frames"):
integer(record.get(key), f"{task_id} demo {key}", 1)
fps = record.get("fps")
require(type(fps) in (int, float) and math.isfinite(fps) and fps > 0,
f"Invalid training demo frame rate: {task_id}")
duration = record.get("durationSeconds")
require(type(duration) in (int, float) and math.isfinite(duration)
and abs(duration - record["frames"] / fps) < .001,
f"Training demo duration/frame mismatch: {task_id}")
require(isinstance(record.get("cameras"), list) and record["cameras"] and
all(isinstance(value, str) and value.strip() for value in record["cameras"]),
f"Missing training camera labels: {task_id}")
for key, suffix in (("video", ".mp4"), ("poster", ".jpg")):
relative = f"media/demos/{benchmark_id}/{task_id}{suffix}"
require(record.get(key) == relative and relative not in media_paths,
f"Incorrect training demo asset mapping: {task_id}")
path = local_asset(root, relative, f"training {key}")
check_digest(record.get(f"{key}Sha256"), f"training {key}")
require(sha256(path) == record[f"{key}Sha256"], f"Training media digest mismatch: {relative}")
if key == "video" or f"{key}Bytes" in record:
require(path.stat().st_size == record.get(f"{key}Bytes"),
f"Training media byte count mismatch: {relative}")
media_paths.add(relative)
videos.append((root / record["video"], {key: record[key] for key in ("width", "height", "frames", "fps")}))
require(catalog.get("coverage") == coverage, "Training demo coverage summary differs from catalog")
return coverage, videos, media_paths
def validate_blog_media(root):
"""Allow only explicit, complete essay clip references into the media tree.
The separate blog validator checks experiment claims and clip fingerprints.
This inventory check also supports references to existing baseline media.
"""
path = root / "data/libero-blog-media.json"
if not path.exists():
return set()
catalog = read_json(path)
public_metadata(catalog, "data/libero-blog-media.json")
require(isinstance(catalog, dict) and catalog.get("schemaVersion") == 1,
"Unsupported blog media schema")
require(catalog.get("complete") is True, "The blog media export is not complete")
clips = catalog.get("clips")
require(isinstance(clips, dict) and clips, "Blog media must declare its clips")
paths = set()
for identifier, clip in clips.items():
require(isinstance(clip, dict) and clip.get("id") == identifier,
"Invalid blog clip identity")
for key, suffix in (("video", ".mp4"), ("poster", ".jpg")):
relative = clip.get(key)
asset = local_asset(root, relative, f"blog {key}")
require(PurePosixPath(relative).parts[0] == "media" and asset.suffix == suffix,
f"Blog {key} must reference a {suffix} file under media")
paths.add(relative)
return paths
def validate_media_inventory(root, media_paths, external_media=()):
actual_media = {path.relative_to(root).as_posix() for path in (root / "media").rglob("*") if path.is_file()}
require(media_paths - set(external_media) <= actual_media <= media_paths,
"The media directory contains missing or unlisted files")
def validate_gallery(root, *, check_interface=True, require_robocasa=False, release_assets=None,
require_demos=False, require_robodojo=False):
required_files = ("index.html", "app.js", "styles.css", ".nojekyll", "METHODOLOGY.md") if check_interface else ()
for filename in required_files:
require((root / filename).is_file(), f"Required public file is missing: {filename}")
expected_data_files = {"gallery.json", "episodes.csv", "export-report.json"}
if (root / "data/task-demos.json").exists():
expected_data_files.add("task-demos.json")
for name in ("libero-blog-media.json", "libero-alignment.json", "libero-alignment-episodes.csv",
"libero-human-review.json", "robotwin-alignment-summary.json"):
if (root / "data" / name).exists():
expected_data_files.add(name)
require({path.relative_to(root / "data").as_posix() for path in (root / "data").rglob("*") if path.is_file()}
== expected_data_files,
"The public data directory contains missing or unexpected files")
gallery = read_json(root / "data/gallery.json")
report = read_json(root / "data/export-report.json")
for path in (root / "data").rglob("*.json"):
public_metadata(read_json(path), path.relative_to(root).as_posix())
require(gallery.get("schemaVersion") == 1 and report.get("schemaVersion") == 1, "Unsupported schema version")
require(gallery.get("model") == "gpt-6-astra", "Unexpected model")
require(report.get("complete") is True, "The media export is not complete")
benchmarks = gallery["benchmarks"]
benchmark_ids = {item["id"] for item in benchmarks}
has_robocasa = "robocasa" in benchmark_ids
has_robodojo = "robodojo" in benchmark_ids
has_robotwin_nvidia10 = "robotwin_nvidia10" in benchmark_ids
require(release_assets is None or has_robocasa, "Release validation requires RoboCasa365")
expected_benchmarks = ({"libero", "robotwin"} | ({"robocasa"} if has_robocasa else set())
| ({"robodojo"} if has_robodojo else set())
| ({"robotwin_nvidia10"} if has_robotwin_nvidia10 else set()))
require(benchmark_ids == expected_benchmarks and len(benchmarks) == len(expected_benchmarks),
"Expected LIBERO and RoboTwin, with complete optional benchmark collections")
require(not require_robocasa or has_robocasa, "The complete RoboCasa365 collection is required")
require(not require_robodojo or has_robodojo, "The complete RoboDojo42 collection is required")
expected_episodes, expected_tasks_total = (815, 455) if has_robocasa else (450, 90)
if has_robodojo:
expected_episodes += 42
expected_tasks_total += 42
if has_robotwin_nvidia10:
expected_episodes += 479
expected_tasks_total += 50
require(gallery.get("evaluationDate") == ("2026-09-22" if has_robotwin_nvidia10 else "2026-09-19" if has_robodojo else "2026-09-18" if has_robocasa else "2026-09-17"),
"Unexpected evaluation date")
if has_robocasa or has_robodojo or has_robotwin_nvidia10:
dates = ["2026-09-17"] + (["2026-09-18"] if has_robocasa else []) + (["2026-09-19"] if has_robodojo else []) + (["2026-09-22"] if has_robotwin_nvidia10 else [])
require(gallery.get("evaluationDates") == dates, "Incorrect evaluation dates")
require(report.get("expectedEpisodes") == expected_episodes and report.get("verifiedEpisodes") == expected_episodes,
f"The export must contain {expected_episodes} verified episodes")
episodes, task_ids, media_paths, summaries = {}, set(), set(), {}
benchmark_by_id = {item["id"]: item for item in benchmarks}
external_media = set()
if has_robocasa:
from release_media import RELEASE_HOSTING, remote_video_url
require(report.get("videoHosting") == {"robocasa": RELEASE_HOSTING}
and benchmark_by_id["robocasa"].get("videoHosting") == RELEASE_HOSTING,
"Incorrect RoboCasa video hosting declaration")
external_media = {episode["video"] for task in benchmark_by_id["robocasa"]["tasks"] for episode in task["episodes"]}
tree = check_tree(root, external_media)
for benchmark in benchmarks:
benchmark_id = benchmark["id"]
is_libero = benchmark_id == "libero"
is_robocasa = benchmark_id == "robocasa"
is_robodojo = benchmark_id == "robodojo"
is_robotwin_nvidia10 = benchmark_id == "robotwin_nvidia10"
if is_robodojo:
public_fields(benchmark, {"id", "name", "evaluationDate", "summary", "suites", "protocol", "provenance", "tasks"},
"RoboDojo benchmark")
public_fields(benchmark["protocol"], {"label", "description", "cameras", "videoNote", "stepsLabel", "episodesPerTask", "trainingDemoNote"},
"RoboDojo protocol")
public_fields(benchmark["provenance"], ROBODOJO_PROVENANCE_KEYS, "RoboDojo provenance")
public_fields(benchmark["summary"], set(counts([], 0)), "RoboDojo summary")
require(is_robocasa or "videoHosting" not in benchmark, "Unexpected external hosting for a historical benchmark")
if has_robocasa or has_robodojo or has_robotwin_nvidia10:
require(benchmark.get("evaluationDate") == ("2026-09-22" if is_robotwin_nvidia10 else "2026-09-19" if is_robodojo else "2026-09-18" if is_robocasa else "2026-09-17"),
f"Incorrect evaluation date: {benchmark_id}")
expected_tasks, per_task = {"libero": (40, 10), "robotwin": (50, 1), "robocasa": (365, 1), "robodojo": (42, 1),
"robotwin_nvidia10": (50, 10)}[benchmark_id]
tasks = benchmark["tasks"]
require(len(tasks) == expected_tasks, f"Wrong task count for {benchmark_id}")
require(benchmark["protocol"]["episodesPerTask"] == per_task, f"Wrong sampling for {benchmark_id}")
check_digest(benchmark["provenance"]["manifestSha256"], f"{benchmark_id} manifest")
require((benchmark["provenance"]["run"], benchmark["provenance"]["manifestSha256"]) == EXPECTED_RUNS[benchmark_id],
f"Unexpected frozen source run for {benchmark_id}")
expected_suites = set(EXPECTED_SUITES) if is_libero else ({"robocasa_atomic", "robocasa_composite"} if is_robocasa else {"robotwin_nvidia10"} if is_robotwin_nvidia10 else {"robotwin"})
if is_robodojo:
expected_suites = set(ROBODOJO_SUITES)
suites = benchmark["suites"]
require(len(suites) == len(expected_suites) and {item["id"] for item in suites} == expected_suites,
f"Wrong suites for {benchmark_id}")
all_episodes = []
for task in tasks:
task_id = task["id"]
require(isinstance(task_id, str) and re.fullmatch(r"[a-z0-9_]+", task_id), "Invalid task ID")
require(task_id not in task_ids, f"Duplicate task ID: {task_id}")
task_ids.add(task_id)
require(task["suite"] in expected_suites, f"Wrong suite for {task_id}")
require(isinstance(task["instruction"], str) and (is_robodojo or task["instruction"].strip()), f"Missing instruction: {task_id}")
if is_robodojo:
validate_robodojo_task(task)
task_episodes = task["episodes"]
if is_robotwin_nvidia10:
require(1 <= len(task_episodes) <= per_task, f"Wrong episode count for {task_id}")
require({item["index"] for item in task_episodes}.issubset(set(range(per_task))), f"Wrong rollout indices: {task_id}")
else:
require(len(task_episodes) == per_task, f"Wrong episode count for {task_id}")
require({item["index"] for item in task_episodes} == set(range(per_task)), f"Wrong rollout indices: {task_id}")
if is_libero:
require({item["initStateId"] for item in task_episodes} == set(range(10)), f"Wrong initial states: {task_id}")
for episode in task_episodes:
episode_id = episode["id"]
require(is_robocasa or "remoteVideo" not in episode, "Unexpected remote video for a historical benchmark")
require(isinstance(episode_id, str) and re.fullmatch(r"[a-z0-9_]+", episode_id), "Invalid episode ID")
require(episode_id not in episodes, f"Duplicate episode ID: {episode_id}")
require(is_libero or task_id.startswith(f"{benchmark_id}_"), f"Unexpected {benchmark_id} task identifier")
expected_prefix = task_id if is_libero or is_robocasa or is_robodojo else task_id.removeprefix("robotwin_")
if is_robotwin_nvidia10:
expected_prefix = task_id
require(episode_id == f"{expected_prefix}_r{episode['index']:02d}", f"Episode/task identity mismatch: {episode_id}")
require(episode["status"] in {"success", "failure", "timeout"}, f"Unscored episode: {episode_id}")
if is_robotwin_nvidia10:
require(isinstance(episode.get("instruction"), str) and episode["instruction"].strip(),
f"Missing episode instruction: {episode_id}")
for key in ("index", "seed", "steps", "maxSteps", "toolCalls", "width", "height", "frames"):
integer(episode[key], f"{episode_id}.{key}", 1 if key in {"width", "height", "frames", "maxSteps"} else 0)
require(episode["steps"] <= episode["maxSteps"] and episode["toolCalls"] <= (1500 if is_robocasa or is_robodojo else 750),
f"Recorded controls exceed the protocol: {episode_id}")
require(episode["steps"] > 0, f"Episode has no recorded action: {episode_id}")
fps = 25 if is_robodojo else 20 if is_libero or is_robocasa else 10
if is_libero:
require(episode["maxSteps"] == 500 and episode["seed"] == episode["index"] == episode["initStateId"],
f"Incorrect LIBERO episode protocol: {episode_id}")
require(episode["frames"] == episode["steps"] + 11, f"LIBERO warmup/frame mismatch: {episode_id}")
elif is_robotwin_nvidia10:
require(episode["initStateId"] is None and 0 <= episode["index"] < 10,
f"Incorrect RoboTwin 10-rollout protocol: {episode_id}")
require(episode["seed"] >= 100000,
f"Incorrect RoboTwin 10-rollout seed: {episode_id}")
require(episode["frames"] == episode["steps"] + 1, f"{benchmark_id} action/frame mismatch: {episode_id}")
require(episode["video"] == f"media/robotwin_nvidia10/{episode_id}.mp4"
and episode["poster"] == f"media/robotwin_nvidia10/{episode_id}.jpg",
f"Incorrect RoboTwin 10-rollout local media paths: {episode_id}")
else:
require(episode["index"] == 0 and episode["initStateId"] is None,
f"Incorrect {benchmark_id} episode protocol: {episode_id}")
require(episode["frames"] == episode["steps"] + 1, f"{benchmark_id} action/frame mismatch: {episode_id}")
if is_robodojo:
validate_robodojo_episode(episode, task)
require((episode["width"], episode["height"]) == (1920, 480),
f"Incorrect RoboDojo native camera layout: {episode_id}")
require(episode["video"] == f"media/robodojo/{episode_id}.mp4"
and episode["poster"] == f"media/robodojo/{episode_id}.jpg",
f"Incorrect RoboDojo local media paths: {episode_id}")
if is_robocasa:
require(episode["seed"] >= 100000 and episode["maxSteps"] <= 7200,
f"Incorrect RoboCasa365 seed or step budget: {episode_id}")
require(episode["width"] == 3 * episode["height"], f"Incorrect three-camera layout: {episode_id}")
require(episode.get("remoteVideo") == remote_video_url(episode_id),
f"Incorrect release video URL: {episode_id}")
require(episode["video"] == f"media/robocasa/{episode_id}.mp4"
and episode["poster"] == f"media/robocasa/{episode_id}.jpg",
f"Incorrect RoboCasa local media paths: {episode_id}")
for key in ("wallSeconds", "durationSeconds"):
require(type(episode[key]) in (int, float) and math.isfinite(episode[key]) and episode[key] >= 0,
f"Invalid duration: {episode_id}.{key}")
require(abs(episode["durationSeconds"] - episode["frames"] / fps) < 1e-6,
f"Playback duration/frame mismatch: {episode_id}")
for key, suffix in (("video", ".mp4"), ("poster", ".jpg")):
relative = episode[key]
require(relative not in media_paths, f"Duplicate media reference: {relative}")
media_paths.add(relative)
allow_missing = is_robocasa and key == "video" and release_assets is not None
require(local_asset(root, relative, key, allow_missing=allow_missing).suffix == suffix,
f"Unexpected media format: {relative}")
episodes[episode_id] = {"episode": episode, "benchmark": benchmark_id, "task": task, "fps": fps}
all_episodes.append(episode)
check_summary(task, counts(task_episodes), task_id, ("successes", "failures", "successRate"))
if is_libero:
require({task["id"] for task in tasks} == {f"{suite}_t{index:02d}" for suite in EXPECTED_SUITES for index in range(10)},
"LIBERO must contain tasks 0–9 in each official suite")
expected = counts(all_episodes, len(tasks))
fixed = {"tasks": 40, "episodes": 400, "successes": 322, "failures": 78, "timeouts": 0, "successRate": .805} if is_libero else {
"tasks": 50, "episodes": 50, "successes": 36, "failures": 14, "timeouts": 1, "successRate": .72}
if not is_robocasa and not is_robotwin_nvidia10:
if is_robodojo:
fixed = {"tasks": 42, "episodes": 42, "successes": 6, "failures": 36, "timeouts": 0, "successRate": 6 / 42}
require(expected == fixed, f"Wrong native outcome aggregate for {benchmark_id}")
check_summary(benchmark["summary"], expected, f"{benchmark_id} summary")
summaries[benchmark_id] = expected
for suite in suites:
suite_tasks = [task for task in tasks if task["suite"] == suite["id"]]
selected = [episode for task in suite_tasks for episode in task["episodes"]]
expected = counts(selected, len(suite_tasks))
if is_libero:
require(expected["tasks"] == 10 and expected["episodes"] == 100 and expected["successes"] == EXPECTED_SUITES[suite["id"]],
f"Wrong LIBERO suite outcome: {suite['id']}")
elif is_robocasa:
require(expected["tasks"] == expected["episodes"] == (65 if suite["id"] == "robocasa_atomic" else 300),
f"Wrong RoboCasa365 task group: {suite['id']}")
elif is_robodojo:
public_fields(suite, {"id", "name", *counts([], 0)}, "RoboDojo suite")
require({task["id"] for task in suite_tasks} ==
{"robodojo_" + name.lower() for name in ROBODOJO_SUITES[suite["id"]]},
f"Wrong RoboDojo standard task matrix: {suite['id']}")
check_summary(suite, expected, f"{suite['id']} summary")
require(len(episodes) == expected_episodes and len(task_ids) == expected_tasks_total,
f"Expected {expected_episodes} unique episodes across {expected_tasks_total} tasks")
runs = report["runs"]
require(len(runs) == len(expected_benchmarks) and {run["benchmark"] for run in runs} == expected_benchmarks,
"Wrong provenance runs")
for run in runs:
benchmark_id = run["benchmark"]
is_robocasa = benchmark_id == "robocasa"
is_robodojo = benchmark_id == "robodojo"
is_robotwin_nvidia10 = benchmark_id == "robotwin_nvidia10"
expected_ids = {key for key, item in episodes.items() if item["benchmark"] == benchmark_id}
provenance = benchmark_by_id[benchmark_id]["provenance"]
require(run["run"] == provenance["run"] and run["manifestSha256"] == provenance["manifestSha256"],
f"Provenance mismatch: {benchmark_id}")
sources = {}
if is_robodojo:
public_fields(run, ROBODOJO_PROVENANCE_KEYS | {"benchmark", "selectedEpisodes", "excludedAttemptCount", "attemptSelection"},
"RoboDojo run")
require(run.get("auditKind") == provenance.get("auditKind") == "robodojo_firstpass",
"RoboDojo requires the complete first-pass audits")
for key in ("auditSha256", "policyBoundaryAuditSha256"):
check_digest(run.get(key), f"RoboDojo {key}")
require(run[key] == provenance.get(key), f"RoboDojo {key} mismatch")
if is_robocasa:
require(run.get("auditKind") == provenance.get("auditKind") == "robocasa_collection",
"RoboCasa requires a complete collection audit")
for key in ("auditSha256", "planSha256"):
check_digest(run.get(key), f"RoboCasa {key}")
require(run[key] == provenance.get(key), f"RoboCasa {key} mismatch")
require(run.get("sources") == provenance.get("sources") and isinstance(run.get("sources"), list),
"RoboCasa source registry mismatch")
for source in run["sources"]:
name = source.get("run")
require(isinstance(name, str) and re.fullmatch(r"[A-Za-z0-9_.-]+", name) and name not in sources,
"Invalid or duplicate RoboCasa source run")
check_digest(source.get("manifestSha256"), "RoboCasa source manifest")
integer(source.get("seedBase"), "RoboCasa source seed", 100000)
sources[name] = source
require(run["run"] in sources and sources[run["run"]]["manifestSha256"] == run["manifestSha256"]
and sources[run["run"]]["seedBase"] == 100000, "RoboCasa primary source mismatch")
selections = run["attemptSelection"]
require(len(selections) == len(expected_ids) and {item["episode"] for item in selections} == expected_ids,
f"Incomplete source selection mapping: {benchmark_id}")
require(run["selectedEpisodes"] == len(expected_ids), f"Incorrect selected episode count: {benchmark_id}")
excluded_count = 0
for selection in selections:
episode = episodes[selection["episode"]]["episode"]
check_digest(selection["resultSha256"], "selected result")
selected_attempt = selection["selectedAttempt"]
if is_robotwin_nvidia10:
require(re.fullmatch(r"attempt_\d+_nvidia_goal_spec_full_v1", selected_attempt),
"Invalid selected attempt identifier")
else:
require(re.fullmatch(r"attempt_\d+", selected_attempt), "Invalid selected attempt identifier")
if is_robodojo:
public_fields(selection, ROBODOJO_SELECTION_KEYS, "RoboDojo selection")
task = episodes[selection["episode"]]["task"]
require(selection.get("sourceEpisodeKey") == task["nativeTaskName"] + "_r00"
and selected_attempt == "attempt_001" and selection["excludedAttempts"] == [],
"RoboDojo first-pass source identity mismatch")
require(selection.get("seed") == episode["seed"]
and selection.get("status") == episode["status"]
and selection.get("steps") == episode["steps"], "RoboDojo selected outcome mismatch")
require(type(selection.get("nativeSuccess")) is bool
and selection["nativeSuccess"] == (episode["status"] == "success"),
"RoboDojo native success mismatch")
require(type(selection.get("nativeScore")) in (int, float)
and selection["nativeScore"] == episode["nativeScore"], "RoboDojo native score mismatch")
require(selection.get("instructionSha256") == task["instructionSha256"],
"RoboDojo selected native instruction fingerprint mismatch")
check_digest(selection.get("sourceVideoSha256"), "RoboDojo selected source video")
check_digest(selection.get("nativeStateSha256"), "RoboDojo selected native state")
if is_robocasa:
source = sources.get(selection.get("run"))
require(source is not None and source["manifestSha256"] == selection.get("manifestSha256")
and source["seedBase"] == selection.get("seed") == episode["seed"],
"RoboCasa selected source or seed mismatch")
original_key = selection.get("sourceEpisodeKey")
require(isinstance(original_key, str) and re.fullmatch(r"[A-Za-z][A-Za-z0-9_]*_r00", original_key)
and selection["episode"] == f"robocasa_{original_key.lower()}", "RoboCasa source episode mismatch")
require(selection.get("status") == episode["status"] and selection.get("steps") == episode["steps"],
"RoboCasa selected outcome mismatch")
require(type(selection.get("nativeScore")) is int
and selection["nativeScore"] == int(episode["status"] == "success"),
"RoboCasa native score mismatch")
check_digest(selection.get("sourceVideoSha256"), "RoboCasa selected source video")
if is_robotwin_nvidia10:
require(selection.get("sourceEpisodeKey") == episode.get("sourceEpisodeKey")
and selection.get("status") == episode["status"]
and selection["excludedAttempts"] == [],
"RoboTwin 10-rollout source identity mismatch")
check_digest(selection.get("sourceVideoSha256"), "RoboTwin 10-rollout selected source video")
excluded_attempts = set()
for excluded in selection["excludedAttempts"]:
require(excluded["status"] in {"error", "interrupted"}, "A policy outcome was excluded by a retry")
require(re.fullmatch(r"attempt_\d+", excluded["attempt"]), "Invalid excluded attempt identifier")
identity = (excluded.get("run"), excluded["attempt"]) if is_robocasa else excluded["attempt"]
require(identity not in excluded_attempts, "Duplicate excluded attempt")
excluded_attempts.add(identity)
if not is_robocasa or excluded.get("run") == selection["run"]:
require(int(excluded["attempt"].split("_")[-1]) < int(selected_attempt.split("_")[-1]),
"The selected result is not the latest attempt")
if is_robocasa:
source = sources.get(excluded.get("run"))
require(source is not None and excluded.get("manifestSha256") == source["manifestSha256"]
and excluded.get("seed") == source["seedBase"] <= selection["seed"],
"RoboCasa excluded source or seed mismatch")
require(excluded.get("sourceEpisodeKey") == selection["sourceEpisodeKey"],
"RoboCasa excluded task mismatch")
require(excluded.get("countsAsValidEpisode") is False and excluded.get("countedAsValidSuccess") is False,
"RoboCasa infrastructure attempts must not count as policy episodes")
require(excluded["status"] == "error" and excluded.get("reason") in {
"simulation_error", "codex_error", "video_recording_error"}, "Unsupported RoboCasa excluded outcome")
integer(excluded.get("steps"), "RoboCasa excluded steps")
integer(excluded.get("visualObservations"), "RoboCasa excluded RGB observations")
if excluded["reason"] == "simulation_error":
require(excluded["steps"] == excluded["visualObservations"] == 0
and excluded.get("nativeSuccess") is True, "Unproved initial-scene exclusion")
else:
require(excluded.get("nativeSuccess") is False and excluded["steps"] > 0
and excluded["visualObservations"] > 0, "Unproved retained API failure")
require(excluded["seed"] == selection["seed"], "An API failure cannot advance the seed")
check_digest(excluded.get("videoSha256"), "RoboCasa excluded video")
check_digest(excluded["resultSha256"], "excluded result")
excluded_count += 1
if not is_robotwin_nvidia10:
require(excluded_count == run["excludedAttemptCount"],
f"Wrong infrastructure/interruption retry count: {benchmark_id}")
if not is_robocasa and not is_robotwin_nvidia10:
require(excluded_count == {"libero": 4, "robotwin": 6, "robodojo": 0}[benchmark_id],
f"Unexpected historical retry count: {benchmark_id}")
if is_robotwin_nvidia10:
require(run["excludedAttemptCount"] == 21, "Unexpected RoboTwin 10-rollout excluded episode count")
media = report["media"]
require(len(media) == expected_episodes and {item["episode"] for item in media} == set(episodes), "Incomplete media provenance mapping")
videos, video_bytes, poster_bytes = [], 0, 0
if release_assets is not None:
require(set(release_assets) == {f"{key}.mp4" for key, item in episodes.items() if item["benchmark"] == "robocasa"},
"Release must contain exactly the 365 selected RoboCasa videos")
for record in media:
item = episodes[record["episode"]]
episode = item["episode"]
if item["benchmark"] == "robodojo":
public_fields(record, {"episode", "benchmark", "sourceSha256", "sourceBytes", "video", "videoSha256", "videoBytes",
"poster", "posterSha256", "posterBytes", "posterFrame", "codec", "pixelFormat", "width",
"height", "fps", "frames", "durationSeconds", "fastStart", "verified"}, "RoboDojo media")
require(record["benchmark"] == item["benchmark"], "Wrong media benchmark mapping")
for key in ("video", "poster", "width", "height", "frames"):
require(record[key] == episode[key], f"Media/source mapping mismatch: {record['episode']}.{key}")
require(record["codec"] == "h264" and record["pixelFormat"] == "yuv420p"
and record["fastStart"] is True and record["verified"] is True, "Unverified or incompatible video export")
require(Fraction(record["fps"]) == item["fps"], "Video frame-rate mapping differs from the source protocol")
require(abs(record["durationSeconds"] - episode["durationSeconds"]) < 1e-3, "Video duration mapping differs from source")
require(0 <= integer(record["posterFrame"], "poster frame") < episode["frames"], "Poster references a missing source frame")
check_digest(record["sourceSha256"], "source video")
if item["benchmark"] in {"robocasa", "robodojo", "robotwin_nvidia10"}:
selection = next(selection for run in runs if run["benchmark"] == item["benchmark"]
for selection in run["attemptSelection"] if selection["episode"] == record["episode"])
label = {"robocasa": "RoboCasa", "robodojo": "RoboDojo", "robotwin_nvidia10": "RoboTwin 10-rollout"}[item["benchmark"]]
require(record["sourceSha256"] == selection["sourceVideoSha256"], f"{label} audit/source video mismatch")
integer(record["sourceBytes"], "source video size", 1)
for key in ("video", "poster"):
check_digest(record[f"{key}Sha256"], f"{key} export")
is_release_video = item["benchmark"] == "robocasa" and key == "video" and release_assets is not None
path = local_asset(root, record[key], key, allow_missing=is_release_video)
if is_release_video:
asset = release_assets[f"{record['episode']}.mp4"]
require(asset.get("state") == "uploaded" and asset.get("size") == record["videoBytes"]
and asset.get("digest") == f"sha256:{record['videoSha256']}"
and asset.get("browser_download_url") == episode["remoteVideo"],
f"Release asset fingerprint mismatch: {record['episode']}")
if path.exists():
require(path.is_file() and path.stat().st_size == record["videoBytes"]
and sha256(path) == record["videoSha256"],
f"Local release video fingerprint mismatch: {record['episode']}")
else:
require(path.stat().st_size == record[f"{key}Bytes"], f"Media byte count mismatch: {record[key]}")
require(sha256(path) == record[f"{key}Sha256"], f"Media digest mismatch: {record[key]}")
video_bytes += record["videoBytes"]
poster_bytes += record["posterBytes"]
video_source = episode["remoteVideo"] if item["benchmark"] == "robocasa" and release_assets is not None else root / record["video"]
videos.append((video_source, {"width": record["width"], "height": record["height"],
"frames": record["frames"], "fps": item["fps"]}))
require(report["videoBytes"] == video_bytes and report["posterBytes"] == poster_bytes
and report["publishedMediaBytes"] == video_bytes + poster_bytes, "Incorrect published media size total")
require(report["sourceBytes"] == sum(record["sourceBytes"] for record in media), "Incorrect source media size total")
if has_robocasa:
external_bytes = sum(record["videoBytes"] for record in media if record["benchmark"] == "robocasa")
pages_bytes = video_bytes + poster_bytes - external_bytes
require(report.get("externalVideoBytes") == external_bytes and report.get("pagesMediaBytes") == pages_bytes,
"Incorrect release/Pages media size totals")
require(pages_bytes < 1_500_000_000, "Pages media exceeds 1.5 GB")
require(report["validation"]["decodedFrameCount"] == sum(record["frames"] for record in media),
"Incorrect mapped frame total")
demo_coverage, demo_videos, demo_paths = validate_training_demos(root, gallery, required=require_demos)
require(not media_paths.intersection(demo_paths), "Training and evaluation media must have distinct assets")
demo_bytes = sum((root / relative).stat().st_size for relative in demo_paths)
if has_robocasa:
require(pages_bytes + demo_bytes < 1_500_000_000, "Pages media including training demos exceeds 1.5 GB")
media_paths.update(demo_paths)
videos.extend(demo_videos)
blog_paths = validate_blog_media(root)
additional_blog_paths = blog_paths - media_paths
blog_bytes = sum((root / relative).stat().st_size for relative in additional_blog_paths)
media_paths.update(blog_paths)
if has_robocasa:
require(pages_bytes + demo_bytes + blog_bytes < 1_500_000_000,
"Pages media including training demos and blog clips exceeds 1.5 GB")
validate_media_inventory(root, media_paths, external_media if release_assets is not None else ())
with (root / "data/episodes.csv").open(newline="", encoding="utf-8") as stream:
csv_rows = list(csv.DictReader(stream))
require(len(csv_rows) == expected_episodes and {row["episode"] for row in csv_rows} == set(episodes), "CSV episode mapping is incomplete")
for row in csv_rows:
public_metadata(row, "episodes.csv")
item = episodes[row["episode"]]
expected_row = {"benchmark": item["benchmark"], "suite": item["task"]["suite"], "task": item["task"]["id"],
"instruction": item["episode"].get("instruction", item["task"]["instruction"]), "episode": row["episode"],
**{key: item["episode"][key] for key in ("status", "seed", "initStateId", "steps", "maxSteps",
"toolCalls", "wallSeconds", "durationSeconds", "frames", "video", "poster")},
"rolloutIndex": item["episode"]["index"]}
if has_robocasa:
expected_row["remoteVideo"] = item["episode"].get("remoteVideo")
require(row == {key: "" if value is None else str(value) for key, value in expected_row.items()},
f"CSV/JSON mismatch: {row['episode']}")
return {"benchmarks": summaries, "episodes": len(episodes), "tasks": len(task_ids),
"training_demos": demo_coverage,
"training_demo_bytes": demo_bytes,
"blog_media_files": len(additional_blog_paths), "blog_media_bytes": blog_bytes,
"media_files": len(media_paths), "media_bytes": video_bytes + poster_bytes + demo_bytes + blog_bytes,
"repository_files": tree["files"], "repository_bytes": tree["bytes"], "pages_bytes": tree["siteBytes"]}, videos
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--root", type=Path, default=DEFAULT_ROOT, help="Gallery repository root")
parser.add_argument("--videos", action="store_true", help="Decode and validate every video with ffprobe")
parser.add_argument("--require-robocasa", action="store_true", help="Require all 365 RoboCasa episodes in addition to LIBERO and RoboTwin")
parser.add_argument("--require-robodojo", action="store_true", help="Require all 42 standard RoboDojo episodes")
parser.add_argument("--require-demos", action="store_true", help="Require the training catalog; RoboDojo may remain wholly unimported")
parser.add_argument("--release-media", action="store_true", help="Verify RoboCasa videos through the public Release asset SHA256/size registry")
parser.add_argument("--workers", type=int, default=4, help="Concurrent ffprobe processes (default: 4)")
args = parser.parse_args()
try:
require(args.root.is_dir(), "Gallery root is missing")
require(1 <= args.workers <= 32, "Workers must be between 1 and 32")
release_assets = None
if args.release_media:
from release_media import fetch_release_assets
release_assets = fetch_release_assets()
result, videos = validate_gallery(args.root.resolve(), require_robocasa=args.require_robocasa,
release_assets=release_assets, require_demos=args.require_demos,
require_robodojo=args.require_robodojo)
if args.videos:
require(shutil.which("ffprobe") is not None, "Install ffprobe to use --videos")
with ThreadPoolExecutor(max_workers=args.workers) as pool:
frames = sum(pool.map(probe_video, videos))
result.update(videos_decoded=len(videos), frames_decoded=frames)
result["passed"] = True
print(json.dumps(result, indent=2, sort_keys=True))
except (ValidationError, OSError, KeyError, TypeError, ValueError, subprocess.TimeoutExpired) as exc:
print(f"Gallery validation failed: {exc}", file=sys.stderr)
return 1
return 0
if __name__ == "__main__":
raise SystemExit(main())
|