Upload folder using huggingface_hub

DOCKERFILE.dockerfile

@@ -0,0 +1,111 @@
+ARG COSMOS_RL_BUILD_MODE=efa
+ARG COSMOS_RL_EXTRAS=""
+
+ARG CUDA_VERSION=12.8.1
+
+FROM nvcr.io/nvidia/cuda:${CUDA_VERSION}-devel-ubuntu22.04 AS no-efa-base
+
+# System dependencies for eval
+RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \
+    python3.10-dev python3-dev python3-distutils \
+    libgl1 libglib2.0-0 libglu1-mesa \
+    curl git ca-certificates \
+    tmux zip unzip vim \
+    && rm -rf /var/lib/apt/lists/*
+
+# Install uv package manager
+RUN curl -LsSf https://astral.sh/uv/install.sh | sh
+ENV PATH="/root/.local/bin:$PATH"
+
+# Install Miniconda
+RUN curl -fsSL -o /tmp/miniconda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh \
+    && bash /tmp/miniconda.sh -b -p /opt/miniconda3 \
+    && rm /tmp/miniconda.sh
+ENV PATH="/opt/miniconda3/bin:$PATH"
+
+ENV EVAL_ROOT=/opt/eval
+
+# Clone b1k-eval-utils
+RUN mkdir -p ${EVAL_ROOT} \
+    && git clone https://github.com/littlespray/b1k-eval-utils.git ${EVAL_ROOT}/b1k-eval-utils
+
+# Clone and setup openpi-comet
+SHELL ["/bin/bash", "-c"]
+RUN cd ${EVAL_ROOT} \
+    && git clone https://github.com/mli0603/openpi-comet.git --depth=1 \
+    && cd openpi-comet \
+    && cp ${EVAL_ROOT}/b1k-eval-utils/eval_openpi.sh ${EVAL_ROOT}/openpi-comet/eval_openpi.sh \
+    && cp ${EVAL_ROOT}/b1k-eval-utils/patch_comet_safetensors.sh ${EVAL_ROOT}/openpi-comet/patch_comet_safetensors.sh \
+    && GIT_LFS_SKIP_SMUDGE=1 uv sync \
+    && GIT_LFS_SKIP_SMUDGE=1 uv pip install -e . \
+    && uv pip install transformers==4.53.2 \
+    && cp -r ./src/openpi/models_pytorch/transformers_replace/* .venv/lib/python3.11/site-packages/transformers/ \
+    && bash patch_comet_safetensors.sh
+
+
+# Clone BEHAVIOR-1K and install bddl/OmniGibson into openpi-comet venv
+RUN git clone -b v3.7.2 --single-branch --depth=1 \
+    https://github.com/StanfordVL/BEHAVIOR-1K.git ${EVAL_ROOT}/BEHAVIOR-1K
+
+RUN cd ${EVAL_ROOT}/BEHAVIOR-1K \
+    && source ${EVAL_ROOT}/openpi-comet/.venv/bin/activate \
+    && uv pip install -e bddl3 \
+    && uv pip install -e "OmniGibson[eval]" \
+    && deactivate
+
+# Prepare b1k environment
+# RUN cd ${EVAL_ROOT}/BEHAVIOR-1K \
+#     && cp ${EVAL_ROOT}/b1k-eval-utils/patch_setup.sh patch_setup.sh \
+#     && bash patch_setup.sh \
+
+WORKDIR ${EVAL_ROOT}/BEHAVIOR-1K
+
+# Run b1k setup.sh (separate layer — most likely to fail, easy to retry)
+ENV TORCH_CUDA_ARCH_LIST="8.0 9.0+PTX"
+RUN cd ${EVAL_ROOT}/BEHAVIOR-1K \
+    && source /opt/miniconda3/etc/profile.d/conda.sh \
+    && conda deactivate \
+    && ./setup.sh --new-env --omnigibson --bddl --joylo --eval --primitives \
+        --accept-conda-tos --accept-nvidia-eula
+        
+RUN source /opt/miniconda3/etc/profile.d/conda.sh \
+    && conda activate behavior \
+    && pip install scipy==1.11.4 \
+    && pip uninstall -y numpy \
+    && pip uninstall -y numpy \
+    && pip install numpy==1.26.4 opencv-contrib-python==4.10.0.84
+
+
+
+# RUN cd ${EVAL_ROOT}/BEHAVIOR-1K \
+#     && source /opt/miniconda3/etc/profile.d/conda.sh \
+#     && conda activate behavior \
+#     && python -c "from omnigibson.utils.asset_utils import download_omnigibson_robot_assets; download_omnigibson_robot_assets()"
+#     && python -c "from omnigibson.utils.asset_utils import download_behavior_1k_assets; download_behavior_1k_assets(accept_license=True)"
+#     && python -c "from omnigibson.utils.asset_utils import download_2025_challenge_task_instances; download_2025_challenge_task_instances()"
+
+# Copy eval script
+RUN cp ${EVAL_ROOT}/b1k-eval-utils/eval_b1k.sh ${EVAL_ROOT}/BEHAVIOR-1K/eval_b1k.sh
+
+# Download HF models into the image for offline eval.
+# Append new model repos here, separated by spaces.
+
+# ENV HF_MODEL_REPOS="shangkuns/pt12-3w-comet2-20260204022222 shangkuns/pt12-2w-comet2-20260205151209 shangkuns/cosmos-pi05-sft-pt12-20260203145532 shangkuns/pt12-3w-20260204011934"
+# RUN pip install --no-cache-dir "huggingface_hub[cli]" \
+#     && mkdir -p /opt/eval/models \
+#     && for repo in $HF_MODEL_REPOS; do \
+#         name="${repo#*/}"; \
+#         hf download "$repo" --repo-type model --local-dir "/opt/eval/models/$name"; \
+#     done
+
+
+RUN cd ${EVAL_ROOT} \
+&& cp -r openpi-comet/src/behavior/learning/* BEHAVIOR-1K/OmniGibson/omnigibson/learning/
+
+# 
+ENV OPENPI_DATA_HOME=/opt/openpi-cache
+RUN mkdir -p /opt/openpi-cache/big_vision \
+    && cp ${EVAL_ROOT}/b1k-eval-utils/tokenizer.model /opt/openpi-cache/big_vision/paligemma_tokenizer.model
+
+# Install huggingface-hub in the default Python environment
+RUN python -m pip install huggingface-hub wandb

batch_eval_lep.sh

@@ -0,0 +1,99 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+task_names=(
+  "turning_on_radio"
+  # "picking_up_trash"
+  # "picking_up_toys"
+  # "rearranging_kitchen_furniture"
+  # "putting_up_Christmas_decorations_inside"
+  # "sorting_vegetables"
+  # "putting_shoes_on_rack"
+  # "boxing_books_up_for_storage"
+  # "storing_food"
+  # "sorting_household_items"
+  # "wash_a_baseball_cap"
+  # "wash_dog_toys"
+  # "hanging_pictures"
+  # "attach_a_camera_to_a_tripod"
+  # "clean_a_trumpet"
+  # "spraying_for_bugs"
+  # "make_microwave_popcorn"
+  # "freeze_pies"
+)
+
+node_per_task=2
+gpu_per_node=4
+episodes_per_task=10
+num_test_per_episode=2
+ckpt_basename="pi05-b1kpt12-cs32"
+
+total_work=$((episodes_per_task * num_test_per_episode))
+base=$((total_work / node_per_task))
+rem=$((total_work % node_per_task))
+
+submitted=0
+
+for task_name in "${task_names[@]}"; do
+  for (( node = 0; node < node_per_task; node++ )); do
+    if (( node < rem )); then
+      node_trials=$((base + 1))
+      node_start=$((node * (base + 1)))
+    else
+      node_trials=$base
+      node_start=$((rem * (base + 1) + (node - rem) * base))
+    fi
+
+    job_command="$(cat <<EOF
+export TASK_NAME="${task_name}"
+export CKPT_BASENAME="${ckpt_basename}"
+export PATH_TO_CKPT=/tmp/\${CKPT_BASENAME}
+export OPENPI_DATA_HOME=/opt/openpi-cache
+export EPISODES_PER_TASK=${episodes_per_task}
+export EVAL_START=${node_start}
+export EVAL_COUNT=${node_trials}
+export NUM_GPU=${gpu_per_node}
+
+EVAL_ROOT=/opt/eval
+
+cd \${EVAL_ROOT}/BEHAVIOR-1K
+source /opt/miniconda3/etc/profile.d/conda.sh
+conda activate behavior
+python -c "from omnigibson.utils.asset_utils import download_omnigibson_robot_assets; download_omnigibson_robot_assets()"
+python -c "from omnigibson.utils.asset_utils import download_behavior_1k_assets; download_behavior_1k_assets(accept_license=True)"
+python -c "from omnigibson.utils.asset_utils import download_2025_challenge_task_instances; download_2025_challenge_task_instances()"
+conda deactivate
+
+hf download shangkuns/\${CKPT_BASENAME} --local-dir \${PATH_TO_CKPT}
+mkdir -p \${PATH_TO_CKPT}/assets/behavior-1k/2025-challenge-demos
+cp \${EVAL_ROOT}/b1k-eval-utils/norm_stats.json \${PATH_TO_CKPT}/assets/behavior-1k/2025-challenge-demos/
+
+cd \${EVAL_ROOT}/b1k-eval-utils
+bash patch_walltime.sh \${EVAL_ROOT}/BEHAVIOR-1K/OmniGibson/omnigibson/learning
+bash run_dual_eval.sh
+
+hf upload shangkuns/\${CKPT_BASENAME} /opt/eval/eval_output \${TASK_NAME} --repo-type dataset
+EOF
+)"
+
+    lep job create \
+      --resource-shape "my.${gpu_per_node}xl40s" \
+      --node-group oci-ord-lepton-001 \
+      --num-workers 1 \
+      --container-image littlespray/b1k-eval-light:v4 \
+      --command "${job_command}" \
+      --intra-job-communication=true \
+      --env NVIDIA_DRIVER_CAPABILITIES=all \
+      --env DEBIAN_FRONTEND=noninteractive \
+      --secret HUGGING_FACE_HUB_TOKEN=SHANGKUN_HF_NV.shangkuns \
+      --ttl-seconds-after-finished 259200 \
+      --log-collection true \
+      --queue-priority 8 \
+      --can-preempt \
+      --name "${task_name//_/-}-n${node}"
+
+    submitted=$((submitted + 1))
+  done
+done
+
+echo "Submitted ${submitted} jobs."

eval_b1k.sh

@@ -0,0 +1,39 @@
+export ROOT_DIR=${ROOT_DIR:-/opt/eval}
+export OMNIGIBSON_HEADLESS=${OMNIGIBSON_HEADLESS:-1}
+export NUM_GPU=${NUM_GPU:-1}
+export BASE_PORT=${BASE_PORT:-8000}
+export TASK_NAME=${TASK_NAME:-turning_on_radio}
+export LOG_PATH=${LOG_PATH:-${ROOT_DIR}/eval_output}
+mkdir -p "$LOG_PATH"
+
+TOTAL=${EVAL_COUNT:-4}
+START=${EVAL_START_IDX:-0}
+
+BASE=$((TOTAL / NUM_GPU))
+REM=$((TOTAL % NUM_GPU))
+
+for ((gpu = 0; gpu < NUM_GPU; gpu++)); do
+  if ((gpu < REM)); then
+    count=$((BASE + 1))
+    s=$((START + gpu * (BASE + 1)))
+  else
+    count=$BASE
+    s=$((START + REM * (BASE + 1) + (gpu - REM) * BASE))
+  fi
+  (( count == 0 )) && continue
+
+  CUDA_VISIBLE_DEVICES="$gpu" \
+    python "${ROOT_DIR}/BEHAVIOR-1K/OmniGibson/omnigibson/learning/eval_custom.py" \
+      policy=websocket \
+      task.name=$TASK_NAME \
+      log_path=$LOG_PATH \
+      env_wrapper._target_=omnigibson.learning.wrappers.RGBWrapper \
+      save_rollout=false \
+      perturb_pose=false \
+      use_parallel_evaluator=true \
+      parallel_evaluator_start_idx=$s \
+      parallel_evaluator_end_idx=$((s + count)) \
+      model.port=$((BASE_PORT + gpu)) \
+      > "$LOG_PATH/${gpu}_stdout.log" 2>&1 &
+done
+wait

eval_b1k_json.py

@@ -0,0 +1,195 @@
+#!/usr/bin/env python3
+"""
+统计 Hugging Face dataset 仓库中所有 exp 的逐轮成功率。
+
+目录结构约定：task/exp/metrics/*.json
+示例文件：
+  turning-on-radio/turning_on_radio_xxx_20260216085234/metrics/turning_on_radio_109_0.json
+
+规则：
+1. 对同一个 episode id，收集所有 exp 中对应 json；
+2. 按 exp 目录末尾时间戳升序排序；
+3. 第 i 轮平均值 = 所有 episode 各自第 i 个结果的 q_score.final 平均；
+4. 当任一 episode 不足第 i 个结果时停止；
+5. 输出每轮平均值，以及这些平均值的平均值。
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import re
+from collections import defaultdict
+from dataclasses import dataclass
+from statistics import mean
+from typing import DefaultDict, Dict, List, Tuple
+
+from huggingface_hub import HfApi, hf_hub_download
+
+
+EXP_TIMESTAMP_RE = re.compile(r"_(\d{14})$")
+EPISODE_ID_RE = re.compile(r"_(\d+)_\d+\.json$")
+METRICS_JSON_RE = re.compile(r"^[^/]+/[^/]+/metrics/[^/]+\.json$")
+
+
+@dataclass(frozen=True)
+class EpisodeEval:
+    task: str
+    episode_id: str
+    timestamp: str
+    q_score: float
+    wall_time: float | None
+    path: str
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(
+        description="计算 HF dataset 仓库中逐轮 episode 成功率均值。"
+    )
+    parser.add_argument(
+        "repo_id",
+        help="HF dataset 仓库名，例如 shangkuns/pt12-3w-comet2-20260204022222",
+    )
+    parser.add_argument(
+        "--repo-type",
+        default="dataset",
+        choices=["dataset", "model", "space"],
+        help="HF 仓库类型，默认 dataset。",
+    )
+    return parser.parse_args()
+
+
+def extract_timestamp_from_exp(exp_name: str) -> str | None:
+    match = EXP_TIMESTAMP_RE.search(exp_name)
+    return match.group(1) if match else None
+
+
+def extract_episode_id_from_filename(filename: str) -> str | None:
+    match = EPISODE_ID_RE.search(filename)
+    return match.group(1) if match else None
+
+
+def collect_episode_evals(
+    repo_id: str, repo_type: str
+) -> Dict[Tuple[str, str], List[EpisodeEval]]:
+    api = HfApi()
+    repo_files = api.list_repo_files(repo_id=repo_id, repo_type=repo_type)
+
+    episode_to_evals: DefaultDict[Tuple[str, str], List[EpisodeEval]] = defaultdict(list)
+
+    for path in repo_files:
+        if not METRICS_JSON_RE.match(path):
+            continue
+
+        parts = path.split("/")
+        # task / exp / metrics / xxx.json
+        if len(parts) != 4:
+            continue
+
+        task_name = parts[0]
+        exp_name = parts[1]
+        file_name = parts[3]
+
+        timestamp = extract_timestamp_from_exp(exp_name)
+        if timestamp is None:
+            continue
+
+        episode_id = extract_episode_id_from_filename(file_name)
+        if episode_id is None:
+            continue
+
+        local_path = hf_hub_download(repo_id=repo_id, repo_type=repo_type, filename=path)
+        with open(local_path, "r", encoding="utf-8") as f:
+            content = json.load(f)
+
+        q_score = float(content["q_score"]["final"])
+        wall_time = None
+        if isinstance(content.get("time"), dict) and "wall_time" in content["time"]:
+            wall_time = float(content["time"]["wall_time"])
+
+        episode_to_evals[(task_name, episode_id)].append(
+            EpisodeEval(
+                task=task_name,
+                episode_id=episode_id,
+                timestamp=timestamp,
+                q_score=q_score,
+                wall_time=wall_time,
+                path=path,
+            )
+        )
+
+    return dict(episode_to_evals)
+
+
+def compute_round_means(
+    episode_to_evals: Dict[Tuple[str, str], List[EpisodeEval]]
+) -> Tuple[List[float], List[EpisodeEval]]:
+    if not episode_to_evals:
+        return [], []
+
+    for evals in episode_to_evals.values():
+        evals.sort(key=lambda x: x.timestamp)
+
+    max_rounds = min(len(evals) for evals in episode_to_evals.values())
+    if max_rounds == 0:
+        return [], []
+
+    round_means: List[float] = []
+    used_evals: List[EpisodeEval] = []
+    episode_keys = sorted(episode_to_evals.keys(), key=lambda x: (x[0], int(x[1])))
+    for round_idx in range(max_rounds):
+        this_round = [episode_to_evals[key][round_idx] for key in episode_keys]
+        values = [item.q_score for item in this_round]
+        used_evals.extend(this_round)
+        round_means.append(mean(values))
+    return round_means, used_evals
+
+
+def compute_wall_time_stats(used_evals: List[EpisodeEval]) -> Tuple[Dict[str, float], float | None]:
+    task_to_times: DefaultDict[str, List[float]] = defaultdict(list)
+    for item in used_evals:
+        if item.wall_time is not None:
+            task_to_times[item.task].append(item.wall_time)
+
+    task_means = {task: mean(times) for task, times in task_to_times.items() if times}
+    if not task_means:
+        return task_means, None
+    overall_task_mean = mean(task_means.values())
+    return task_means, overall_task_mean
+
+
+def main() -> None:
+    args = parse_args()
+
+    episode_to_evals = collect_episode_evals(args.repo_id, args.repo_type)
+    if not episode_to_evals:
+        print("未找到符合 task/exp/metrics/*.json 结构且可解析 episode/timestamp 的文件。")
+        return
+
+    round_means, used_evals = compute_round_means(episode_to_evals)
+    if not round_means:
+        print("没有可计算的轮次（可能某些 episode 无有效结果）。")
+        return
+
+    print(f"repo: {args.repo_id}")
+    print(f"episode 数量: {len(episode_to_evals)}")
+    print(f"可计算轮次: {len(round_means)}")
+    for idx, value in enumerate(round_means, start=1):
+        print(f"第{idx}轮平均成功率: {value:.6f}")
+
+    overall = mean(round_means)
+    print(f"各轮平均值的平均值: {overall:.6f}")
+
+    task_means, overall_task_mean = compute_wall_time_stats(used_evals)
+    if not task_means:
+        print("参与 q-score 统计的样本中未找到 time.wall_time，跳过用时统计。")
+        return
+
+    print("各 task 平均 episode 用时:")
+    for task in sorted(task_means.keys()):
+        print(f"- {task}: {task_means[task]:.6f}")
+    print(f"所有 task 平均用时: {overall_task_mean:.6f}")
+
+
+if __name__ == "__main__":
+    main()

eval_openpi.sh

@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+TASK_NAME="${TASK_NAME:-cook_hot_dogs}"
+CONFIG="${CONFIG:-pi05_b1k-base}"
+PATH_TO_CKPT="${PATH_TO_CKPT:-/workspace/comet-pi05-b1k-pt50-ptm}"
+PORT="${PORT:-8000}"
+
+exec uv run scripts/serve_b1k.py \
+  --task_name "$TASK_NAME" \
+  --port "$PORT" \
+  policy:checkpoint \
+  --policy.config "$CONFIG" \
+  --policy.dir "$PATH_TO_CKPT"

norm_stats.json

@@ -0,0 +1,280 @@
+{
+  "norm_stats": {
+    "state": {
+      "mean": [
+        -0.0007269753236491301,
+        0.0005871853206764299,
+        -0.00037458819335587837,
+        1.1220268708249688,
+        -1.6177487234510692,
+        -0.7037017488819726,
+        5.178331679376605e-6,
+        -0.3819930547932975,
+        0.0636385481520187,
+        -0.036330391163538304,
+        -0.5923406613420917,
+        0.27300850717924735,
+        0.06392431067313378,
+        -0.23112970703914087,
+        -0.4207707978561216,
+        -0.06831169401385709,
+        0.0243751039006989,
+        -0.6469659968323186,
+        -0.12143957181667113,
+        0.14976853360672077,
+        0.130979475995902,
+        0.08751846492897486,
+        0.07864204959670715,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0
+      ],
+      "std": [
+        0.02199286521964398,
+        0.01975387071601238,
+        0.027931866359868447,
+        0.43276966334356803,
+        0.7793389811993767,
+        0.4685643576682486,
+        0.00007484946359270099,
+        0.3495479212084949,
+        0.14741056781059605,
+        0.4086500808617923,
+        0.45792477411954097,
+        0.4472871307419175,
+        0.4255421558256086,
+        0.5117453304941143,
+        0.3626599950258924,
+        0.14969143472339538,
+        0.3820732235768937,
+        0.5161227600903063,
+        0.42373140636873546,
+        0.4839893318550682,
+        0.4912556634102691,
+        0.026452244464230175,
+        0.03343202113475432,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0
+      ],
+      "q01": [
+        -0.2613118174761534,
+        -0.25247000222802163,
+        -0.29782283776104446,
+        0.4487143921405077,
+        -2.5600294193029405,
+        -1.8321708536148071,
+        -0.0007136256152298301,
+        -2.105691260957718,
+        -0.17450003325939178,
+        -2.230193532180786,
+        -1.6330286006212233,
+        -1.411822907948494,
+        -1.0471969878673553,
+        -1.5708000659942627,
+        -1.91510467287302,
+        -0.849857434040308,
+        -1.6666959735155105,
+        -1.6168319140672684,
+        -2.3132422157287595,
+        -1.0375005970597269,
+        -1.570798397064209,
+        0.0,
+        6.501830672824839e-8,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0
+      ],
+      "q99": [
+        0.2542599085897208,
+        0.2542742739796637,
+        0.30030812568068505,
+        1.716256618249416,
+        -0.3974635741829872,
+        0.1548894247069954,
+        0.0005543326894694459,
+        0.37031316382586876,
+        0.9278406740725034,
+        1.91316024565696,
+        0.09516809616684914,
+        2.356194269657135,
+        1.0472097456455232,
+        1.517247679293155,
+        0.36055924194753,
+        0.17450004816055298,
+        1.6652137512683844,
+        0.0914144726134836,
+        1.6950346724390943,
+        1.0471995235800744,
+        1.56818338637352,
+        0.10000000149011612,
+        0.10000000149011612,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0
+      ]
+    },
+    "actions": {
+      "mean": [
+        0.04972944409549112,
+        0.003690394849812747,
+        -0.0029349690138718285,
+        1.122085165280863,
+        -1.6180965905064797,
+        -0.7043162896930203,
+        0.0,
+        -0.382348926450185,
+        0.06370661786532403,
+        -0.03661990100703703,
+        -0.5924590200206378,
+        0.2750886004936484,
+        0.06431002005410705,
+        -0.23144970536531229,
+        0.4210819724256232,
+        -0.4212541587202455,
+        -0.06839742236782122,
+        0.024488948505712194,
+        -0.6468751837828239,
+        -0.12300583051861715,
+        0.1507738855467879,
+        0.13119771996817328,
+        0.20474606868389103,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0
+      ],
+      "std": [
+        0.13018670744434768,
+        0.07378928098443516,
+        0.11732054405741633,
+        0.4326509056935839,
+        0.7791573756824512,
+        0.46843066554707175,
+        0.0,
+        0.34964716759678244,
+        0.14751595777952248,
+        0.41012746451839815,
+        0.4580261645466588,
+        0.4486994719029271,
+        0.4276847015578234,
+        0.5130942257930428,
+        0.9070220542734125,
+        0.3626863499954174,
+        0.14981404106277435,
+        0.3835292477509873,
+        0.5164924719696999,
+        0.4256839962396164,
+        0.48620749808715086,
+        0.49307611404915425,
+        0.9788150111676108,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0
+      ],
+      "q01": [
+        -0.34868216374218464,
+        -0.3418492655992508,
+        -0.2999999523162842,
+        0.44999998807907104,
+        -2.560204291343689,
+        -1.8325998783111572,
+        0.0,
+        -2.1070283545494077,
+        -0.1745000034570694,
+        -2.230065962147713,
+        -1.6336875317335129,
+        -1.4130705307483673,
+        -1.0471980571746826,
+        -1.5707999467849731,
+        -1.0,
+        -1.9151124985814094,
+        -0.8498951392710209,
+        -1.6674389212727547,
+        -1.616815806877613,
+        -2.3132428766012194,
+        -1.0385202980041504,
+        -1.5707999467849731,
+        -1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0
+      ],
+      "q99": [
+        0.34921015488803386,
+        0.34948091855645175,
+        0.2999999523162842,
+        1.7161086392402647,
+        -0.4000000059604645,
+        0.1548384608328342,
+        0.0,
+        0.3706086640655987,
+        0.9287866185903546,
+        1.9131654405593803,
+        0.09510681201517582,
+        2.3561956882476807,
+        1.0471980571746826,
+        1.5173104754924756,
+        1.0,
+        0.3605582716763002,
+        0.1745000034570694,
+        1.6659219603061668,
+        0.09357282519340515,
+        1.6951719859242371,
+        1.0471980571746826,
+        1.5707999467849731,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0
+      ]
+    }
+  }
+}

patch_comet_safetensors.sh

@@ -0,0 +1,51 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+ROOT_DIR="${ROOT_DIR:-$SCRIPT_DIR}"
+
+POLICY_CONFIG_FILE="${ROOT_DIR}/src/openpi/policies/policy_config.py"
+MODEL_FILE="${ROOT_DIR}/src/openpi/models/model.py"
+
+python3 - "$POLICY_CONFIG_FILE" "$MODEL_FILE" <<'PY'
+import pathlib
+import sys
+
+policy_config_file = pathlib.Path(sys.argv[1])
+model_file = pathlib.Path(sys.argv[2])
+
+
+def replace_or_exit(path: pathlib.Path, old: str, new: str, label: str) -> None:
+    text = path.read_text()
+    if new in text:
+        print(f"[skip] {path.name}: {label} already applied")
+        return
+    if old not in text:
+        raise SystemExit(f"[error] Could not find expected block for {label} in {path}")
+    path.write_text(text.replace(old, new))
+    print(f"[ok] {path.name}: {label}")
+
+
+replace_or_exit(
+    policy_config_file,
+    """    # Check if this is a PyTorch model by looking for model.safetensors\n    weight_path = os.path.join(checkpoint_dir, "model.safetensors")\n    is_pytorch = os.path.exists(weight_path)\n""",
+    """    # Check if this is a PyTorch model by looking for either a monolithic\n    # safetensors file or a sharded safetensors index.\n    single_weight_path = os.path.join(checkpoint_dir, "model.safetensors")\n    sharded_index_path = os.path.join(checkpoint_dir, "model.safetensors.index.json")\n    if os.path.exists(single_weight_path):\n        weight_path = single_weight_path\n    elif os.path.exists(sharded_index_path):\n        weight_path = sharded_index_path\n    else:\n        weight_path = single_weight_path\n    is_pytorch = os.path.exists(single_weight_path) or os.path.exists(sharded_index_path)\n""",
+    "detect sharded safetensors checkpoints",
+)
+
+replace_or_exit(
+    model_file,
+    """import abc\nfrom collections.abc import Sequence\nimport dataclasses\nimport enum\nimport logging\nimport pathlib\nfrom typing import Generic, TypeVar\n""",
+    """import abc\nfrom collections.abc import Sequence\nimport dataclasses\nimport enum\nimport json\nimport logging\nimport pathlib\nfrom typing import Generic, TypeVar\n""",
+    "add json import",
+)
+
+replace_or_exit(
+    model_file,
+    """    def load_pytorch(self, train_config, weight_path: str):\n        logger.info(f"train_config: {train_config}")\n        model = pi0_pytorch.PI0Pytorch(config=train_config.model)\n        safetensors.torch.load_model(model, weight_path)\n        return model\n""",
+    """    def load_pytorch(self, train_config, weight_path: str):\n        logger.info(f"train_config: {train_config}")\n        model = pi0_pytorch.PI0Pytorch(config=train_config.model)\n\n        weight_path_obj = pathlib.Path(weight_path)\n        if weight_path_obj.name == "model.safetensors.index.json":\n            if not weight_path_obj.exists():\n                raise FileNotFoundError(f"Sharded index file not found at {weight_path_obj}")\n\n            with weight_path_obj.open("r", encoding="utf-8") as f:\n                index_data = json.load(f)\n\n            weight_map = index_data.get("weight_map", {})\n            if not isinstance(weight_map, dict) or not weight_map:\n                raise ValueError(\n                    f"Invalid sharded checkpoint index at {weight_path_obj}: missing non-empty weight_map"\n                )\n\n            shard_names = sorted(set(weight_map.values()))\n            missing_shards = [\n                shard_name for shard_name in shard_names if not (weight_path_obj.parent / shard_name).exists()\n            ]\n            if missing_shards:\n                raise FileNotFoundError(\n                    f"Missing safetensors shards under {weight_path_obj.parent}: {missing_shards}"\n                )\n\n            state_dict = {}\n            for shard_name in shard_names:\n                shard_path = weight_path_obj.parent / shard_name\n                shard_state = safetensors.torch.load_file(str(shard_path))\n                duplicated_keys = set(state_dict).intersection(shard_state)\n                if duplicated_keys:\n                    raise ValueError(\n                        f"Duplicated tensor keys detected across shards: {sorted(duplicated_keys)[:10]}"\n                    )\n                state_dict.update(shard_state)\n\n            missing_keys, unexpected_keys = model.load_state_dict(state_dict, strict=False)\n            if unexpected_keys:\n                raise ValueError(\n                    f"Unexpected checkpoint keys when loading sharded safetensors: {unexpected_keys[:10]}"\n                )\n            if missing_keys:\n                logger.warning(\n                    "Missing model keys when loading sharded safetensors (showing first 10): %s",\n                    missing_keys[:10],\n                )\n        else:\n            safetensors.torch.load_model(model, str(weight_path_obj))\n\n        return model\n""",
+    "load sharded safetensors index",
+)
+PY
+
+echo "Patch applied successfully."

patch_walltime.sh

@@ -0,0 +1,66 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+# Usage:
+#   bash apply_eval_custom_wall_time_patch.sh /path/to/dir/containing/eval_custom.py
+#   bash apply_eval_custom_wall_time_patch.sh /path/to/eval_custom.py
+
+TARGET="${1:-.}"
+if [[ -d "$TARGET" ]]; then
+  FILE="$TARGET/eval_custom.py"
+else
+  FILE="$TARGET"
+fi
+
+if [[ ! -f "$FILE" ]]; then
+  echo "[ERROR] eval_custom.py not found: $FILE"
+  exit 1
+fi
+
+python3 - "$FILE" <<'PY'
+import re
+import sys
+from pathlib import Path
+
+path = Path(sys.argv[1])
+text = path.read_text()
+original = text
+
+if 'metrics["time"]["wall_time"]' in text:
+    print(f"[INFO] Already patched: {path}")
+    sys.exit(0)
+
+# 1) Initialize wall_time once per instance loop.
+pat1 = r"(\n\s*done = False\n)(\s*if config\.write_video:)"
+rep1 = r"\1                wall_time = 0.0\n\2"
+text, n1 = re.subn(pat1, rep1, text, count=1)
+
+# 2) Accumulate per-step wall time.
+pat2 = (
+    r"(\n\s*time_start = time\.time\(\)\n"
+    r"\s*terminated, truncated, info = evaluator\.step\(\)\n"
+    r"\s*time_step = time\.time\(\) - time_start\n)"
+)
+rep2 = r"\1                    wall_time += time_step\n"
+text, n2 = re.subn(pat2, rep2, text, count=1)
+
+# 3) Persist wall_time in metrics["time"].
+pat3 = (
+    r"(\n\s*for metric in evaluator\.metrics:\n"
+    r"\s*metrics\.update\(metric\.gather_results\(\)\)\n)"
+)
+rep3 = r'\1                metrics["time"]["wall_time"] = wall_time\n'
+text, n3 = re.subn(pat3, rep3, text, count=1)
+
+if not (n1 == 1 and n2 == 1 and n3 == 1):
+    print("[ERROR] Patch failed: target code pattern not found (file may differ from expected).")
+    print(f"[DEBUG] replacements: n1={n1}, n2={n2}, n3={n3}")
+    sys.exit(2)
+
+if text == original:
+    print(f"[INFO] No changes made: {path}")
+    sys.exit(0)
+
+path.write_text(text)
+print(f"[OK] Patched: {path}")
+PY

run_dual_eval.sh

@@ -0,0 +1,72 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+TASK_NAME="${TASK_NAME:-turning_on_radio}"
+PATH_TO_CKPT="${PATH_TO_CKPT:-/opt/eval/models/cosmos-pi05-sft-pt12-20260203145532}"
+CONDA_ROOT="${CONDA_ROOT:-/opt/miniconda3}"
+CONDA_ENV="${CONDA_ENV:-behavior}"
+OPENPI_DATA_HOME="${OPENPI_DATA_HOME:-/opt/openpi-cache}"
+BASE_PORT="${BASE_PORT:-8000}"
+EPISODES_PER_TASK="${EPISODES_PER_TASK:-10}"
+EVAL_START="${EVAL_START:-0}"
+EVAL_COUNT="${EVAL_COUNT:-10}"
+NUM_GPU="${NUM_GPU:-1}"
+
+
+LOG_DIR="${LOG_DIR:-/opt/eval/eval_output}"
+mkdir -p "$OPENPI_DATA_HOME" "$LOG_DIR"
+export OPENPI_DATA_HOME
+
+echo "[INFO] task=$TASK_NAME gpus=$NUM_GPU eval_start=$EVAL_START eval_count=$EVAL_COUNT"
+
+# Launch one openpi policy server per GPU
+OPENPI_PIDS=()
+for ((gpu = 0; gpu < NUM_GPU; gpu++)); do
+  (
+    cd /opt/eval/openpi-comet
+    CUDA_VISIBLE_DEVICES="$gpu" \
+    TASK_NAME="$TASK_NAME" \
+    PATH_TO_CKPT="$PATH_TO_CKPT" \
+    PORT="$((BASE_PORT + gpu))" \
+      bash /opt/eval/openpi-comet/eval_openpi.sh
+  ) >"$LOG_DIR/openpi_gpu${gpu}.log" 2>&1 &
+  OPENPI_PIDS+=("$!")
+done
+
+# Run eval with episode wrapping
+(
+  source "$CONDA_ROOT/etc/profile.d/conda.sh"
+  conda activate "$CONDA_ENV"
+  cd /opt/eval/BEHAVIOR-1K
+
+  _remaining=$EVAL_COUNT
+  _cursor=$EVAL_START
+
+  while (( _remaining > 0 )); do
+    _s=$((_cursor % EPISODES_PER_TASK))
+    _chunk=$((EPISODES_PER_TASK - _s))
+    (( _chunk > _remaining )) && _chunk=$_remaining
+
+    echo "[INFO] eval chunk: episodes ${_s}..$((_s + _chunk - 1)) (${_chunk} trials)"
+
+    TASK_NAME="$TASK_NAME" \
+    LOG_PATH="$LOG_DIR" \
+    NUM_GPU="$NUM_GPU" \
+    BASE_PORT="$BASE_PORT" \
+    EVAL_START_IDX="$_s" \
+    EVAL_COUNT="$_chunk" \
+      bash /opt/eval/BEHAVIOR-1K/eval_b1k.sh
+
+    _cursor=$((_cursor + _chunk))
+    _remaining=$((_remaining - _chunk))
+  done
+) >"$LOG_DIR/behavior_eval.log" 2>&1 &
+B1K_PID=$!
+
+wait "$B1K_PID" || true
+for pid in "${OPENPI_PIDS[@]}"; do
+  kill "$pid" 2>/dev/null || true
+  wait "$pid" || true
+done
+
+echo "[INFO] done."

tokenizer.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8986bb4f423f07f8c7f70d0dbe3526fb2316056c17bae71b1ea975e77a168fc6
+size 4264023

tokenizer_config.json

@@ -0,0 +1,7 @@
+{
+  "tokenizer_class": "PaligemmaTokenizer",
+  "max_len": 200,
+  "auto_map": {
+    "AutoTokenizer": ["tokenizer_pi05.PaligemmaTokenizer", null]
+  }
+}