File size: 7,802 Bytes
67c4f10
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1871410
 
67c4f10
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
#!/usr/bin/env python3
"""Single-task and multi-trial LIBERO evaluation runner for Parvulus.

Evaluates the Parvulus policy against the canonical LIBERO benchmark suites.
Supports saving rollout visualization MP4 videos and reporting step-level metrics.
"""

import argparse
import os
import sys
import time
from pathlib import Path
from typing import Optional, Tuple

import imageio
import numpy as np
import torch

SCRIPT_DIR = Path(__file__).resolve().parent
if str(SCRIPT_DIR) not in sys.path:
    sys.path.insert(0, str(SCRIPT_DIR))

if "LIBERO_ROOT" in os.environ and os.environ["LIBERO_ROOT"] not in sys.path:
    sys.path.insert(0, os.environ["LIBERO_ROOT"])

from infer import ParvulusInference

try:
    from libero.libero import benchmark, get_libero_path
    from libero.libero.envs import OffScreenRenderEnv
except ImportError as e:
    raise ImportError(
        f"Could not import LIBERO ({e}). Ensure LIBERO and its dependencies are installed."
    )

TASK_SUITES = ["libero_spatial", "libero_object", "libero_goal", "libero_10"]
MAX_STEPS = {
    "libero_spatial": 500,
    "libero_object": 500,
    "libero_goal": 500,
    "libero_10": 600,
}


def run_trial(
    infer: ParvulusInference,
    env: OffScreenRenderEnv,
    task_desc: str,
    init_state: Optional[np.ndarray] = None,
    max_steps: int = 500,
    settle_steps: int = 5,
    save_video_path: Optional[str] = None,
) -> Tuple[bool, int, float]:
    """Runs a single simulation rollout trial."""
    infer.reset(task_description=task_desc)
    obs = env.reset()
    if init_state is not None:
        obs = env.set_init_state(init_state)

    # Initial settling with open gripper
    settle_noop = np.array([0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -1.0])
    for _ in range(settle_steps):
        obs, _, _, _ = env.step(settle_noop)

    frames = []
    if save_video_path:
        frames.append(obs["agentview_image"][::-1])

    success = False
    step_count = 0
    t0 = time.time()

    while step_count < max_steps and not success:
        agent_img = obs["agentview_image"]
        wrist_img = obs["robot0_eye_in_hand_image"]

        action = infer.step(agent_img, wrist_img)
        obs, reward, done, info = env.step(action)
        step_count += 1

        if save_video_path:
            frames.append(obs["agentview_image"][::-1])

        if done:
            # Probe steps to verify stability
            for _ in range(10):
                obs, _, d, _ = env.step(action)
                if save_video_path:
                    frames.append(obs["agentview_image"][::-1])
                if d:
                    done = True

            # Retreat steps
            retreat_act = np.array([0.0, 0.0, 0.6, 0.0, 0.0, 0.0, -1.0])
            for _ in range(15):
                obs, _, d, _ = env.step(retreat_act)
                if save_video_path:
                    frames.append(obs["agentview_image"][::-1])

            # End settle frames
            for _ in range(5):
                obs, _, d, _ = env.step(np.zeros(7))
                if save_video_path:
                    frames.append(obs["agentview_image"][::-1])

            success = bool(done)
            break

    elapsed = time.time() - t0

    if save_video_path and frames:
        Path(save_video_path).parent.mkdir(parents=True, exist_ok=True)
        imageio.mimwrite(save_video_path, frames, fps=20)
        print(f"Saved trial video: {save_video_path}")

    return success, step_count, elapsed


def parse_args():
    parser = argparse.ArgumentParser(description="Parvulus LIBERO Benchmark Evaluation")
    parser.add_argument("--suite", type=str, default="libero_spatial", choices=TASK_SUITES)
    parser.add_argument("--task_id", type=int, default=0, help="Task index in suite (default: 0)")
    parser.add_argument("--num_trials", type=int, default=1, help="Number of trials to evaluate")
    parser.add_argument("--seed", type=int, default=42, help="Random seed (default: 42)")
    parser.add_argument("--device", type=str, default="cuda", choices=["cuda", "cpu", "xpu", "auto"])
    parser.add_argument("--backend", type=str, default="pytorch", choices=["pytorch", "cuda_c", "cuda_graph", "pure_cuda"], help="Inference backend engine")
    parser.add_argument("--use_kv_cache", action="store_true", default=True, help="Enable fast KV caching")
    parser.add_argument("--num_ode_steps", type=int, default=10, help="Euler integration steps")
    parser.add_argument("--chunk_size", type=int, default=8, help="Action chunk horizon")
    parser.add_argument("--replan_stride", type=int, default=8, help="Receding horizon replan stride")
    parser.add_argument("--resolution", type=int, default=256, help="Camera resolution")
    parser.add_argument("--save_video", action="store_true", help="Save rollout video as MP4")
    parser.add_argument("--output_dir", type=str, default="./eval_results", help="Directory to save videos/results")
    return parser.parse_args()


def main():
    args = parse_args()
    print("=" * 70)
    print("  PARVULUS Physical AI Model - LIBERO Evaluation")
    print("=" * 70)
    print(f"  Suite:       {args.suite}")
    print(f"  Task ID:     {args.task_id}")
    print(f"  Trials:      {args.num_trials}")
    print(f"  Device:      {args.device}")
    print(f"  KV Cache:    {args.use_kv_cache}")
    print(f"  ODE Steps:   {args.num_ode_steps}")
    print("=" * 70)

    # Initialize policy
    infer = ParvulusInference.from_pretrained(
        pretrained_path=str(SCRIPT_DIR),
        device=args.device,
        backend=args.backend,
        use_kv_cache=args.use_kv_cache,
        num_ode_steps=args.num_ode_steps,
        chunk_size=args.chunk_size,
        replan_stride=args.replan_stride,
        resolution=args.resolution,
    )

    # Initialize task suite
    suite_dict = benchmark.get_benchmark_dict()
    if args.suite not in suite_dict:
        raise ValueError(f"Unknown suite {args.suite}. Available: {list(suite_dict.keys())}")
    suite = suite_dict[args.suite]()

    task = suite.get_task(args.task_id)
    task_desc = task.language
    print(f"\n[Task {args.task_id}]: \"{task_desc}\"")

    bddl_file = Path(get_libero_path("bddl_files")) / task.problem_folder / task.bddl_file
    env = OffScreenRenderEnv(
        bddl_file_name=str(bddl_file),
        camera_heights=args.resolution,
        camera_widths=args.resolution,
    )

    with torch.serialization.safe_globals([np.core.multiarray._reconstruct, np.ndarray, np.dtype, np.dtypes.Float64DType]):
        init_states = suite.get_task_init_states(args.task_id)

    max_steps = MAX_STEPS.get(args.suite, 500)
    successes = 0

    for trial in range(args.num_trials):
        env.seed(args.seed + trial)
        torch.manual_seed(args.seed + trial)
        init_state = init_states[trial % len(init_states)]

        video_path = None
        if args.save_video:
            video_path = os.path.join(
                args.output_dir, f"{args.suite}_task{args.task_id}_trial{trial}.mp4"
            )

        print(f"\n--- Running Trial {trial + 1}/{args.num_trials} ---")
        succ, steps, elapsed = run_trial(
            infer=infer,
            env=env,
            task_desc=task_desc,
            init_state=init_state,
            max_steps=max_steps,
            save_video_path=video_path,
        )

        status_str = "SUCCESS" if succ else "FAILURE"
        print(f"  Result:  {status_str}")
        print(f"  Steps:   {steps}/{max_steps}")
        print(f"  Latency: {elapsed:.2f}s ({steps / max(1e-5, elapsed):.1f} steps/s)")

        if succ:
            successes += 1

    env.close()

    print("\n" + "=" * 70)
    print(f"  SUMMARY: {successes}/{args.num_trials} ({successes / args.num_trials * 100:.1f}%) SUCCESS RATE")
    print("=" * 70)


if __name__ == "__main__":
    main()