code-Psi0 / baselines /act /act_inference.py
Tinghong-Ye22's picture
Add files using upload-large-folder tool
2d3265e verified
Raw History Blame Contribute Delete
8.34 kB
import os
# Set working directory to project root
os.chdir('/home/jliu/we_learn')
os.environ['PWD'] = '/home/jliu/we_learn'
import dotenv
dotenv.load_dotenv('/home/jliu/we_learn/.env')
import torch
import numpy as np
from pathlib import Path
from safetensors.torch import load_file
from we.utils import parse_args_to_tyro_config, seed_everything, move_to_device, batchify
from we.config.config import LaunchConfig
from we.config.data import LerobotDataConfig
from we.config.model import ACT_ModelConfig
from we.learn.models.act import ACTConfig, ACTPolicy
# ============ Configuration ============
ckpt_step = 50000
run_dir = Path(".runs/act/act.vlt.cosin.lr1.0e-04.b256.gpus4.2601111414")
# ============ Load Config ============
launch_config: LaunchConfig = parse_args_to_tyro_config(run_dir / "argv.txt") # type: ignore
conf = (run_dir / "run_config.json").open("r").read()
config = launch_config.model_validate_json(conf)
data_cfg: LerobotDataConfig = config.data # type: ignore
model_cfg: ACT_ModelConfig = config.model # type: ignore
seed_everything(config.seed or 42)
# ============ Device Setup ============
DEVICE = "cuda:0"
print(f"Using device: {DEVICE}")
print(f"GPU name: {torch.cuda.get_device_name(0)}")
# ============ Load Model ============
def load_model(model_cfg: ACT_ModelConfig, run_dir: Path, ckpt_step: int | str = "latest"):
ckpt_path = run_dir / "checkpoints" / f"ckpt_{ckpt_step}" / "model.safetensors"
if not ckpt_path.exists():
# Try .pth format
ckpt_path = run_dir / "checkpoints" / f"ckpt_{ckpt_step}.pth"
if not ckpt_path.exists():
raise FileNotFoundError(f"Checkpoint not found: {ckpt_path}")
act_config = ACTConfig(
n_obs_steps=model_cfg.n_obs_steps,
chunk_size=model_cfg.chunk_size,
n_action_steps=model_cfg.n_action_steps,
action_dim=model_cfg.action_dim,
state_dim=model_cfg.state_dim,
dim_model=model_cfg.dim_model,
n_heads=model_cfg.n_heads,
dim_feedforward=model_cfg.dim_feedforward,
feedforward_activation=model_cfg.feedforward_activation,
n_encoder_layers=model_cfg.n_encoder_layers,
n_decoder_layers=model_cfg.n_decoder_layers,
pre_norm=model_cfg.pre_norm,
dropout=model_cfg.dropout,
use_vae=model_cfg.use_vae,
latent_dim=model_cfg.latent_dim,
n_vae_encoder_layers=model_cfg.n_vae_encoder_layers,
kl_weight=model_cfg.kl_weight,
temporal_ensemble_coeff=model_cfg.temporal_ensemble_coeff,
)
model = ACTPolicy(config=act_config, dataset_stats=None)
print(f"Loading checkpoint from {ckpt_path}")
if ckpt_path.suffix == ".safetensors":
state_dict = load_file(ckpt_path)
else:
state_dict = torch.load(ckpt_path, map_location="cpu")
model.load_state_dict(state_dict)
return model
model = load_model(model_cfg, run_dir, ckpt_step)
model = model.to(DEVICE)
model.eval()
print("Model loaded successfully!")
num_params = sum(p.numel() for p in model.parameters())
print(f"Parameters (in millions): {num_params * 1e-6:.3f}")
# ============ Load Dataset ============
transform_kwargs = {
"no_aug": True, # No augmentation for evaluation
}
train_dataset = data_cfg(split="train", transform_kwargs=transform_kwargs)
print(f"Train dataset size: {len(train_dataset)}")
val_dataset = data_cfg(split="val", transform_kwargs=transform_kwargs)
print(f"Validation dataset size: {len(val_dataset)}")
# ============ Load Normalization Stats ============
maxmin = data_cfg.transform.action_state
# ============ Evaluation on One Episode ============
np.set_printoptions(precision=4, suppress=True)
dataset = train_dataset
start_frame_idx = train_dataset.raw_dataset.base_dataset.episode_data_index["from"][515].item()
# tensor(16261)
end_frame_idx = train_dataset.raw_dataset.base_dataset.episode_data_index["to"][515].item()
# tensor(16301)
print(f"\n{'='*50}")
print(f"Frame range: {start_frame_idx} - {end_frame_idx}")
print(f"Episode length: {end_frame_idx - start_frame_idx} frames")
print(f"{'='*50}\n")
# Collect all action errors for RMSE calculation (same as evaluate)
all_action_l1_errs = [] # Will be (N, Da) where N = num_frames * Tp
l2_xyz_errors = []
l2_rpy_errors = []
l2_gripper_errors = []
for i in range(start_frame_idx, end_frame_idx):
batch = dataset[i]
batch = move_to_device(batchify(batch), DEVICE)
gt_actions = batch["action"] # (B, Tp, Da)
B, Tp, Da = gt_actions.shape
# ACT dataset uses "observation.images" and "observation.state" keys
images = batch["observation.images"] # (B, N*To, C, H, W)
if len(images.shape) == 5:
# (B, N*To, C, H, W) -> keep batch dim for ACT
pass
states = batch["observation.state"] # (B, state_dim) or (B, obs_horizon, state_dim)
if len(states.shape) == 3:
states = states.squeeze(1) # (B, 1, S) -> (B, S)
# Run inference with ACT model
# ACT expects batch dict with "observation.images" and "observation.state"
with torch.inference_mode():
# Prepare batch dict for ACT model (already in correct format)
act_batch = {
"observation.images": images, # (B, N*To, C, H, W)
"observation.state": states, # (B, S)
}
# Use predict_action for full chunk prediction
pred_actions = model.predict_action(act_batch)
# pred_actions is tensor (B, Tp, Da) or (Tp, Da)
if len(pred_actions.shape) == 3:
pred_actions = pred_actions[0] # (Tp, Da)
# Convert to numpy
pred_actions = pred_actions.cpu().numpy()
# Denormalize actions
denorm_gt_actions = maxmin.denormalize(gt_actions[0]).cpu().numpy() # (Tp, Da)
denorm_pred_actions = maxmin.denormalize(torch.from_numpy(pred_actions)).numpy() # (Tp, Da)
# Only compare the first action step for fair comparison with act_client
# (act_client only has single-step gt_action from raw dataset)
action_l1_errs = np.abs(denorm_pred_actions[0] - denorm_gt_actions[0]) # (Da,)
all_action_l1_errs.append(action_l1_errs[np.newaxis, :])
# Also compute per-frame average for display
avg_action_errors = action_l1_errs # (Da,)
# Split into xyz, rpy, gripper
labels_denormed = [
"denorm_err_l1_xyz",
"denorm_err_l1_rpy",
"denorm_err_l1_gripper",
]
avg_l1_action_err = np.split(avg_action_errors, [3, 6], axis=-1)
metric = {**dict(zip(labels_denormed, map(np.linalg.norm, avg_l1_action_err)))}
print(metric)
l2_xyz_errors.append(metric["denorm_err_l1_xyz"])
l2_rpy_errors.append(metric["denorm_err_l1_rpy"])
l2_gripper_errors.append(metric["denorm_err_l1_gripper"])
# ============ Summary Statistics ============
print(f"\n{'='*50}")
print("Episode Summary")
print(f"{'='*50}")
l2_xyz_errors = np.array(l2_xyz_errors)
l2_rpy_errors = np.array(l2_rpy_errors)
l2_gripper_errors = np.array(l2_gripper_errors)
from we.utils import rmse
print(f"XYZ Error: Mean={np.mean(l2_xyz_errors):.4f} ± {np.std(l2_xyz_errors):.4f}, RMSE={rmse(l2_xyz_errors):.4f}")
print(f"RPY Error: Mean={np.mean(l2_rpy_errors):.4f} ± {np.std(l2_rpy_errors):.4f}, RMSE={rmse(l2_rpy_errors):.4f}")
print(f"Gripper Error: Mean={np.mean(l2_gripper_errors):.4f} ± {np.std(l2_gripper_errors):.4f}, RMSE={rmse(l2_gripper_errors):.4f}")
print(f"\nTotal frames evaluated: {len(l2_xyz_errors)}")
# ============ Quality Assessment ============
print(f"\n{'='*50}")
print("Quality Assessment")
print(f"{'='*50}")
xyz_mean = np.mean(l2_xyz_errors)
rpy_mean = np.mean(l2_rpy_errors)
# Position error assessment (in meters)
if xyz_mean < 0.01:
xyz_quality = "✅ Good (< 1cm)"
elif xyz_mean < 0.03:
xyz_quality = "⚠️ Acceptable (1-3cm)"
else:
xyz_quality = "❌ Poor (> 3cm)"
# Rotation error assessment (in radians)
if rpy_mean < 0.1:
rpy_quality = "✅ Good (< 5.7°)"
elif rpy_mean < 0.3:
rpy_quality = "⚠️ Acceptable (5.7-17°)"
else:
rpy_quality = "❌ Poor (> 17°)"
print(f"Position (XYZ): {xyz_quality} - Mean error: {xyz_mean*100:.2f} cm")
print(f"Rotation (RPY): {rpy_quality} - Mean error: {np.degrees(rpy_mean):.2f}°")
print(f"\n💡 Tip: Compare with other checkpoints and validation set for relative assessment.")