Nadeem-Shoukath115113's picture
download
raw
23.2 kB
import os
import numpy as np
from pathlib import Path
from omegaconf import OmegaConf
from datetime import datetime
import torch
import torch.nn as nn
import torch.nn.functional as F
from torch.optim.lr_scheduler import SequentialLR, LinearLR, CosineAnnealingLR
from tqdm import tqdm
import wandb
from lerobot.common.datasets.rm_lerobot_dataset import FrameGapLeRobotDataset
from utils.data_utils import get_valid_episodes, split_train_eval_episodes, adapt_lerobot_batch_rewind
from utils.train_utils import set_seed, save_ckpt, get_normalizer_from_calculated, plot_episode_result, plot_episode_result_raw_data
from utils.raw_data_utils import get_frame_num, get_frame_data_fast, get_traj_data, normalize_dense
from models.rewind_reward_model import RewardTransformer
from models.clip_encoder import FrozenCLIPEncoder
from utils.make_demo_video import produce_video
os.environ["TOKENIZERS_PARALLELISM"] = "false"
os.environ["WANDB_IGNORE_GLOBS"] = "**/rollout/**"
# os.environ["WANDB_MODE"] = "disabled"
class ReWiNDWorkspace:
def __init__(self, cfg):
self.cfg = cfg
self.device = torch.device(cfg.general.device if torch.cuda.is_available() else "cpu")
print(f"[Init] Using device: {self.device}")
set_seed(cfg.general.seed)
self.camera_names = cfg.general.camera_names
self.save_dir = Path(f'{cfg.general.project_name}/{cfg.general.task_name}')
self.save_dir.mkdir(parents=True, exist_ok=True)
print(f"[Init] Logging & ckpts to: {self.save_dir}")
def train(self):
cfg = self.cfg
OmegaConf.save(cfg, self.save_dir / "config.yaml")
# --- wandb ---
wandb.init(
project=f'{cfg.general.project_name}-{cfg.general.task_name}',
name=f'{datetime.now().strftime("%Y.%m.%d-%H.%M.%S")}',
config=cfg,
)
# --- data ---
valid_episodes = get_valid_episodes(cfg.general.repo_id)
train_eps, val_eps = split_train_eval_episodes(valid_episodes, 1 - cfg.train.val_portion, seed=cfg.general.seed)
dataset_train = FrameGapLeRobotDataset(repo_id=cfg.general.repo_id,
episodes=train_eps,
n_obs_steps=cfg.model.n_obs_steps,
frame_gap=cfg.model.frame_gap,
max_rewind_steps=cfg.model.max_rewind_steps,
image_names=cfg.general.camera_names,
annotation_list=cfg.model.annotation_list,
task_name=cfg.general.task_name)
dataset_val = FrameGapLeRobotDataset(repo_id=cfg.general.repo_id,
episodes=val_eps,
n_obs_steps=cfg.model.n_obs_steps,
frame_gap=cfg.model.frame_gap,
max_rewind_steps=cfg.model.max_rewind_steps,
image_names=cfg.general.camera_names,
annotation_list=cfg.model.annotation_list,
task_name=cfg.general.task_name)
dataloader_train = torch.utils.data.DataLoader(dataset_train, **cfg.dataloader)
dataloader_val = torch.utils.data.DataLoader(dataset_val, **cfg.val_dataloader)
state_normalizer = get_normalizer_from_calculated(cfg.general.state_norm_path, self.device)
# CLIP encoder
clip_encoder = FrozenCLIPEncoder(cfg.encoders.vision_ckpt, self.device)
vis_dim = 512
txt_dim = 512
# --- reward_model ---
reward_model = RewardTransformer(d_model=cfg.model.d_model,
vis_emb_dim=vis_dim,
text_emb_dim=txt_dim,
state_dim=cfg.model.state_dim,
n_layers=cfg.model.n_layers,
n_heads=cfg.model.n_heads,
dropout=cfg.model.dropout,
num_cameras=len(self.camera_names),
).to(self.device)
if cfg.model.resume_training:
reward_model_path = Path(cfg.model.model_path)
reward_ckpt = torch.load(reward_model_path, map_location=self.device)
reward_model.load_state_dict(reward_ckpt["model"])
reward_model.to(self.device)
reward_model.train()
# Optimizer
reward_optimizer = torch.optim.AdamW(
reward_model.parameters(),
lr=cfg.optim.lr,
betas=tuple(cfg.optim.betas),
eps=cfg.optim.eps,
weight_decay=cfg.optim.weight_decay,
)
# Schedulers
reward_warmup_scheduler = LinearLR(
reward_optimizer,
start_factor=1e-6 / cfg.optim.lr,
end_factor=1.0,
total_iters=cfg.optim.warmup_steps
)
reward_cosine_scheduler = CosineAnnealingLR(
reward_optimizer,
T_max=cfg.optim.total_steps - cfg.optim.warmup_steps,
eta_min=0.0
)
reward_scheduler = SequentialLR(
reward_optimizer,
schedulers=[reward_warmup_scheduler, reward_cosine_scheduler],
milestones=[cfg.optim.warmup_steps]
)
# ==================== training loop ==================================
best_val = float("inf")
step = 0
for epoch in range(1, cfg.train.num_epochs + 1):
reward_model.train()
with tqdm(dataloader_train, desc=f"Epoch {epoch}") as pbar:
for batch in pbar:
batch = adapt_lerobot_batch_rewind(batch, camera_names=cfg.general.camera_names)
B, T = batch["image_frames"][self.camera_names[0]].shape[:2]
img_list = []
for key in self.camera_names:
imgs = batch["image_frames"][key].flatten(0, 1).to(self.device) # (B*T, C, H, W)
img_list.append(imgs)
lang_strs = batch["tasks"]
trg = batch["targets"].to(self.device)
lens = batch["lengths"].to(self.device)
state = batch["state"].to(self.device)
with torch.no_grad():
state = state_normalizer.normalize(state)
# CLIP encoding
imgs_all = torch.cat(img_list, dim=0) # (N * B * T, C, H, W)
img_emb = clip_encoder.encode_image(imgs_all) # (N * B * T, D)
img_emb = img_emb.view(len(img_list), B, T, -1).permute(1, 0, 2, 3) # (B, N, T, D)
lang_emb = clip_encoder.encode_text(lang_strs) # lang_emb: (B, txt_dim)
if cfg.model.no_state:
state = torch.zeros_like(state, device=self.device)
reward_pred = reward_model(img_emb, lang_emb, state, lens)
reward_loss = F.mse_loss(reward_pred, trg, reduction="mean")
reward_optimizer.zero_grad()
reward_loss.backward()
reward_unclipped = nn.utils.clip_grad_norm_(reward_model.parameters(), float("inf")).item()
_ = nn.utils.clip_grad_norm_(reward_model.parameters(), cfg.train.grad_clip)
reward_optimizer.step()
reward_scheduler.step()
if step % cfg.train.log_every == 0:
wandb.log({
"train/total_loss": reward_loss.item(),
"train/lr": reward_scheduler.get_last_lr()[0],
"train/reward_grad_norm": reward_unclipped,
"epoch": epoch,
}, step=step)
pbar.set_postfix(loss=f"{(reward_loss.item()):.4f}")
if step % cfg.train.save_every == 0:
save_ckpt(reward_model, reward_optimizer, epoch, self.save_dir, input_name=f"reward_step_{step:06d}_loss_{reward_loss.item():.3f}")
step += 1
# --- validation ---
if epoch % cfg.train.eval_every == 0:
reward_model.eval()
total_loss, num = 0.0, 0
print("running validation...")
with torch.no_grad():
for batch in dataloader_val:
batch = adapt_lerobot_batch_rewind(batch, camera_names=cfg.general.camera_names)
B, T = batch["image_frames"][self.camera_names[0]].shape[:2]
img_list = []
for key in self.camera_names:
imgs = batch["image_frames"][key].flatten(0, 1).to(self.device) # (B*T, C, H, W)
img_list.append(imgs)
lang_strs = batch["tasks"]
trg = batch["targets"].to(self.device)
lens = batch["lengths"].to(self.device)
state = batch["state"].to(self.device)
state = state_normalizer.normalize(state)
# CLIP encoding
imgs_all = torch.cat(img_list, dim=0) # (N * B * T, C, H, W)
img_emb = clip_encoder.encode_image(imgs_all) # (N * B * T, D)
img_emb = img_emb.view(len(img_list), B, T, -1).permute(1, 0, 2, 3) # (B, N, T, D)
lang_emb = clip_encoder.encode_text(lang_strs) # lang_emb: (B, txt_dim)
if cfg.model.no_state:
state = torch.zeros_like(state, device=self.device)
reward_pred = reward_model(img_emb, lang_emb, state, lens)
reward_loss = F.mse_loss(reward_pred, trg, reduction="mean")
total_loss += reward_loss.item()
num += 1
val_loss = total_loss / num
print(f"[Eval] Epoch {epoch} Val L1: {val_loss:.6f}")
wandb.log({"val/loss": val_loss}, step=step)
torch.cuda.empty_cache()
# --- save checkpoints ---
save_ckpt(reward_model, reward_optimizer, epoch, self.save_dir, input_name="reward_latest")
if epoch == cfg.train.num_epochs:
save_ckpt(reward_model, reward_optimizer, epoch, self.save_dir, input_name="reward_final")
if val_loss < best_val:
best_val = val_loss
save_ckpt(reward_model, reward_optimizer, epoch, self.save_dir, input_name="reward_best")
print(f"Training done. Best val_loss MSE = {best_val}")
wandb.finish()
# Evaluate whole trajectory from demo data, generating video
def eval(self):
import random
cfg = self.cfg
repo_id = cfg.general.repo_id
valid_episodes = get_valid_episodes(repo_id)
train_eps, val_eps = split_train_eval_episodes(valid_episodes, 1 - cfg.train.val_portion, seed=cfg.general.seed)
dataset_val = FrameGapLeRobotDataset(repo_id=repo_id,
episodes=val_eps,
n_obs_steps=cfg.model.n_obs_steps,
frame_gap=cfg.model.frame_gap,
max_rewind_steps=cfg.model.max_rewind_steps,
image_names=cfg.general.camera_names,
annotation_list=cfg.model.annotation_list,
task_name=cfg.general.task_name,
video_eval=True)
state_normalizer = get_normalizer_from_calculated(cfg.general.state_norm_path, self.device)
# CLIP encoder
clip_encoder = FrozenCLIPEncoder(cfg.encoders.vision_ckpt, self.device)
vis_dim = 512
txt_dim = 512
# Create model instances
reward_model = RewardTransformer(d_model=cfg.model.d_model,
vis_emb_dim=vis_dim,
text_emb_dim=txt_dim,
state_dim=cfg.model.state_dim,
n_layers=cfg.model.n_layers,
n_heads=cfg.model.n_heads,
dropout=cfg.model.dropout,
num_cameras=len(self.camera_names))
# Load checkpoints
reward_model_path = Path(cfg.eval.ckpt_path) / cfg.eval.reward_model_name
reward_ckpt = torch.load(reward_model_path, map_location=self.device)
reward_model.load_state_dict(reward_ckpt["model"])
reward_model.to(self.device)
reward_model.eval()
# save path
rollout_save_dir = Path(self.save_dir) / "eval_video"
rollout_save_dir.mkdir(parents=True, exist_ok=True)
OmegaConf.save(cfg, rollout_save_dir / "config.yaml")
evaled_list = []
for i in range(cfg.eval.run_times):
ep_index = random.choice([idx for idx in valid_episodes if idx not in evaled_list])
global_idx = valid_episodes.index(ep_index)
evaled_list.append(ep_index)
start_idx = dataset_val.episode_data_index["from"][global_idx].item()
end_idx = dataset_val.episode_data_index["to"][global_idx].item()
gt_ep_result = []
pred_ep_result = []
pred_ep_smoothed = []
x_offset = 0
# x_offset = cfg.model.frame_gap * cfg.model.n_obs_steps
eval_frame_gap = cfg.eval.eval_frame_gap
print(f"[Eval Video] Evaluating episode_{ep_index}, progress: {i} / {cfg.eval.run_times}")
for idx in tqdm(range(start_idx, end_idx, eval_frame_gap), desc=f"Processing episode {ep_index}"):
data_point = dataset_val[idx]
batch = adapt_lerobot_batch_rewind(data_point, camera_names=cfg.general.camera_names, eval_video=True)
B, T = batch["image_frames"][self.camera_names[0]].shape[:2]
img_list = []
for key in self.camera_names:
imgs = batch["image_frames"][key].flatten(0, 1).to(self.device) # (B*T, C, H, W)
img_list.append(imgs)
lang_strs = batch["tasks"]
trg = batch["targets"].to(self.device)
lens = batch["lengths"].to(self.device)
state = batch["state"].to(self.device)
state = state_normalizer.normalize(state)
# CLIP encoding
imgs_all = torch.cat(img_list, dim=0) # (N * B * T, C, H, W)
img_emb = clip_encoder.encode_image(imgs_all) # (N * B * T, D)
img_emb = img_emb.view(len(img_list), B, T, -1).permute(1, 0, 2, 3) # (B, N, T, D)
lang_emb = clip_encoder.encode_text(lang_strs) # lang_emb: (B, txt_dim)
if cfg.model.no_state:
state = torch.zeros_like(state, device=self.device)
reward_pred = reward_model(img_emb, lang_emb, state, lens) # (B, T)
pred = torch.clip(reward_pred, 0, 1) # (B, T)
raw_item = pred[0, cfg.model.n_obs_steps].item()
smoothed_item = raw_item
pred_ep_result.append(raw_item)
gt_ep_result.append(normalize_dense(trg[0, cfg.model.n_obs_steps].item()))
pred_ep_smoothed.append(smoothed_item)
# save results
save_dir = plot_episode_result(ep_index, pred_ep_smoothed, gt_ep_result, x_offset, rollout_save_dir, frame_gap=eval_frame_gap)
np.save(Path(save_dir) / "pred.npy", np.array(pred_ep_result))
np.save(Path(save_dir) / "gt.npy", np.array(gt_ep_result))
np.save(Path(save_dir) / "smoothed.npy", np.array(pred_ep_smoothed))
print(f"[Eval Video] episode_{ep_index} making video...")
chunk_id = ep_index // 1000
root = Path.home() / ".cache" / "huggingface" / "lerobot" / repo_id # or change to your LEROBOT_LOCAL_DIR
middle_video_dir = root / f"videos/chunk-{chunk_id:03d}/top_camera-images-rgb"
try:
produce_video(save_dir=rollout_save_dir,
middle_video=middle_video_dir,
episode_num=ep_index,
x_offset=x_offset,
frame_gap=eval_frame_gap)
except Exception as e:
print(f"[Eval Video] episode_{ep_index} video production failed: {e}")
print(f"[Eval Video] episode_{ep_index} results saved to: {save_dir}, progress: {i+1} / {cfg.eval.run_times}")
def eval_raw_data(self):
import random
cfg = self.cfg
state_normalizer = get_normalizer_from_calculated(cfg.general.state_norm_path, self.device)
# CLIP encoding
clip_encoder = FrozenCLIPEncoder(cfg.encoders.vision_ckpt, self.device)
vis_dim = 512
txt_dim = 512
# Create model instances
reward_model = RewardTransformer(d_model=cfg.model.d_model,
vis_emb_dim=vis_dim,
text_emb_dim=txt_dim,
state_dim=cfg.model.state_dim,
n_layers=cfg.model.n_layers,
n_heads=cfg.model.n_heads,
dropout=cfg.model.dropout,
num_cameras=len(self.camera_names),
)
# Load checkpoints
reward_model_path = Path(cfg.eval.ckpt_path) / cfg.eval.reward_model_name
reward_ckpt = torch.load(reward_model_path, map_location=self.device)
reward_model.load_state_dict(reward_ckpt["model"])
reward_model.to(self.device)
reward_model.eval()
# Eval save path
datetime_str = datetime.now().strftime("%Y.%m.%d-%H.%M.%S")
rollout_save_dir = Path(self.save_dir) / "eval_video" / f"{datetime_str}" # convert to Path first
rollout_save_dir.mkdir(parents=True, exist_ok=True)
OmegaConf.save(cfg, rollout_save_dir / "config.yaml")
x_offset = 0
# x_offset = cfg.model.frame_gap * cfg.model.n_obs_steps
data_dir = cfg.eval.raw_data_dir
run_times = cfg.eval.raw_data_run_times
# Get all valid episode paths
all_episodes = [
os.path.join(data_dir, f)
for f in os.listdir(data_dir)
if f.startswith("episode_")
]
eval_list = all_episodes
random.seed(cfg.general.seed)
# randomly select eval_list
if len(all_episodes) >= run_times:
eval_list = random.sample(all_episodes, run_times)
else:
raise ValueError(f"Not enough episodes in {data_dir} to sample {run_times} items.")
for i in range(run_times):
data_path = eval_list[i]
pred_ep_result = []
pred_ep_smoothed = []
# randomly select
ep_index = os.path.basename(data_path)
frame_num = get_frame_num(data_path)
traj_joint_data = get_traj_data(data_path)
eval_frame_gap = cfg.eval.eval_frame_gap
print(f"[EVAL_RAW]: process {i+1}/{run_times} episode: {ep_index}")
for idx in tqdm(range(0, frame_num, eval_frame_gap), desc=f"Processing data"):
batch = get_frame_data_fast(path=data_path,
traj_joint_data=traj_joint_data,
idx=idx,
n_obs_steps=cfg.model.n_obs_steps,
frame_gap=cfg.model.frame_gap,
max_rewind_steps=cfg.model.max_rewind_steps,
camera_names=cfg.general.camera_names,
device=self.device)
B, T = batch["image_frames"][self.camera_names[0]].shape[:2]
img_list = []
for key in self.camera_names:
imgs = batch["image_frames"][key].flatten(0, 1).to(self.device) # (B*T, C, H, W)
img_list.append(imgs)
lang_strs = ["fold the tshirt"]
lens = torch.tensor([1+cfg.model.n_obs_steps], dtype=torch.int32, device=self.device)
state = batch["state"].to(self.device)
state = state_normalizer.normalize(state)
# CLIP encoding
imgs_all = torch.cat(img_list, dim=0) # (N * B * T, C, H, W)
img_emb = clip_encoder.encode_image(imgs_all) # (N * B * T, D)
img_emb = img_emb.view(len(img_list), B, T, -1).permute(1, 0, 2, 3) # (B, N, T, D)
lang_emb = clip_encoder.encode_text(lang_strs) # lang_emb: (B, txt_dim)
if cfg.model.no_state:
state = torch.zeros_like(state, device=self.device)
reward_pred = reward_model(img_emb, lang_emb, state, lens) # (B, T)
pred = torch.clip(reward_pred, 0, 1) # (B, T)
raw_item = pred[0, cfg.model.n_obs_steps].item()
smoothed_item = raw_item
pred_ep_result.append(raw_item)
pred_ep_smoothed.append(smoothed_item)
# save results
save_dir = plot_episode_result_raw_data(ep_index, pred_ep_result, x_offset, rollout_save_dir, frame_gap=eval_frame_gap, ep_smoothed=None)
np.save(Path(save_dir) / "pred.npy", np.array(pred_ep_result))
print(f"[Eval Video] episode_{ep_index} making video...")
middle_video_dir = Path(f"{data_path}/top_camera-images-rgb.mp4")
try:
produce_video(save_dir=rollout_save_dir,
middle_video=middle_video_dir,
episode_num=ep_index,
raw_data=True,
x_offset=x_offset,
frame_gap=eval_frame_gap)
except Exception as e:
print(f"[Eval Video] episode_{ep_index} video production failed: {e}")
print(f"[Eval Video] episode_{ep_index} results saved to: {save_dir}")

Xet Storage Details

Size:
23.2 kB
·
Xet hash:
04b7fb4641a6e7232fda51a43325d2bfd4b5a07df1fc99985c90f2539ebebf08

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.