code-Psi0-yth-v21 / src /psi /data /seed /seed_motion_dataset.py
Tinghong-Ye22's picture
Add files using upload-large-folder tool
c4f8e4a verified
Raw History Blame Contribute Delete
15.9 kB
from __future__ import annotations
import hashlib
import json
import os
import pickle
import time
from pathlib import Path
from typing import Any, Dict, List, Optional
import numpy as np
import pandas as pd
import torch
from tqdm import tqdm
from psi.data.seed.text_prompt import (
DEFAULT_SEED_TEXT_FIELDS,
collect_seed_text_candidates,
compose_seed_motion_prompt_from_candidates,
)
from psi.data.seed.progress import build_progress_conditions
_DEFAULT_TEXT_FIELDS: List[str] = list(DEFAULT_SEED_TEXT_FIELDS)
def _rank0_tqdm(iterable, **kwargs):
rank = int(os.environ.get("RANK", "0"))
return tqdm(iterable, disable=(rank != 0), **kwargs)
class SeedMotionDataset(torch.utils.data.Dataset):
def __init__(
self,
root_dir: str,
metadata_csv: str = "metadata/seed_metadata_v004_100.csv",
split: str = "train",
val_ratio: float = 0.1,
split_seed: int = 42,
action_chunk_size: int = 16,
upsample_rate: int = 1,
use_delta_actions: bool = False,
num_past_frames: int = 0,
state_history_stride: int = 30,
motion_cache_size: int = 8,
text_fields: Optional[List[str]] = None,
records_cache_dir: str = "",
) -> None:
self.root_dir = Path(root_dir)
self.metadata_csv = self._resolve_path(metadata_csv)
self.split = split
self.val_ratio = float(val_ratio)
self.split_seed = int(split_seed)
self.action_chunk_size = int(action_chunk_size)
self.upsample_rate = int(upsample_rate)
self.use_delta_actions = bool(use_delta_actions)
self.num_past_frames = int(num_past_frames)
self.state_history_stride = int(state_history_stride)
self.motion_cache_size = int(motion_cache_size)
self.text_fields = text_fields or list(_DEFAULT_TEXT_FIELDS)
self.records_cache_dir = str(records_cache_dir).strip()
if not self.root_dir.exists():
raise FileNotFoundError(f"Seed root dir does not exist: {self.root_dir}")
if not self.metadata_csv.exists():
raise FileNotFoundError(f"Seed metadata csv does not exist: {self.metadata_csv}")
if self.action_chunk_size <= 0:
raise ValueError(f"action_chunk_size must be positive, got {self.action_chunk_size}")
if self.upsample_rate <= 0:
raise ValueError(f"upsample_rate must be positive, got {self.upsample_rate}")
if self.num_past_frames < 0:
raise ValueError(f"num_past_frames must be non-negative, got {self.num_past_frames}")
if self.state_history_stride <= 0:
raise ValueError(f"state_history_stride must be positive, got {self.state_history_stride}")
self.motion_records = self._load_or_build_motion_records()
if not self.motion_records:
raise RuntimeError("No valid seed motion records found after filtering.")
sample_counts = np.fromiter(
(
max(1, (int(record["n_frames"]) + self.upsample_rate - 1) // self.upsample_rate)
for record in _rank0_tqdm(
self.motion_records,
desc=f"Build text index {self.split}",
unit="motion",
)
),
dtype=np.int64,
count=len(self.motion_records),
)
self.sample_offsets = np.zeros(len(self.motion_records) + 1, dtype=np.int64)
self.sample_offsets[1:] = np.cumsum(sample_counts, dtype=np.int64)
self.num_samples = int(self.sample_offsets[-1])
self._motion_cache: Dict[str, np.ndarray] = {}
def _resolve_path(self, maybe_rel_path: str) -> Path:
p = Path(maybe_rel_path)
if p.is_absolute():
return p
return self.root_dir / p
def _default_records_cache_dir(self) -> Path:
if self.records_cache_dir:
return Path(self.records_cache_dir)
return self.root_dir / ".index_cache_multimodal_yth"
def _records_cache_path(self) -> Path:
stat = self.metadata_csv.stat()
key_data = {
"version": 2,
"kind": "text_records",
"metadata_csv": str(self.metadata_csv.resolve()),
"metadata_size": stat.st_size,
"metadata_mtime_ns": stat.st_mtime_ns,
"root_dir": str(self.root_dir.resolve()),
"split": self.split,
"val_ratio": self.val_ratio,
"split_seed": self.split_seed,
"text_fields": self.text_fields,
}
digest = hashlib.sha1(json.dumps(key_data, sort_keys=True).encode("utf-8")).hexdigest()[:20]
return self._default_records_cache_dir() / f"records.seed_text.{self.split}.{digest}.pkl"
def _load_or_build_motion_records(self) -> List[Dict[str, Any]]:
cache_path = self._records_cache_path()
rank = int(os.environ.get("RANK", "0"))
cached = self._try_load_pickle(cache_path)
if cached is not None:
return cached
if rank != 0:
waited = self._wait_for_pickle(cache_path)
if waited is not None:
return waited
records = self._build_motion_records()
self._atomic_save_pickle(cache_path, records)
return records
def _try_load_pickle(self, path: Path) -> Any | None:
if not path.exists():
return None
try:
rank = int(os.environ.get("RANK", "0"))
if rank == 0:
size_mb = path.stat().st_size / (1024 ** 2)
print(f"[TEXT CACHE] loading {path} ({size_mb:.1f} MB)", flush=True)
start = time.time()
with path.open("rb") as f:
obj = pickle.load(f)
if isinstance(obj, dict) and "data" in obj:
obj = obj["data"]
if rank == 0:
print(f"[TEXT CACHE] loaded {path} in {time.time() - start:.1f}s", flush=True)
return obj
except Exception:
return None
def _wait_for_pickle(self, path: Path, timeout_sec: int = 7200) -> Any | None:
rank = int(os.environ.get("RANK", "0"))
if rank == 0:
print(f"[TEXT CACHE] waiting for {path}", flush=True)
start = time.time()
while time.time() - start < timeout_sec:
obj = self._try_load_pickle(path)
if obj is not None:
return obj
time.sleep(2.0)
return None
def _atomic_save_pickle(self, path: Path, data: Any) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
tmp_path = path.with_name(f".{path.name}.{os.getpid()}.tmp")
rank = int(os.environ.get("RANK", "0"))
if rank == 0:
print(f"[TEXT CACHE] saving {path}", flush=True)
start = time.time()
with tmp_path.open("wb") as f:
pickle.dump({"data": data}, f, protocol=pickle.HIGHEST_PROTOCOL)
os.replace(tmp_path, path)
if rank == 0:
size_mb = path.stat().st_size / (1024 ** 2)
print(f"[TEXT CACHE] saved {path} ({size_mb:.1f} MB) in {time.time() - start:.1f}s", flush=True)
def _select_row_ids(self, n: int, split: str) -> np.ndarray:
if n == 0:
return np.array([], dtype=np.int64)
rng = np.random.RandomState(self.split_seed)
perm = rng.permutation(n)
n_val = int(round(n * self.val_ratio))
if n > 1 and self.val_ratio > 0 and n_val == 0:
n_val = 1
if n > 1 and n_val >= n:
n_val = n - 1
split_norm = (split or "train").lower()
if split_norm == "train":
return perm[n_val:]
if split_norm in ("val", "valid", "validation", "test"):
return perm[:n_val]
if split_norm == "all":
return perm
raise ValueError(f"Unsupported split='{split}', expected train/val/all")
def _build_motion_records(self) -> List[Dict[str, Any]]:
usecols = ["move_g1_path", "move_name", "move_duration_frames", *self.text_fields]
usecols_set = set(usecols)
metadata = pd.read_csv(self.metadata_csv, usecols=lambda c: c in usecols_set)
if "move_g1_path" not in metadata.columns:
raise ValueError("metadata csv must contain column 'move_g1_path'")
selected_row_ids = self._select_row_ids(len(metadata), split=self.split)
motion_records: List[Dict[str, Any]] = []
for row_id in _rank0_tqdm(
selected_row_ids,
desc=f"Build text records {self.split}",
total=len(selected_row_ids),
unit="row",
):
row = metadata.iloc[int(row_id)]
motion_dir = self._map_motion_dir(str(row["move_g1_path"]))
if not motion_dir.exists():
raise FileNotFoundError(f"Mapped motion dir does not exist: {motion_dir}")
metadata_n_frames = 0
if "move_duration_frames" in row and pd.notna(row["move_duration_frames"]):
metadata_n_frames = int(row["move_duration_frames"])
n_frames = metadata_n_frames if metadata_n_frames > 0 else self._peek_num_frames(motion_dir)
if n_frames <= 0:
continue
instruction_candidates = self._collect_instruction_candidates(row)
motion_records.append(
{
"row_id": int(row_id),
"move_name": str(row.get("move_name", "")),
"motion_dir": motion_dir,
"instruction_candidates": instruction_candidates,
"n_frames": int(n_frames),
}
)
return motion_records
def _map_motion_dir(self, move_g1_path: str) -> Path:
rel = str(move_g1_path).strip().replace("\\", "/")
rel = rel.replace("g1/csv/", "g1/csv_sonic/")
rel = rel.replace("/csv/", "/csv_sonic/")
if rel.endswith(".csv"):
rel = rel[:-4]
if rel.startswith("dataset/seed/"):
rel = rel[len("dataset/seed/") :]
if rel.startswith("seed/"):
rel = rel[len("seed/") :]
p = Path(rel)
if p.is_absolute():
return p
return self.root_dir / p
def _collect_instruction_candidates(self, row: pd.Series) -> List[str]:
return collect_seed_text_candidates(row, self.text_fields)
def _compose_instruction(self, record: Dict[str, Any]) -> str:
fallback = str(record.get("move_name", "seed motion")).strip() or "seed motion"
return compose_seed_motion_prompt_from_candidates(
record.get("instruction_candidates", []),
fallback=fallback,
)
def _peek_num_frames(self, motion_dir: Path) -> int:
joint_path = motion_dir / "joint_pos.csv"
if not joint_path.exists():
raise FileNotFoundError(f"joint_pos.csv not found: {joint_path}")
return int(len(pd.read_csv(joint_path)))
def _load_motion_array(self, motion_dir: Path) -> np.ndarray:
key = str(motion_dir)
if key in self._motion_cache:
return self._motion_cache[key]
joint = pd.read_csv(motion_dir / "joint_pos.csv").to_numpy(dtype=np.float32) # (T,29)
body_pos = pd.read_csv(motion_dir / "body_pos.csv").to_numpy(dtype=np.float32) # (T,3)
body_quat = pd.read_csv(motion_dir / "body_quat.csv").to_numpy(dtype=np.float32) # (T,4)
lengths = {joint.shape[0], body_pos.shape[0], body_quat.shape[0]}
if len(lengths) != 1:
raise ValueError(
f"Frame length mismatch at {motion_dir}: "
f"joint={joint.shape[0]}, body_pos={body_pos.shape[0]}, body_quat={body_quat.shape[0]}"
)
if joint.shape[1] != 29:
raise ValueError(f"Expected joint_pos.csv with 29 cols at {motion_dir}, got {joint.shape[1]}")
if body_pos.shape[1] != 3:
raise ValueError(f"Expected body_pos.csv with 3 cols at {motion_dir}, got {body_pos.shape[1]}")
if body_quat.shape[1] != 4:
raise ValueError(f"Expected body_quat.csv with 4 cols at {motion_dir}, got {body_quat.shape[1]}")
action = np.concatenate([joint, body_pos, body_quat], axis=1).astype(np.float32) # (T,36)
if self.motion_cache_size > 0:
if len(self._motion_cache) >= self.motion_cache_size:
self._motion_cache.pop(next(iter(self._motion_cache)))
self._motion_cache[key] = action
return action
def __len__(self) -> int:
return self.num_samples
def _pad_index(self, index: int, length: int) -> int:
if index < 0:
return 0
if index >= length:
return length - 1
return index
def __getitem__(self, idx: int) -> Dict[str, Any]:
if idx < 0 or idx >= self.num_samples:
raise IndexError(f"Index out of range: {idx}")
motion_id = int(np.searchsorted(self.sample_offsets, idx, side="right") - 1)
local_index = int(idx - self.sample_offsets[motion_id])
frame_id = int(local_index * self.upsample_rate)
record = self.motion_records[motion_id]
motion_dir: Path = record["motion_dir"]
instruction = self._compose_instruction(record)
motion = self._load_motion_array(motion_dir) # (T,36)
total_frames = motion.shape[0]
requested_frame_id = frame_id
frame_id = self._pad_index(frame_id, total_frames)
action_len = self.action_chunk_size + (1 if self.use_delta_actions else 0)
actions: List[np.ndarray] = []
action_mask: List[float] = []
for step in range(action_len):
# <modify> 非delta训练预测下一帧开始的绝对姿态,避免多chunk拼接时重复当前帧
# 原逻辑:raw_index = frame_id + step * self.upsample_rate
if self.use_delta_actions:
raw_index = requested_frame_id + step * self.upsample_rate
else:
raw_index = requested_frame_id + (step + 1) * self.upsample_rate
valid = 0 <= raw_index < total_frames
src_index = self._pad_index(raw_index, total_frames)
actions.append(motion[src_index])
action_mask.append(1.0 if valid else 0.0)
action_arr = np.stack(actions, axis=0).astype(np.float32) # (Tp or Tp+1,36)
action_mask_arr = np.asarray(action_mask, dtype=np.float32) # (Tp or Tp+1,)
# <modify_states> SEED训练加入过去几帧state,和HE数据路径保持一致。
# 原逻辑:
# states = motion[frame_id:frame_id + 1].astype(np.float32)
obs_indices = [
self._pad_index(frame_id - past_i * self.state_history_stride, total_frames)
for past_i in range(self.num_past_frames, 0, -1)
]
obs_indices.append(self._pad_index(frame_id, total_frames))
states = motion[obs_indices].astype(np.float32)
progress, future_progress = build_progress_conditions(
frame_id=frame_id,
total_frames=total_frames,
action_horizon=action_arr.shape[0],
step_stride=self.upsample_rate,
use_delta_actions=self.use_delta_actions,
)
return {
"actions": action_arr,
"action_mask": action_mask_arr,
"states": states,
"progress": progress,
"future_progress": future_progress,
"instruction": instruction,
"dataset_name": "seed-g1",
# <modify_temp> 临时透传样本来源信息,用于定位大梯度batch。
"row_id": int(record["row_id"]),
"move_name": str(record.get("move_name", "")),
"motion_dir": str(motion_dir),
"frame_id": int(frame_id),
"total_frames": int(total_frames),
}