from __future__ import annotations import hashlib import json import os import pickle import time from pathlib import Path from typing import Any, Dict, List, Optional import numpy as np import pandas as pd import torch from tqdm import tqdm from psi.data.seed.text_prompt import ( DEFAULT_SEED_TEXT_FIELDS, collect_seed_text_candidates, compose_seed_motion_prompt_from_candidates, ) from psi.data.seed.progress import build_progress_conditions _DEFAULT_TEXT_FIELDS: List[str] = list(DEFAULT_SEED_TEXT_FIELDS) def _rank0_tqdm(iterable, **kwargs): rank = int(os.environ.get("RANK", "0")) return tqdm(iterable, disable=(rank != 0), **kwargs) class SeedMotionDataset(torch.utils.data.Dataset): def __init__( self, root_dir: str, metadata_csv: str = "metadata/seed_metadata_v004_100.csv", split: str = "train", val_ratio: float = 0.1, split_seed: int = 42, action_chunk_size: int = 16, upsample_rate: int = 1, use_delta_actions: bool = False, num_past_frames: int = 0, state_history_stride: int = 30, motion_cache_size: int = 8, text_fields: Optional[List[str]] = None, records_cache_dir: str = "", ) -> None: self.root_dir = Path(root_dir) self.metadata_csv = self._resolve_path(metadata_csv) self.split = split self.val_ratio = float(val_ratio) self.split_seed = int(split_seed) self.action_chunk_size = int(action_chunk_size) self.upsample_rate = int(upsample_rate) self.use_delta_actions = bool(use_delta_actions) self.num_past_frames = int(num_past_frames) self.state_history_stride = int(state_history_stride) self.motion_cache_size = int(motion_cache_size) self.text_fields = text_fields or list(_DEFAULT_TEXT_FIELDS) self.records_cache_dir = str(records_cache_dir).strip() if not self.root_dir.exists(): raise FileNotFoundError(f"Seed root dir does not exist: {self.root_dir}") if not self.metadata_csv.exists(): raise FileNotFoundError(f"Seed metadata csv does not exist: {self.metadata_csv}") if self.action_chunk_size <= 0: raise ValueError(f"action_chunk_size must be positive, got {self.action_chunk_size}") if self.upsample_rate <= 0: raise ValueError(f"upsample_rate must be positive, got {self.upsample_rate}") if self.num_past_frames < 0: raise ValueError(f"num_past_frames must be non-negative, got {self.num_past_frames}") if self.state_history_stride <= 0: raise ValueError(f"state_history_stride must be positive, got {self.state_history_stride}") self.motion_records = self._load_or_build_motion_records() if not self.motion_records: raise RuntimeError("No valid seed motion records found after filtering.") sample_counts = np.fromiter( ( max(1, (int(record["n_frames"]) + self.upsample_rate - 1) // self.upsample_rate) for record in _rank0_tqdm( self.motion_records, desc=f"Build text index {self.split}", unit="motion", ) ), dtype=np.int64, count=len(self.motion_records), ) self.sample_offsets = np.zeros(len(self.motion_records) + 1, dtype=np.int64) self.sample_offsets[1:] = np.cumsum(sample_counts, dtype=np.int64) self.num_samples = int(self.sample_offsets[-1]) self._motion_cache: Dict[str, np.ndarray] = {} def _resolve_path(self, maybe_rel_path: str) -> Path: p = Path(maybe_rel_path) if p.is_absolute(): return p return self.root_dir / p def _default_records_cache_dir(self) -> Path: if self.records_cache_dir: return Path(self.records_cache_dir) return self.root_dir / ".index_cache_multimodal_yth" def _records_cache_path(self) -> Path: stat = self.metadata_csv.stat() key_data = { "version": 2, "kind": "text_records", "metadata_csv": str(self.metadata_csv.resolve()), "metadata_size": stat.st_size, "metadata_mtime_ns": stat.st_mtime_ns, "root_dir": str(self.root_dir.resolve()), "split": self.split, "val_ratio": self.val_ratio, "split_seed": self.split_seed, "text_fields": self.text_fields, } digest = hashlib.sha1(json.dumps(key_data, sort_keys=True).encode("utf-8")).hexdigest()[:20] return self._default_records_cache_dir() / f"records.seed_text.{self.split}.{digest}.pkl" def _load_or_build_motion_records(self) -> List[Dict[str, Any]]: cache_path = self._records_cache_path() rank = int(os.environ.get("RANK", "0")) cached = self._try_load_pickle(cache_path) if cached is not None: return cached if rank != 0: waited = self._wait_for_pickle(cache_path) if waited is not None: return waited records = self._build_motion_records() self._atomic_save_pickle(cache_path, records) return records def _try_load_pickle(self, path: Path) -> Any | None: if not path.exists(): return None try: rank = int(os.environ.get("RANK", "0")) if rank == 0: size_mb = path.stat().st_size / (1024 ** 2) print(f"[TEXT CACHE] loading {path} ({size_mb:.1f} MB)", flush=True) start = time.time() with path.open("rb") as f: obj = pickle.load(f) if isinstance(obj, dict) and "data" in obj: obj = obj["data"] if rank == 0: print(f"[TEXT CACHE] loaded {path} in {time.time() - start:.1f}s", flush=True) return obj except Exception: return None def _wait_for_pickle(self, path: Path, timeout_sec: int = 7200) -> Any | None: rank = int(os.environ.get("RANK", "0")) if rank == 0: print(f"[TEXT CACHE] waiting for {path}", flush=True) start = time.time() while time.time() - start < timeout_sec: obj = self._try_load_pickle(path) if obj is not None: return obj time.sleep(2.0) return None def _atomic_save_pickle(self, path: Path, data: Any) -> None: path.parent.mkdir(parents=True, exist_ok=True) tmp_path = path.with_name(f".{path.name}.{os.getpid()}.tmp") rank = int(os.environ.get("RANK", "0")) if rank == 0: print(f"[TEXT CACHE] saving {path}", flush=True) start = time.time() with tmp_path.open("wb") as f: pickle.dump({"data": data}, f, protocol=pickle.HIGHEST_PROTOCOL) os.replace(tmp_path, path) if rank == 0: size_mb = path.stat().st_size / (1024 ** 2) print(f"[TEXT CACHE] saved {path} ({size_mb:.1f} MB) in {time.time() - start:.1f}s", flush=True) def _select_row_ids(self, n: int, split: str) -> np.ndarray: if n == 0: return np.array([], dtype=np.int64) rng = np.random.RandomState(self.split_seed) perm = rng.permutation(n) n_val = int(round(n * self.val_ratio)) if n > 1 and self.val_ratio > 0 and n_val == 0: n_val = 1 if n > 1 and n_val >= n: n_val = n - 1 split_norm = (split or "train").lower() if split_norm == "train": return perm[n_val:] if split_norm in ("val", "valid", "validation", "test"): return perm[:n_val] if split_norm == "all": return perm raise ValueError(f"Unsupported split='{split}', expected train/val/all") def _build_motion_records(self) -> List[Dict[str, Any]]: usecols = ["move_g1_path", "move_name", "move_duration_frames", *self.text_fields] usecols_set = set(usecols) metadata = pd.read_csv(self.metadata_csv, usecols=lambda c: c in usecols_set) if "move_g1_path" not in metadata.columns: raise ValueError("metadata csv must contain column 'move_g1_path'") selected_row_ids = self._select_row_ids(len(metadata), split=self.split) motion_records: List[Dict[str, Any]] = [] for row_id in _rank0_tqdm( selected_row_ids, desc=f"Build text records {self.split}", total=len(selected_row_ids), unit="row", ): row = metadata.iloc[int(row_id)] motion_dir = self._map_motion_dir(str(row["move_g1_path"])) if not motion_dir.exists(): raise FileNotFoundError(f"Mapped motion dir does not exist: {motion_dir}") metadata_n_frames = 0 if "move_duration_frames" in row and pd.notna(row["move_duration_frames"]): metadata_n_frames = int(row["move_duration_frames"]) n_frames = metadata_n_frames if metadata_n_frames > 0 else self._peek_num_frames(motion_dir) if n_frames <= 0: continue instruction_candidates = self._collect_instruction_candidates(row) motion_records.append( { "row_id": int(row_id), "move_name": str(row.get("move_name", "")), "motion_dir": motion_dir, "instruction_candidates": instruction_candidates, "n_frames": int(n_frames), } ) return motion_records def _map_motion_dir(self, move_g1_path: str) -> Path: rel = str(move_g1_path).strip().replace("\\", "/") rel = rel.replace("g1/csv/", "g1/csv_sonic/") rel = rel.replace("/csv/", "/csv_sonic/") if rel.endswith(".csv"): rel = rel[:-4] if rel.startswith("dataset/seed/"): rel = rel[len("dataset/seed/") :] if rel.startswith("seed/"): rel = rel[len("seed/") :] p = Path(rel) if p.is_absolute(): return p return self.root_dir / p def _collect_instruction_candidates(self, row: pd.Series) -> List[str]: return collect_seed_text_candidates(row, self.text_fields) def _compose_instruction(self, record: Dict[str, Any]) -> str: fallback = str(record.get("move_name", "seed motion")).strip() or "seed motion" return compose_seed_motion_prompt_from_candidates( record.get("instruction_candidates", []), fallback=fallback, ) def _peek_num_frames(self, motion_dir: Path) -> int: joint_path = motion_dir / "joint_pos.csv" if not joint_path.exists(): raise FileNotFoundError(f"joint_pos.csv not found: {joint_path}") return int(len(pd.read_csv(joint_path))) def _load_motion_array(self, motion_dir: Path) -> np.ndarray: key = str(motion_dir) if key in self._motion_cache: return self._motion_cache[key] joint = pd.read_csv(motion_dir / "joint_pos.csv").to_numpy(dtype=np.float32) # (T,29) body_pos = pd.read_csv(motion_dir / "body_pos.csv").to_numpy(dtype=np.float32) # (T,3) body_quat = pd.read_csv(motion_dir / "body_quat.csv").to_numpy(dtype=np.float32) # (T,4) lengths = {joint.shape[0], body_pos.shape[0], body_quat.shape[0]} if len(lengths) != 1: raise ValueError( f"Frame length mismatch at {motion_dir}: " f"joint={joint.shape[0]}, body_pos={body_pos.shape[0]}, body_quat={body_quat.shape[0]}" ) if joint.shape[1] != 29: raise ValueError(f"Expected joint_pos.csv with 29 cols at {motion_dir}, got {joint.shape[1]}") if body_pos.shape[1] != 3: raise ValueError(f"Expected body_pos.csv with 3 cols at {motion_dir}, got {body_pos.shape[1]}") if body_quat.shape[1] != 4: raise ValueError(f"Expected body_quat.csv with 4 cols at {motion_dir}, got {body_quat.shape[1]}") action = np.concatenate([joint, body_pos, body_quat], axis=1).astype(np.float32) # (T,36) if self.motion_cache_size > 0: if len(self._motion_cache) >= self.motion_cache_size: self._motion_cache.pop(next(iter(self._motion_cache))) self._motion_cache[key] = action return action def __len__(self) -> int: return self.num_samples def _pad_index(self, index: int, length: int) -> int: if index < 0: return 0 if index >= length: return length - 1 return index def __getitem__(self, idx: int) -> Dict[str, Any]: if idx < 0 or idx >= self.num_samples: raise IndexError(f"Index out of range: {idx}") motion_id = int(np.searchsorted(self.sample_offsets, idx, side="right") - 1) local_index = int(idx - self.sample_offsets[motion_id]) frame_id = int(local_index * self.upsample_rate) record = self.motion_records[motion_id] motion_dir: Path = record["motion_dir"] instruction = self._compose_instruction(record) motion = self._load_motion_array(motion_dir) # (T,36) total_frames = motion.shape[0] requested_frame_id = frame_id frame_id = self._pad_index(frame_id, total_frames) action_len = self.action_chunk_size + (1 if self.use_delta_actions else 0) actions: List[np.ndarray] = [] action_mask: List[float] = [] for step in range(action_len): # 非delta训练预测下一帧开始的绝对姿态,避免多chunk拼接时重复当前帧 # 原逻辑:raw_index = frame_id + step * self.upsample_rate if self.use_delta_actions: raw_index = requested_frame_id + step * self.upsample_rate else: raw_index = requested_frame_id + (step + 1) * self.upsample_rate valid = 0 <= raw_index < total_frames src_index = self._pad_index(raw_index, total_frames) actions.append(motion[src_index]) action_mask.append(1.0 if valid else 0.0) action_arr = np.stack(actions, axis=0).astype(np.float32) # (Tp or Tp+1,36) action_mask_arr = np.asarray(action_mask, dtype=np.float32) # (Tp or Tp+1,) # SEED训练加入过去几帧state,和HE数据路径保持一致。 # 原逻辑: # states = motion[frame_id:frame_id + 1].astype(np.float32) obs_indices = [ self._pad_index(frame_id - past_i * self.state_history_stride, total_frames) for past_i in range(self.num_past_frames, 0, -1) ] obs_indices.append(self._pad_index(frame_id, total_frames)) states = motion[obs_indices].astype(np.float32) progress, future_progress = build_progress_conditions( frame_id=frame_id, total_frames=total_frames, action_horizon=action_arr.shape[0], step_stride=self.upsample_rate, use_delta_actions=self.use_delta_actions, ) return { "actions": action_arr, "action_mask": action_mask_arr, "states": states, "progress": progress, "future_progress": future_progress, "instruction": instruction, "dataset_name": "seed-g1", # 临时透传样本来源信息,用于定位大梯度batch。 "row_id": int(record["row_id"]), "move_name": str(record.get("move_name", "")), "motion_dir": str(motion_dir), "frame_id": int(frame_id), "total_frames": int(total_frames), }