Download src/psi/config/config.py from Tinghong-Ye22/code-Psi0-yth-v21: direct link, hf CLI and curl.
- Browser
- Download file 8.32 kB
-
https://huggingface.co/Tinghong-Ye22/code-Psi0-yth-v21/resolve/main/src/psi/config/config.py
- Command line
-
hf download hf://Tinghong-Ye22/code-Psi0-yth-v21/src/psi/config/config.py
-
curl -L -o config.py https://huggingface.co/Tinghong-Ye22/code-Psi0-yth-v21/resolve/main/src/psi/config/config.py
8.32 kB
| from __future__ import annotations | |
| import os | |
| from typing_extensions import Self, Annotated | |
| from pydantic import BaseModel, Field, model_validator | |
| import os | |
| import tyro | |
| from typing import Union, Any, TYPE_CHECKING | |
| from tyro.conf import subcommand as cmd | |
| from pathlib import Path | |
| import datetime | |
| from psi.config.transform import DataTransform | |
| class LoggingConfig(BaseModel): | |
| logging_dir: str = "logs" | |
| report_to: str | None = None | |
| log_freq: int = 100 | |
| class WandbConfig(BaseModel): | |
| project: str = "psi" | |
| entity: str | None = None | |
| group: str | None = None | |
| id: str | None = None | |
| name: str | None = None | |
| resume: str = "allow" # allow, must, never | |
| def model_post_init(self, __context: Any) -> None: | |
| if self.entity is None: | |
| self.entity = os.getenv("WANDB_ENTITY", None) | |
| # TextOp分组loss权重:支持39D新布局,同时保留70D字段兼容旧实验。 | |
| class LossGroupWeightConfig(BaseModel): | |
| joint_abs: float = 1.0 | |
| root_delta_xy: float = 1.0 | |
| root_rot5_delta: float = 1.0 | |
| root_height: float = 1.0 | |
| foot_contact: float = 1.0 | |
| # 70D legacy fields. | |
| joint_delta: float = 1.0 | |
| root_delta_local: float = 1.0 | |
| root_rot6d_delta: float = 1.0 | |
| class TrainConfig(BaseModel): | |
| num_workers: int = 8 | |
| val_num_workers: int | None = None | |
| overfit_single_batch: bool = False | |
| name: str = "human3d" # "vqvae" | |
| resume_from_checkpoint: str | None = None | |
| resume_load_full_state: bool = False | |
| skip_resumed_steps: bool = False | |
| # HF Hub Credentials (for any gated models) | |
| hf_token: str | Path = Path(".hf_token") # Environment variable or Path to HF Token | |
| lora: bool = False | |
| output_dir: str = ".runs" | |
| gradient_accumulation_steps: int = 1 | |
| mixed_precision: str = "bf16" # "no" for fp32, "fp16", "bf16" | |
| max_grad_norm: float | None = None # 1.0 | |
| # <modify>+梯度保护:逐元素梯度裁剪阈值,默认开启一个较保守的保护范围。 | |
| max_grad_value: float | None = 5.0 | |
| # <modify>+梯度保护:loss 或梯度出现非有限值时,直接跳过本次参数更新。 | |
| skip_step_on_nonfinite_grad: bool = True | |
| # <modify>+70D分组loss权重:训练时对不同动作组单独加权,默认保持原行为不变。 | |
| loss_group_weight: LossGroupWeightConfig = Field(default_factory=LossGroupWeightConfig) | |
| optimizer_foreach: bool | None = None | |
| train_batch_size: int = 16 | |
| val_batch_size: int = 16 | |
| val_num_batches: int = 20 | |
| checkpointing_steps: int = 5000 | |
| max_checkpoints_to_keep: int | None = None | |
| validation_steps: int = 50 | |
| learning_rate: float = 1e-5 | |
| # linear, cosine, cosine_with_restarts, polynomial, constant, constant_with_warmup | |
| lr_scheduler_type: str = "cosine_with_min_lr" | |
| lr_scheduler_kwargs: dict[str, float | tuple[float, ...]] = Field( | |
| default_factory=lambda: { | |
| # "num_warmup_steps": 0, | |
| "betas": (0.9, 0.95), | |
| "weight_decay": 1e-8, | |
| "eps": 1e-8, | |
| } | |
| ) | |
| scheduler_specific_kwargs: dict[str, float | tuple[float, ...]] = Field( | |
| default_factory=lambda: {"min_lr": 5.0e-07} | |
| ) | |
| ## FSDP or DDP | |
| data_parallel: str = "ddp" # "deepspeed", "ddp" or "fsdp" | |
| sharding_strategy: str = "full-shard" | |
| deepspeed_config: str = "src/InternVLA/config/deepseeds/zero3.json" | |
| enable_gradient_checkpointing: bool = True | |
| enable_mixed_precision_training: bool = True | |
| reduce_in_full_precision: bool = True | |
| max_training_steps: int | None = 100_000 | |
| num_train_epochs: int | None = None | |
| warmup_steps: int | None = 0 # if > 0, overrides warmup_ratio | |
| warmup_ratio: float | None = 0.05 # used if warmup_steps is not set or 0 | |
| def check_warmup(self) -> Self: | |
| steps, ratio = self.warmup_steps, self.warmup_ratio | |
| if (steps is not None and steps > 0) and (ratio is not None and ratio > 0): | |
| raise ValueError("Only one of warmup_steps or warmup_ratio can be set") | |
| steps, ratio = self.max_training_steps, self.num_train_epochs | |
| if (steps is None or steps == 0) and (ratio is None or ratio == 0): | |
| raise ValueError( | |
| "At least one of max_training_steps or num_train_epochs must be set" | |
| ) | |
| if steps is not None and ratio is not None and steps > 0 and ratio > 0: | |
| raise ValueError( | |
| "Only one of max_training_steps or num_train_epochs can be set" | |
| ) | |
| return self | |
| def model_post_init(self, __context: Any) -> None: | |
| if self.lr_scheduler_type != "cosine_with_min_lr": | |
| self.scheduler_specific_kwargs = {} | |
| if not os.path.isabs(self.deepspeed_config): | |
| from psi.utils import resolve_path | |
| self.deepspeed_config = os.path.abspath( | |
| resolve_path(self.deepspeed_config) | |
| ) | |
| class ServerConfig(BaseModel): | |
| host: str = "0.0.0.0" | |
| port: int = 21074 | |
| device: str = "cuda:0" | |
| policy: str | None = None | |
| action_exec_horizon: int | None = None | |
| rtc: bool = False | |
| run_dir: str | |
| ckpt_step: int | str | |
| def set_policy(self): | |
| if self.policy is None: | |
| run_dir_path = Path(self.run_dir) | |
| self.policy = run_dir_path.parts[1] | |
| return self | |
| class DataConfig(BaseModel): | |
| transform: DataTransform | |
| def __call__(self, split: str = "train", transform_kwargs={}, **kwargs) -> Any: | |
| raise NotImplementedError | |
| class ModelConfig(BaseModel): | |
| ... | |
| class DummyModelConfig(ModelConfig): | |
| # boilerplate model config | |
| ... | |
| class LaunchConfig(BaseModel): | |
| # NOTE: This class is used for type hinting only! | |
| # The real implmentation is DynamicLaunchConfig . | |
| exp: str = Field(..., description="Name of the experiment", frozen=True) | |
| seed: int | None = None | |
| auto_tag_run: bool = False | |
| eval: bool = False | |
| debug: bool = False | |
| timestamp: str | None = None | |
| log: LoggingConfig | |
| wandb: WandbConfig | |
| train: TrainConfig | |
| data: DataConfig | |
| model: ModelConfig | |
| def model_post_init(self, __context: Any) -> None: | |
| is_multinode = ( | |
| "SLURM_NODELIST" in os.environ and | |
| len(os.environ["SLURM_NODELIST"].split(",")) > 1 | |
| ) | |
| if is_multinode: | |
| assert self.timestamp is not None, "Timestamp must be provided for multi-node training, eg., --timestamp=$(date +\"%y%m%d%H%M\")" | |
| def extract_timestamp(folder_name): | |
| parts = folder_name.split('.') | |
| return parts[-1] if len(parts) > 1 else '' | |
| if self.train.resume_from_checkpoint == "latest": | |
| """ auto resume by looking up timestamp or latest run folder """ | |
| auto_resume_success = False | |
| trainer_dir = os.path.join(self.train.output_dir, self.train.name) | |
| if os.path.exists(trainer_dir): | |
| # Sort folders by timestamp (assumed to be last part after a dot) | |
| runs = dict(sorted({ | |
| extract_timestamp(f): os.path.join(trainer_dir, f) for f in os.listdir(trainer_dir) | |
| }.items(), reverse=True)) | |
| if self.timestamp is not None and self.timestamp in runs: | |
| print(f"Will resume latest run with specified timestamp: {self.timestamp}") | |
| self.train.resume_from_checkpoint = runs[self.timestamp] | |
| auto_resume_success = True | |
| """ elif len(runs) > 0: | |
| latest_timestamp = next(iter(runs)) | |
| print(f"Will auto-resume from latest run with timestamp: {latest_timestamp}") | |
| if self.timestamp is not None and self.timestamp != latest_timestamp: | |
| print(f"Overriding timestamp {self.timestamp} with latest timestamp {latest_timestamp}") | |
| self.timestamp = latest_timestamp | |
| self.train.resume_from_checkpoint = runs[latest_timestamp] | |
| auto_resume_success = True """ | |
| if not auto_resume_success: | |
| self.train.resume_from_checkpoint = None | |
| if self.timestamp is None: | |
| self.timestamp = datetime.datetime.now().strftime("%y%m%d%H%M") | |