Download src/psi/trainers/trainer.py from Tinghong-Ye22/code-HFD: direct link, hf CLI and curl.
- Browser
- Download file 30.1 kB
-
https://huggingface.co/Tinghong-Ye22/code-HFD/resolve/main/src/psi/trainers/trainer.py
- Command line
-
hf download hf://Tinghong-Ye22/code-HFD/src/psi/trainers/trainer.py
-
curl -L -o trainer.py https://huggingface.co/Tinghong-Ye22/code-HFD/resolve/main/src/psi/trainers/trainer.py
30.1 kB
| from __future__ import annotations | |
| from typing import Union, Optional, List, Any, TYPE_CHECKING | |
| from pathlib import Path | |
| import torch | |
| import math | |
| from copy import copy | |
| from abc import ABC, abstractmethod | |
| if TYPE_CHECKING: | |
| from psi.config.config import LaunchConfig, TrainConfig, LoggingConfig | |
| from psi.config.config import DataConfig | |
| from accelerate import Accelerator | |
| from transformers.trainer_utils import PredictionOutput | |
| from transformers.optimization import get_scheduler | |
| from torch.optim import Optimizer | |
| from psi.utils import snake_to_pascal | |
| from psi.checkpoint_lease import remove_checkpoint_unless_in_use | |
| import os | |
| import re | |
| import importlib | |
| from torch.utils.data import DataLoader, Dataset | |
| import torch.nn as nn | |
| import numpy as np | |
| import datetime | |
| import accelerate | |
| import random | |
| import shutil | |
| import json | |
| import time | |
| from psi.utils import initialize_overwatch | |
| overwatch = initialize_overwatch(__name__) | |
| def _is_meaningful_eval_improvement( | |
| metric_value: float, | |
| best_metric: float, | |
| min_relative_delta: float, | |
| ) -> tuple[bool, float]: | |
| """Return whether a lower metric beats the configured noise margin.""" | |
| current = float(metric_value) | |
| best = float(best_metric) | |
| relative = float(min_relative_delta) | |
| if ( | |
| not np.isfinite(current) | |
| or not np.isfinite(relative) | |
| or relative < 0.0 | |
| ): | |
| raise ValueError( | |
| "Best-checkpoint metric and relative delta must be finite; " | |
| f"metric={current} relative_delta={relative}" | |
| ) | |
| required = abs(best) * relative if np.isfinite(best) else 0.0 | |
| return current < best - required, required | |
| def _remove_checkpoint_tree_best_effort( | |
| path: str, | |
| *, | |
| attempts: int = 3, | |
| delay_seconds: float = 0.2, | |
| ) -> bool: | |
| """Remove an obsolete checkpoint without making training depend on cleanup.""" | |
| for attempt in range(max(int(attempts), 1)): | |
| try: | |
| shutil.rmtree(path) | |
| return True | |
| except FileNotFoundError: | |
| return True | |
| except OSError as exc: | |
| if attempt + 1 >= max(int(attempts), 1): | |
| overwatch.warning( | |
| "Checkpoint cleanup was deferred because the directory is " | |
| f"still busy: {path} ({type(exc).__name__}: {exc})" | |
| ) | |
| return False | |
| time.sleep(max(float(delay_seconds), 0.0)) | |
| return False | |
| def _atomic_publish_checkpoint_directory( | |
| tmp_checkpoint_dir: str, | |
| checkpoint_dir: str, | |
| *, | |
| global_step: int, | |
| ) -> str | None: | |
| """Publish a completed checkpoint without deleting its live path in place. | |
| Readers may still have files open under ``checkpoint_dir``. Renaming that | |
| immutable snapshot out of the way is safe on POSIX/shared filesystems; | |
| deleting it in place is not, because a concurrent reader can race with | |
| ``shutil.rmtree`` and make checkpoint saving terminate the training job. | |
| """ | |
| retired_dir: str | None = None | |
| if os.path.lexists(checkpoint_dir): | |
| retired_dir = ( | |
| f"{checkpoint_dir}_retired_{int(global_step)}_" | |
| f"{os.getpid()}_{time.time_ns()}" | |
| ) | |
| os.replace(checkpoint_dir, retired_dir) | |
| try: | |
| os.replace(tmp_checkpoint_dir, checkpoint_dir) | |
| except BaseException: | |
| if ( | |
| retired_dir is not None | |
| and os.path.lexists(retired_dir) | |
| and not os.path.lexists(checkpoint_dir) | |
| ): | |
| os.replace(retired_dir, checkpoint_dir) | |
| raise | |
| if retired_dir is not None: | |
| _remove_checkpoint_tree_best_effort(retired_dir) | |
| return retired_dir | |
| def worker_init_fn(worker_id): | |
| # print(f"worker_init_fn called by worker {worker_id}") | |
| worker_seed = torch.initial_seed() % 2**32 | |
| random.seed(worker_seed) | |
| np.random.seed(worker_seed) | |
| torch.manual_seed(worker_seed) | |
| try: | |
| worker_threads = max(int(os.environ.get("PSI_WORKER_TORCH_THREADS", "1")), 1) | |
| torch.set_num_threads(worker_threads) | |
| except (TypeError, ValueError, RuntimeError): | |
| pass | |
| class Trainer(ABC): | |
| cfg: LaunchConfig | |
| model: Any | |
| optimizer: torch.optim.Optimizer | |
| lr_scheduler: torch.optim.lr_scheduler.LRScheduler | |
| train_dataset: Dataset | |
| val_dataset: Optional[Dataset] | |
| def __init__(self, cfg: LaunchConfig, device: Union[torch.device, int]): | |
| self.device = torch.device(device) | |
| self.cfg = cfg | |
| # For mixed precision training we cast all non-trainable weights (vae, non-lora text_encoder and non-lora unet) to half-precision | |
| if cfg.train.mixed_precision == "fp16": | |
| self.dtype = torch.float16 | |
| elif cfg.train.mixed_precision == "bf16": | |
| self.dtype = torch.bfloat16 | |
| else: | |
| self.dtype = torch.float32 | |
| # avoid duplicate run names in a real training run | |
| # should read the timestamp from command line args instead | |
| # self.timestamp = datetime.datetime.now().strftime("%y%m%d%H%M") | |
| self.timestamp = self.cfg.timestamp | |
| def instantiate( | |
| # 把对应输入的具体的trainer实例化 | |
| cls, cfg: LaunchConfig, device: Union[torch.device, int] | |
| ) -> "Trainer": | |
| trainer_name = cfg.train.name # "human-3d" | |
| try: | |
| parts = re.split(r"[-_]", trainer_name) # ["human", "3d"] | |
| trainer_name = "_".join([p.lower() for p in parts]) # "human_3d" | |
| module = importlib.import_module(f"psi.trainers.{trainer_name}") | |
| trainer_clazz = getattr(module, f"{snake_to_pascal(trainer_name)}Trainer") # 帕斯卡命名法 | |
| except Exception as e: | |
| raise ValueError( | |
| f"fail to import {trainer_name} from psi.trainers" | |
| ) from e | |
| return trainer_clazz(cfg, device) | |
| def train_cfg(self) -> TrainConfig: | |
| return self.cfg.train | |
| def log_cfg(self) -> LoggingConfig: | |
| return self.cfg.log | |
| def data_cfg(self) -> DataConfig: | |
| return self.cfg.data | |
| def hf_token(self): | |
| return self.cfg.train.hf_token.read_text().strip() \ | |
| if isinstance(self.cfg.train.hf_token, Path) \ | |
| else os.environ[self.cfg.train.hf_token] | |
| def get_fsdp_plugin(self) -> accelerate.utils.FullyShardedDataParallelPlugin | None: ... | |
| def create_datasets(self) -> tuple[Dataset, Dataset|None]: | |
| ... | |
| def create_dataloaders( | |
| self, train_dataset, val_dataset | |
| ) -> tuple[DataLoader, DataLoader|None]: ... | |
| def create_optimizer(self): | |
| optimizer_kwargs = dict(self.cfg.train.lr_scheduler_kwargs) | |
| if self.cfg.train.optimizer_foreach is not None: | |
| optimizer_kwargs["foreach"] = self.cfg.train.optimizer_foreach | |
| self.optimizer = torch.optim.AdamW( | |
| self.model.parameters(), | |
| lr=self.cfg.train.learning_rate, | |
| **optimizer_kwargs, # type: ignore | |
| ) | |
| def create_scheduler( | |
| self, num_training_steps: int | None = None, optimizer: Optimizer | None = None | |
| ): | |
| if num_training_steps is None: | |
| num_training_steps = self.max_training_steps | |
| # Creates Dummy Scheduler if `scheduler` was specified in the config file else creates `args.lr_scheduler_type` Scheduler | |
| if ( | |
| self.accelerator.state.deepspeed_plugin is None | |
| or "scheduler" not in self.accelerator.state.deepspeed_plugin.deepspeed_config | |
| ): | |
| self.lr_scheduler = get_scheduler( | |
| name=self.cfg.train.lr_scheduler_type, | |
| optimizer=optimizer if optimizer is not None else self.optimizer, | |
| num_warmup_steps=self.num_warmup_steps * self.world_size, | |
| num_training_steps=num_training_steps * self.world_size, | |
| scheduler_specific_kwargs=self.cfg.train.scheduler_specific_kwargs, | |
| ) | |
| else: | |
| self.lr_scheduler = accelerate.utils.DummyScheduler( | |
| optimizer, total_num_steps=self.max_training_steps, warmup_num_steps=self.num_warmup_steps | |
| ) # type: ignore | |
| return self.lr_scheduler | |
| def log( | |
| self, metrics: dict[str, Any], start_time: Optional[float] = None | |
| ) -> None: | |
| wandb_dict = copy(metrics) | |
| for k,v in list(wandb_dict.items()): | |
| if isinstance(v, torch.Tensor) or isinstance(v, np.ndarray): | |
| wandb_dict[k] = v.item() | |
| # wandb_dict.update({"train/grad_norm": grad_norm}) | |
| # wandb_dict.update(self.get_log_kv()) | |
| self.accelerator.log(wandb_dict, step=self.global_step) # WANDB logging | |
| def create_optimizer_and_scheduler(self, num_training_steps: int | None = None): | |
| optimizer = self.create_optimizer() | |
| self.create_scheduler( | |
| num_training_steps=num_training_steps, optimizer=optimizer | |
| ) | |
| def compute_loss( | |
| self, batch#, return_outputs=False#, num_items_in_batch=None | |
| ) -> dict: ... | |
| def training_step( | |
| self, | |
| # model: nn.Module, | |
| batch: dict[str, Union[torch.Tensor, Any]], | |
| ) -> tuple[bool, dict[str, Any]]: ... | |
| def prediction_step( | |
| self, | |
| model: nn.Module, | |
| inputs: dict[str, Union[torch.Tensor, Any]], | |
| prediction_loss_only: bool, | |
| ignore_keys: Optional[list[str]] = None, | |
| ) -> tuple[ | |
| Optional[torch.Tensor], Optional[torch.Tensor], Optional[torch.Tensor] | |
| ]: ... | |
| def evaluate( | |
| self, | |
| # eval_dataset: Optional[Union[Dataset, dict[str, Dataset]]] = None, | |
| # ignore_keys: Optional[list[str]] = None, | |
| # metric_key_prefix: str = "eval", | |
| ) -> dict[str, float] | None: | |
| ... | |
| def predict( | |
| self, | |
| test_dataset: Dataset, | |
| ignore_keys: Optional[list[str]] = None, | |
| metric_key_prefix: str = "test", | |
| ) -> PredictionOutput: ... | |
| def world_size(self) -> int: | |
| """Returns the number of processes in the current distributed group.""" | |
| return overwatch.world_size() | |
| def device_train_batch_size(self) -> int: | |
| """Returns the training batch size per process.""" | |
| return self.cfg.train.train_batch_size | |
| def gradient_accumulation_steps(self) -> int: | |
| """Returns the number of gradient accumulation steps.""" | |
| return self.cfg.train.gradient_accumulation_steps | |
| def global_train_batch_size(self) -> int: | |
| """Returns the global training batch size.""" | |
| return ( | |
| self.device_train_batch_size | |
| * self.world_size | |
| * self.gradient_accumulation_steps | |
| ) | |
| def len_train_dataset(self) -> int: | |
| """Returns the length of the training dataset.""" | |
| if hasattr(self, "_len_train_dataset") and self._len_train_dataset: # type: ignore | |
| return self._len_train_dataset # type: ignore | |
| return len(self.train_dataset) # type: ignore | |
| def len_train_dataset(self, length: int): | |
| self._len_train_dataset = length | |
| def len_val_dataset(self) -> int: | |
| """Returns the length of the training dataset.""" | |
| if hasattr(self, "_len_train_dataset") and self._len_val_dataset: # type: ignore | |
| return self._len_val_dataset # type: ignore | |
| if isinstance(self.val_dataset, dict): | |
| return sum(len(dataset) for dataset in self.val_dataset.values()) # type: ignore | |
| return len(self.val_dataset) # type: ignore | |
| def len_val_dataset(self, length: int): | |
| self._len_val_dataset = length | |
| def len_train_dataloader(self) -> int: | |
| total = self.len_train_dataset / ( | |
| self.device_train_batch_size * self.world_size | |
| ) | |
| if hasattr(self, "train_dataloader_drop_last") and self.train_dataloader_drop_last: # type: ignore | |
| return math.floor(total) # drop the last incomplete batch | |
| else: | |
| return math.ceil(total) # include the last incomplete batch | |
| def len_val_dataloader(self) -> int: | |
| if not hasattr(self, "val_dataloader") or self.val_dataloader is None: | |
| return 0 | |
| if isinstance(self.val_dataloader, dict): | |
| return sum(len(dataloader) for dataloader in self.val_dataloader.values()) | |
| total = self.len_val_dataset / ( | |
| self.device_train_batch_size * self.world_size | |
| ) | |
| if hasattr(self, "val_dataloader_drop_last") and self.val_dataloader_drop_last: # type: ignore | |
| return math.floor(total) # drop the last incomplete batch | |
| else: | |
| return math.ceil(total) # include the last incomplete batch | |
| def num_steps_per_epoch(self) -> int: | |
| """number of global steps (sync gradients) per epoch""" | |
| # assert self.len_train_dataloader > self.gradient_accumulation_steps | |
| return max(self.len_train_dataloader // self.gradient_accumulation_steps, 1) | |
| def max_training_steps(self) -> int: | |
| """Returns the maximum number of training steps.""" | |
| if self.cfg.train.max_training_steps is not None: | |
| return self.cfg.train.max_training_steps | |
| else: | |
| assert self.cfg.train.num_train_epochs is not None | |
| return self.num_steps_per_epoch * self.world_size * self.cfg.train.num_train_epochs | |
| def max_training_epochs(self) -> int: | |
| return math.ceil(self.max_training_steps / self.num_steps_per_epoch) | |
| def num_warmup_steps(self) -> int: | |
| """ | |
| Get number of steps used for a linear warmup. | |
| """ | |
| warmup_steps = ( | |
| self.cfg.train.warmup_steps | |
| if self.cfg.train.warmup_steps is not None | |
| else math.ceil(self.max_training_steps * self.cfg.train.warmup_ratio) # type: ignore | |
| ) | |
| return warmup_steps # because accelerate divides the steps by world_size when preparing the scheduler | |
| def task_run_name(self) -> str: | |
| """Returns the task-specific run name.""" | |
| def run_name(self) -> str: | |
| run_name = ( | |
| f"{self.cfg.exp}{self.task_run_name}" | |
| f".b{self.global_train_batch_size}.gpus{overwatch.world_size()}" | |
| ) | |
| run_name = f"{run_name}.{self.timestamp}" | |
| if self.cfg.debug: | |
| run_name = f"debug-{run_name}" | |
| return run_name | |
| def project_dir(self) -> str: | |
| """Returns the project directory for saving checkpoints and logs.""" | |
| return os.path.join( | |
| self.cfg.train.output_dir, self.cfg.train.name, self.run_name | |
| ) | |
| def next_epoch(self, epoch): | |
| """called between epochs, e.g. for resetting the distributed samplers""" | |
| if hasattr(self, "train_sampler") and self.train_sampler is not None: # type: ignore | |
| self.train_sampler.set_epoch(epoch) # type: ignore | |
| def init_models(self): | |
| """Initialize the models for training.""" | |
| # @abstractmethod | |
| def set_train(self): | |
| self.model.train() | |
| # @abstractmethod | |
| def set_eval(self): | |
| self.model.eval() | |
| # @abstractmethod | |
| # def create_optimizers(self): | |
| # ... | |
| # @abstractmethod | |
| # def create_lr_schedulers(self): | |
| # ... | |
| # @abstractmethod | |
| # def train_one_step(self, batch_input, global_step, local_step, accelerator=None): | |
| # ... | |
| def step(self, batch_input, global_step, local_step) -> tuple[bool, dict[str, Any]]: | |
| """ Perform a single training step. """ | |
| self.local_step = local_step | |
| self.global_step = global_step | |
| sync_gradients, losses = self.training_step(batch_input) | |
| return sync_gradients, losses | |
| def prepare(self, accelerator: Accelerator) -> DataLoader: | |
| self.optimizer, self.lr_scheduler = accelerator.prepare( | |
| self.optimizer, self.lr_scheduler | |
| ) | |
| self.train_dataloader = accelerator.prepare(self.train_dataloader) | |
| if self.cfg.train.overfit_single_batch: | |
| overwatch.warning("Overfitting a single batch: reusing first batch every step. set cfg.data.image_aug = False for true memorization.") | |
| first_batch = next(iter(self.train_dataloader)) | |
| class SingleBatchLoader: | |
| def __iter__(self): | |
| while True: | |
| yield first_batch | |
| def __len__(self): | |
| return 1 | |
| self.train_dataloader = SingleBatchLoader() | |
| # FIXME if self.train_dataloader.dataset is IterableDataset: | |
| # assert ( | |
| # abs(len(self.train_dataloader) - self.len_train_dataloader) | |
| # <= 1 # because of drop_last option is lost in prepare. | |
| # ), f"check calculations again, {len(self.train_dataloader)} != {self.len_train_dataloader}" | |
| val_dataloader = getattr(self, "val_dataloader", None) | |
| if val_dataloader is not None: # not using if self.val_dataloader to avoid DataLoader.__len__() being called on iterable dataset | |
| self.val_dataloader = accelerator.prepare(self.val_dataloader) | |
| """ NOTE: | |
| We have to manually record how many steps we have accumulated gradients | |
| because the accelerator does not do it for us. | |
| """ | |
| self.accelerator = accelerator | |
| return self.train_dataloader # type: ignore | |
| def save_checkpoint(self, global_step: int) -> str | None: | |
| save_dir = os.path.join(self.project_dir, "checkpoints") | |
| os.makedirs(save_dir, exist_ok=True) | |
| ckpt_dir = os.path.join(save_dir, f"ckpt_{global_step}") | |
| self.accelerator.save_state(ckpt_dir) | |
| self.accelerator.wait_for_everyone() | |
| if self.accelerator.is_main_process: | |
| # Keep only the latest max_checkpoints_to_keep checkpoints | |
| max_to_keep = self.train_cfg.max_checkpoints_to_keep or 100 | |
| if max_to_keep is not None and max_to_keep > 0: | |
| # List all checkpoint directories matching ckpt_* | |
| ckpt_dirs = [d for d in os.listdir(save_dir) if d.startswith("ckpt_") and os.path.isdir(os.path.join(save_dir, d))] | |
| # Extract step numbers and sort by step (assume ckpt_{step}) | |
| def extract_step(d): | |
| try: | |
| return int(d.split("ckpt_")[-1]) | |
| except Exception: | |
| return -1 | |
| ckpt_dirs_sorted = sorted(ckpt_dirs, key=extract_step, reverse=True) | |
| # Remove older checkpoints if exceeding max_to_keep | |
| for old_ckpt in ckpt_dirs_sorted[max_to_keep:]: | |
| old_ckpt_path = os.path.join(save_dir, old_ckpt) | |
| # if "0000" in old_ckpt: | |
| # # force keeping ckpt saved every 10k | |
| # continue | |
| try: | |
| if remove_checkpoint_unless_in_use(Path(old_ckpt_path)): | |
| overwatch.info(f"Removed old checkpoint: {old_ckpt_path}") | |
| else: | |
| overwatch.info(f"Retained checkpoint in active inference: {old_ckpt_path}") | |
| except Exception as e: | |
| overwatch.warning(f"Failed to remove old checkpoint {old_ckpt_path}: {e}") | |
| self.accelerator.wait_for_everyone() | |
| return ckpt_dir | |
| def save_best_checkpoint(self, eval_metrics: dict[str, float], global_step: int) -> str | None: | |
| # <modify> 根据eval loss保存当前最优ckpt,独立于按步数保存的ckpt,不参与max_checkpoints_to_keep清理。 | |
| metric_name = "loss" | |
| if metric_name not in eval_metrics: | |
| return None | |
| save_dir = os.path.join(self.project_dir, "checkpoints") | |
| os.makedirs(save_dir, exist_ok=True) | |
| best_info_path = os.path.join(save_dir, "best_eval_metric.json") | |
| best_ckpt_dir = os.path.join(save_dir, "best_ckpt") | |
| tmp_ckpt_dir = os.path.join(save_dir, f"best_ckpt_tmp_{global_step}") | |
| metric_value = float(eval_metrics[metric_name]) | |
| min_relative_delta = float( | |
| getattr( | |
| self.train_cfg, | |
| "best_checkpoint_min_relative_delta", | |
| 0.0, | |
| ) | |
| ) | |
| best_metric = float("inf") | |
| if os.path.exists(best_info_path): | |
| try: | |
| with open(best_info_path, "r") as f: | |
| best_info = json.load(f) | |
| best_metric = float(best_info.get("metric_value", best_metric)) | |
| except Exception as e: | |
| overwatch.warning(f"Failed to read best eval metric from {best_info_path}: {e}") | |
| improved, required_improvement = _is_meaningful_eval_improvement( | |
| metric_value, | |
| best_metric, | |
| min_relative_delta, | |
| ) | |
| if not improved: | |
| return None | |
| if self.accelerator.is_main_process and os.path.exists(tmp_ckpt_dir): | |
| shutil.rmtree(tmp_ckpt_dir) | |
| self.accelerator.wait_for_everyone() | |
| self.accelerator.save_state(tmp_ckpt_dir) | |
| self.accelerator.wait_for_everyone() | |
| # Subclasses can place their architecture/data compatibility marker in | |
| # the temporary directory before it becomes visible as ``best_ckpt``. | |
| self._prepare_best_checkpoint_for_publish(tmp_ckpt_dir, global_step) | |
| self.accelerator.wait_for_everyone() | |
| if self.accelerator.is_main_process: | |
| _atomic_publish_checkpoint_directory( | |
| tmp_ckpt_dir, | |
| best_ckpt_dir, | |
| global_step=global_step, | |
| ) | |
| best_info = { | |
| "metric_name": metric_name, | |
| "metric_value": metric_value, | |
| "min_relative_delta": min_relative_delta, | |
| "required_improvement": required_improvement, | |
| "global_step": global_step, | |
| "checkpoint_dir": best_ckpt_dir, | |
| } | |
| with open(best_info_path, "w") as f: | |
| json.dump(best_info, f, indent=2) | |
| overwatch.info( | |
| f"Saved best checkpoint: {best_ckpt_dir} " | |
| f"({metric_name}={metric_value:.6f}, previous_best={best_metric:.6f})" | |
| ) | |
| self.accelerator.wait_for_everyone() | |
| return best_ckpt_dir | |
| def _prepare_best_checkpoint_for_publish( | |
| self, | |
| checkpoint_dir: str, | |
| global_step: int, | |
| ) -> None: | |
| """Hook for writing readiness metadata before atomic publication.""" | |
| del checkpoint_dir, global_step | |
| def resume_from_checkpoint(self) -> tuple[int, Optional[str]]: | |
| """ resume from a checkpoint if specified in the config. | |
| the checkpoint path can be either: | |
| 1) full path to a checkpoint folder, e.g. | |
| .runs/trainer_name/run_name/checkpoints/ckpt_xxxxxx | |
| 3) or .runs/trainer_name/run_name (latest) | |
| """ | |
| if self.cfg.train.resume_from_checkpoint is None: | |
| return 0, None | |
| # <modify> 传入run目录时默认读取best_ckpt;普通ckpt只保留ckpt_数字目录参与排序。 | |
| # 原逻辑: | |
| # if "ckpt_" in self.cfg.train.resume_from_checkpoint: | |
| # path = os.path.basename(self.cfg.train.resume_from_checkpoint) | |
| # else: | |
| # if os.path.exists(f"{self.cfg.train.resume_from_checkpoint}/checkpoints"): | |
| # # Get the most recent checkpoint | |
| # dirs = os.listdir(f"{self.cfg.train.resume_from_checkpoint}/checkpoints") | |
| # dirs = sorted(dirs, key=lambda x: int(x.split("_")[1])) | |
| # path = dirs[-1] if len(dirs) > 0 else None | |
| # else: | |
| # path = None | |
| resume_path = self.cfg.train.resume_from_checkpoint | |
| best_info_path = None | |
| if os.path.exists(os.path.join(resume_path, "model.safetensors")): | |
| load_path = resume_path | |
| path = os.path.basename(load_path) | |
| best_info_path = os.path.join(os.path.dirname(load_path), "best_eval_metric.json") | |
| elif os.path.exists(os.path.join(resume_path, "checkpoints")): | |
| checkpoint_dir = os.path.join(resume_path, "checkpoints") | |
| best_path = os.path.join(checkpoint_dir, "best_ckpt") | |
| best_info_path = os.path.join(checkpoint_dir, "best_eval_metric.json") | |
| if os.path.exists(best_path): | |
| path = "best_ckpt" | |
| load_path = best_path | |
| else: | |
| dirs = os.listdir(checkpoint_dir) | |
| dirs = [d for d in dirs if re.match(r"^ckpt_\d+$", d)] | |
| dirs = sorted(dirs, key=lambda x: int(x.split("_")[1])) | |
| path = dirs[-1] if len(dirs) > 0 else None | |
| load_path = os.path.join(checkpoint_dir, path) if path is not None else None | |
| else: | |
| path = None | |
| load_path = None | |
| if path is None: | |
| overwatch.critical( | |
| f"Checkpoint '{self.cfg.train.resume_from_checkpoint}' does not exist. Starting a new training run." | |
| ) | |
| self.cfg.train.resume_from_checkpoint = None | |
| initial_global_step = 0 | |
| else: | |
| overwatch.info(f"Resuming from checkpoint {load_path}") | |
| if self.cfg.train.resume_load_full_state: | |
| self.accelerator.load_state(load_path) | |
| overwatch.info(f"Loaded full training state from {load_path}") | |
| else: | |
| # <modify> 默认 resume 只加载模型权重,optimizer/scheduler 按当前脚本配置重新开始。 | |
| from safetensors.torch import load_file | |
| model_path = os.path.join(load_path, "model.safetensors") | |
| state_dict = load_file(model_path, device="cpu") | |
| is_hfd_delta = os.path.isfile( | |
| os.path.join(load_path, "hfd_fused_delta_manifest.json") | |
| ) | |
| self.accelerator.unwrap_model(self.model).load_state_dict( | |
| state_dict, | |
| strict=not is_hfd_delta, | |
| ) | |
| overwatch.info(f"Loaded model weights from {model_path}") | |
| if path == "best_ckpt" and best_info_path is not None and os.path.exists(best_info_path): | |
| with open(best_info_path, "r") as f: | |
| best_info = json.load(f) | |
| # Checkpoint names/metadata store the number of optimizer | |
| # steps already completed (save_checkpoint receives | |
| # ``global_step + 1``). That count is exactly the zero-based | |
| # index of the next step; adding another one skips a batch and | |
| # desynchronizes restored sampler/RNG state. | |
| initial_global_step = int(best_info["global_step"]) | |
| else: | |
| initial_global_step = int(path.split("_")[1]) | |
| return initial_global_step, load_path | |
| def lr(self): | |
| return self.get_lr() | |
| def get_lr(self): | |
| return self.lr_scheduler.get_last_lr()[0] | |
| def get_total_grad_norm(self, params=None, norm_type=2): | |
| if "DeepSpeedEngine" in self.model.__class__.__name__: | |
| grad_norm = self.model.get_global_grad_norm() | |
| return grad_norm | |
| else: | |
| if params is None: | |
| params = self.model.parameters() | |
| params = list(params) | |
| use_foreach = os.environ.get( | |
| "PSI_FOREACH_GRAD_NORM", "0" | |
| ).strip().lower() in {"1", "true", "yes", "y", "on"} | |
| if use_foreach: | |
| grads = [p.grad.detach() for p in params if p.grad is not None] | |
| if not grads: | |
| return 0.0 | |
| # foreach avoids one host synchronization per parameter. It is | |
| # logging-only: gradients are read, never clipped or modified. | |
| grouped_grads = {} | |
| for grad in grads: | |
| grouped_grads.setdefault((grad.device, grad.dtype), []).append(grad) | |
| parameter_norms = [] | |
| for grouped in grouped_grads.values(): | |
| if any(grad.is_sparse for grad in grouped): | |
| parameter_norms.extend( | |
| torch.linalg.vector_norm(grad, norm_type) for grad in grouped | |
| ) | |
| else: | |
| parameter_norms.extend(torch._foreach_norm(grouped, norm_type)) | |
| first_device = grads[0].device | |
| return torch.linalg.vector_norm( | |
| torch.stack( | |
| [norm.float().to(first_device) for norm in parameter_norms] | |
| ), | |
| norm_type, | |
| ).item() | |
| total_norm = 0.0 | |
| for p in params: | |
| if p.grad is not None: | |
| param_norm = p.grad.data.norm(norm_type) | |
| total_norm += param_norm.item() ** norm_type | |
| total_norm = total_norm ** (1. / norm_type) | |
| return total_norm | |
| def unwrap_model(self): | |
| # Function for unwrapping if model was compiled with `torch.compile`. | |
| model = self.accelerator.unwrap_model(self.model) | |
| from diffusers.utils.torch_utils import is_compiled_module | |
| model = model._orig_mod if is_compiled_module(model) else model | |
| # songlin: revert original forward which is changed by accelerate.prepare to always return fp32 | |
| # if hasattr(model, "_original_forward"): | |
| # model.forward = model._original_forward | |
| return model | |
| # def log( | |
| # self, ): | |
| # ... | |
| # def evaluate(self, global_step: int, accelerator: Accelerator): | |
| # ... | |
| def finalize(self): | |
| ... | |