AnyMo / tools /train_deepspeed.py

8b3f571 5 months ago

6.57 kB

# Copyright (c) OpenMMLab. All rights reserved.
import sys, os
sys.path.append(os.path.join(os.path.abspath(os.path.dirname(__file__)), '..'))
import src
import copy
import argparse
import os.path as osp

from mmengine.config import Config, DictAction
from mmengine.registry import RUNNERS
from mmengine.runner._flexible_runner import FlexibleRunner

# import torch
# torch.autograd.set_detect_anomaly(True) 
# import faulthandler
# faulthandler.enable()
# import numpy as np
# np.seterr(divide='raise', invalid='raise', over='raise') 

def parse_args():
    parser = argparse.ArgumentParser(description='Train a detector')
    parser.add_argument('config', help='train config file path')
    parser.add_argument('--work-dir', help='the dir to save logs and models')
    parser.add_argument(
        '--amp',
        action='store_true',
        default=False,
        help='enable automatic-mixed-precision training')
    parser.add_argument(
        '--auto-scale-lr',
        action='store_true',
        help='enable automatically scaling LR.')
    parser.add_argument(
        '--resume',
        nargs='?',
        type=str,
        const='auto',
        help='If specify checkpoint path, resume from it, while if not '
        'specify, try to auto resume from the latest checkpoint '
        'in the work directory.')
    parser.add_argument(
        '--cfg-options',
        nargs='+',
        action=DictAction,
        help='override some settings in the used config, the key-value pair '
        'in xxx=yyy format will be merged into config file. If the value to '
        'be overwritten is a list, it should be like key="[a,b]" or key=a,b '
        'It also allows nested list/tuple values, e.g. key="[(a,b),(c,d)]" '
        'Note that the quotation marks are necessary and that no white space '
        'is allowed.')
    parser.add_argument(
        '--launcher',
        choices=['none', 'pytorch', 'slurm', 'mpi'],
        default='none',
        help='job launcher')
    # When using PyTorch version >= 2.0.0, the `torch.distributed.launch`
    # will pass the `--local-rank` parameter to `tools/train.py` instead
    # of `--local_rank`.
    parser.add_argument('--local_rank', '--local-rank', type=int, default=0)
    args = parser.parse_args()
    if 'LOCAL_RANK' not in os.environ:
        os.environ['LOCAL_RANK'] = str(args.local_rank)

    return args


def main():
    args = parse_args()

    # load config
    cfg = Config.fromfile(args.config)
    cfg.launcher = args.launcher
    if args.cfg_options is not None:
        cfg.merge_from_dict(args.cfg_options)

    # work_dir is determined in this priority: CLI > segment in file > filename
    if args.work_dir is not None:
        # update configs according to CLI args if args.work_dir is not None
        cfg.work_dir = args.work_dir
    elif cfg.get('work_dir', None) is None:
        # use config filename as default work_dir if cfg.work_dir is None
        cfg.work_dir = osp.join('./work_dirs',
                                osp.splitext(osp.basename(args.config))[0])

    # enable automatic-mixed-precision training
    if args.amp is True:
        cfg.optim_wrapper.type = 'AmpOptimWrapper'
        cfg.optim_wrapper.loss_scale = 'dynamic'

    # enable automatically scaling LR
    if args.auto_scale_lr:
        if 'auto_scale_lr' in cfg and \
                'enable' in cfg.auto_scale_lr and \
                'base_batch_size' in cfg.auto_scale_lr:
            cfg.auto_scale_lr.enable = True
        else:
            raise RuntimeError('Can not find "auto_scale_lr" or '
                               '"auto_scale_lr.enable" or '
                               '"auto_scale_lr.base_batch_size" in your'
                               ' configuration file.')

    # resume is determined in this priority: resume from > auto_resume
    if args.resume == 'auto':
        cfg.resume = True
        cfg.load_from = None
    elif args.resume is not None:
        cfg.resume = True
        cfg.load_from = args.resume

    # 指定 DeepSpeedStrategy 并配置参数
    strategy = dict(
        type='DeepSpeedStrategy',
        bf16=dict(
            enabled=True,
        ),
        inputs_to_half=['motion', 'traj_data', 'audio_data', 'mean', 'std'],

        zero_optimization=dict(
            stage=3,
            ignore_unused_parameters=True, 
            allgather_partitions=True,
            reduce_scatter=True,
            allgather_bucket_size=10000000,
            reduce_bucket_size=10000000,
            # overlap_comm=True,
            overlap_comm=False,
            contiguous_gradients=True,
            cpu_offload=False),

        # zero_optimization = dict(
        #     stage=2,
        #     ignore_unused_parameters=True,   # 你这个场景需要保留
        #     reduce_scatter=True,
        #     allgather_partitions=True,
        #     overlap_comm=True,
        #     contiguous_gradients=True,
        #     reduce_bucket_size=5e7,
        #     allgather_bucket_size=5e7,
        # )
    )


    optim_wrapper = dict(type='DeepSpeedOptimWrapper')
    optim_wrapper['optimizer'] = cfg.optim_wrapper.optimizer
    cfg.optim_wrapper = optim_wrapper

    cfg = copy.deepcopy(cfg)
    runner = FlexibleRunner(
        model=cfg['model'],
        work_dir=cfg.get('work_dir', 'work_dirs'),
        experiment_name=cfg.get('experiment_name'),
        train_dataloader=cfg.get('train_dataloader'),
        optim_wrapper=cfg.get('optim_wrapper'),
        param_scheduler=cfg.get('param_scheduler'),
        train_cfg=cfg.get('train_cfg'),
        val_dataloader=cfg.get('val_dataloader'),
        val_evaluator=cfg.get('val_evaluator'),
        val_cfg=cfg.get('val_cfg'),
        test_dataloader=cfg.get('test_dataloader'),
        test_evaluator=cfg.get('test_evaluator'),
        test_cfg=cfg.get('test_cfg'),
        strategy=strategy,
        auto_scale_lr=cfg.get('auto_scale_lr'),
        default_hooks=cfg.get('default_hooks'),
        custom_hooks=cfg.get('custom_hooks'),
        data_preprocessor=cfg.get('data_preprocessor'),
        load_from=cfg.get('load_from'),
        resume=cfg.get('resume', False),
        launcher=cfg.get('launcher'),
        env_cfg=cfg.get('env_cfg'),  # type: ignore
        log_processor=cfg.get('log_processor'),
        log_level=cfg.get('log_level', 'INFO'),
        visualizer=cfg.get('visualizer'),
        default_scope=cfg.get('default_scope', 'mmengine'),
        randomness=cfg.get('randomness', dict(seed=None)),
        cfg=cfg,
    )
    runner.train()


if __name__ == '__main__':
    main()

	# Copyright (c) OpenMMLab. All rights reserved.
	import sys, os
	sys.path.append(os.path.join(os.path.abspath(os.path.dirname(__file__)), '..'))
	import src
	import copy
	import argparse
	import os.path as osp

	from mmengine.config import Config, DictAction
	from mmengine.registry import RUNNERS
	from mmengine.runner._flexible_runner import FlexibleRunner

	# import torch
	# torch.autograd.set_detect_anomaly(True)
	# import faulthandler
	# faulthandler.enable()
	# import numpy as np
	# np.seterr(divide='raise', invalid='raise', over='raise')

	def parse_args():
	parser = argparse.ArgumentParser(description='Train a detector')
	parser.add_argument('config', help='train config file path')
	parser.add_argument('--work-dir', help='the dir to save logs and models')
	parser.add_argument(
	'--amp',
	action='store_true',
	default=False,
	help='enable automatic-mixed-precision training')
	parser.add_argument(
	'--auto-scale-lr',
	action='store_true',
	help='enable automatically scaling LR.')
	parser.add_argument(
	'--resume',
	nargs='?',
	type=str,
	const='auto',
	help='If specify checkpoint path, resume from it, while if not '
	'specify, try to auto resume from the latest checkpoint '
	'in the work directory.')
	parser.add_argument(
	'--cfg-options',
	nargs='+',
	action=DictAction,
	help='override some settings in the used config, the key-value pair '
	'in xxx=yyy format will be merged into config file. If the value to '
	'be overwritten is a list, it should be like key="[a,b]" or key=a,b '
	'It also allows nested list/tuple values, e.g. key="[(a,b),(c,d)]" '
	'Note that the quotation marks are necessary and that no white space '
	'is allowed.')
	parser.add_argument(
	'--launcher',
	choices=['none', 'pytorch', 'slurm', 'mpi'],
	default='none',
	help='job launcher')
	# When using PyTorch version >= 2.0.0, the `torch.distributed.launch`
	# will pass the `--local-rank` parameter to `tools/train.py` instead
	# of `--local_rank`.
	parser.add_argument('--local_rank', '--local-rank', type=int, default=0)
	args = parser.parse_args()
	if 'LOCAL_RANK' not in os.environ:
	os.environ['LOCAL_RANK'] = str(args.local_rank)

	return args


	def main():
	args = parse_args()

	# load config
	cfg = Config.fromfile(args.config)
	cfg.launcher = args.launcher
	if args.cfg_options is not None:
	cfg.merge_from_dict(args.cfg_options)

	# work_dir is determined in this priority: CLI > segment in file > filename
	if args.work_dir is not None:
	# update configs according to CLI args if args.work_dir is not None
	cfg.work_dir = args.work_dir
	elif cfg.get('work_dir', None) is None:
	# use config filename as default work_dir if cfg.work_dir is None
	cfg.work_dir = osp.join('./work_dirs',
	osp.splitext(osp.basename(args.config))[0])

	# enable automatic-mixed-precision training
	if args.amp is True:
	cfg.optim_wrapper.type = 'AmpOptimWrapper'
	cfg.optim_wrapper.loss_scale = 'dynamic'

	# enable automatically scaling LR
	if args.auto_scale_lr:
	if 'auto_scale_lr' in cfg and \
	'enable' in cfg.auto_scale_lr and \
	'base_batch_size' in cfg.auto_scale_lr:
	cfg.auto_scale_lr.enable = True
	else:
	raise RuntimeError('Can not find "auto_scale_lr" or '
	'"auto_scale_lr.enable" or '
	'"auto_scale_lr.base_batch_size" in your'
	' configuration file.')

	# resume is determined in this priority: resume from > auto_resume
	if args.resume == 'auto':
	cfg.resume = True
	cfg.load_from = None
	elif args.resume is not None:
	cfg.resume = True
	cfg.load_from = args.resume

	# 指定 DeepSpeedStrategy 并配置参数
	strategy = dict(
	type='DeepSpeedStrategy',
	bf16=dict(
	enabled=True,
	),
	inputs_to_half=['motion', 'traj_data', 'audio_data', 'mean', 'std'],

	zero_optimization=dict(
	stage=3,
	ignore_unused_parameters=True,
	allgather_partitions=True,
	reduce_scatter=True,
	allgather_bucket_size=10000000,
	reduce_bucket_size=10000000,
	# overlap_comm=True,
	overlap_comm=False,
	contiguous_gradients=True,
	cpu_offload=False),

	# zero_optimization = dict(
	# stage=2,
	# ignore_unused_parameters=True, # 你这个场景需要保留
	# reduce_scatter=True,
	# allgather_partitions=True,
	# overlap_comm=True,
	# contiguous_gradients=True,
	# reduce_bucket_size=5e7,
	# allgather_bucket_size=5e7,
	# )
	)


	optim_wrapper = dict(type='DeepSpeedOptimWrapper')
	optim_wrapper['optimizer'] = cfg.optim_wrapper.optimizer
	cfg.optim_wrapper = optim_wrapper

	cfg = copy.deepcopy(cfg)
	runner = FlexibleRunner(
	model=cfg['model'],
	work_dir=cfg.get('work_dir', 'work_dirs'),
	experiment_name=cfg.get('experiment_name'),
	train_dataloader=cfg.get('train_dataloader'),
	optim_wrapper=cfg.get('optim_wrapper'),
	param_scheduler=cfg.get('param_scheduler'),
	train_cfg=cfg.get('train_cfg'),
	val_dataloader=cfg.get('val_dataloader'),
	val_evaluator=cfg.get('val_evaluator'),
	val_cfg=cfg.get('val_cfg'),
	test_dataloader=cfg.get('test_dataloader'),
	test_evaluator=cfg.get('test_evaluator'),
	test_cfg=cfg.get('test_cfg'),
	strategy=strategy,
	auto_scale_lr=cfg.get('auto_scale_lr'),
	default_hooks=cfg.get('default_hooks'),
	custom_hooks=cfg.get('custom_hooks'),
	data_preprocessor=cfg.get('data_preprocessor'),
	load_from=cfg.get('load_from'),
	resume=cfg.get('resume', False),
	launcher=cfg.get('launcher'),
	env_cfg=cfg.get('env_cfg'), # type: ignore
	log_processor=cfg.get('log_processor'),
	log_level=cfg.get('log_level', 'INFO'),
	visualizer=cfg.get('visualizer'),
	default_scope=cfg.get('default_scope', 'mmengine'),
	randomness=cfg.get('randomness', dict(seed=None)),
	cfg=cfg,
	)
	runner.train()


	if __name__ == '__main__':
	main()