Upload folder using huggingface_hub

f1c1609 verified over 1 year ago

22.6 kB

# coding:utf-8
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import json
import time
import torch
import os
import sys
import collections
import numpy as np
from tqdm import tqdm
import torch.optim as optim
from torch.utils.data import DataLoader
from os.path import dirname, abspath

pdvc_dir = dirname(abspath(__file__))
sys.path.insert(0, pdvc_dir)
sys.path.insert(0, os.path.join(pdvc_dir, 'densevid_eval3'))
sys.path.insert(0, os.path.join(pdvc_dir, 'densevid_eval3/SODA'))
# print(sys.path)


os.environ["TOKENIZERS_PARALLELISM"] = "false" # To avoid warning of tokenizer
from eval_utils import evaluate
import opts
from tensorboardX import SummaryWriter
from misc.utils import print_alert_message, build_floder, create_logger, backup_envir, print_opt, set_seed
from data.video_dataset import PropSeqDataset, collate_fn
from pdvc.pdvc import build
from collections import OrderedDict
from transformers import AutoTokenizer, get_linear_schedule_with_warmup, get_cosine_schedule_with_warmup
import copy

a100_folder = ['/cpfs01/shared/Gvlab-A100/Gvlab-A100_hdd/wuhao/youcook2', '/cpfs01/shared/Gvlab-A100/Gvlab-A100_hdd/wuhao/Tasty/features', '/cpfs01/shared/Gvlab-A100/Gvlab-A100_hdd/huabin/dataset/Tasty/UniVL_feature', '/cpfs01/shared/Gvlab-A100/Gvlab-A100_hdd/huabin/dataset/Anet', '/cpfs01/shared/Gvlab-A100/Gvlab-A100_hdd/wuhao/howto100m/features']
r3090_folder = ['/mnt/data/Gvlab/wuhao/features/yc2', '/mnt/data/Gvlab/wuhao/features/tasty', '/mnt/data/Gvlab/wuhao/features/tasty/univl', '/mnt/data/Gvlab/wuhao/features/anet', '/mnt/data/Gvlab/wuhao/features/howto100m']

# /cpfs01/shared/Gvlab-A100/Gvlab-A100_hdd/wuhao/howto100m/features -> /mnt/data/Gvlab/wuhao/features/howto100m
# /cpfs01/shared/Gvlab-A100/Gvlab-A100_hdd/wuhao/howto100m/features/clip -> /mnt/data/Gvlab/wuhao/features/howto100m/clip_features
# /cpfs01/shared/Gvlab-A100/Gvlab-A100_hdd/wuhao/howto100m/features/UniVL -> /mnt/data/Gvlab/wuhao/features/howto100m/univl_features

def _init_fn(worker_id):
    np.random.seed(12 + worker_id)

def map_path(path):
    path_backup = copy.deepcopy(path)
    # breakpoint()
    for i, folder in enumerate(a100_folder):
        if folder in path:
            path = path.replace(folder, r3090_folder[i])
            return path
    if path == path_backup:
        print('map failed')
        exit(1)


def train(opt):
    set_seed(opt.seed)
    save_folder = build_floder(opt)
    logger = create_logger(save_folder, 'train.log')
    tf_writer = SummaryWriter(os.path.join(save_folder, 'tf_summary'))

    if not opt.start_from:
        backup_envir(save_folder)
        logger.info('backup evironment completed !')

    saved_info = {'best': {}, 'last': {}, 'history': {}, 'eval_history': {}}

    # continue training
    if opt.start_from:
        opt.pretrain = False
        infos_path = os.path.join(save_folder, 'info.json')
        with open(infos_path) as f:
            logger.info('Load info from {}'.format(infos_path))
            saved_info = json.load(f)
            prev_opt = saved_info[opt.start_from_mode[:4]]['opt']

            exclude_opt = ['start_from', 'start_from_mode', 'pretrain']
            for opt_name in prev_opt.keys():
                if opt_name not in exclude_opt:
                    vars(opt).update({opt_name: prev_opt.get(opt_name)})
                if prev_opt.get(opt_name) != vars(opt).get(opt_name):
                    logger.info('Change opt {} : {} --> {}'.format(opt_name, prev_opt.get(opt_name),
                                                                   vars(opt).get(opt_name)))
    if len(opt.visual_feature_folder) == 2:
        train_dataset_pretrain = PropSeqDataset(opt.train_caption_file[0],
                                            [opt.visual_feature_folder[0]],
                                            [opt.text_feature_folder[0]],
                                            opt.dict_file, True, 'gt',
                                            opt)
        train_dataset_target = PropSeqDataset(opt.train_caption_file[1],
                                            [opt.visual_feature_folder[1]],
                                            [opt.text_feature_folder[1]],
                                            opt.dict_file, True, 'gt',
                                            opt)
        train_loader_pretrain = DataLoader(train_dataset_pretrain, batch_size=opt.batch_size,
                              shuffle=True, num_workers=opt.nthreads, collate_fn=collate_fn, worker_init_fn=_init_fn)
        train_loader_target = DataLoader(train_dataset_target, batch_size=opt.batch_size,
                                    shuffle=True, num_workers=opt.nthreads, collate_fn=collate_fn, worker_init_fn=_init_fn)
        
        train_dataloaders = [train_loader_pretrain, train_loader_target]
        # train_dataset = torch.utils.data.ConcatDataset([train_dataset_1, train_dataset_2])
        # train_dataset.translator = train_dataset_1.translator

    else:
        train_dataset_target = PropSeqDataset(opt.train_caption_file,
                                    opt.visual_feature_folder,
                                    opt.text_feature_folder,
                                    opt.dict_file, True, 'gt',
                                    opt)
        train_loader_target = DataLoader(train_dataset_target, batch_size=opt.batch_size,
                                shuffle=True, num_workers=opt.nthreads, collate_fn=collate_fn, worker_init_fn=_init_fn)
        train_dataloaders = [train_loader_target]

    # val_dataset = PropSeqDataset(opt.val_caption_file,
    #                              opt.visual_feature_folder,
    #                              opt.text_feature_folder,
    #                              opt.dict_file, False, 'gt',
    #                              opt)
    if not hasattr(opt, 'dict_file_val'):
        opt.dict_file_val = opt.dict_file
        opt.vocab_size_val = opt.vocab_size

    val_dataset = PropSeqDataset(opt.val_caption_file,
                                opt.visual_feature_folder_val,
                                opt.text_feature_folder_val,
                                opt.dict_file, False, 'gt',
                                opt)


    val_loader = DataLoader(val_dataset, batch_size=opt.batch_size_for_eval,
                            shuffle=False, num_workers=opt.nthreads, collate_fn=collate_fn, worker_init_fn=_init_fn)

    epoch = saved_info[opt.start_from_mode[:4]].get('epoch', 0)
    iteration = saved_info[opt.start_from_mode[:4]].get('iter', 0)
    best_val_score = saved_info[opt.start_from_mode[:4]].get('best_val_score', -1e5)
    val_result_history = saved_info['history'].get('val_result_history', {})
    loss_history = saved_info['history'].get('loss_history', {})
    lr_history = saved_info['history'].get('lr_history', {})
    opt.current_lr = vars(opt).get('current_lr', opt.lr)

    # Build model

    model, criterion, contrastive_criterion, postprocessors = build(opt)
    model.translator = train_dataset_target.translator
    model.train()

    # Recover the parameters
    if opt.start_from and (not opt.pretrain):
        if opt.start_from_mode == 'best':
            model_pth = torch.load(os.path.join(save_folder, 'model-best.pth'))
        elif opt.start_from_mode == 'last':
            model_pth = torch.load(os.path.join(save_folder, 'model-last.pth'))
        logger.info('Loading pth from {}, iteration:{}'.format(save_folder, iteration))
        model.load_state_dict(model_pth['model'])

    # Load the pre-trained model
    if opt.pretrain and (not opt.start_from):
        logger.info('Load pre-trained parameters from {}'.format(opt.pretrain_path))
        model_pth = torch.load(opt.pretrain_path, map_location=torch.device(opt.device))
        # query_weight = model_pth['model'].pop('query_embed.weight')
        if opt.pretrain == 'encoder':
            encoder_filter = model.get_filter_rule_for_encoder()
            encoder_pth = {k:v for k,v in model_pth['model'].items() if encoder_filter(k)}
            model.load_state_dict(encoder_pth, strict=True)
        elif opt.pretrain == 'decoder':
            encoder_filter = model.get_filter_rule_for_encoder()
            decoder_pth = {k:v for k,v in model_pth['model'].items() if not encoder_filter(k)}
            model.load_state_dict(decoder_pth, strict=True)
            pass
        elif opt.pretrain == 'full':
            # model_pth = transfer(model, model_pth)
            model.load_state_dict(model_pth['model'], strict=True)
        else:
            raise ValueError("wrong value of opt.pretrain")
        

    model.to(opt.device)

    # Decide which parameters need to be trained
    # if (opt.matcher_type =='DTW' or opt.use_pseudo_box) and opt.text_encoder_learning_strategy == 'frozen':
    #     for _, p in model.text_encoder.named_parameters():
    #         p.requires_grad = False
    #         text_encoder_params = list(map(id, model.text_encoder.parameters()))
    #         other_params = filter(lambda p: id(p) not in text_encoder_params, model.parameters())
    # else:
    #     other_params = model.parameters()
    other_params = model.parameters()

    training_params = [{'params': other_params, 'lr': opt.lr}]

    if opt.optimizer_type == 'adam':
        optimizer = optim.Adam(training_params, weight_decay=opt.weight_decay)

    elif opt.optimizer_type == 'adamw':
        optimizer = optim.AdamW(training_params, weight_decay=opt.weight_decay)

    milestone = [opt.learning_rate_decay_start + opt.learning_rate_decay_every * _ for _ in range(int((opt.epoch - opt.learning_rate_decay_start) / opt.learning_rate_decay_every))]
    lr_scheduler = torch.optim.lr_scheduler.MultiStepLR(optimizer, milestone, gamma=opt.learning_rate_decay_rate)

    # Load tokenizer for text encoder
    # for i in range(10):
    #     try:
    #         if opt.pretrained_language_model == 'UniVL':
    #             tokenizer = AutoTokenizer.from_pretrained('bert-base-uncased')
    #         else:
    #             tokenizer = AutoTokenizer.from_pretrained(opt.pretrained_language_model)
    #         break
    #     except:
    #         print('download error in AutoTokenizer, retry...')
    #         time.sleep(1)

    if opt.start_from:
        optimizer.load_state_dict(model_pth['optimizer'])
        lr_scheduler.step(epoch-1)

    # print the args for debugging  
    print_opt(opt, model, logger)
    print_alert_message('Strat training !', logger)

    loss_sum = OrderedDict()
    bad_video_num = 0

    start = time.time()

    weight_dict = criterion.weight_dict
    logger.info('loss type: {}'.format(weight_dict.keys()))
    logger.info('loss weights: {}'.format(weight_dict.values()))

    # Epoch-level iteration

    while True:
        if True:
            # scheduled sampling rate update
            if epoch > opt.scheduled_sampling_start >= 0:
                frac = (epoch - opt.scheduled_sampling_start) // opt.scheduled_sampling_increase_every
                opt.ss_prob = min(opt.basic_ss_prob + opt.scheduled_sampling_increase_prob * frac,
                                  opt.scheduled_sampling_max_prob)
                model.caption_head.ss_prob = opt.ss_prob

            print('lr:{}'.format(float(opt.current_lr)))
            pass

        # breakpoint()
        # Batch-level iteration
        for train_loader in train_dataloaders:
            trained_samples = 0
            for dt in tqdm(train_loader, disable=opt.disable_tqdm):
                # # for fast debugging
                # if trained_samples > 25:
                #     break
                # else:
                #     trained_samples += 1
                if opt.device=='cuda':
                    torch.cuda.synchronize(opt.device)
                if opt.debug:
                    # each epoch contains less mini-batches for debugging
                    if (iteration + 1) % 5 == 0:
                        iteration += 1
                        break
                iteration += 1

                optimizer.zero_grad()
                dt = {key: _.to(opt.device) if isinstance(_, torch.Tensor) else _ for key, _ in dt.items()}
                dt['video_target'] = [
                    {key: _.to(opt.device) if isinstance(_, torch.Tensor) else _ for key, _ in vid_info.items()} for vid_info in
                    dt['video_target']]

                # Add text encoder
                # if opt.matcher_type == 'DTW' or opt.use_pseudo_box:
                #     captions = list()
                #     for video_sents in dt['cap_raw']:  # dt['cap_raw']: [[sent_1, sent_2, ..., sent_n]]
                #         captions.extend(video_sents) 
                #     text_encoder_input = tokenizer(captions, return_tensors='pt', truncation=True, padding=True, max_length=opt.max_text_input_len)
                #     text_encoder_input = {key: _.to(opt.device) if isinstance(_, torch.Tensor) else _ for key, _ in text_encoder_input.items()} 
                #     # text_encoder_input: {'input_ids': tensor([[  101,  1996,  2307,  ...,     0,     0,     0],...]),  'attention_mask': tensor([[1, 1, 1,  ..., 0, 0, 0],...])}
                #     # len(text_encoder_input['input_ids']) = n * max_text_input_len
                #     dt['text_encoder_input'] = text_encoder_input

                # dt = collections.defaultdict(lambda: None, dt) # Commented to 

                output, loss = model(dt, criterion, contrastive_criterion)
                final_loss = sum(loss[k] * weight_dict[k] for k in loss.keys() if k in weight_dict)
                # breakpoint()
                final_loss.backward()
                torch.nn.utils.clip_grad_norm_(model.parameters(), opt.grad_clip)

                optimizer.step()

                for loss_k,loss_v in loss.items():
                    loss_sum[loss_k] = loss_sum.get(loss_k, 0)+ loss_v.item()
                loss_sum['total_loss'] = loss_sum.get('total_loss', 0) + final_loss.item()

                if opt.device=='cuda':
                    torch.cuda.synchronize()

                losses_log_every = int(len(train_loader) / 10)

                if opt.debug:
                    losses_log_every = 6

                if iteration % losses_log_every == 0:
                    end = time.time()
                    for k in loss_sum.keys():
                        loss_sum[k] = np.round(loss_sum[k] /losses_log_every, 3).item()

                    logger.info(
                        "ID {} iter {} (epoch {}), \nloss = {}, \ntime/iter = {:.3f}, bad_vid = {:.3f}"
                            .format(opt.id, iteration, epoch, loss_sum,
                                    (end - start) / losses_log_every, bad_video_num))

                    tf_writer.add_scalar('lr', opt.current_lr, iteration)
                    for loss_type in loss_sum.keys():
                        tf_writer.add_scalar(loss_type, loss_sum[loss_type], iteration)
                    loss_history[iteration] = loss_sum
                    lr_history[iteration] = opt.current_lr
                    loss_sum = OrderedDict()
                    start = time.time()
                    bad_video_num = 0
                    torch.cuda.empty_cache()

        # evaluation
        if (epoch % opt.save_checkpoint_every == 0) and (epoch >= opt.min_epoch_when_save):
            
            # Save model
            saved_pth = {'epoch': epoch,
                         'model': model.state_dict(),
                         'optimizer': optimizer.state_dict()}

            if opt.save_all_checkpoint:
                checkpoint_path = os.path.join(save_folder, 'model_iter_{}.pth'.format(iteration))
            else:
                checkpoint_path = os.path.join(save_folder, 'model-last.pth')

            torch.save(saved_pth, checkpoint_path)

            model.eval()
            result_json_path = os.path.join(save_folder, 'prediction',
                                         'num{}_epoch{}.json'.format(
                                             len(val_dataset), epoch))
            #eval_score, eval_loss = evaluate(model, criterion,  postprocessors, val_loader, result_json_path, logger=logger, args=opt, alpha=opt.ec_alpha, device=opt.device, debug=opt.debug)
            eval_score, _ = evaluate(model, criterion,  postprocessors, val_loader, result_json_path, logger=logger, args=opt, alpha=opt.ec_alpha, device=opt.device, debug=opt.debug)
            if opt.caption_decoder_type == 'none':
                current_score = 2./(1./eval_score['Precision'] + 1./eval_score['Recall'])
            else:
                if opt.criteria_for_best_ckpt == 'dvc':
                    current_score = np.array(eval_score['METEOR']).mean() + np.array(eval_score['soda_c']).mean()
                else:
                    current_score = np.array(eval_score['para_METEOR']).mean() + np.array(eval_score['para_CIDEr']).mean() + np.array(eval_score['para_Bleu_4']).mean()

            # add to tf summary
            for key in eval_score.keys():
                tf_writer.add_scalar(key, np.array(eval_score[key]).mean(), iteration)

            # Huabin comment this part for avoiding reporting losses during evaluation
            # for loss_type in eval_loss.keys():
            #     tf_writer.add_scalar('eval_' + loss_type, eval_loss[loss_type], iteration)

            _ = [item.append(np.array(item).mean()) for item in eval_score.values() if isinstance(item, list)]
            print_info = '\n'.join([key + ":" + str(eval_score[key]) for key in eval_score.keys()])
            logger.info('\nValidation results of iter {}:\n'.format(iteration) + print_info)
            logger.info('\noverall score of iter {}: {}\n'.format(iteration, current_score))
            val_result_history[epoch] = {'eval_score': eval_score}
            logger.info('Save model at iter {} to {}.'.format(iteration, checkpoint_path))

            # save the model parameter and  of best epoch
            if current_score >= best_val_score:
                best_val_score = current_score
                best_epoch = epoch
                saved_info['best'] = {'opt': vars(opt),
                                      'iter': iteration,
                                      'epoch': best_epoch,
                                      'best_val_score': best_val_score,
                                      'result_json_path': result_json_path,
                                      'avg_proposal_num': eval_score['avg_proposal_number'],
                                      'Precision': eval_score['Precision'],
                                      'Recall': eval_score['Recall']
                                      }

                # suffix = "RL" if sc_flag else "CE"
                torch.save(saved_pth, os.path.join(save_folder, 'model-best.pth'))
                logger.info('Save Best-model at iter {} to checkpoint file.'.format(iteration))

            saved_info['last'] = {'opt': vars(opt),
                                  'iter': iteration,
                                  'epoch': epoch,
                                  'best_val_score': best_val_score,
                                  }
            saved_info['history'] = {'val_result_history': val_result_history,
                                     'loss_history': loss_history,
                                     'lr_history': lr_history,
                                     # 'query_matched_fre_hist': query_matched_fre_hist,
                                     }
            with open(os.path.join(save_folder, 'info.json'), 'w') as f:
                json.dump(saved_info, f)
            logger.info('Save info to info.json')

            model.train()

        epoch += 1
        lr_scheduler.step()
        opt.current_lr = optimizer.param_groups[0]['lr']
        torch.cuda.empty_cache()
        # Stop criterion
        if epoch >= opt.epoch:
            # load Best model and conduct evaluation
            print('====== Conduct the Final Evaluation to test Best Checkpoint ======')
            val_logger = create_logger(save_folder, 'val.log')
            loaded_pth = torch.load(os.path.join(save_folder, 'model-best.pth'), map_location='cuda')
            model.load_state_dict(loaded_pth['model'], strict=True)
            model.eval()
            result_json_path = saved_info['best']['result_json_path']
            eval_score, _ = evaluate(model, criterion,  postprocessors, val_loader, result_json_path, logger=logger, args=opt, alpha=opt.ec_alpha, device=opt.device, debug=opt.debug)
            if opt.caption_decoder_type == 'none':
                current_score = 2./(1./eval_score['Precision'] + 1./eval_score['Recall'])
            else:
                if opt.criteria_for_best_ckpt == 'dvc':
                    current_score = np.array(eval_score['METEOR']).mean() + np.array(eval_score['soda_c']).mean()
                else:
                    current_score = np.array(eval_score['para_METEOR']).mean() + np.array(eval_score['para_CIDEr']).mean() + np.array(eval_score['para_Bleu_4']).mean()

            _ = [item.append(np.array(item).mean()) for item in eval_score.values() if isinstance(item, list)]
            print_info = '\n'.join([key + ":" + str(eval_score[key]) for key in eval_score.keys()])
            val_logger.info('Best-model is saved at iter {}.\n'.format(saved_info['best']['iter']))
            val_logger.info('\nBest Model Performance:\n' + print_info)
            val_logger.info('\nBest Overall Score {}: {}\n'.format(iteration, current_score))

            tf_writer.close()
            break

    return saved_info


if __name__ == '__main__':
    opt = opts.parse_opts()
    opt.id = 'seq-train'

    if not hasattr(opt, 'visual_feature_folder_val'):
        opt.visual_feature_folder_val = opt.visual_feature_folder
        opt.text_feature_folder_val = opt.text_feature_folder

    if opt.map:
        opt.visual_feature_folder = [map_path(path) for path in opt.visual_feature_folder]
        opt.text_feature_folder = [map_path(path) for path in opt.text_feature_folder]
        opt.visual_feature_folder_val = [map_path(path) for path in opt.visual_feature_folder_val]
        opt.text_feature_folder_val = [map_path(path) for path in opt.text_feature_folder_val]

    if opt.gpu_id:
        os.environ["CUDA_VISIBLE_DEVICES"] = ",".join([str(i) for i in opt.gpu_id])
    if opt.disable_cudnn:
        torch.backends.cudnn.enabled = False

    os.environ['KMP_DUPLICATE_LIB_OK'] = 'True' # to avoid OMP problem on macos
    # breakpoint()
    train(opt)

	# coding:utf-8
	from __future__ import absolute_import
	from __future__ import division
	from __future__ import print_function

	import json
	import time
	import torch
	import os
	import sys
	import collections
	import numpy as np
	from tqdm import tqdm
	import torch.optim as optim
	from torch.utils.data import DataLoader
	from os.path import dirname, abspath

	pdvc_dir = dirname(abspath(__file__))
	sys.path.insert(0, pdvc_dir)
	sys.path.insert(0, os.path.join(pdvc_dir, 'densevid_eval3'))
	sys.path.insert(0, os.path.join(pdvc_dir, 'densevid_eval3/SODA'))
	# print(sys.path)


	os.environ["TOKENIZERS_PARALLELISM"] = "false" # To avoid warning of tokenizer
	from eval_utils import evaluate
	import opts
	from tensorboardX import SummaryWriter
	from misc.utils import print_alert_message, build_floder, create_logger, backup_envir, print_opt, set_seed
	from data.video_dataset import PropSeqDataset, collate_fn
	from pdvc.pdvc import build
	from collections import OrderedDict
	from transformers import AutoTokenizer, get_linear_schedule_with_warmup, get_cosine_schedule_with_warmup
	import copy

	a100_folder = ['/cpfs01/shared/Gvlab-A100/Gvlab-A100_hdd/wuhao/youcook2', '/cpfs01/shared/Gvlab-A100/Gvlab-A100_hdd/wuhao/Tasty/features', '/cpfs01/shared/Gvlab-A100/Gvlab-A100_hdd/huabin/dataset/Tasty/UniVL_feature', '/cpfs01/shared/Gvlab-A100/Gvlab-A100_hdd/huabin/dataset/Anet', '/cpfs01/shared/Gvlab-A100/Gvlab-A100_hdd/wuhao/howto100m/features']
	r3090_folder = ['/mnt/data/Gvlab/wuhao/features/yc2', '/mnt/data/Gvlab/wuhao/features/tasty', '/mnt/data/Gvlab/wuhao/features/tasty/univl', '/mnt/data/Gvlab/wuhao/features/anet', '/mnt/data/Gvlab/wuhao/features/howto100m']

	# /cpfs01/shared/Gvlab-A100/Gvlab-A100_hdd/wuhao/howto100m/features -> /mnt/data/Gvlab/wuhao/features/howto100m
	# /cpfs01/shared/Gvlab-A100/Gvlab-A100_hdd/wuhao/howto100m/features/clip -> /mnt/data/Gvlab/wuhao/features/howto100m/clip_features
	# /cpfs01/shared/Gvlab-A100/Gvlab-A100_hdd/wuhao/howto100m/features/UniVL -> /mnt/data/Gvlab/wuhao/features/howto100m/univl_features

	def _init_fn(worker_id):
	np.random.seed(12 + worker_id)

	def map_path(path):
	path_backup = copy.deepcopy(path)
	# breakpoint()
	for i, folder in enumerate(a100_folder):
	if folder in path:
	path = path.replace(folder, r3090_folder[i])
	return path
	if path == path_backup:
	print('map failed')
	exit(1)


	def train(opt):
	set_seed(opt.seed)
	save_folder = build_floder(opt)
	logger = create_logger(save_folder, 'train.log')
	tf_writer = SummaryWriter(os.path.join(save_folder, 'tf_summary'))

	if not opt.start_from:
	backup_envir(save_folder)
	logger.info('backup evironment completed !')

	saved_info = {'best': {}, 'last': {}, 'history': {}, 'eval_history': {}}

	# continue training
	if opt.start_from:
	opt.pretrain = False
	infos_path = os.path.join(save_folder, 'info.json')
	with open(infos_path) as f:
	logger.info('Load info from {}'.format(infos_path))
	saved_info = json.load(f)
	prev_opt = saved_info[opt.start_from_mode[:4]]['opt']

	exclude_opt = ['start_from', 'start_from_mode', 'pretrain']
	for opt_name in prev_opt.keys():
	if opt_name not in exclude_opt:
	vars(opt).update({opt_name: prev_opt.get(opt_name)})
	if prev_opt.get(opt_name) != vars(opt).get(opt_name):
	logger.info('Change opt {} : {} --> {}'.format(opt_name, prev_opt.get(opt_name),
	vars(opt).get(opt_name)))
	if len(opt.visual_feature_folder) == 2:
	train_dataset_pretrain = PropSeqDataset(opt.train_caption_file[0],
	[opt.visual_feature_folder[0]],
	[opt.text_feature_folder[0]],
	opt.dict_file, True, 'gt',
	opt)
	train_dataset_target = PropSeqDataset(opt.train_caption_file[1],
	[opt.visual_feature_folder[1]],
	[opt.text_feature_folder[1]],
	opt.dict_file, True, 'gt',
	opt)
	train_loader_pretrain = DataLoader(train_dataset_pretrain, batch_size=opt.batch_size,
	shuffle=True, num_workers=opt.nthreads, collate_fn=collate_fn, worker_init_fn=_init_fn)
	train_loader_target = DataLoader(train_dataset_target, batch_size=opt.batch_size,
	shuffle=True, num_workers=opt.nthreads, collate_fn=collate_fn, worker_init_fn=_init_fn)

	train_dataloaders = [train_loader_pretrain, train_loader_target]
	# train_dataset = torch.utils.data.ConcatDataset([train_dataset_1, train_dataset_2])
	# train_dataset.translator = train_dataset_1.translator

	else:
	train_dataset_target = PropSeqDataset(opt.train_caption_file,
	opt.visual_feature_folder,
	opt.text_feature_folder,
	opt.dict_file, True, 'gt',
	opt)
	train_loader_target = DataLoader(train_dataset_target, batch_size=opt.batch_size,
	shuffle=True, num_workers=opt.nthreads, collate_fn=collate_fn, worker_init_fn=_init_fn)
	train_dataloaders = [train_loader_target]

	# val_dataset = PropSeqDataset(opt.val_caption_file,
	# opt.visual_feature_folder,
	# opt.text_feature_folder,
	# opt.dict_file, False, 'gt',
	# opt)
	if not hasattr(opt, 'dict_file_val'):
	opt.dict_file_val = opt.dict_file
	opt.vocab_size_val = opt.vocab_size

	val_dataset = PropSeqDataset(opt.val_caption_file,
	opt.visual_feature_folder_val,
	opt.text_feature_folder_val,
	opt.dict_file, False, 'gt',
	opt)


	val_loader = DataLoader(val_dataset, batch_size=opt.batch_size_for_eval,
	shuffle=False, num_workers=opt.nthreads, collate_fn=collate_fn, worker_init_fn=_init_fn)

	epoch = saved_info[opt.start_from_mode[:4]].get('epoch', 0)
	iteration = saved_info[opt.start_from_mode[:4]].get('iter', 0)
	best_val_score = saved_info[opt.start_from_mode[:4]].get('best_val_score', -1e5)
	val_result_history = saved_info['history'].get('val_result_history', {})
	loss_history = saved_info['history'].get('loss_history', {})
	lr_history = saved_info['history'].get('lr_history', {})
	opt.current_lr = vars(opt).get('current_lr', opt.lr)

	# Build model

	model, criterion, contrastive_criterion, postprocessors = build(opt)
	model.translator = train_dataset_target.translator
	model.train()

	# Recover the parameters
	if opt.start_from and (not opt.pretrain):
	if opt.start_from_mode == 'best':
	model_pth = torch.load(os.path.join(save_folder, 'model-best.pth'))
	elif opt.start_from_mode == 'last':
	model_pth = torch.load(os.path.join(save_folder, 'model-last.pth'))
	logger.info('Loading pth from {}, iteration:{}'.format(save_folder, iteration))
	model.load_state_dict(model_pth['model'])

	# Load the pre-trained model
	if opt.pretrain and (not opt.start_from):
	logger.info('Load pre-trained parameters from {}'.format(opt.pretrain_path))
	model_pth = torch.load(opt.pretrain_path, map_location=torch.device(opt.device))
	# query_weight = model_pth['model'].pop('query_embed.weight')
	if opt.pretrain == 'encoder':
	encoder_filter = model.get_filter_rule_for_encoder()
	encoder_pth = {k:v for k,v in model_pth['model'].items() if encoder_filter(k)}
	model.load_state_dict(encoder_pth, strict=True)
	elif opt.pretrain == 'decoder':
	encoder_filter = model.get_filter_rule_for_encoder()
	decoder_pth = {k:v for k,v in model_pth['model'].items() if not encoder_filter(k)}
	model.load_state_dict(decoder_pth, strict=True)
	pass
	elif opt.pretrain == 'full':
	# model_pth = transfer(model, model_pth)
	model.load_state_dict(model_pth['model'], strict=True)
	else:
	raise ValueError("wrong value of opt.pretrain")


	model.to(opt.device)

	# Decide which parameters need to be trained
	# if (opt.matcher_type =='DTW' or opt.use_pseudo_box) and opt.text_encoder_learning_strategy == 'frozen':
	# for _, p in model.text_encoder.named_parameters():
	# p.requires_grad = False
	# text_encoder_params = list(map(id, model.text_encoder.parameters()))
	# other_params = filter(lambda p: id(p) not in text_encoder_params, model.parameters())
	# else:
	# other_params = model.parameters()
	other_params = model.parameters()

	training_params = [{'params': other_params, 'lr': opt.lr}]

	if opt.optimizer_type == 'adam':
	optimizer = optim.Adam(training_params, weight_decay=opt.weight_decay)

	elif opt.optimizer_type == 'adamw':
	optimizer = optim.AdamW(training_params, weight_decay=opt.weight_decay)

	milestone = [opt.learning_rate_decay_start + opt.learning_rate_decay_every * _ for _ in range(int((opt.epoch - opt.learning_rate_decay_start) / opt.learning_rate_decay_every))]
	lr_scheduler = torch.optim.lr_scheduler.MultiStepLR(optimizer, milestone, gamma=opt.learning_rate_decay_rate)

	# Load tokenizer for text encoder
	# for i in range(10):
	# try:
	# if opt.pretrained_language_model == 'UniVL':
	# tokenizer = AutoTokenizer.from_pretrained('bert-base-uncased')
	# else:
	# tokenizer = AutoTokenizer.from_pretrained(opt.pretrained_language_model)
	# break
	# except:
	# print('download error in AutoTokenizer, retry...')
	# time.sleep(1)

	if opt.start_from:
	optimizer.load_state_dict(model_pth['optimizer'])
	lr_scheduler.step(epoch-1)

	# print the args for debugging
	print_opt(opt, model, logger)
	print_alert_message('Strat training !', logger)

	loss_sum = OrderedDict()
	bad_video_num = 0

	start = time.time()

	weight_dict = criterion.weight_dict
	logger.info('loss type: {}'.format(weight_dict.keys()))
	logger.info('loss weights: {}'.format(weight_dict.values()))

	# Epoch-level iteration

	while True:
	if True:
	# scheduled sampling rate update
	if epoch > opt.scheduled_sampling_start >= 0:
	frac = (epoch - opt.scheduled_sampling_start) // opt.scheduled_sampling_increase_every
	opt.ss_prob = min(opt.basic_ss_prob + opt.scheduled_sampling_increase_prob * frac,
	opt.scheduled_sampling_max_prob)
	model.caption_head.ss_prob = opt.ss_prob

	print('lr:{}'.format(float(opt.current_lr)))
	pass

	# breakpoint()
	# Batch-level iteration
	for train_loader in train_dataloaders:
	trained_samples = 0
	for dt in tqdm(train_loader, disable=opt.disable_tqdm):
	# # for fast debugging
	# if trained_samples > 25:
	# break
	# else:
	# trained_samples += 1
	if opt.device=='cuda':
	torch.cuda.synchronize(opt.device)
	if opt.debug:
	# each epoch contains less mini-batches for debugging
	if (iteration + 1) % 5 == 0:
	iteration += 1
	break
	iteration += 1

	optimizer.zero_grad()
	dt = {key: _.to(opt.device) if isinstance(_, torch.Tensor) else _ for key, _ in dt.items()}
	dt['video_target'] = [
	{key: _.to(opt.device) if isinstance(_, torch.Tensor) else _ for key, _ in vid_info.items()} for vid_info in
	dt['video_target']]

	# Add text encoder
	# if opt.matcher_type == 'DTW' or opt.use_pseudo_box:
	# captions = list()
	# for video_sents in dt['cap_raw']: # dt['cap_raw']: [[sent_1, sent_2, ..., sent_n]]
	# captions.extend(video_sents)
	# text_encoder_input = tokenizer(captions, return_tensors='pt', truncation=True, padding=True, max_length=opt.max_text_input_len)
	# text_encoder_input = {key: _.to(opt.device) if isinstance(_, torch.Tensor) else _ for key, _ in text_encoder_input.items()}
	# # text_encoder_input: {'input_ids': tensor([[ 101, 1996, 2307, ..., 0, 0, 0],...]), 'attention_mask': tensor([[1, 1, 1, ..., 0, 0, 0],...])}
	# # len(text_encoder_input['input_ids']) = n * max_text_input_len
	# dt['text_encoder_input'] = text_encoder_input

	# dt = collections.defaultdict(lambda: None, dt) # Commented to

	output, loss = model(dt, criterion, contrastive_criterion)
	final_loss = sum(loss[k] * weight_dict[k] for k in loss.keys() if k in weight_dict)
	# breakpoint()
	final_loss.backward()
	torch.nn.utils.clip_grad_norm_(model.parameters(), opt.grad_clip)

	optimizer.step()

	for loss_k,loss_v in loss.items():
	loss_sum[loss_k] = loss_sum.get(loss_k, 0)+ loss_v.item()
	loss_sum['total_loss'] = loss_sum.get('total_loss', 0) + final_loss.item()

	if opt.device=='cuda':
	torch.cuda.synchronize()

	losses_log_every = int(len(train_loader) / 10)

	if opt.debug:
	losses_log_every = 6

	if iteration % losses_log_every == 0:
	end = time.time()
	for k in loss_sum.keys():
	loss_sum[k] = np.round(loss_sum[k] /losses_log_every, 3).item()

	logger.info(
	"ID {} iter {} (epoch {}), \nloss = {}, \ntime/iter = {:.3f}, bad_vid = {:.3f}"
	.format(opt.id, iteration, epoch, loss_sum,
	(end - start) / losses_log_every, bad_video_num))

	tf_writer.add_scalar('lr', opt.current_lr, iteration)
	for loss_type in loss_sum.keys():
	tf_writer.add_scalar(loss_type, loss_sum[loss_type], iteration)
	loss_history[iteration] = loss_sum
	lr_history[iteration] = opt.current_lr
	loss_sum = OrderedDict()
	start = time.time()
	bad_video_num = 0
	torch.cuda.empty_cache()

	# evaluation
	if (epoch % opt.save_checkpoint_every == 0) and (epoch >= opt.min_epoch_when_save):

	# Save model
	saved_pth = {'epoch': epoch,
	'model': model.state_dict(),
	'optimizer': optimizer.state_dict()}

	if opt.save_all_checkpoint:
	checkpoint_path = os.path.join(save_folder, 'model_iter_{}.pth'.format(iteration))
	else:
	checkpoint_path = os.path.join(save_folder, 'model-last.pth')

	torch.save(saved_pth, checkpoint_path)

	model.eval()
	result_json_path = os.path.join(save_folder, 'prediction',
	'num{}_epoch{}.json'.format(
	len(val_dataset), epoch))
	#eval_score, eval_loss = evaluate(model, criterion, postprocessors, val_loader, result_json_path, logger=logger, args=opt, alpha=opt.ec_alpha, device=opt.device, debug=opt.debug)
	eval_score, _ = evaluate(model, criterion, postprocessors, val_loader, result_json_path, logger=logger, args=opt, alpha=opt.ec_alpha, device=opt.device, debug=opt.debug)
	if opt.caption_decoder_type == 'none':
	current_score = 2./(1./eval_score['Precision'] + 1./eval_score['Recall'])
	else:
	if opt.criteria_for_best_ckpt == 'dvc':
	current_score = np.array(eval_score['METEOR']).mean() + np.array(eval_score['soda_c']).mean()
	else:
	current_score = np.array(eval_score['para_METEOR']).mean() + np.array(eval_score['para_CIDEr']).mean() + np.array(eval_score['para_Bleu_4']).mean()

	# add to tf summary
	for key in eval_score.keys():
	tf_writer.add_scalar(key, np.array(eval_score[key]).mean(), iteration)

	# Huabin comment this part for avoiding reporting losses during evaluation
	# for loss_type in eval_loss.keys():
	# tf_writer.add_scalar('eval_' + loss_type, eval_loss[loss_type], iteration)

	_ = [item.append(np.array(item).mean()) for item in eval_score.values() if isinstance(item, list)]
	print_info = '\n'.join([key + ":" + str(eval_score[key]) for key in eval_score.keys()])
	logger.info('\nValidation results of iter {}:\n'.format(iteration) + print_info)
	logger.info('\noverall score of iter {}: {}\n'.format(iteration, current_score))
	val_result_history[epoch] = {'eval_score': eval_score}
	logger.info('Save model at iter {} to {}.'.format(iteration, checkpoint_path))

	# save the model parameter and of best epoch
	if current_score >= best_val_score:
	best_val_score = current_score
	best_epoch = epoch
	saved_info['best'] = {'opt': vars(opt),
	'iter': iteration,
	'epoch': best_epoch,
	'best_val_score': best_val_score,
	'result_json_path': result_json_path,
	'avg_proposal_num': eval_score['avg_proposal_number'],
	'Precision': eval_score['Precision'],
	'Recall': eval_score['Recall']
	}

	# suffix = "RL" if sc_flag else "CE"
	torch.save(saved_pth, os.path.join(save_folder, 'model-best.pth'))
	logger.info('Save Best-model at iter {} to checkpoint file.'.format(iteration))

	saved_info['last'] = {'opt': vars(opt),
	'iter': iteration,
	'epoch': epoch,
	'best_val_score': best_val_score,
	}
	saved_info['history'] = {'val_result_history': val_result_history,
	'loss_history': loss_history,
	'lr_history': lr_history,
	# 'query_matched_fre_hist': query_matched_fre_hist,
	}
	with open(os.path.join(save_folder, 'info.json'), 'w') as f:
	json.dump(saved_info, f)
	logger.info('Save info to info.json')

	model.train()

	epoch += 1
	lr_scheduler.step()
	opt.current_lr = optimizer.param_groups[0]['lr']
	torch.cuda.empty_cache()
	# Stop criterion
	if epoch >= opt.epoch:
	# load Best model and conduct evaluation
	print('====== Conduct the Final Evaluation to test Best Checkpoint ======')
	val_logger = create_logger(save_folder, 'val.log')
	loaded_pth = torch.load(os.path.join(save_folder, 'model-best.pth'), map_location='cuda')
	model.load_state_dict(loaded_pth['model'], strict=True)
	model.eval()
	result_json_path = saved_info['best']['result_json_path']
	eval_score, _ = evaluate(model, criterion, postprocessors, val_loader, result_json_path, logger=logger, args=opt, alpha=opt.ec_alpha, device=opt.device, debug=opt.debug)
	if opt.caption_decoder_type == 'none':
	current_score = 2./(1./eval_score['Precision'] + 1./eval_score['Recall'])
	else:
	if opt.criteria_for_best_ckpt == 'dvc':
	current_score = np.array(eval_score['METEOR']).mean() + np.array(eval_score['soda_c']).mean()
	else:
	current_score = np.array(eval_score['para_METEOR']).mean() + np.array(eval_score['para_CIDEr']).mean() + np.array(eval_score['para_Bleu_4']).mean()

	_ = [item.append(np.array(item).mean()) for item in eval_score.values() if isinstance(item, list)]
	print_info = '\n'.join([key + ":" + str(eval_score[key]) for key in eval_score.keys()])
	val_logger.info('Best-model is saved at iter {}.\n'.format(saved_info['best']['iter']))
	val_logger.info('\nBest Model Performance:\n' + print_info)
	val_logger.info('\nBest Overall Score {}: {}\n'.format(iteration, current_score))

	tf_writer.close()
	break

	return saved_info


	if __name__ == '__main__':
	opt = opts.parse_opts()
	opt.id = 'seq-train'

	if not hasattr(opt, 'visual_feature_folder_val'):
	opt.visual_feature_folder_val = opt.visual_feature_folder
	opt.text_feature_folder_val = opt.text_feature_folder

	if opt.map:
	opt.visual_feature_folder = [map_path(path) for path in opt.visual_feature_folder]
	opt.text_feature_folder = [map_path(path) for path in opt.text_feature_folder]
	opt.visual_feature_folder_val = [map_path(path) for path in opt.visual_feature_folder_val]
	opt.text_feature_folder_val = [map_path(path) for path in opt.text_feature_folder_val]

	if opt.gpu_id:
	os.environ["CUDA_VISIBLE_DEVICES"] = ",".join([str(i) for i in opt.gpu_id])
	if opt.disable_cudnn:
	torch.backends.cudnn.enabled = False

	os.environ['KMP_DUPLICATE_LIB_OK'] = 'True' # to avoid OMP problem on macos
	# breakpoint()
	train(opt)