diff --git a/data/fairseq/examples/MMPT/.gitignore b/data/fairseq/examples/MMPT/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..70a255dc918bf4a4242dd7ea5ded63e6848264db --- /dev/null +++ b/data/fairseq/examples/MMPT/.gitignore @@ -0,0 +1,139 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +pip-wheel-metadata/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +.python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ +runs +data +pretrained_models +projects/mmfusion_* +log_test +third-party +python_log +slurm_snapshot_code +lightning_logs +demos diff --git a/data/fairseq/examples/MMPT/CONFIG.md b/data/fairseq/examples/MMPT/CONFIG.md new file mode 100644 index 0000000000000000000000000000000000000000..bbd1403dfafc79ab581c691281ca49de28a4656f --- /dev/null +++ b/data/fairseq/examples/MMPT/CONFIG.md @@ -0,0 +1,41 @@ +### Config Files Explained + +Taking `projects/mfmmlm.yaml` for example, which run pretraining using masked frame model (MFM) and masked language model (MLM) on a single BERT: + +```yaml +project_dir: mfmmlm # specify the project dir for this baseline. +run_task: + - how2.yaml # run pretraining on how2 when launching `projects/taskmfmmlm.yaml` + - [vtt.yaml, vttcap.yaml, vttqa.yaml, youcook.yaml, youcookcap.yaml, crosstask.yaml, coin.yaml] # run fine-tuning tasks. +base_dir: task # a global template folder to specify each training task. +task_group: + pretrain: # section for pretraining. Most baselines differs in this section. + task_list: + - how2.yaml # reconfig `projects/task/how2.yaml` + dataset: + aligner: MFMMLMAligner # overwrite the aligner for MFMMLM training task. + model: + model_cls: MMFusionMFMMLM # overwrite the model, which constructs negative examples for MFM on-the-fly. + loss: + loss_cls: MFMMLM # overwrite the loss as MFMMLM, which combines MFM and MLM together. + fairseq: # all fairseq args can be expecified under this name. + dataset: + batch_size: 128 + finetune: # section for fine-tuning tasks, we don't need to change anything here mostly since we want to see how pretraining can contribute to finetuning. + task_list: # specify the list of downstream tasks, e.g., copy `projects/task/vtt.yaml` to `projects/mfmmlm`. + - vtt.yaml + - vttqa.yaml + - youcook.yaml + - youcookcap.yaml + - crosstask.yaml + - coin.yaml + test: # section for testing. + task_list: + - test_vtt.yaml + - test_vttqa.yaml + - test_youcook.yaml + - test_youcookcap.yaml + - test_crosstask.yaml + - test_crosstask_zs.yaml + - test_coin.yaml +``` diff --git a/data/fairseq/examples/MMPT/DATASET.md b/data/fairseq/examples/MMPT/DATASET.md new file mode 100644 index 0000000000000000000000000000000000000000..930403eb3651a14dfcd015d094383faeb052c8ac --- /dev/null +++ b/data/fairseq/examples/MMPT/DATASET.md @@ -0,0 +1,34 @@ +# Dataset + +We understand video data are challenging to download and process. For videos, we provide our preprocessing scripts under `scripts/video_feature_extractor` (deeply adapted from `https://github.com/antoine77340/video_feature_extractor`); for text, we pre-tokenizing scripts under `scripts/text_token_extractor`. + +### S3D Feature Extraction +We use pre-trained [S3D](https://github.com/antoine77340/S3D_HowTo100M) for video feature extraction. Please place the models as `pretrained_models/s3d_dict.npy` and `pretrained_models/s3d_howto100m.pth`. + +We implement a `PathBuilder` to automatically track video ids, source video paths to their feature locations (you may need `conda install -c anaconda pandas`). Decoding may need `pip install ffmpeg-python`. + +### Howto100M +[Howto100M](https://www.di.ens.fr/willow/research/howto100m/) is a large-scale video pre-training datasets. You may download videos by yourself and run preprocessing of our scripts. + +Several key differences of our preprocessing from existing papers: (1) we use `raw_caption.json` instead of `caption.json` to have pure self-supervision on text (`caption.json` has manual removal of stop words); (2) we remove partially duplicated texts that are originally designed for real-time readability (see `mmpt/processors/dedupprocessor.py`); (3) then we shard video/text features using `SharedTensor` in `mmpt/utils/shardedtensor.py` for fast loading during training (faster than `h5py`). + +#### Steps +##### video +To extract video features: edit and run `bash scripts/video_feature_extractor/how2/s3d.sh`. (consider to run this on multiple machines; by default, we store features in fp16 to save space and also for faster training). + +Split available video ids as `data/how2/how2_s3d_train.lst` and `data/how2/how2_s3d_val.lst`. + +Lastly, pack video features into `ShardedTensor` using `python scripts/video_feature_extractor/shard_feature.py`. + +##### text +Clean captions using `python -m mmpt.processors.dedupprocessor`. + +Tokenize dedupped captions `data/how2/raw_caption_dedup.pkl` into sharded numpy arrays: +``` +python scripts/text_token_extractor/pretokenization.py scripts/text_token_extractor/configs/bert-base-uncased.yaml +``` + +### Youcook, MSRVTT etc. +We use the version of Youcook and MSRVTT come with Howto100M and MILNCE. Please download the data to `data/youcook` and `data/msrvtt` accordingly, you can also check `projects/task/youcook.yaml` and `projects/task/vtt.yaml` etc. in details. +We extract features for Youcook, MSRVTT similar to the first step of Howto100M but we read text from meta data directly and perform on-the-fly tokenization. + diff --git a/data/fairseq/examples/MMPT/README.md b/data/fairseq/examples/MMPT/README.md new file mode 100644 index 0000000000000000000000000000000000000000..4a84819d9ddc992170296958a9431ff0538a85be --- /dev/null +++ b/data/fairseq/examples/MMPT/README.md @@ -0,0 +1,166 @@ +# VideoCLIP and VLM + +You just find this toolkit for multimodal video understanding! It contains implementation of two recent multi-modal video understanding papers [VideoCLIP](https://arxiv.org/pdf/2109.14084.pdf) (EMNLP, 2021) and [VLM](https://aclanthology.org/2021.findings-acl.370.pdf) (ACL Findings, 2021), along with high-performance toolkits that are typically lacking in existing codebase. The toolkit is desigend to contain generic performance-tuned components that can be potentially adapted to other frameworks (we initially use fairseq). + +VideoCLIP is a contrastive learning model for zero-shot transfer to retrieval/classification/sequence labeling style tasks. + + + +VLM is a masked language model style pre-training using only one encoder with masked modality model (MMM) for retrieval/generation/sequence labeling style tasks. + + + +### News +[Oct. 2021] Initial release of implementation for the following papers: +[VideoCLIP: Contrastive Pre-training for Zero-shot Video-Text Understanding](https://arxiv.org/pdf/2109.14084.pdf) (Xu et. al., EMNLP 2021) +[VLM: Task-agnostic Video-Language Model Pre-training for Video Understanding](https://aclanthology.org/2021.findings-acl.370.pdf) (Xu et. al., ACL Findings 2021) + + +### Installation +We aim to minimize the dependency of this repo on other packages. +We use fairseq as the main trainer (no models/datasets dependency on fairseq. We will support other trainer in future): +``` +git clone https://github.com/pytorch/fairseq +cd fairseq +pip install -e . # also optionally follow fairseq README for apex installation for fp16 training. +export MKL_THREADING_LAYER=GNU # fairseq may need this for numpy. +``` + +Then install this toolkit: +``` +cd examples/MMPT # MMPT can be in any folder, not necessarily under fairseq/examples. +pip install -e . +``` + +The code is developed under Python=3.8.8, Pytorch=1.8, cuda=11.0 with fairseq=1.0.0a0+af0389f and tested under Python=3.8.8 pytorch=1.9 cuda=11.0 fairseq=1.0.0a0+8e7bc73 during code release. +Most models require `transformers==3.4` for API compatibility `pip install transformers==3.4`. +In addition, some downstream tasks may need `conda install pandas`. + + +### Usage +#### Download Checkpoints +We use pre-trained [S3D](https://github.com/antoine77340/S3D_HowTo100M) for video feature extraction. Please place the models as `pretrained_models/s3d_dict.npy` and `pretrained_models/s3d_howto100m.pth`. + +Download VideoCLIP checkpoint `https://dl.fbaipublicfiles.com/MMPT/retri/videoclip/checkpoint_best.pt` to `runs/retri/videoclip` or VLM checkpoint `https://dl.fbaipublicfiles.com/MMPT/mtm/vlm/checkpoint_best.pt` to `runs/mtm/vlm`. + +#### Demo of Inference +run `python locallaunch.py projects/retri/videoclip.yaml --dryrun` to get all `.yaml`s for VideoCLIP. + +```python +import torch + +from mmpt.models import MMPTModel + + +model, tokenizer, aligner = MMPTModel.from_pretrained( + "projects/retri/videoclip/how2.yaml") + +model.eval() + + +# B, T, FPS, H, W, C (VideoCLIP is trained on 30 fps of s3d) +video_frames = torch.randn(1, 2, 30, 224, 224, 3) +caps, cmasks = aligner._build_text_seq( + tokenizer("some text", add_special_tokens=False)["input_ids"] +) + +caps, cmasks = caps[None, :], cmasks[None, :] # bsz=1 + +with torch.no_grad(): + output = model(video_frames, caps, cmasks, return_score=True) +print(output["score"]) # dot-product +``` + +#### Data Preparation +See [dataset](DATASET.md) for each dataset. + +#### Global Config for Training Pipeline +We organize a global config file for a training/testing pipeline under projects (see a detailed [explanation](CONFIG.md)). For example, VideoCLIP in `projects/retri/videoclip.yaml` and VLM is in `projects/mtm/vlm.yaml`. + +We wrap all cmds into `locallaunch.py` and `mmpt_cli/localjob.py`. You can check concrete cmds by `--dryrun` and then drop it for actual run. + +First, run `python locallaunch.py projects/retri/videoclip.yaml --dryrun` will generate configs for all configs of pre-training, zero-shot evaluation, fine-tuning and testing, for VideoCLIP under `projects/retri/videoclip`. + +Then each (either training or evaluation) process will be configed by a concrete config file (we save all complex arguments into the concrete config file for reproducibility, including fairseq args). For example, run zero-shot evaluation on youcook, +``` +python locallaunch.py projects/retri/videoclip/test_youcook_zs.yaml --jobtype local_predict # zero-shot evaluation. +python locallaunch.py projects/retri/videoclip/youcook_videoclip.yaml --jobtype local_single --dryrun # fine-tuning: use --dryrun to check cmds and drop it to make an actual run; local_small will run on two gpus (as in paper). +python locallaunch.py projects/retri/videoclip/test_youcook_videoclip.yaml --jobtype local_predict # testing on fine-tuned model. +``` + +Pretraining can be run as: +``` +python locallaunch.py projects/retri/videoclip/how2.yaml --jobtype local_single --dryrun # check then drop dryrun; paper is ran on local_big as 8 gpus. +``` +You may need to change `--jobtype`, check/extend `LocalJob` in `mmpt_cli/localjob.py` for multi-gpu/multi-node pre-training. + +The detailed instructions of pretraining and fine-tuning can be found at [pretraining instruction](pretraining.md) and [finetuning instruction](endtask.md). + + +### Development +Several components of this toolkit can be re-used for future research (and also our ongoing research). + +#### Framework Wrapper +We currently only support fairseq, but most components can be easily fit into other frameworks like huggingface. This repo is a `--user-dir` of fairseq with fairseq wrapper. For example, `mmpt/tasks` includes a `FairseqMMTTask`, which manages `mmpt/datasets` with `FairseqDataset`, `mmpt/models` with `FairseqModel`, `mmpt/losses` with `FairseqCriterion`. + +#### Processors +**Multi**modal research introduces the complexity on modality alignment from different input sources to losses. Inspired by [MMF](https://github.com/facebookresearch/mmf), this toolkit leverages `mmpt/processors` to handle various needs of data preprocessing and loading, **alleviating** the needs of multiple `torch.data.utils.Dataset` (that can be tricky for ablation study). +Processors can also be decoupled from `torch.data.utils.Dataset` for offline preprocessing instead of on-the-fly data preprocessing. + +We decouple a `mmpt.MMDataset` as 3 types of processors: `MetaProcessor`, `VideoProcessor`, `TextProcessor` and `Aligner`. They can be configed in `dataset` field of a config file (e.g., see `projects/task/how2.yaml`). +`MetaProcessor` is used to load the meta data about a dataset, aka, all video_ids of how2 dataset. +`VideoProcessor` is used to load the video features about a dataset. For example, S3D features for each second of a video. +`TextProcessor` is used to load the text (feature). For example, BERT pre-tokenized text clips for how2 dataset (with `start`s, `end`s of timestamps and `cap` for `token_ids`). +`Aligner` is the core class for different baselines that prepares the training data. For example, sampling a clip, masking tokens for MLM, etc. + +#### Performance-tuned Components +To speed up pre-training, this toolkit uses sharded features stored in mmaped numpy, backed by `ShardedTensor` in `mmpt/utils/shardedtensor.py` (adopted from MARGE paper). This reduces the loads of IO for multi-GPU training without loading all features for a video into the memory each time and `ShardedTensor` ensure features are stored in continuous disk space for near random access. This is used for both How2 video features and texts in `mmpt/processors/how2processor.py`. + + +### Citation +If this codebase is useful for your work, please cite the following papers: + +```BibTeX +@inproceedings{xu-etal-2021-videoclip, + title = "{VideoCLIP}: Contrastive Pre-training for\\Zero-shot Video-Text Understanding", + author = "Xu, Hu and + Ghosh, Gargi and + Huang, Po-Yao and + Okhonko, Dmytro and + Aghajanyan, Armen and + Metze, Florian and + Zettlemoyer, Luke and + Feichtenhofer, Christoph", + booktitle = "Proceedings of the 2021 Conference on Empirical Methods in Natural Language Processing (EMNLP)", + month = nov, + year = "2021", + address = "Online", + publisher = "Association for Computational Linguistics", +} + +@inproceedings{xu-etal-2021-vlm, + title = "{VLM}: Task-agnostic Video-Language Model Pre-training for Video Understanding", + author = "Xu, Hu and + Ghosh, Gargi and + Huang, Po-Yao and + Arora, Prahal and + Aminzadeh, Masoumeh and + Feichtenhofer, Christoph and + Metze, Florian and + Zettlemoyer, Luke", + booktitle = "Findings of the Association for Computational Linguistics: ACL-IJCNLP 2021", + month = aug, + year = "2021", + address = "Online", + publisher = "Association for Computational Linguistics", + url = "https://aclanthology.org/2021.findings-acl.370", + doi = "10.18653/v1/2021.findings-acl.370", + pages = "4227--4239", +} +``` + +### Bug Reports +This repo is in its initial stage, welcome bug reports to huxu@fb.com + +### Copyright +The majority of Multimodal Pre-training (MMPT) is licensed under CC-BY-NC, however portions of the project are available under separate license terms: Evaluation Codes/Models: Howto100M and HuggingFace Transformers are licensed under the Apache2.0 license; COIN and NLG-eval are licensed under the MIT license; CrossTask is licensed under the BSD-3; DiDeMo is licensed under the BSD-2 license. diff --git a/data/fairseq/examples/MMPT/endtask.md b/data/fairseq/examples/MMPT/endtask.md new file mode 100644 index 0000000000000000000000000000000000000000..7690955327283ded4b37857a4a7b78463e0eb0f8 --- /dev/null +++ b/data/fairseq/examples/MMPT/endtask.md @@ -0,0 +1,41 @@ +# Zero-shot Transfer and Finetuning + +(If you are new to the ideas of `mmpt.processors`, see [README](README.md) first.) +All finetuning datasets (specifically `processors`) are defined in `mmpt.processors.dsprocessor`. +Given the complexity of different types of finetuning tasks, each task may have their own meta/video/text/aligner processors and `mmpt/evaluators/{Predictor,Metric}`. + +### Tasks + +Currently, we support 5 end datasets: `MSRVTT`, `Youcook`, `COIN`, `Crosstask` and `DiDeMo` with the following tasks: +text-video retrieval: `MSRVTT`, `Youcook`, `DiDeMo`; +video captioning: `Youcook`; +Video Question and Answering: `MSRVTT-QA`. + +To add your own dataset, you can specify the corresponding processors and config them in the `dataset` field of a config file, such as `projects/task/vtt.yaml`. + +### Zero-shot Transfer (no Training) +Zero-shot transfer will run the pre-trained model (e.g., VideoCLIP) directly on testing data. Configs with pattern: `projects/task/*_zs_*.yaml` are dedicated for zero-shot transfer. + +### Fine-tuning + +The training of a downstream task is similar to pretraining, execept you may need to specify the `restore_file` in `fairseq.checkpoint` and reset optimizers, see `projects/task/ft.yaml` that is included by `projects/task/vtt.yaml`. + +We typically do finetuning on 2 gpus (`local_small`). + +### Testing +For each finetuning dataset, you may need to specify a testing config, similar to `projects/task/test_vtt.yaml`. + +We define `mmpt.evaluators.Predictor` for different types of prediction. For example, `MSRVTT` and `Youcook` are video-retrieval tasks and expecting to use `RetrievalPredictor`. You may need to define your new type of predictors and specify that in `predictor` field of a testing config. + +Each task may also have their own metric for evaluation. This can be created in `mmpt.evaluators.Metric` and specified in the `metric` field of a testing config. + +Launching a testing is as simple as training by specifying the path of a testing config: +```python locallaunch.py projects/mfmmlm/test_vtt.yaml``` +Testing will be launched locally by default since prediction is computationally less expensive. + +### Third-party Libraries +We list the following finetuning tasks that require third-party libraries. + +Youcook captioning: `https://github.com/Maluuba/nlg-eval` + +CrossTask: `https://github.com/DmZhukov/CrossTask`'s `dp` under `third-party/CrossTask` (`python setup.py build_ext --inplace`) diff --git a/data/fairseq/examples/MMPT/locallaunch.py b/data/fairseq/examples/MMPT/locallaunch.py new file mode 100644 index 0000000000000000000000000000000000000000..e20fd816fa3bed8b1af8f6a4d1a07ccb69a1fffa --- /dev/null +++ b/data/fairseq/examples/MMPT/locallaunch.py @@ -0,0 +1,148 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +# +# This source code is licensed under the MIT license found in the +# LICENSE file in the root directory of this source tree. +import argparse +import os + +from omegaconf import OmegaConf + +from mmpt.utils import recursive_config, overwrite_dir +from mmpt_cli.localjob import LocalJob + + +class JobLauncher(object): + JOB_CONFIG = { + "local": LocalJob, + } + + def __init__(self, yaml_file): + self.yaml_file = yaml_file + job_key = "local" + + if yaml_file.endswith(".yaml"): + config = recursive_config(yaml_file) + if config.task_type is not None: + job_key = config.task_type.split("_")[0] + else: + raise ValueError("unknown extension of job file:", yaml_file) + self.job_key = job_key + + def __call__(self, job_type=None, dryrun=False): + if job_type is not None: + self.job_key = job_type.split("_")[0] + print("[JobLauncher] job_key", self.job_key) + job = JobLauncher.JOB_CONFIG[self.job_key]( + self.yaml_file, job_type=job_type, dryrun=dryrun) + return job.submit() + + +class Pipeline(object): + """a job that loads yaml config.""" + + def __init__(self, fn): + """ + load a yaml config of a job and save generated configs as yaml for each task. + return: a list of files to run as specified by `run_task`. + """ + if fn.endswith(".py"): + # a python command. + self.backend = "python" + self.run_yamls = [fn] + return + + job_config = recursive_config(fn) + if job_config.base_dir is None: # single file job config. + self.run_yamls = [fn] + return + + self.project_dir = os.path.join("projects", job_config.project_dir) + self.run_dir = os.path.join("runs", job_config.project_dir) + + if job_config.run_task is not None: + run_yamls = [] + for stage in job_config.run_task: + # each stage can have multiple tasks running in parallel. + if OmegaConf.is_list(stage): + stage_yamls = [] + for task_file in stage: + stage_yamls.append( + os.path.join(self.project_dir, task_file)) + run_yamls.append(stage_yamls) + else: + run_yamls.append(os.path.join(self.project_dir, stage)) + self.run_yamls = run_yamls + configs_to_save = self._overwrite_task(job_config) + self._save_configs(configs_to_save) + + def __getitem__(self, idx): + yaml_files = self.run_yamls[idx] + if isinstance(yaml_files, list): + return [JobLauncher(yaml_file) for yaml_file in yaml_files] + return [JobLauncher(yaml_files)] + + def __len__(self): + return len(self.run_yamls) + + def _save_configs(self, configs_to_save: dict): + # save + os.makedirs(self.project_dir, exist_ok=True) + for config_file in configs_to_save: + config = configs_to_save[config_file] + print("saving", config_file) + OmegaConf.save(config=config, f=config_file) + + def _overwrite_task(self, job_config): + configs_to_save = {} + self.base_project_dir = os.path.join("projects", job_config.base_dir) + self.base_run_dir = os.path.join("runs", job_config.base_dir) + + for config_sets in job_config.task_group: + overwrite_config = job_config.task_group[config_sets] + if ( + overwrite_config.task_list is None + or len(overwrite_config.task_list) == 0 + ): + print( + "[warning]", + job_config.task_group, + "has no task_list specified.") + # we don't want this added to a final config. + task_list = overwrite_config.pop("task_list", None) + for config_file in task_list: + config_file_path = os.path.join( + self.base_project_dir, config_file) + config = recursive_config(config_file_path) + # overwrite it. + if overwrite_config: + config = OmegaConf.merge(config, overwrite_config) + overwrite_dir(config, self.run_dir, basedir=self.base_run_dir) + save_file_path = os.path.join(self.project_dir, config_file) + configs_to_save[save_file_path] = config + return configs_to_save + + +def main(args): + job_type = args.jobtype if args.jobtype else None + # parse multiple pipelines. + pipelines = [Pipeline(fn) for fn in args.yamls.split(",")] + + for pipe_id, pipeline in enumerate(pipelines): + if not hasattr(pipeline, "project_dir"): + for job in pipeline[0]: + job(job_type=job_type, dryrun=args.dryrun) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("yamls", type=str) + parser.add_argument( + "--dryrun", + action="store_true", + help="run config and prepare to submit without launch the job.", + ) + parser.add_argument( + "--jobtype", type=str, default="", + help="force to run jobs as specified.") + args = parser.parse_args() + main(args) diff --git a/data/fairseq/examples/MMPT/mmpt/__init__.py b/data/fairseq/examples/MMPT/mmpt/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..6ff86ddd5ce0c454281e6568c628bd3c49ea5024 --- /dev/null +++ b/data/fairseq/examples/MMPT/mmpt/__init__.py @@ -0,0 +1,12 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +# +# This source code is licensed under the MIT license found in the +# LICENSE file in the root directory of this source tree. +try: + # fairseq user dir + from .datasets import FairseqMMDataset + from .losses import FairseqCriterion + from .models import FairseqMMModel + from .tasks import FairseqMMTask +except ImportError: + pass diff --git a/data/fairseq/examples/MMPT/mmpt/evaluators/__init__.py b/data/fairseq/examples/MMPT/mmpt/evaluators/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..2d06b9d7974db251143025124468f48cd230e89a --- /dev/null +++ b/data/fairseq/examples/MMPT/mmpt/evaluators/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +# +# This source code is licensed under the MIT license found in the +# LICENSE file in the root directory of this source tree. +from .metric import * +from .evaluator import * + + +# experimental. +try: + from .expmetric import * +except ImportError: + pass diff --git a/data/fairseq/examples/MMPT/mmpt/evaluators/predictor.py b/data/fairseq/examples/MMPT/mmpt/evaluators/predictor.py new file mode 100644 index 0000000000000000000000000000000000000000..2ffef6ab474a7a275d90784250d04222bd6dc70f --- /dev/null +++ b/data/fairseq/examples/MMPT/mmpt/evaluators/predictor.py @@ -0,0 +1,595 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +# +# This source code is licensed under the MIT license found in the +# LICENSE file in the root directory of this source tree. +import os +import random +import json +import numpy as np +import torch +import pickle +import math + +from tqdm import tqdm + + +class Predictor(object): + """this base class is used to save predictions to disk + (and being called by a evaluator later). + Predictor has minimum support of single gpu prediction. + """ + def __init__(self, config): + self.pred_dir = None # on-the-fly eval does not save the results. + if hasattr(config, "eval") and config.eval is not None: + self.pred_dir = config.eval.save_path + os.makedirs(self.pred_dir, exist_ok=True) + + def __call__(self, outputs): + """extract the prediction and save it.""" + raise NotImplementedError + + def predict_loop(self, model, eval_dataloader, output_file=None): + """on-the-fly prediction on a single gpu.""" + self.full_scores = [] + model.eval() + model = model.to(0) + with torch.no_grad(): + for data in eval_dataloader: + data = self.to_ctx(data) + outputs = model(**data) + outputs.update(data) + self(outputs) + return self.finalize(output_file) + + def finalize(self, output_file): + pass + + def to_ctx(self, data, ctx=0, dtype=None): + if isinstance(data, dict): + for key in data: + if torch.is_tensor(data[key]): + if dtype is not None and data[key].dtype == torch.float32: + data[key] = data[key].to(dtype) + data[key] = data[key].to(ctx) + return data + else: + raise ValueError("non-dict type of batch is not supported yet.") + + +class NLGPredictor(Predictor): + """Predicting Text from MMFusion models.""" + """TODO: make a context.""" + def __init__(self, config): + super().__init__(config) + from transformers import AutoTokenizer + + self.tokenizer = AutoTokenizer.from_pretrained( + config.dataset.bert_name, + bos_token="[CLS]", eos_token="[SEP]") + self.bos_token_id = self.tokenizer.bos_token_id + self.eos_token_id = self.tokenizer.eos_token_id + + def predict_loop(self, model, eval_dataloader, output_file=None): + """TODO: refactor base classes.""" + ctx = 0 + outputs = {"outputs": [], "targets": [[]]} + model.eval() + model = model.to(ctx) + with torch.no_grad(): + for data in tqdm(eval_dataloader): + data = self.to_ctx(data, ctx) + self(data, model, outputs) + return self.finalize(outputs, output_file) + + def __call__(self, data, model, outputs): + data.update({ + "bos_token_id": self.bos_token_id, + "eos_token_id": self.eos_token_id + }) + + output = model.generate(**data) + assert len(output) == len(data["ref"]) + for idx, _output in enumerate(output): + generated_text = self.tokenizer.decode( + _output, skip_special_tokens=True) + if generated_text == "": + generated_text = "none" + outputs["outputs"].append(generated_text) + outputs["targets"][0].append(data["ref"][idx]) + if random.random() < 0.001: + print("_output", _output) + print("generated_text", generated_text) + print("ref", data["ref"][idx]) + + def finalize(self, outputs, output_file=None): + if output_file is not None: + with open(os.path.join( + self.pred_dir, output_file + ".json"), "w") as fw: + json.dump(outputs, fw, indent=4) + return outputs + + +class RetrievalPredictor(Predictor): + """generated `pooled_video` and `pooled_text`.""" + def __init__(self, config): + super().__init__(config) + from transformers import AutoTokenizer + self.tokenizer = AutoTokenizer.from_pretrained( + config.dataset.bert_name) + + def predict_loop( + self, + model, + eval_dataloader, + output_file="retrieval.npy" + ): + """on-the-fly prediction on a single gpu.""" + full_scores = [] + texts = [] + model.eval() + model = model.cuda() + with torch.no_grad(): + for data in eval_dataloader: + # convert to dict. + if not isinstance(data, dict): + data = { + "caps": data[0], + "cmasks": data[1], + "vfeats": data[2], + "vmasks": data[3], + "video_id": data[4] + } + data = self.to_ctx(data) + outputs = model(**data) + outputs.update(data) + self(outputs, full_scores) + for _cap in data["caps"]: + texts.append( + self.tokenizer.decode(_cap, skip_special_tokens=True) + ) + + return self.finalize(full_scores, texts, output_file) + + def __call__(self, sample, full_scores): + scores = self._get_pooled_outputs(sample) + self._append_scores(scores, full_scores) + + def finalize(self, full_scores, texts, output_file=None): + outputs = self._aggregate_scores(full_scores) + if output_file is not None: + np.save(os.path.join(self.pred_dir, output_file + ".npy"), outputs) + return {"outputs": outputs, "texts": texts} + + def _get_pooled_outputs(self, outputs): + if "pooled_video" in outputs: + return outputs["pooled_video"], outputs["pooled_text"] + else: + raise ValueError("unknown format of outputs.") + + def _append_scores(self, scores, full_scores): + assert len(scores) == 2 + if len(full_scores) == 0: + full_scores.append([]) + full_scores.append([]) + full_scores[0].append(scores[0].cpu().detach().numpy()) + full_scores[1].append(scores[1].cpu().detach().numpy()) + + def _aggregate_scores(self, scores): + assert len(scores) == 2 + video_hidden = np.concatenate(scores[0], axis=0) + text_hidden = np.concatenate(scores[1], axis=0) + # clear up. + self.full_scores = [] + return np.matmul(text_hidden, video_hidden.T) + + +class QAPredictor(Predictor): + """generated `pooled_video` and `pooled_text`.""" + def __init__(self, config): + super().__init__(config) + """predictor maintains scores and aggregate them.""" + + def predict_loop(self, model, eval_dataloader, output_file="qa.npy"): + """on-the-fly prediction on a single gpu.""" + self.full_scores = [] + model.eval() + model = model.cuda() + with torch.no_grad(): + for data in eval_dataloader: + # reshape ans and dup video 5 times. + v_len = data["vfeats"].size(1) + hidden_size = data["vfeats"].size(2) + data["vfeats"] = data["vfeats"].unsqueeze(1).repeat(1, 5, 1, 1).view(-1, v_len, hidden_size) + data["vmasks"] = data["vmasks"].unsqueeze(1).repeat(1, 5, 1).view(-1, v_len) + + t_len = data["caps"].size(-1) + data["caps"] = data["caps"].view(-1, t_len) + data["cmasks"] = data["cmasks"].view(-1, t_len) + + data = self.to_ctx(data) + outputs = model(**data) + outputs.update(data) + self(outputs) + return self.finalize(output_file) + + def __call__(self, sample): + hidden_size = sample["pooled_video"].size(-1) + pooled_video = sample["pooled_video"].view(-1, 5, hidden_size) + pooled_text = sample["pooled_text"].view(-1, 5, hidden_size) + scores = torch.bmm(pooled_video, pooled_text.transpose(2, 1)) + scores = scores.argmax(-1) + self._append_scores(scores[:, 0], sample["answers"], self.full_scores) + + def finalize(self, output_file=None): + outputs, targets = self._aggregate_scores(self.full_scores) + if output_file is not None: + np.save(os.path.join(self.pred_dir, output_file + ".npy"), outputs) + return {"outputs": outputs, "targets": targets} + + def _append_scores(self, scores, answers, full_scores): + if len(full_scores) == 0: + full_scores.append([]) + full_scores.append([]) + full_scores[0].append(scores.cpu().detach().numpy()) + full_scores[1].append(answers.cpu().detach().numpy()) + + def _aggregate_scores(self, scores): + assert len(scores) == 2 + outputs = np.concatenate(scores[0], axis=0) + targets = np.concatenate(scores[1], axis=0) + # clear up. + self.full_scores = [] + return outputs, targets + + +class CrossTaskPredictor(Predictor): + """ + CrossTaskPredictor needs to compute the average of logits + for overlapped sliding-window. + """ + def __init__(self, config): + super().__init__(config) + self.lsm = torch.nn.LogSoftmax(dim=1) + self.max_video_len = config.dataset.max_video_len + self.sliding_window = config.dataset.sliding_window + self.sliding_window_size = config.dataset.sliding_window_size + self.annotation_path = config.dataset.annotation_path + + def predict_loop(self, model, eval_dataloader, output_file="result.pkl"): + """refactored from line 144: + https://github.com/DmZhukov/CrossTask/blob/master/train.py + """ + ctx = 0 + model.eval() + model = model.to(ctx) + # this is not a loss but just compute neg_log_prob. + Y_pred = {} + Y_true = {} + with torch.no_grad(): + for batch in eval_dataloader: + self(batch, model, Y_pred, Y_true) + return self.finalize(Y_pred, Y_true, output_file) + + def __call__(self, sample, model, Y_pred, Y_true): + # please install dp from `https://github.com/DmZhukov/CrossTask` + from dp import dp + vid, task = sample['video_id'][0], sample['task'][0] + sample = self.to_ctx(sample) + # compute the average logits over sliding windows. + output = model(**sample) + batch_logits = output["logits"].cpu() + + video_len = sample["video_len"][0] + + # the following version is slow. + logits = torch.zeros((video_len, batch_logits.size(1))) + logits_counts = torch.zeros((video_len, 1), dtype=torch.long) + # use the same loop as aligner to recover. + batch_logit_idx = 0 + for window_start in range(0, video_len, self.sliding_window): + video_end = min(video_len - window_start, self.sliding_window_size) + logits[window_start: window_start + video_end] += batch_logits[ + batch_logit_idx: batch_logit_idx + video_end] + batch_logit_idx += video_end + logits_counts[window_start: window_start + video_end] += torch.ones((video_end, 1), dtype=torch.long) + + if (video_len - window_start) <= self.sliding_window_size: + break + + logits /= logits_counts + assert logits.size() == (video_len, batch_logits.size(1)), "{}, {}".format(logits.size(), video_len) + + O = self.lsm(logits) + y = np.zeros(O.size(), dtype=np.float32) + dp(y, -O.detach().cpu().numpy()) + if task not in Y_pred: + Y_pred[task] = {} + Y_pred[task][vid] = y + annot_path = os.path.join( + self.annotation_path, task+'_'+vid+'.csv') + if os.path.exists(annot_path): + if task not in Y_true: + Y_true[task] = {} + Y_true[task][vid] = self._read_assignment( + *y.shape, annot_path) + + def finalize(self, Y_pred, Y_true, output_file=None): + if output_file is not None: + with open( + os.path.join(self.pred_dir, output_file + ".pkl"), + "wb") as fw: + pickle.dump( + {"Y_pred": Y_pred, "Y_true": Y_true}, fw, + protocol=pickle.HIGHEST_PROTOCOL) + return {"outputs": Y_pred, "targets": Y_true} + + def _read_assignment(self, T, K, path): + """ + refactored from https://github.com/DmZhukov/CrossTask/blob/master/data.py + Howto interpret contraints on loss that is going to be minimized: + lambd is a big number; + self.lambd * C is a big number for all valid position (csv stores invalids) + + def forward(self, O, Y, C): + return (Y*(self.lambd * C - self.lsm(O))).mean(dim=0).sum() + + This will load the csv file and fill-in the step col from start to end rows. + """ + + Y = np.zeros([T, K], dtype=np.uint8) + with open(path, 'r') as f: + for line in f: + step, start, end = line.strip().split(',') + start = int(math.floor(float(start))) + end = int(math.ceil(float(end))) + step = int(step) - 1 + Y[start:end, step] = 1 + return Y + + +class COINPredictor(Predictor): + """ + COINPredictor is similar to CrossTask on sliding windows. + """ + def __init__(self, config): + super().__init__(config) + self.max_video_len = config.dataset.max_video_len + self.sliding_window = config.dataset.sliding_window + self.sliding_window_size = config.dataset.sliding_window_size + + def predict_loop(self, model, eval_dataloader, output_file="result.pkl"): + """refactored from line 144: + https://github.com/DmZhukov/CrossTask/blob/master/train.py + """ + ctx = 0 + model.eval() + model = model.to(ctx) + # this is not a loss but just compute neg_log_prob. + Y_pred = [] + Y_true = [] + with torch.no_grad(): + for batch in eval_dataloader: + self(batch, model, Y_pred, Y_true) + return self.finalize(Y_pred, Y_true, output_file) + + def __call__(self, sample, model, Y_pred, Y_true): + sample = self.to_ctx(sample) + # compute the average logits over sliding windows. + output = model(**sample) + logits = self._merge_windows(sample, output) + Y_pred.append(logits.argmax(dim=1)) + Y_true.append(sample["video_targets"].squeeze(0).cpu()) + + def _merge_windows(self, sample, output): + targets = sample["targets"].reshape(-1).cpu() + valid_mask = targets != -100 + targets = targets[valid_mask] + batch_logits = output["logits"].cpu() + batch_logits = batch_logits.reshape(-1, batch_logits.size(-1)) + batch_logits = batch_logits[valid_mask] + + video_len = sample["video_len"][0] + + # the following version is slow. + logits = torch.zeros((video_len, batch_logits.size(1))) + logits_counts = torch.zeros((video_len, 1), dtype=torch.long) + # use the same loop as aligner to recover. + batch_logit_idx = 0 + for window_start in range(0, video_len, self.sliding_window): + video_end = min(video_len - window_start, self.sliding_window_size) + logits[window_start: window_start + video_end] += batch_logits[ + batch_logit_idx: batch_logit_idx + video_end] + batch_logit_idx += video_end + logits_counts[window_start: window_start + video_end] += torch.ones((video_end, 1), dtype=torch.long) + if (video_len - window_start) <= self.sliding_window_size: + break + logits /= logits_counts + assert logits.size() == (video_len, batch_logits.size(1)), "{}, {}".format(logits.size(), video_len) + return logits + + def finalize(self, Y_pred, Y_true, output_file=None): + Y_pred = torch.cat(Y_pred, dim=0).numpy() + Y_true = torch.cat(Y_true, dim=0).numpy() + assert len(Y_pred) == len(Y_true) + + error_mask = Y_pred != Y_true + print("sample error", Y_pred[error_mask][:10], Y_true[error_mask][:10]) + print("sample error", Y_pred[error_mask][10:20], Y_true[error_mask][10:20]) + + if output_file is not None: + with open( + os.path.join(self.pred_dir, output_file + ".pkl"), + "wb") as fw: + pickle.dump( + {"Y_pred": Y_pred, "Y_true": Y_true}, fw, + protocol=pickle.HIGHEST_PROTOCOL) + return {"outputs": Y_pred, "targets": Y_true} + + +class COINZSPredictor(COINPredictor): + """ + COINZSPredictor for COIN zero-shot prediction. + """ + + def __init__(self, config): + super().__init__(config) + self.dataset_config = config.dataset + + def predict_loop(self, model, eval_dataloader, output_file="result.pkl"): + """refactored from line 144: + https://github.com/DmZhukov/CrossTask/blob/master/train.py + """ + ctx = 0 + model.eval() + model = model.to(ctx) + + with torch.no_grad(): + outputs = eval_dataloader.dataset.meta_processor.meta_text_labels( + self.dataset_config) + outputs = self.to_ctx(outputs, ctx) + label_hidden_states = model.forward_text(**outputs).cpu() + label_sim = label_hidden_states @ label_hidden_states.t() + num_labels = label_sim.size(0) + eye_mask = ~torch.eye(num_labels, dtype=torch.bool) + label_sim = label_sim.masked_select(eye_mask).view(num_labels, num_labels - 1) + lbd = label_sim.max() + + # this is not a loss but just compute neg_log_prob. + Y_pred = [] + Y_true = [] + with torch.no_grad(): + for batch in eval_dataloader: + self(batch, label_hidden_states, model, lbd, Y_pred, Y_true) + return self.finalize(Y_pred, Y_true, output_file) + + def reshape_subsample(self, sample): + for key in sample: + if torch.is_tensor(sample[key]): + sample[key] = self.flat_subsample(sample[key]) + return sample + + def flat_subsample(self, tensor): + if len(tensor.size()) > 1 and tensor.size(0) == 1: + tensor = tensor.squeeze(0) + return tensor + + def __call__(self, sample, label_hidden_states, model, lbd, Y_pred, Y_true): + sample = self.reshape_subsample(sample) + sample = self.to_ctx(sample) + # compute the average logits over sliding windows. + sample["output_hidden_states"] = True + video_outputs = model.forward_video(**sample).cpu() + output = {"logits": video_outputs[:, 1:sample["vmasks"].size(1)+1] @ label_hidden_states.t()} + logits = self._merge_windows(sample, output) + # logic of zero-shot for sequence labeling. + logits_argmax = logits.argmax(dim=1) + 1 # 0 is "O" label. + logits_max = logits.max(dim=1)[0] + + pred = torch.zeros_like(logits_argmax) + label_select = logits_max > lbd # 73 or 74 + pred[label_select] = logits_argmax[label_select] + + Y_pred.append(pred) + Y_true.append(sample["video_targets"].squeeze(0).cpu()) + + def finalize(self, Y_pred, Y_true, output_file=None): + Y_pred = torch.cat(Y_pred, dim=0).numpy() + Y_true = torch.cat(Y_true, dim=0).numpy() + assert len(Y_pred) == len(Y_true) + + error_mask = Y_pred != Y_true + print("sample error", Y_pred[error_mask][:10], Y_true[error_mask][:10]) + print("sample error", Y_pred[error_mask][10:20], Y_true[error_mask][10:20]) + + if output_file is not None: + with open( + os.path.join(self.pred_dir, output_file + ".pkl"), + "wb") as fw: + pickle.dump( + {"Y_pred": Y_pred, "Y_true": Y_true}, fw, + protocol=pickle.HIGHEST_PROTOCOL) + return {"outputs": Y_pred, "targets": Y_true} + + +class DiDeMoPredictor(Predictor): + """reference: https://github.com/LisaAnne/LocalizingMoments/blob/master/utils/eval.py + https://github.com/LisaAnne/LocalizingMoments/blob/master/utils/data_processing.py + """ + def __init__(self, config): + super().__init__(config) + # load targets. + with open(config.dataset.test_path) as data_file: + self.test_data = json.load(data_file) + + def predict_loop(self, model, eval_dataloader, output_file="didemo.npy"): + """ + TODO: two solutions here. + """ + import itertools + # 21 chunks. + self.possible_segments = [(0,0), (1,1), (2,2), (3,3), (4,4), (5,5)] + for i in itertools.combinations(range(6), 2): + self.possible_segments.append(i) + # pick segments from a video. + + """on-the-fly prediction on a single gpu.""" + self.full_scores = [] + model.eval() + model = model.cuda() + with torch.no_grad(): + for data in eval_dataloader: + # TODO special forwarding logic here. + data = self.to_ctx(data) + data["output_hidden_states"] = True + hidden_video = model.forward_video(**data) + data["output_hidden_states"] = False + pooled_text = model.forward_text(**data) + outputs = { + "hidden_video": hidden_video, + "pooled_text": pooled_text + } + outputs.update(data) + self(outputs) + return self.finalize(output_file) + + def __call__(self, sample): + # TODO: make an index select from self.possible_segments. + hidden_video = sample["hidden_video"] + pooled_text = sample["pooled_text"] + vmasks = sample["vmasks"] + # probably maintain valid results here. + + hidden_video = hidden_video[:, 1:-1, :] + # probably maintain valid results here. + pooled_video = [] + for s, e in self.possible_segments: + pooled_video.append( + torch.mean( + hidden_video[:, int(s*5):int((e+1)*5), :], + dim=1, keepdim=True) + ) + pooled_video = torch.cat(pooled_video, dim=1) + scores = torch.bmm( + pooled_video, pooled_text.unsqueeze(-1)).squeeze(-1).cpu() + + ranks = scores.argsort(dim=-1, descending=True) + + for batch_idx, rank in enumerate(ranks): + rank_of_moment = [] + for m_idx, moment in enumerate(rank): + s, e = self.possible_segments[moment.item()] + if torch.any( + vmasks[batch_idx, int(s*5):int((e+1)*5)] + ): + rank_of_moment.append((s, e)) + self.full_scores.append(rank_of_moment) + + def finalize(self, output_file=None): + outputs = self._aggregate_scores(self.full_scores) + if output_file is not None: + np.save(os.path.join(self.pred_dir, output_file + ".npy"), outputs) + return {"outputs": outputs, "targets": self.test_data} + + def _aggregate_scores(self, scores): + self.full_scores = [] + return scores diff --git a/data/fairseq/examples/MMPT/mmpt/utils/__init__.py b/data/fairseq/examples/MMPT/mmpt/utils/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..2429ee3757353e768f71b27d129eb3ca3bcbec73 --- /dev/null +++ b/data/fairseq/examples/MMPT/mmpt/utils/__init__.py @@ -0,0 +1,68 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +# +# This source code is licensed under the MIT license found in the +# LICENSE file in the root directory of this source tree. +import random +import numpy as np +import torch + +from .shardedtensor import * +from .load_config import * + + +def set_seed(seed=43211): + random.seed(seed) + np.random.seed(seed) + torch.manual_seed(seed) + torch.cuda.manual_seed_all(seed) + if torch.backends.cudnn.enabled: + torch.backends.cudnn.benchmark = False + torch.backends.cudnn.deterministic = True + + +def get_world_size(): + if torch.distributed.is_initialized(): + world_size = torch.distributed.get_world_size() + else: + world_size = 1 + return world_size + + +def get_local_rank(): + return torch.distributed.get_rank() \ + if torch.distributed.is_initialized() else 0 + + +def print_on_rank0(func): + local_rank = get_local_rank() + if local_rank == 0: + print("[INFO]", func) + + +class RetriMeter(object): + """ + Statistics on whether retrieval yields a better pair. + """ + def __init__(self, freq=1024): + self.freq = freq + self.total = 0 + self.replace = 0 + self.updates = 0 + + def __call__(self, data): + if isinstance(data, np.ndarray): + self.replace += data.shape[0] - int((data[:, 0] == -1).sum()) + self.total += data.shape[0] + elif torch.is_tensor(data): + self.replace += int(data.sum()) + self.total += data.size(0) + else: + raise ValueError("unsupported RetriMeter data type.", type(data)) + + self.updates += 1 + if get_local_rank() == 0 and self.updates % self.freq == 0: + print("[INFO]", self) + + def __repr__(self): + return "RetriMeter (" + str(self.replace / self.total) \ + + "/" + str(self.replace) + "/" + str(self.total) + ")" diff --git a/data/fairseq/examples/MMPT/mmpt/utils/load_config.py b/data/fairseq/examples/MMPT/mmpt/utils/load_config.py new file mode 100644 index 0000000000000000000000000000000000000000..ede4f94117b197e5bdb551c5140604c6e47c91fb --- /dev/null +++ b/data/fairseq/examples/MMPT/mmpt/utils/load_config.py @@ -0,0 +1,81 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +# +# This source code is licensed under the MIT license found in the +# LICENSE file in the root directory of this source tree. +import os +import omegaconf +from omegaconf import OmegaConf + + +def load_config(args=None, config_file=None, overwrite_fairseq=False): + """TODO (huxu): move fairseq overwrite to another function.""" + if args is not None: + config_file = args.taskconfig + config = recursive_config(config_file) + + if config.dataset.subsampling is not None: + batch_size = config.fairseq.dataset.batch_size // config.dataset.subsampling + print( + "adjusting batch_size to {} due to subsampling {}.".format( + batch_size, config.dataset.subsampling + ) + ) + config.fairseq.dataset.batch_size = batch_size + + is_test = config.dataset.split is not None and config.dataset.split == "test" + if not is_test: + if ( + config.fairseq.checkpoint is None + or config.fairseq.checkpoint.save_dir is None + ): + raise ValueError("fairseq save_dir or save_path must be specified.") + + save_dir = config.fairseq.checkpoint.save_dir + os.makedirs(save_dir, exist_ok=True) + if config.fairseq.common.tensorboard_logdir is not None: + tb_run_dir = suffix_rundir( + save_dir, config.fairseq.common.tensorboard_logdir + ) + config.fairseq.common.tensorboard_logdir = tb_run_dir + print( + "update tensorboard_logdir as", config.fairseq.common.tensorboard_logdir + ) + os.makedirs(save_dir, exist_ok=True) + OmegaConf.save(config=config, f=os.path.join(save_dir, "config.yaml")) + + if overwrite_fairseq and config.fairseq is not None and args is not None: + # flatten fields. + for group in config.fairseq: + for field in config.fairseq[group]: + print("overwrite args." + field, "as", config.fairseq[group][field]) + setattr(args, field, config.fairseq[group][field]) + return config + + +def recursive_config(config_path): + """allows for stacking of configs in any depth.""" + config = OmegaConf.load(config_path) + if config.includes is not None: + includes = config.includes + config.pop("includes") + base_config = recursive_config(includes) + config = OmegaConf.merge(base_config, config) + return config + + +def suffix_rundir(save_dir, run_dir): + max_id = -1 + for search_dir in os.listdir(save_dir): + if search_dir.startswith(run_dir): + splits = search_dir.split("_") + cur_id = int(splits[1]) if len(splits) > 1 else 0 + max_id = max(max_id, cur_id) + return os.path.join(save_dir, run_dir + "_" + str(max_id + 1)) + + +def overwrite_dir(config, replace, basedir): + for key in config: + if isinstance(config[key], str) and config[key].startswith(basedir): + config[key] = config[key].replace(basedir, replace) + if isinstance(config[key], omegaconf.dictconfig.DictConfig): + overwrite_dir(config[key], replace, basedir) diff --git a/data/fairseq/examples/MMPT/mmpt/utils/shardedtensor.py b/data/fairseq/examples/MMPT/mmpt/utils/shardedtensor.py new file mode 100644 index 0000000000000000000000000000000000000000..2424f360ef2a93f0f2783e38f0bd966086e92fd4 --- /dev/null +++ b/data/fairseq/examples/MMPT/mmpt/utils/shardedtensor.py @@ -0,0 +1,46 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +# +# This source code is licensed under the MIT license found in the +# LICENSE file in the root directory of this source tree. +import os +import pickle +import numpy as np + + +class ShardedTensor(object): + def __init__(self, data, starts): + self.data = data + self.starts = starts + assert self.starts[0] == 0 + assert self.starts[-1] == len(self.data) + assert (self.starts[1:] >= self.starts[:-1]).all() + assert (self.starts > -1).all() + + @staticmethod + def from_list(xs): + starts = np.full((len(xs) + 1,), -1, dtype=np.long) + data = np.concatenate(xs, axis=0) + starts[0] = 0 + for i, x in enumerate(xs): + starts[i + 1] = starts[i] + x.shape[0] + assert (starts > -1).all() + return ShardedTensor(data, starts) + + def __getitem__(self, i): + return self.data[self.starts[i] : self.starts[i + 1]] + + def __len__(self): + return len(self.starts) - 1 + + def lengths(self): + return self.starts[1:] - self.starts[:-1] + + def save(self, path): + np.save(path + "_starts", self.starts) + np.save(path + "_data", self.data) + + @staticmethod + def load(path, mmap_mode=None): + starts = np.load(path + "_starts.npy", mmap_mode) + data = np.load(path + "_data.npy", mmap_mode) + return ShardedTensor(data, starts) diff --git a/data/fairseq/examples/MMPT/pretraining.md b/data/fairseq/examples/MMPT/pretraining.md new file mode 100644 index 0000000000000000000000000000000000000000..8f8e6d0facaa47141342294d7eb26c4232a9677b --- /dev/null +++ b/data/fairseq/examples/MMPT/pretraining.md @@ -0,0 +1,29 @@ +# Pretraining + +(If you are new to the ideas of `mmpt.processors`, see [README](README.md) first.) +We mostly use [howto100M](https://github.com/antoine77340/howto100m) dataset for pretraining (other datasets are coming). So you are less likely to write a new `MetaProcessor`, `VideoProcessor` or `TextProcessor` but only working on a new `Aligner`, a new model and loss. + +### Data Sharding +Pretraining on Howto100M is heavy on IO since we have millions of videos or captions on the hard disk that cannot be fit into the memory. +It is desirable to have an optimized preprocessing step before the actual dataloading. + +We support data sharding to pack multiple videos into a shards of training data for both videos and captions. (see [dataset](DATASET.md) for preprocessing). +These shards will be mapped into memory to reduce the frequency of IO access on millions of files. See (processors starting with `Sharded*`). +This will be the default config for a how2 dataset `projects/task/how2.yaml`. + +Great thanks to Dmytro Okhonko for sharing the code from MARGE project. + +### Training +Pretraining on Howto100m is expected on one or multiple nodes, where each node has 8 GPUS with 32 GB mem. +launching a pretraing on MFM+MLM can be done, via: +```python locallaunch.py projects/mfmmlm/how2.yaml``` + +### Pre-training with a Retrieval Model (VideoCLIP) +This projects now support alternatively run a retrieval model and pre-training. +We implement a basic retrieval model that is built on the hidden states of a video and faiss. + +You may need to install faiss via `conda install faiss-cpu -c pytorch`. + +Right now, the hidden states of a video is computed as the average of 8 clips of their pooled visual/text hidden states. +See `mmpt/tasks/retritask.py` for more details. +The `.yaml` config for running pre-training with a retrieval model can be found at `projects/retri/videoretri.yaml`. diff --git a/data/fairseq/examples/MMPT/projects/mfmmlm.yaml b/data/fairseq/examples/MMPT/projects/mfmmlm.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0f3450a1e091517de19422b0c09cea6a01a02729 --- /dev/null +++ b/data/fairseq/examples/MMPT/projects/mfmmlm.yaml @@ -0,0 +1,59 @@ +project_dir: mfmmlm +run_task: + - how2.yaml + - [vtt.yaml, vttcap.yaml, vttqa.yaml, youcook.yaml, youcookcap.yaml, crosstask.yaml, coin.yaml] +base_dir: task +task_group: + pretrain: + task_list: + - how2.yaml + dataset: + subsampling: 32 + sampled_min_len: 10 + sampled_max_len: 64 + max_video_len: 32 + max_len: 96 + aligner: MFMMLMAligner + lazy_vfeat_mask: True + mfm_probability: 0.15 + mlm_probability: 0.15 + mm_prob: 0.5 + model: + model_cls: MMFusionMFMMLM + mm_encoder_cls: MMFusionForMFMMLM + loss: + loss_cls: MFMMLM + fairseq: + common: + fp16: true + dataset: + batch_size: 256 + optimization: + max_epoch: 15 + finetune: + task_list: + - vtt.yaml + - vttqa.yaml + - youcook.yaml + - youcookcap.yaml + - crosstask.yaml + - coin.yaml + dataset: + max_video_len: 32 + max_len: 96 + fairseq: + common: + fp16: true + # do not write any model or loss here (they are expected to be fixed in mmfusion). + test: + task_list: + - test_vtt.yaml + - test_vttqa.yaml + - test_youcook.yaml + - test_youcookcap.yaml + - test_crosstask.yaml + - test_crosstask_zs.yaml + - test_coin.yaml + dataset: + max_video_len: 32 + max_len: 96 diff --git a/data/fairseq/examples/MMPT/projects/mtm/mmfusionmtm.yaml b/data/fairseq/examples/MMPT/projects/mtm/mmfusionmtm.yaml new file mode 100644 index 0000000000000000000000000000000000000000..337d66a2aadcff86da033d6656bdc570a20c32e6 --- /dev/null +++ b/data/fairseq/examples/MMPT/projects/mtm/mmfusionmtm.yaml @@ -0,0 +1,19 @@ +includes: projects/mfmmlm.yaml +project_dir: mtm/mmfusionmtm +task_group: + pretrain: + task: VLMTask # reproducible + dataset: + aligner: MFMMLMAligner + model: + use_seg_emb: True # reproducible + model_cls: MMFusionMTM + mm_encoder_cls: MMBertForMFMMLM + loss: + loss_cls: MTM + finetune: + model: + use_seg_emb: True # reproducible + test: + model: + use_seg_emb: True # reproducible diff --git a/data/fairseq/examples/MMPT/projects/mtm/vlm.yaml b/data/fairseq/examples/MMPT/projects/mtm/vlm.yaml new file mode 100644 index 0000000000000000000000000000000000000000..022a2623c5fdee1da51f211bef25a0dfebd7cbb0 --- /dev/null +++ b/data/fairseq/examples/MMPT/projects/mtm/vlm.yaml @@ -0,0 +1,8 @@ +includes: projects/mtm/mmfusionmtm.yaml +project_dir: mtm/vlm +task_group: + pretrain: + dataset: + sampled_min_len: 8 + loss: + loss_cls: MTM diff --git a/data/fairseq/examples/MMPT/projects/mtm/vlm/coin.yaml b/data/fairseq/examples/MMPT/projects/mtm/vlm/coin.yaml new file mode 100644 index 0000000000000000000000000000000000000000..48fd64a5f4996a03b412c418e83c4672b7252e8d --- /dev/null +++ b/data/fairseq/examples/MMPT/projects/mtm/vlm/coin.yaml @@ -0,0 +1,47 @@ +dataset: + video_processor: VideoProcessor + bert_name: bert-base-uncased + meta_processor: COINActionSegmentationMetaProcessor + train_path: data/coin/COIN.json + val_path: data/coin/COIN.json + vfeat_dir: data/feat/feat_coin_s3d + text_processor: COINActionSegmentationTextProcessor + aligner: COINActionSegmentationAligner + num_iso_layer: 12 + sliding_window: 8 + sliding_window_size: 32 + max_video_len: 32 + max_len: 96 +fairseq: + common: + tensorboard_logdir: run + log_interval: 1000 + fp16: true + dataset: + num_workers: 4 + batch_size: 1 + optimization: + lr: + - 5.0e-05 + clip_norm: 2.0 + optimizer: adam + adam_betas: (0.9, 0.98) + lr_scheduler: polynomial_decay + total_num_update: 1000000 + warmup_updates: 122 + weight_decay: 0.0 + ddp_backend: no_c10d + max_epoch: 8 + checkpoint: + restore_file: runs/mtm/vlm/checkpoint_best.pt + reset_optimizer: true + reset_dataloader: true + reset_meters: true + save_dir: runs/mtm/vlm/coin +task_type: sweep_big +model: + model_cls: MMFusionActionSegmentation + mm_encoder_cls: MMBertForTokenClassification + use_seg_emb: true +loss: + loss_cls: CrossEntropy diff --git a/data/fairseq/examples/MMPT/projects/mtm/vlm/crosstask.yaml b/data/fairseq/examples/MMPT/projects/mtm/vlm/crosstask.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4e706b549e6085c9823049afaff3285aa2b88f4f --- /dev/null +++ b/data/fairseq/examples/MMPT/projects/mtm/vlm/crosstask.yaml @@ -0,0 +1,53 @@ +dataset: + video_processor: CrossTaskVideoProcessor + bert_name: bert-base-uncased + meta_processor: CrossTaskMetaProcessor + train_path: data/crosstask/crosstask_release/videos.csv + train_csv_path: data/crosstask/crosstask_release/videos.csv + val_path: data/crosstask/crosstask_release/videos_val.csv + val_csv_path: data/crosstask/crosstask_release/videos_val.csv + primary_path: data/crosstask/crosstask_release/tasks_primary.txt + related_path: data/crosstask/crosstask_release/tasks_related.txt + vfeat_dir: data/feat/feat_crosstask_s3d + annotation_path: data/crosstask/crosstask_release/annotations + n_train: 30 + text_processor: CrossTaskTextProcessor + aligner: CrossTaskAligner + num_iso_layer: 12 + sliding_window: 16 + sliding_window_size: 32 + max_video_len: 32 + max_len: 96 +fairseq: + common: + tensorboard_logdir: run + log_interval: 1000 + fp16: true + dataset: + num_workers: 4 + batch_size: 1 + optimization: + lr: + - 5.0e-05 + clip_norm: 2.0 + optimizer: adam + adam_betas: (0.9, 0.98) + lr_scheduler: polynomial_decay + total_num_update: 1000000 + warmup_updates: 122 + weight_decay: 0.0 + ddp_backend: no_c10d + max_epoch: 5 + checkpoint: + restore_file: runs/mtm/vlm/checkpoint11.pt + reset_optimizer: true + reset_dataloader: true + reset_meters: true + save_dir: runs/mtm/vlm/crosstask +task_type: sweep_small +model: + model_cls: MMFusionActionLocalization + mm_encoder_cls: MMBertForJoint + use_seg_emb: true +loss: + loss_cls: BCE diff --git a/data/fairseq/examples/MMPT/projects/mtm/vlm/how2.yaml b/data/fairseq/examples/MMPT/projects/mtm/vlm/how2.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7ca40ad815d9393bc9dc4d248cf7ec8810f1011f --- /dev/null +++ b/data/fairseq/examples/MMPT/projects/mtm/vlm/how2.yaml @@ -0,0 +1,55 @@ +dataset: + video_processor: ShardedVideoProcessor + bert_name: bert-base-uncased + meta_processor: ShardedHow2MetaProcessor + train_path: data/how2/how2_s3d_train.lst + val_path: data/how2/how2_s3d_val.lst + vfeat_dir: data/feat/feat_how2_s3d_shard_small + text_processor: ShardedTextProcessor + tfeat_dir: data/feat/feat_how2_s3d_shard_small/raw_caption_dedup.bert-base-uncased. + aligner: MFMMLMAligner + subsampling: 32 + sampled_min_len: 8 + sampled_max_len: 64 + max_video_len: 32 + max_len: 96 + lazy_vfeat_mask: true + mfm_probability: 0.15 + mlm_probability: 0.15 + mm_prob: 0.5 +fairseq: + common: + tensorboard_logdir: run + log_interval: 1000 + fp16: true + dataset: + num_workers: 4 + batch_size: 256 + optimization: + lr: + - 5.0e-05 + clip_norm: 2.0 + optimizer: adam + adam_betas: (0.9, 0.98) + lr_scheduler: polynomial_decay + total_num_update: 1000000 + warmup_updates: 1000 + weight_decay: 0.0 + ddp_backend: no_c10d + max_epoch: 15 + checkpoint: + save_dir: runs/mtm/vlm + save_interval_updates: 1024 + keep_interval_updates: 2 + keep_last_epochs: 30 +task_type: sweep_big +slurm_config: big +eval: + save_path: runs/mtm/vlm +model: + model_cls: MMFusionMTM + mm_encoder_cls: MMBertForMFMMLM + use_seg_emb: true +loss: + loss_cls: MTM +task: VLMTask diff --git a/data/fairseq/examples/MMPT/projects/mtm/vlm/test_coin.yaml b/data/fairseq/examples/MMPT/projects/mtm/vlm/test_coin.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8df2e66ad175529e0e86f69ade1e392849507036 --- /dev/null +++ b/data/fairseq/examples/MMPT/projects/mtm/vlm/test_coin.yaml @@ -0,0 +1,31 @@ +slurm_config: big +task_type: local_predict +dataset: + split: test + video_processor: VideoProcessor + aligner: COINActionSegmentationAligner + bert_name: bert-base-uncased + test_path: data/coin/COIN.json + meta_processor: COINActionSegmentationMetaProcessor + vfeat_dir: data/feat/feat_coin_s3d + text_processor: COINActionSegmentationTextProcessor + num_iso_layer: 12 + sliding_window: 16 + sliding_window_size: 32 + max_video_len: 32 + max_len: 96 +fairseq: + dataset: + batch_size: 1 + valid_subset: test + num_workers: 2 + common_eval: + path: runs/mtm/vlm/coin/checkpoint_best.pt +model: + model_cls: MMFusionActionSegmentation + mm_encoder_cls: MMBertForTokenClassification + use_seg_emb: true +eval: + save_path: runs/mtm/vlm/coin/eval +metric: COINActionSegmentationMetric +predictor: COINPredictor diff --git a/data/fairseq/examples/MMPT/projects/mtm/vlm/test_crosstask.yaml b/data/fairseq/examples/MMPT/projects/mtm/vlm/test_crosstask.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d159847875f74050fbae5f0c223fa6d56bbd3d38 --- /dev/null +++ b/data/fairseq/examples/MMPT/projects/mtm/vlm/test_crosstask.yaml @@ -0,0 +1,38 @@ +slurm_config: big +task_type: local_predict +dataset: + split: test + video_processor: CrossTaskVideoProcessor + aligner: CrossTaskAligner + bert_name: bert-base-uncased + meta_processor: CrossTaskMetaProcessor + test_path: data/crosstask/crosstask_release/videos_val.csv + train_csv_path: data/crosstask/crosstask_release/videos.csv + val_path: data/crosstask/crosstask_release/videos_val.csv + val_csv_path: data/crosstask/crosstask_release/videos_val.csv + primary_path: data/crosstask/crosstask_release/tasks_primary.txt + related_path: data/crosstask/crosstask_release/tasks_related.txt + vfeat_dir: data/feat/feat_crosstask_s3d + annotation_path: data/crosstask/crosstask_release/annotations + n_train: 30 + text_processor: CrossTaskTextProcessor + num_iso_layer: 12 + sliding_window: 16 + sliding_window_size: 32 + max_video_len: 32 + max_len: 96 +fairseq: + dataset: + batch_size: 1 + valid_subset: test + num_workers: 2 + common_eval: + path: runs/mtm/vlm/crosstask/checkpoint_best.pt +model: + model_cls: MMFusionActionLocalization + mm_encoder_cls: MMBertForJoint + use_seg_emb: true +eval: + save_path: runs/mtm/vlm/crosstask/eval +metric: CrossTaskMetric +predictor: CrossTaskPredictor diff --git a/data/fairseq/examples/MMPT/projects/mtm/vlm/test_crosstask_zs.yaml b/data/fairseq/examples/MMPT/projects/mtm/vlm/test_crosstask_zs.yaml new file mode 100644 index 0000000000000000000000000000000000000000..59833c55406ce53225d704b46eb0c141ad4d5e9d --- /dev/null +++ b/data/fairseq/examples/MMPT/projects/mtm/vlm/test_crosstask_zs.yaml @@ -0,0 +1,38 @@ +slurm_config: big +task_type: local_predict +dataset: + split: test + video_processor: CrossTaskVideoProcessor + aligner: CrossTaskAligner + bert_name: bert-base-uncased + meta_processor: CrossTaskMetaProcessor + test_path: data/crosstask/crosstask_release/videos_val.csv + train_csv_path: data/crosstask/crosstask_release/videos.csv + val_path: data/crosstask/crosstask_release/videos_val.csv + val_csv_path: data/crosstask/crosstask_release/videos_val.csv + primary_path: data/crosstask/crosstask_release/tasks_primary.txt + related_path: data/crosstask/crosstask_release/tasks_related.txt + vfeat_dir: data/feat/feat_crosstask_s3d + annotation_path: data/crosstask/crosstask_release/annotations + n_train: 30 + text_processor: CrossTaskTextProcessor + num_iso_layer: 12 + sliding_window: 16 + sliding_window_size: 32 + max_video_len: 32 + max_len: 96 +fairseq: + dataset: + batch_size: 1 + valid_subset: test + num_workers: 2 + common_eval: + path: runs/mtm/vlm/checkpoint_best.pt +model: + model_cls: MMFusionActionLocalization + mm_encoder_cls: MMBertForJoint + use_seg_emb: true +eval: + save_path: runs/mtm/vlm/crosstask_zs/eval +metric: CrossTaskMetric +predictor: CrossTaskPredictor diff --git a/data/fairseq/examples/MMPT/projects/mtm/vlm/test_vtt.yaml b/data/fairseq/examples/MMPT/projects/mtm/vlm/test_vtt.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a41557df6af496c755b0b32345a736b1cdc01ce4 --- /dev/null +++ b/data/fairseq/examples/MMPT/projects/mtm/vlm/test_vtt.yaml @@ -0,0 +1,29 @@ +slurm_config: big +task_type: local_predict +dataset: + split: test + video_processor: VideoProcessor + aligner: DSAligner + bert_name: bert-base-uncased + meta_processor: MSRVTTMetaProcessor + test_path: data/msrvtt/MSRVTT_JSFUSION_test.csv + vfeat_dir: data/feat/feat_vtt_s3d + text_processor: MSRVTTTextProcessor + num_iso_layer: 12 + max_video_len: 32 + max_len: 96 +fairseq: + dataset: + batch_size: 256 + valid_subset: test + num_workers: 2 + common_eval: + path: runs/mtm/vlm/vtt/checkpoint_last.pt +model: + model_cls: MMFusionJoint + mm_encoder_cls: MMBertForJoint + use_seg_emb: true +eval: + save_path: runs/mtm/vlm/vtt/eval +metric: RetrievalMetric +predictor: RetrievalPredictor diff --git a/data/fairseq/examples/MMPT/projects/mtm/vlm/test_vttqa.yaml b/data/fairseq/examples/MMPT/projects/mtm/vlm/test_vttqa.yaml new file mode 100644 index 0000000000000000000000000000000000000000..abf3309f7072c88a6b7c72e2540d4950d0e55575 --- /dev/null +++ b/data/fairseq/examples/MMPT/projects/mtm/vlm/test_vttqa.yaml @@ -0,0 +1,29 @@ +slurm_config: big +task_type: local_predict +dataset: + split: test + video_processor: VideoProcessor + aligner: MSRVTTQAAligner + bert_name: bert-base-uncased + meta_processor: MSRVTTQAMetaProcessor + test_path: data/msrvtt-qa/MSR_MC_test.csv + vfeat_dir: data/feat/feat_vtt_s3d + text_processor: MSRVTTQATextProcessor + num_iso_layer: 12 + max_video_len: 32 + max_len: 96 +fairseq: + dataset: + batch_size: 256 + valid_subset: test + num_workers: 2 + common_eval: + path: runs/mtm/vlm/vttqa/checkpoint_last.pt +model: + model_cls: MMFusionJoint + mm_encoder_cls: MMBertForJoint + use_seg_emb: true +eval: + save_path: runs/mtm/vlm/vttqa/eval +metric: QAMetric +predictor: QAPredictor diff --git a/data/fairseq/examples/MMPT/projects/mtm/vlm/test_youcook.yaml b/data/fairseq/examples/MMPT/projects/mtm/vlm/test_youcook.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3a57d25c240758cf989ad0ec538562d046446232 --- /dev/null +++ b/data/fairseq/examples/MMPT/projects/mtm/vlm/test_youcook.yaml @@ -0,0 +1,31 @@ +slurm_config: big +task_type: local_predict +dataset: + split: test + video_processor: YoucookVideoProcessor + aligner: DSAligner + bert_name: bert-base-uncased + meta_processor: YoucookMetaProcessor + test_path: data/youcook/youcook_val.pkl + trainval_annotation: data/youcook/youcookii_annotations_trainval.json + use_annotation_text: true + vfeat_dir: data/feat/feat_youcook_s3d + text_processor: TextProcessor + num_iso_layer: 12 + max_video_len: 32 + max_len: 96 +fairseq: + dataset: + batch_size: 256 + valid_subset: test + num_workers: 2 + common_eval: + path: runs/mtm/vlm/youcook/checkpoint_last.pt +model: + model_cls: MMFusionJoint + mm_encoder_cls: MMBertForJoint + use_seg_emb: true +eval: + save_path: runs/mtm/vlm/youcook/eval +metric: RetrievalMetric +predictor: RetrievalPredictor diff --git a/data/fairseq/examples/MMPT/projects/mtm/vlm/test_youcookcap.yaml b/data/fairseq/examples/MMPT/projects/mtm/vlm/test_youcookcap.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b2595d7c3c633d0faac5e4891457067659f7fafc --- /dev/null +++ b/data/fairseq/examples/MMPT/projects/mtm/vlm/test_youcookcap.yaml @@ -0,0 +1,32 @@ +slurm_config: big +task_type: local_predict +dataset: + split: test + video_processor: YoucookVideoProcessor + aligner: DSNLGAligner + bert_name: bert-base-uncased + meta_processor: YoucookNLGMetaProcessor + test_path: data/youcook/val_list.txt + trainval_annotation: data/youcook/youcookii_annotations_trainval.json + vfeat_dir: data/feat/feat_youcook_s3d + text_processor: NLGTextProcessor + max_video_len: 32 + max_len: 96 +fairseq: + dataset: + batch_size: 256 + valid_subset: test + num_workers: 2 + common_eval: + path: runs/mtm/vlm/youcookcap/checkpoint_best.pt +model: + model_cls: MMFusionNLG + mm_encoder_cls: MMBertForNLG + max_decode_length: 24 + use_seg_emb: true +eval: + save_path: runs/mtm/vlm/youcookcap/eval +metric: NLGMetric +predictor: NLGPredictor +gen_param: + num_beams: 5 diff --git a/data/fairseq/examples/MMPT/projects/mtm/vlm/vtt.yaml b/data/fairseq/examples/MMPT/projects/mtm/vlm/vtt.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c6c5b1ab40cbd2256919b42ea6b6a5fa52fe41a1 --- /dev/null +++ b/data/fairseq/examples/MMPT/projects/mtm/vlm/vtt.yaml @@ -0,0 +1,49 @@ +dataset: + video_processor: VideoProcessor + bert_name: bert-base-uncased + meta_processor: MSRVTTMetaProcessor + train_path: data/msrvtt/MSRVTT_train.csv + jsfusion_path: data/msrvtt/MSRVTT_JSFUSION_test.csv + full_test_path: data/msrvtt/MSRVTT_FULL_test.csv + dup: 20 + val_path: data/msrvtt/MSRVTT_JSFUSION_test.csv + vfeat_dir: data/feat/feat_vtt_s3d + text_processor: MSRVTTTextProcessor + json_path: data/msrvtt/MSRVTT_data.json + aligner: DSAligner + num_iso_layer: 12 + max_video_len: 32 + max_len: 96 +fairseq: + common: + tensorboard_logdir: run + log_interval: 1000 + fp16: true + dataset: + num_workers: 4 + batch_size: 256 + optimization: + lr: + - 5.0e-05 + clip_norm: 2.0 + optimizer: adam + adam_betas: (0.9, 0.98) + lr_scheduler: polynomial_decay + total_num_update: 1000000 + warmup_updates: 122 + weight_decay: 0.0 + ddp_backend: no_c10d + max_epoch: 10 + checkpoint: + restore_file: runs/mtm/vlm/checkpoint_best.pt + reset_optimizer: true + reset_dataloader: true + reset_meters: true + save_dir: runs/mtm/vlm/vtt +task_type: sweep_small +model: + model_cls: MMFusionJoint + mm_encoder_cls: MMBertForJoint + use_seg_emb: true +loss: + loss_cls: T2VContraLoss diff --git a/data/fairseq/examples/MMPT/projects/mtm/vlm/vttqa.yaml b/data/fairseq/examples/MMPT/projects/mtm/vlm/vttqa.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0a440c7dd2f5b324d57367456d22ca5aab8f397f --- /dev/null +++ b/data/fairseq/examples/MMPT/projects/mtm/vlm/vttqa.yaml @@ -0,0 +1,47 @@ +dataset: + video_processor: VideoProcessor + bert_name: bert-base-uncased + meta_processor: MSRVTTMetaProcessor + train_path: data/msrvtt/MSRVTT_train.csv + dup: 20 + val_path: data/msrvtt/MSRVTT_JSFUSION_test.csv + vfeat_dir: data/feat/feat_vtt_s3d + text_processor: MSRVTTTextProcessor + json_path: data/msrvtt/MSRVTT_data.json + aligner: DSAligner + num_iso_layer: 12 + max_video_len: 32 + max_len: 96 +fairseq: + common: + tensorboard_logdir: run + log_interval: 1000 + fp16: true + dataset: + num_workers: 4 + batch_size: 128 + optimization: + lr: + - 5.0e-05 + clip_norm: 2.0 + optimizer: adam + adam_betas: (0.9, 0.98) + lr_scheduler: polynomial_decay + total_num_update: 1000000 + warmup_updates: 122 + weight_decay: 0.0 + ddp_backend: no_c10d + max_epoch: 5 + checkpoint: + restore_file: runs/mtm/vlm/checkpoint_best.pt + reset_optimizer: true + reset_dataloader: true + reset_meters: true + save_dir: runs/mtm/vlm/vttqa +task_type: sweep_small +model: + model_cls: MMFusionJoint + mm_encoder_cls: MMBertForJoint + use_seg_emb: true +loss: + loss_cls: V2TContraLoss diff --git a/data/fairseq/examples/MMPT/projects/mtm/vlm/youcook.yaml b/data/fairseq/examples/MMPT/projects/mtm/vlm/youcook.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9ee82b81b8b14a9f689e73bac22f21b780640984 --- /dev/null +++ b/data/fairseq/examples/MMPT/projects/mtm/vlm/youcook.yaml @@ -0,0 +1,47 @@ +dataset: + video_processor: YoucookVideoProcessor + bert_name: bert-base-uncased + meta_processor: YoucookMetaProcessor + train_path: data/youcook/youcook_train.pkl + val_path: data/youcook/youcook_val.pkl + trainval_annotation: data/youcook/youcookii_annotations_trainval.json + use_annotation_text: true + vfeat_dir: data/feat/feat_youcook_s3d + text_processor: TextProcessor + aligner: DSAligner + num_iso_layer: 12 + max_video_len: 32 + max_len: 96 +fairseq: + common: + tensorboard_logdir: run + log_interval: 1000 + fp16: true + dataset: + num_workers: 4 + batch_size: 128 + optimization: + lr: + - 5.0e-05 + clip_norm: 2.0 + optimizer: adam + adam_betas: (0.9, 0.98) + lr_scheduler: polynomial_decay + total_num_update: 1000000 + warmup_updates: 122 + weight_decay: 0.0 + ddp_backend: no_c10d + max_epoch: 10 + checkpoint: + restore_file: runs/mtm/vlm/checkpoint_best.pt + reset_optimizer: true + reset_dataloader: true + reset_meters: true + save_dir: runs/mtm/vlm/youcook +task_type: sweep_small +model: + model_cls: MMFusionJoint + mm_encoder_cls: MMBertForJoint + use_seg_emb: true +loss: + loss_cls: T2VContraLoss diff --git a/data/fairseq/examples/MMPT/projects/mtm/vlm/youcookcap.yaml b/data/fairseq/examples/MMPT/projects/mtm/vlm/youcookcap.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d29dfad5cd861cc1e8f587c730938ed491d60289 --- /dev/null +++ b/data/fairseq/examples/MMPT/projects/mtm/vlm/youcookcap.yaml @@ -0,0 +1,45 @@ +dataset: + video_processor: YoucookVideoProcessor + bert_name: bert-base-uncased + meta_processor: YoucookNLGMetaProcessor + train_path: data/youcook/train_list.txt + val_path: data/youcook/val_list.txt + trainval_annotation: data/youcook/youcookii_annotations_trainval.json + vfeat_dir: data/feat/feat_youcook_s3d + text_processor: NLGTextProcessor + aligner: DSNLGAligner + max_video_len: 32 + max_len: 96 +fairseq: + common: + tensorboard_logdir: run + log_interval: 1000 + fp16: true + dataset: + num_workers: 4 + batch_size: 128 + optimization: + lr: + - 5.0e-05 + clip_norm: 2.0 + optimizer: adam + adam_betas: (0.9, 0.98) + lr_scheduler: polynomial_decay + total_num_update: 1000000 + warmup_updates: 122 + weight_decay: 0.0 + ddp_backend: no_c10d + max_epoch: 10 + checkpoint: + restore_file: runs/mtm/vlm/checkpoint_best.pt + reset_optimizer: true + reset_dataloader: true + reset_meters: true + save_dir: runs/mtm/vlm/youcookcap +task_type: sweep_small +model: + model_cls: MMFusionNLG + mm_encoder_cls: MMBertForNLG + use_seg_emb: true +loss: + loss_cls: NLGLoss diff --git a/data/fairseq/examples/MMPT/projects/retri/videoclip.yaml b/data/fairseq/examples/MMPT/projects/retri/videoclip.yaml new file mode 100644 index 0000000000000000000000000000000000000000..afd040ab050554aa0ca9c65ab4cf36cf6f3155dd --- /dev/null +++ b/data/fairseq/examples/MMPT/projects/retri/videoclip.yaml @@ -0,0 +1,10 @@ +includes: projects/retri/videoretri.yaml +project_dir: retri/videoclip +task_group: + pretrain: + model: + model_cls: MMFusionSeparate + mm_encoder_cls: + video_encoder_cls: MMBertForEncoder + text_encoder_cls: BertModel + num_hidden_video_layers: 6 diff --git a/data/fairseq/examples/MMPT/projects/retri/videoclip/coin_videoclip.yaml b/data/fairseq/examples/MMPT/projects/retri/videoclip/coin_videoclip.yaml new file mode 100644 index 0000000000000000000000000000000000000000..aaed5e47f62d421e5c434a2f539437831ae387db --- /dev/null +++ b/data/fairseq/examples/MMPT/projects/retri/videoclip/coin_videoclip.yaml @@ -0,0 +1,49 @@ +dataset: + video_processor: VideoProcessor + bert_name: bert-base-uncased + meta_processor: COINActionSegmentationMetaProcessor + train_path: data/coin/COIN.json + val_path: data/coin/COIN.json + vfeat_dir: data/feat/feat_coin_s3d + text_processor: COINActionSegmentationTextProcessor + aligner: COINActionSegmentationAligner + num_iso_layer: 12 + sliding_window: 8 + sliding_window_size: 32 + max_video_len: 32 + max_len: 96 +fairseq: + common: + tensorboard_logdir: run + log_interval: 1000 + fp16: true + dataset: + num_workers: 4 + batch_size: 1 + optimization: + lr: + - 5.0e-05 + clip_norm: 2.0 + optimizer: adam + adam_betas: (0.9, 0.98) + lr_scheduler: polynomial_decay + total_num_update: 1000000 + warmup_updates: 122 + weight_decay: 0.0 + ddp_backend: no_c10d + max_epoch: 8 + checkpoint: + restore_file: runs/retri/videoclip/checkpoint_best.pt + reset_optimizer: true + reset_dataloader: true + reset_meters: true + save_dir: runs/retri/videoclip/coin +task_type: sweep_big +model: + model_cls: MMFusionSeparateActionSegmentation + mm_encoder_cls: null + video_encoder_cls: MMBertForTokenClassification + text_encoder_cls: BertModel + num_hidden_video_layers: 6 +loss: + loss_cls: CrossEntropy diff --git a/data/fairseq/examples/MMPT/projects/retri/videoclip/crosstask_videoclip.yaml b/data/fairseq/examples/MMPT/projects/retri/videoclip/crosstask_videoclip.yaml new file mode 100644 index 0000000000000000000000000000000000000000..758601e3593aa3960fee996dcfdfd71972f9f068 --- /dev/null +++ b/data/fairseq/examples/MMPT/projects/retri/videoclip/crosstask_videoclip.yaml @@ -0,0 +1,55 @@ +dataset: + video_processor: CrossTaskVideoProcessor + bert_name: bert-base-uncased + meta_processor: CrossTaskMetaProcessor + train_path: data/crosstask/crosstask_release/videos.csv + train_csv_path: data/crosstask/crosstask_release/videos.csv + val_path: data/crosstask/crosstask_release/videos_val.csv + val_csv_path: data/crosstask/crosstask_release/videos_val.csv + primary_path: data/crosstask/crosstask_release/tasks_primary.txt + related_path: data/crosstask/crosstask_release/tasks_related.txt + vfeat_dir: data/feat/feat_crosstask_s3d + annotation_path: data/crosstask/crosstask_release/annotations + n_train: 30 + text_processor: CrossTaskTextProcessor + aligner: CrossTaskAligner + num_iso_layer: 12 + sliding_window: 16 + sliding_window_size: 32 + max_video_len: 32 + max_len: 96 +fairseq: + common: + tensorboard_logdir: run + log_interval: 1000 + fp16: true + dataset: + num_workers: 4 + batch_size: 1 + optimization: + lr: + - 5.0e-05 + clip_norm: 2.0 + optimizer: adam + adam_betas: (0.9, 0.98) + lr_scheduler: polynomial_decay + total_num_update: 1000000 + warmup_updates: 122 + weight_decay: 0.0 + ddp_backend: no_c10d + max_epoch: 5 + checkpoint: + restore_file: runs/retri/videoclip/checkpoint_best.pt + reset_optimizer: true + reset_dataloader: true + reset_meters: true + save_dir: runs/retri/videoclip/crosstask +task_type: sweep_small +model: + model_cls: MMFusionSeparateActionLocalization + mm_encoder_cls: null + video_encoder_cls: MMBertForEncoder + text_encoder_cls: BertModel + num_hidden_video_layers: 6 +loss: + loss_cls: BCE diff --git a/data/fairseq/examples/MMPT/projects/retri/videoclip/how2.yaml b/data/fairseq/examples/MMPT/projects/retri/videoclip/how2.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b49581e878390cd8861926988a58a892b5a38606 --- /dev/null +++ b/data/fairseq/examples/MMPT/projects/retri/videoclip/how2.yaml @@ -0,0 +1,65 @@ +dataset: + video_processor: ShardedVideoRetriVideoProcessor + bert_name: bert-base-uncased + meta_processor: ShardedHow2VideoRetriMetaProcessor + train_path: data/how2/how2_s3d_train.lst + val_path: data/how2/how2_s3d_val.lst + vfeat_dir: data/feat/feat_how2_s3d_shard_small + text_processor: ShardedVideoRetriTextProcessor + tfeat_dir: data/feat/feat_how2_s3d_shard_small/raw_caption_dedup.bert-base-uncased. + aligner: VideoRetriOverlappedAligner + subsampling: 1 + sampled_min_len: 8 + sampled_max_len: 64 + max_video_len: 32 + max_len: 96 + lazy_vfeat_mask: true + mfm_probability: 0.15 + mlm_probability: 0.15 + mm_prob: 0.5 + sampled_video_min_len: 3 + sampled_video_max_len: 32 + num_video_per_batch: 32 + clip_per_video: 16 +fairseq: + common: + tensorboard_logdir: run + log_interval: 1000 + fp16: true + dataset: + num_workers: 4 + batch_size: 1 + optimization: + lr: + - 5.0e-05 + clip_norm: 2.0 + optimizer: adam + adam_betas: (0.9, 0.98) + lr_scheduler: polynomial_decay + total_num_update: 1000000 + warmup_updates: 1000 + weight_decay: 0.0 + ddp_backend: no_c10d + max_epoch: 25 + checkpoint: + save_dir: runs/retri/videoclip + save_interval_updates: 1024 + keep_interval_updates: 2 + keep_last_epochs: 30 +task_type: sweep_big +slurm_config: big +eval: + save_path: runs/retri/videoclip +model: + model_cls: MMFusionSeparate + mm_encoder_cls: null + video_encoder_cls: MMBertForEncoder + text_encoder_cls: BertModel + num_hidden_video_layers: 6 +loss: + loss_cls: MMContraLoss +task: VideoRetriTask +retri_epoch: 1 +vectorpool_cls: VideoVectorPool +retriever_cls: VectorRetriever +num_cands: 64 diff --git a/data/fairseq/examples/MMPT/projects/retri/videoclip/test_coin_videoclip.yaml b/data/fairseq/examples/MMPT/projects/retri/videoclip/test_coin_videoclip.yaml new file mode 100644 index 0000000000000000000000000000000000000000..409906203c22d670c361d011d1035874961d4b5b --- /dev/null +++ b/data/fairseq/examples/MMPT/projects/retri/videoclip/test_coin_videoclip.yaml @@ -0,0 +1,33 @@ +slurm_config: big +task_type: local_predict +dataset: + split: test + video_processor: VideoProcessor + aligner: COINActionSegmentationAligner + bert_name: bert-base-uncased + test_path: data/coin/COIN.json + meta_processor: COINActionSegmentationMetaProcessor + vfeat_dir: data/feat/feat_coin_s3d + text_processor: COINActionSegmentationTextProcessor + num_iso_layer: 12 + sliding_window: 16 + sliding_window_size: 32 + max_video_len: 32 + max_len: 96 +fairseq: + dataset: + batch_size: 1 + valid_subset: test + num_workers: 2 + common_eval: + path: runs/retri/videoclip/coin/checkpoint_best.pt +model: + model_cls: MMFusionSeparateActionSegmentation + mm_encoder_cls: null + video_encoder_cls: MMBertForTokenClassification + text_encoder_cls: BertModel + num_hidden_video_layers: 6 +eval: + save_path: runs/retri/videoclip/coin/eval +metric: COINActionSegmentationMetric +predictor: COINPredictor diff --git a/data/fairseq/examples/MMPT/projects/retri/videoclip/test_coin_zs.yaml b/data/fairseq/examples/MMPT/projects/retri/videoclip/test_coin_zs.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b33739c7b6d7c4d935c0e4d78354a02f79e21d91 --- /dev/null +++ b/data/fairseq/examples/MMPT/projects/retri/videoclip/test_coin_zs.yaml @@ -0,0 +1,33 @@ +slurm_config: big +task_type: local_predict +dataset: + split: test + video_processor: VideoProcessor + aligner: COINActionSegmentationAligner + bert_name: bert-base-uncased + test_path: data/coin/COIN.json + meta_processor: COINActionSegmentationMetaProcessor + vfeat_dir: data/feat/feat_coin_s3d + text_processor: COINActionSegmentationTextProcessor + num_iso_layer: 12 + sliding_window: 16 + sliding_window_size: 32 + max_video_len: 32 + max_len: 96 +fairseq: + dataset: + batch_size: 1 + valid_subset: test + num_workers: 2 + common_eval: + path: runs/retri/videoclip/checkpoint_best.pt +model: + model_cls: MMFusionSeparate + mm_encoder_cls: null + video_encoder_cls: MMBertForEncoder + text_encoder_cls: BertModel + num_hidden_video_layers: 6 +eval: + save_path: runs/retri/videoclip/coin_zs/eval +metric: COINActionSegmentationMetric +predictor: COINZSPredictor diff --git a/data/fairseq/examples/MMPT/projects/retri/videoclip/test_crosstask_videoclip.yaml b/data/fairseq/examples/MMPT/projects/retri/videoclip/test_crosstask_videoclip.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e82f54fbe52defa233f1b561415737924dd6ba9c --- /dev/null +++ b/data/fairseq/examples/MMPT/projects/retri/videoclip/test_crosstask_videoclip.yaml @@ -0,0 +1,40 @@ +slurm_config: big +task_type: local_predict +dataset: + split: test + video_processor: CrossTaskVideoProcessor + aligner: CrossTaskAligner + bert_name: bert-base-uncased + meta_processor: CrossTaskMetaProcessor + test_path: data/crosstask/crosstask_release/videos_val.csv + train_csv_path: data/crosstask/crosstask_release/videos.csv + val_path: data/crosstask/crosstask_release/videos_val.csv + val_csv_path: data/crosstask/crosstask_release/videos_val.csv + primary_path: data/crosstask/crosstask_release/tasks_primary.txt + related_path: data/crosstask/crosstask_release/tasks_related.txt + vfeat_dir: data/feat/feat_crosstask_s3d + annotation_path: data/crosstask/crosstask_release/annotations + n_train: 30 + text_processor: CrossTaskTextProcessor + num_iso_layer: 12 + sliding_window: 16 + sliding_window_size: 32 + max_video_len: 32 + max_len: 96 +fairseq: + dataset: + batch_size: 1 + valid_subset: test + num_workers: 2 + common_eval: + path: runs/retri/videoclip/crosstask/checkpoint_best.pt +model: + model_cls: MMFusionSeparateActionLocalization + mm_encoder_cls: null + video_encoder_cls: MMBertForEncoder + text_encoder_cls: BertModel + num_hidden_video_layers: 6 +eval: + save_path: runs/retri/videoclip/crosstask/eval +metric: CrossTaskMetric +predictor: CrossTaskPredictor diff --git a/data/fairseq/examples/MMPT/projects/retri/videoclip/test_crosstask_zs_videoclip.yaml b/data/fairseq/examples/MMPT/projects/retri/videoclip/test_crosstask_zs_videoclip.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6fc357cc1f5285229a508a95dc31c2aae17c977d --- /dev/null +++ b/data/fairseq/examples/MMPT/projects/retri/videoclip/test_crosstask_zs_videoclip.yaml @@ -0,0 +1,40 @@ +slurm_config: big +task_type: local_predict +dataset: + split: test + video_processor: CrossTaskVideoProcessor + aligner: CrossTaskAligner + bert_name: bert-base-uncased + meta_processor: CrossTaskMetaProcessor + test_path: data/crosstask/crosstask_release/videos_val.csv + train_csv_path: data/crosstask/crosstask_release/videos.csv + val_path: data/crosstask/crosstask_release/videos_val.csv + val_csv_path: data/crosstask/crosstask_release/videos_val.csv + primary_path: data/crosstask/crosstask_release/tasks_primary.txt + related_path: data/crosstask/crosstask_release/tasks_related.txt + vfeat_dir: data/feat/feat_crosstask_s3d + annotation_path: data/crosstask/crosstask_release/annotations + n_train: 30 + text_processor: CrossTaskTextProcessor + num_iso_layer: 12 + sliding_window: 16 + sliding_window_size: 32 + max_video_len: 32 + max_len: 96 +fairseq: + dataset: + batch_size: 1 + valid_subset: test + num_workers: 2 + common_eval: + path: runs/retri/videoclip/checkpoint_best.pt +model: + model_cls: MMFusionSeparateActionLocalization + mm_encoder_cls: null + video_encoder_cls: MMBertForEncoder + text_encoder_cls: BertModel + num_hidden_video_layers: 6 +eval: + save_path: runs/retri/videoclip/crosstask_zs/eval +metric: CrossTaskMetric +predictor: CrossTaskPredictor diff --git a/data/fairseq/examples/MMPT/projects/retri/videoclip/test_didemo_zs.yaml b/data/fairseq/examples/MMPT/projects/retri/videoclip/test_didemo_zs.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8dc716815d8d4812237c814fc17a90dd7ac3cf1e --- /dev/null +++ b/data/fairseq/examples/MMPT/projects/retri/videoclip/test_didemo_zs.yaml @@ -0,0 +1,31 @@ +slurm_config: big +task_type: local_predict +dataset: + split: test + video_processor: VideoProcessor + aligner: DiDeMoAligner + bert_name: bert-base-uncased + meta_processor: DiDeMoMetaProcessor + test_path: data/didemo/test_data.json + vfeat_dir: data/feat/feat_didemo_s3d + text_processor: DiDeMoTextProcessor + num_iso_layer: 12 + max_video_len: 32 + max_len: 96 +fairseq: + dataset: + batch_size: 256 + valid_subset: test + num_workers: 2 + common_eval: + path: runs/retri/videoclip/checkpoint_best.pt +model: + model_cls: MMFusionSeparate + mm_encoder_cls: null + video_encoder_cls: MMBertForEncoder + text_encoder_cls: BertModel + num_hidden_video_layers: 6 +eval: + save_path: runs/retri/videoclip/didemo_zs/eval +metric: DiDeMoMetric +predictor: DiDeMoPredictor diff --git a/data/fairseq/examples/MMPT/projects/retri/videoclip/test_vtt_videoclip.yaml b/data/fairseq/examples/MMPT/projects/retri/videoclip/test_vtt_videoclip.yaml new file mode 100644 index 0000000000000000000000000000000000000000..19321ad5f489121381b1b78506b38ba3705b37e4 --- /dev/null +++ b/data/fairseq/examples/MMPT/projects/retri/videoclip/test_vtt_videoclip.yaml @@ -0,0 +1,31 @@ +slurm_config: big +task_type: local_predict +dataset: + split: test + video_processor: VideoProcessor + aligner: DSAligner + bert_name: bert-base-uncased + meta_processor: MSRVTTMetaProcessor + test_path: data/msrvtt/MSRVTT_JSFUSION_test.csv + vfeat_dir: data/feat/feat_vtt_s3d + text_processor: MSRVTTTextProcessor + num_iso_layer: 12 + max_video_len: 32 + max_len: 96 +fairseq: + dataset: + batch_size: 256 + valid_subset: test + num_workers: 2 + common_eval: + path: runs/retri/videoclip/vtt/checkpoint_last.pt +model: + model_cls: MMFusionSeparate + mm_encoder_cls: null + video_encoder_cls: MMBertForEncoder + text_encoder_cls: BertModel + num_hidden_video_layers: 6 +eval: + save_path: runs/retri/videoclip/vtt/eval +metric: RetrievalMetric +predictor: RetrievalPredictor diff --git a/data/fairseq/examples/MMPT/projects/retri/videoclip/test_vtt_zs.yaml b/data/fairseq/examples/MMPT/projects/retri/videoclip/test_vtt_zs.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d149fa3960294f4d9dcac9c4f4f18631eb622435 --- /dev/null +++ b/data/fairseq/examples/MMPT/projects/retri/videoclip/test_vtt_zs.yaml @@ -0,0 +1,31 @@ +slurm_config: big +task_type: local_predict +dataset: + split: test + video_processor: VideoProcessor + aligner: DSAligner + bert_name: bert-base-uncased + meta_processor: MSRVTTMetaProcessor + test_path: data/msrvtt/MSRVTT_JSFUSION_test.csv + vfeat_dir: data/feat/feat_vtt_s3d + text_processor: MSRVTTTextProcessor + num_iso_layer: 12 + max_video_len: 32 + max_len: 96 +fairseq: + dataset: + batch_size: 256 + valid_subset: test + num_workers: 2 + common_eval: + path: runs/retri/videoclip/checkpoint_best.pt +model: + model_cls: MMFusionSeparate + mm_encoder_cls: null + video_encoder_cls: MMBertForEncoder + text_encoder_cls: BertModel + num_hidden_video_layers: 6 +eval: + save_path: runs/retri/videoclip/vtt_zs/eval +metric: RetrievalMetric +predictor: RetrievalPredictor diff --git a/data/fairseq/examples/MMPT/projects/retri/videoclip/test_vttqa_videoclip.yaml b/data/fairseq/examples/MMPT/projects/retri/videoclip/test_vttqa_videoclip.yaml new file mode 100644 index 0000000000000000000000000000000000000000..295aeedbb0f29a28ad9065da0a18e82ac98b6683 --- /dev/null +++ b/data/fairseq/examples/MMPT/projects/retri/videoclip/test_vttqa_videoclip.yaml @@ -0,0 +1,31 @@ +slurm_config: big +task_type: local_predict +dataset: + split: test + video_processor: VideoProcessor + aligner: MSRVTTQAAligner + bert_name: bert-base-uncased + meta_processor: MSRVTTQAMetaProcessor + test_path: data/msrvtt-qa/MSR_MC_test.csv + vfeat_dir: data/feat/feat_vtt_s3d + text_processor: MSRVTTQATextProcessor + num_iso_layer: 12 + max_video_len: 32 + max_len: 96 +fairseq: + dataset: + batch_size: 256 + valid_subset: test + num_workers: 2 + common_eval: + path: runs/retri/videoclip/vttqa/checkpoint_last.pt +model: + model_cls: MMFusionSeparate + mm_encoder_cls: null + video_encoder_cls: MMBertForEncoder + text_encoder_cls: BertModel + num_hidden_video_layers: 6 +eval: + save_path: runs/retri/videoclip/vttqa/eval +metric: QAMetric +predictor: QAPredictor diff --git a/data/fairseq/examples/MMPT/projects/retri/videoclip/test_vttqa_zs.yaml b/data/fairseq/examples/MMPT/projects/retri/videoclip/test_vttqa_zs.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7a876c822ae38cadc03c69544e64025d516e8cca --- /dev/null +++ b/data/fairseq/examples/MMPT/projects/retri/videoclip/test_vttqa_zs.yaml @@ -0,0 +1,31 @@ +slurm_config: big +task_type: local_predict +dataset: + split: test + video_processor: VideoProcessor + aligner: MSRVTTQAAligner + bert_name: bert-base-uncased + meta_processor: MSRVTTQAMetaProcessor + test_path: data/msrvtt-qa/MSR_MC_test.csv + vfeat_dir: data/feat/feat_vtt_s3d + text_processor: MSRVTTQATextProcessor + num_iso_layer: 12 + max_video_len: 32 + max_len: 96 +fairseq: + dataset: + batch_size: 256 + valid_subset: test + num_workers: 2 + common_eval: + path: runs/retri/videoclip/checkpoint_best.pt +model: + model_cls: MMFusionSeparate + mm_encoder_cls: null + video_encoder_cls: MMBertForEncoder + text_encoder_cls: BertModel + num_hidden_video_layers: 6 +eval: + save_path: runs/retri/videoclip/vttqa_zs/eval +metric: QAMetric +predictor: QAPredictor diff --git a/data/fairseq/examples/MMPT/projects/retri/videoclip/test_youcook_videoclip.yaml b/data/fairseq/examples/MMPT/projects/retri/videoclip/test_youcook_videoclip.yaml new file mode 100644 index 0000000000000000000000000000000000000000..86a4ab203e097ec156ce02d88ce9c1b2374cec9f --- /dev/null +++ b/data/fairseq/examples/MMPT/projects/retri/videoclip/test_youcook_videoclip.yaml @@ -0,0 +1,33 @@ +slurm_config: big +task_type: local_predict +dataset: + split: test + video_processor: YoucookVideoProcessor + aligner: DSAligner + bert_name: bert-base-uncased + meta_processor: YoucookMetaProcessor + test_path: data/youcook/youcook_val.pkl + trainval_annotation: data/youcook/youcookii_annotations_trainval.json + use_annotation_text: true + vfeat_dir: data/feat/feat_youcook_s3d + text_processor: TextProcessor + num_iso_layer: 12 + max_video_len: 32 + max_len: 96 +fairseq: + dataset: + batch_size: 256 + valid_subset: test + num_workers: 2 + common_eval: + path: runs/retri/videoclip/youcook/checkpoint_last.pt +model: + model_cls: MMFusionSeparate + mm_encoder_cls: null + video_encoder_cls: MMBertForEncoder + text_encoder_cls: BertModel + num_hidden_video_layers: 6 +eval: + save_path: runs/retri/videoclip/youcook/eval +metric: RetrievalMetric +predictor: RetrievalPredictor diff --git a/data/fairseq/examples/MMPT/projects/retri/videoclip/test_youcook_zs.yaml b/data/fairseq/examples/MMPT/projects/retri/videoclip/test_youcook_zs.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fd2941708be0baacf9e12ca757e0de40676e570d --- /dev/null +++ b/data/fairseq/examples/MMPT/projects/retri/videoclip/test_youcook_zs.yaml @@ -0,0 +1,33 @@ +slurm_config: big +task_type: local_predict +dataset: + split: test + video_processor: YoucookVideoProcessor + aligner: DSAligner + bert_name: bert-base-uncased + meta_processor: YoucookMetaProcessor + test_path: data/youcook/youcook_val.pkl + trainval_annotation: data/youcook/youcookii_annotations_trainval.json + use_annotation_text: true + vfeat_dir: data/feat/feat_youcook_s3d + text_processor: TextProcessor + num_iso_layer: 12 + max_video_len: 32 + max_len: 96 +fairseq: + dataset: + batch_size: 256 + valid_subset: test + num_workers: 2 + common_eval: + path: runs/retri/videoclip/checkpoint_best.pt +model: + model_cls: MMFusionSeparate + mm_encoder_cls: null + video_encoder_cls: MMBertForEncoder + text_encoder_cls: BertModel + num_hidden_video_layers: 6 +eval: + save_path: runs/retri/videoclip/youcook_zs/eval +metric: RetrievalMetric +predictor: RetrievalPredictor diff --git a/data/fairseq/examples/MMPT/projects/retri/videoclip/vtt_videoclip.yaml b/data/fairseq/examples/MMPT/projects/retri/videoclip/vtt_videoclip.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d8b4079ac233af17ee16cfed7f3e2316e0e236c5 --- /dev/null +++ b/data/fairseq/examples/MMPT/projects/retri/videoclip/vtt_videoclip.yaml @@ -0,0 +1,51 @@ +dataset: + video_processor: VideoProcessor + bert_name: bert-base-uncased + meta_processor: MSRVTTMetaProcessor + train_path: data/msrvtt/MSRVTT_train.csv + jsfusion_path: data/msrvtt/MSRVTT_JSFUSION_test.csv + full_test_path: data/msrvtt/MSRVTT_FULL_test.csv + dup: 20 + val_path: data/msrvtt/MSRVTT_JSFUSION_test.csv + vfeat_dir: data/feat/feat_vtt_s3d + text_processor: MSRVTTTextProcessor + json_path: data/msrvtt/MSRVTT_data.json + aligner: DSAligner + num_iso_layer: 12 + max_video_len: 32 + max_len: 96 +fairseq: + common: + tensorboard_logdir: run + log_interval: 1000 + fp16: true + dataset: + num_workers: 4 + batch_size: 224 + optimization: + lr: + - 5.0e-05 + clip_norm: 2.0 + optimizer: adam + adam_betas: (0.9, 0.98) + lr_scheduler: polynomial_decay + total_num_update: 1000000 + warmup_updates: 122 + weight_decay: 0.0 + ddp_backend: no_c10d + max_epoch: 10 + checkpoint: + restore_file: runs/retri/videoclip/checkpoint_best.pt + reset_optimizer: true + reset_dataloader: true + reset_meters: true + save_dir: runs/retri/videoclip/vtt +task_type: sweep_small +model: + model_cls: MMFusionSeparate + mm_encoder_cls: null + video_encoder_cls: MMBertForEncoder + text_encoder_cls: BertModel + num_hidden_video_layers: 6 +loss: + loss_cls: T2VContraLoss diff --git a/data/fairseq/examples/MMPT/projects/retri/videoclip/vttqa_videoclip.yaml b/data/fairseq/examples/MMPT/projects/retri/videoclip/vttqa_videoclip.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f0566d784ab53e0159a1db6b82222ad11ad12add --- /dev/null +++ b/data/fairseq/examples/MMPT/projects/retri/videoclip/vttqa_videoclip.yaml @@ -0,0 +1,49 @@ +dataset: + video_processor: VideoProcessor + bert_name: bert-base-uncased + meta_processor: MSRVTTMetaProcessor + train_path: data/msrvtt/MSRVTT_train.csv + dup: 20 + val_path: data/msrvtt/MSRVTT_JSFUSION_test.csv + vfeat_dir: data/feat/feat_vtt_s3d + text_processor: MSRVTTTextProcessor + json_path: data/msrvtt/MSRVTT_data.json + aligner: DSAligner + num_iso_layer: 12 + max_video_len: 32 + max_len: 96 +fairseq: + common: + tensorboard_logdir: run + log_interval: 1000 + fp16: true + dataset: + num_workers: 4 + batch_size: 128 + optimization: + lr: + - 5.0e-05 + clip_norm: 2.0 + optimizer: adam + adam_betas: (0.9, 0.98) + lr_scheduler: polynomial_decay + total_num_update: 1000000 + warmup_updates: 122 + weight_decay: 0.0 + ddp_backend: no_c10d + max_epoch: 5 + checkpoint: + restore_file: runs/retri/videoclip/checkpoint_best.pt + reset_optimizer: true + reset_dataloader: true + reset_meters: true + save_dir: runs/retri/videoclip/vttqa +task_type: sweep_small +model: + model_cls: MMFusionSeparate + mm_encoder_cls: null + video_encoder_cls: MMBertForEncoder + text_encoder_cls: BertModel + num_hidden_video_layers: 6 +loss: + loss_cls: V2TContraLoss diff --git a/data/fairseq/examples/MMPT/projects/retri/videoclip/youcook_videoclip.yaml b/data/fairseq/examples/MMPT/projects/retri/videoclip/youcook_videoclip.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c2b13e5519fe0839c6b3bc681f7ee97b522077d8 --- /dev/null +++ b/data/fairseq/examples/MMPT/projects/retri/videoclip/youcook_videoclip.yaml @@ -0,0 +1,49 @@ +dataset: + video_processor: YoucookVideoProcessor + bert_name: bert-base-uncased + meta_processor: YoucookMetaProcessor + train_path: data/youcook/youcook_train.pkl + val_path: data/youcook/youcook_val.pkl + trainval_annotation: data/youcook/youcookii_annotations_trainval.json + use_annotation_text: true + vfeat_dir: data/feat/feat_youcook_s3d + text_processor: TextProcessor + aligner: DSAligner + num_iso_layer: 12 + max_video_len: 32 + max_len: 96 +fairseq: + common: + tensorboard_logdir: run + log_interval: 1000 + fp16: true + dataset: + num_workers: 4 + batch_size: 128 + optimization: + lr: + - 5.0e-05 + clip_norm: 2.0 + optimizer: adam + adam_betas: (0.9, 0.98) + lr_scheduler: polynomial_decay + total_num_update: 1000000 + warmup_updates: 122 + weight_decay: 0.0 + ddp_backend: no_c10d + max_epoch: 10 + checkpoint: + restore_file: runs/retri/videoclip/checkpoint_best.pt + reset_optimizer: true + reset_dataloader: true + reset_meters: true + save_dir: runs/retri/videoclip/youcook +task_type: sweep_small +model: + model_cls: MMFusionSeparate + mm_encoder_cls: null + video_encoder_cls: MMBertForEncoder + text_encoder_cls: BertModel + num_hidden_video_layers: 6 +loss: + loss_cls: T2VContraLoss diff --git a/data/fairseq/examples/MMPT/projects/retri/videoretri.yaml b/data/fairseq/examples/MMPT/projects/retri/videoretri.yaml new file mode 100644 index 0000000000000000000000000000000000000000..969e1fb2793d5381e52188c4ad78aea240698969 --- /dev/null +++ b/data/fairseq/examples/MMPT/projects/retri/videoretri.yaml @@ -0,0 +1,51 @@ +includes: projects/mfmmlm.yaml +project_dir: retri/videoretri +run_task: + - how2.yaml +task_group: + pretrain: + task: VideoRetriTask + retri_epoch: 1 + vectorpool_cls: VideoVectorPool + retriever_cls: VectorRetriever + num_cands: 64 + dataset: + train_path: data/how2/how2_s3d_train.lst + meta_processor: ShardedHow2VideoRetriMetaProcessor + video_processor: ShardedVideoRetriVideoProcessor + text_processor: ShardedVideoRetriTextProcessor + aligner: VideoRetriOverlappedAligner + sampled_video_min_len: 3 + sampled_video_max_len: 32 + sampled_min_len: 8 + sampled_max_len: 64 + num_video_per_batch: 32 + # do not use subsampling as it changes fairseq batch_size. + subsampling: 1 # disable subsampling + clip_per_video: 16 + fairseq: + dataset: + batch_size: 1 + optimization: + max_epoch: 25 + model: + model_cls: MMFusionShare + mm_encoder_cls: MMBertForEncoder + loss: + loss_cls: MMContraLoss + finetune: + task_list: [vtt_videoclip.yaml, youcook_videoclip.yaml, vttqa_videoclip.yaml, crosstask_videoclip.yaml, coin_videoclip.yaml] + test: + task_list: + - test_youcook_zs.yaml + - test_vtt_zs.yaml + - test_vttqa_zs.yaml + - test_crosstask_zs_videoclip.yaml + - test_coin_zs.yaml + - test_didemo_zs.yaml + - test_youcook_videoclip.yaml + - test_vtt_videoclip.yaml + - test_vttqa_videoclip.yaml + - test_crosstask_videoclip.yaml + - test_coin_videoclip.yaml + diff --git a/data/fairseq/examples/MMPT/projects/task/coin.yaml b/data/fairseq/examples/MMPT/projects/task/coin.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e7772486e166348deff1f7c9deceb68c7e1bb443 --- /dev/null +++ b/data/fairseq/examples/MMPT/projects/task/coin.yaml @@ -0,0 +1,25 @@ +includes: projects/task/ft.yaml +task_type: sweep_big +dataset: + meta_processor: COINActionSegmentationMetaProcessor + train_path: data/coin/COIN.json + val_path: data/coin/COIN.json + vfeat_dir: data/feat/feat_coin_s3d + video_processor: VideoProcessor + text_processor: COINActionSegmentationTextProcessor + aligner: COINActionSegmentationAligner + num_iso_layer: 12 + sliding_window: 8 + sliding_window_size: 32 +model: + model_cls: MMFusionActionSegmentation + mm_encoder_cls: MMBertForTokenClassification +loss: + loss_cls: CrossEntropy +fairseq: + dataset: + batch_size: 1 + optimization: + max_epoch: 8 + checkpoint: + save_dir: runs/task/coin diff --git a/data/fairseq/examples/MMPT/projects/task/coin_videoclip.yaml b/data/fairseq/examples/MMPT/projects/task/coin_videoclip.yaml new file mode 100644 index 0000000000000000000000000000000000000000..69988bc18a03a8f9bf594266f3bf06aa54f11b6a --- /dev/null +++ b/data/fairseq/examples/MMPT/projects/task/coin_videoclip.yaml @@ -0,0 +1,7 @@ +includes: projects/task/coin.yaml +model: + model_cls: MMFusionSeparateActionSegmentation + mm_encoder_cls: + video_encoder_cls: MMBertForTokenClassification + text_encoder_cls: BertModel # dummy, not used. + num_hidden_video_layers: 6 diff --git a/data/fairseq/examples/MMPT/projects/task/crosstask.yaml b/data/fairseq/examples/MMPT/projects/task/crosstask.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cb4dbb0cb4a07020d1fbba9a810eb30fd81ae96d --- /dev/null +++ b/data/fairseq/examples/MMPT/projects/task/crosstask.yaml @@ -0,0 +1,31 @@ +includes: projects/task/ft.yaml +dataset: + meta_processor: CrossTaskMetaProcessor + train_path: data/crosstask/crosstask_release/videos.csv # dummy + train_csv_path: data/crosstask/crosstask_release/videos.csv + val_path: data/crosstask/crosstask_release/videos_val.csv # dummy + val_csv_path: data/crosstask/crosstask_release/videos_val.csv + primary_path: data/crosstask/crosstask_release/tasks_primary.txt + related_path: data/crosstask/crosstask_release/tasks_related.txt + vfeat_dir: data/feat/feat_crosstask_s3d + annotation_path: data/crosstask/crosstask_release/annotations + n_train: 30 + video_processor: CrossTaskVideoProcessor + text_processor: CrossTaskTextProcessor + aligner: CrossTaskAligner + num_iso_layer: 12 + sliding_window: 16 + sliding_window_size: 32 +model: + model_cls: MMFusionActionLocalization + mm_encoder_cls: MMBertForJoint +loss: + loss_cls: BCE +fairseq: + dataset: + batch_size: 1 + optimization: + max_epoch: 5 + checkpoint: + save_dir: runs/task/crosstask + restore_file: runs/task/checkpoint11.pt # for VLM diff --git a/data/fairseq/examples/MMPT/projects/task/crosstask_videoclip.yaml b/data/fairseq/examples/MMPT/projects/task/crosstask_videoclip.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6ec613c07fcacfa6e6d3c5fc78aaefcb2b33eff5 --- /dev/null +++ b/data/fairseq/examples/MMPT/projects/task/crosstask_videoclip.yaml @@ -0,0 +1,10 @@ +includes: projects/task/crosstask.yaml +model: + model_cls: MMFusionSeparateActionLocalization + mm_encoder_cls: + video_encoder_cls: MMBertForEncoder + text_encoder_cls: BertModel # dummy, not used. + num_hidden_video_layers: 6 +fairseq: + checkpoint: + restore_file: runs/task/checkpoint_best.pt # overwrite the default of VLM. diff --git a/data/fairseq/examples/MMPT/projects/task/default.yaml b/data/fairseq/examples/MMPT/projects/task/default.yaml new file mode 100644 index 0000000000000000000000000000000000000000..087fef71a4b94d81b74c5e9ef647db189626ff36 --- /dev/null +++ b/data/fairseq/examples/MMPT/projects/task/default.yaml @@ -0,0 +1,20 @@ +# this yaml cannot be run alone. you must use `how2.yaml`, `vtt.yaml` etc for training. +dataset: + video_processor: VideoProcessor + bert_name: bert-base-uncased +fairseq: + common: + tensorboard_logdir: run + log_interval: 1000 + dataset: + num_workers: 4 + optimization: + lr: [ 0.00005 ] + clip_norm: 2.0 + optimizer: adam + adam_betas: (0.9, 0.98) + lr_scheduler: polynomial_decay + total_num_update: 1000000 # backward compatible on fairseq 1.0.0a0+af0389f for reproducibility. + warmup_updates: 1000 + weight_decay: 0.0 + ddp_backend: no_c10d diff --git a/data/fairseq/examples/MMPT/projects/task/ft.yaml b/data/fairseq/examples/MMPT/projects/task/ft.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c93b8a73ea93e00da8a0ee76751c9e1bd34ab8a2 --- /dev/null +++ b/data/fairseq/examples/MMPT/projects/task/ft.yaml @@ -0,0 +1,13 @@ +includes: projects/task/default.yaml +# all derived config will be run by fairseq-train. +task_type: sweep_small +fairseq: + optimization: + warmup_updates: 122 # copied from roberta glue: https://github.com/pytorch/fairseq/blob/master/examples/roberta/README.glue.md + checkpoint: + # save_interval_updates: 512 + # borrowed from Roberta script. + restore_file: runs/task/checkpoint_best.pt + reset_optimizer: True + reset_dataloader: True + reset_meters: True diff --git a/data/fairseq/examples/MMPT/projects/task/how2.yaml b/data/fairseq/examples/MMPT/projects/task/how2.yaml new file mode 100644 index 0000000000000000000000000000000000000000..094dd04bfcfdb96699ac80066f874fca954dc56a --- /dev/null +++ b/data/fairseq/examples/MMPT/projects/task/how2.yaml @@ -0,0 +1,22 @@ +includes: projects/task/default.yaml +task_type: sweep_big +slurm_config: big +dataset: + meta_processor: ShardedHow2MetaProcessor + train_path: data/how2/how2_s3d_train.lst + val_path: data/how2/how2_s3d_val.lst + video_processor: ShardedVideoProcessor + vfeat_dir: data/feat/feat_how2_s3d_shard_small + text_processor: ShardedTextProcessor + tfeat_dir: data/feat/feat_how2_s3d_shard_small/raw_caption_dedup.bert-base-uncased. + aligner: FixedLenAligner +# disable direct running of this yaml +eval: + save_path: runs/task +fairseq: + checkpoint: + save_dir: runs/task + save_interval_updates: 1024 + keep_interval_updates: 2 + keep_last_epochs: 30 + diff --git a/data/fairseq/examples/MMPT/projects/task/test.yaml b/data/fairseq/examples/MMPT/projects/task/test.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0a98445241a17bbfda84a3b3be0687d31e49d12f --- /dev/null +++ b/data/fairseq/examples/MMPT/projects/task/test.yaml @@ -0,0 +1,13 @@ +# this yaml cannot be run alone: implement a test_${dataset}.yaml +slurm_config: big +task_type: local_predict +dataset: + split: test + video_processor: VideoProcessor + aligner: DSAligner + bert_name: bert-base-uncased +fairseq: + dataset: + batch_size: 256 + valid_subset: test + num_workers: 2 diff --git a/data/fairseq/examples/MMPT/projects/task/test_coin.yaml b/data/fairseq/examples/MMPT/projects/task/test_coin.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6d919df7c2a69de33debd03a6d9b1a513da39618 --- /dev/null +++ b/data/fairseq/examples/MMPT/projects/task/test_coin.yaml @@ -0,0 +1,24 @@ +includes: projects/task/test.yaml +dataset: + split: test + test_path: data/coin/COIN.json + meta_processor: COINActionSegmentationMetaProcessor + vfeat_dir: data/feat/feat_coin_s3d + video_processor: VideoProcessor + text_processor: COINActionSegmentationTextProcessor + aligner: COINActionSegmentationAligner + num_iso_layer: 12 + sliding_window: 16 + sliding_window_size: 32 +model: + model_cls: MMFusionActionSegmentation + mm_encoder_cls: MMBertForTokenClassification +eval: + save_path: runs/task/coin/eval +fairseq: + dataset: + batch_size: 1 + common_eval: + path: runs/task/coin/checkpoint_best.pt +metric: COINActionSegmentationMetric +predictor: COINPredictor diff --git a/data/fairseq/examples/MMPT/projects/task/test_coin_videoclip.yaml b/data/fairseq/examples/MMPT/projects/task/test_coin_videoclip.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b41f5bc4890405993217859c22653fa77a0dd4c3 --- /dev/null +++ b/data/fairseq/examples/MMPT/projects/task/test_coin_videoclip.yaml @@ -0,0 +1,7 @@ +includes: projects/task/test_coin.yaml +model: + model_cls: MMFusionSeparateActionSegmentation + mm_encoder_cls: + video_encoder_cls: MMBertForTokenClassification + text_encoder_cls: BertModel # dummy, not used. + num_hidden_video_layers: 6 diff --git a/data/fairseq/examples/MMPT/projects/task/test_coin_zs.yaml b/data/fairseq/examples/MMPT/projects/task/test_coin_zs.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5d19b09f1dee1922ffddf5bf030135cb46016dc1 --- /dev/null +++ b/data/fairseq/examples/MMPT/projects/task/test_coin_zs.yaml @@ -0,0 +1,13 @@ +includes: projects/task/test_coin.yaml +model: + model_cls: MMFusionSeparate + mm_encoder_cls: + video_encoder_cls: MMBertForEncoder + text_encoder_cls: BertModel + num_hidden_video_layers: 6 +eval: + save_path: runs/task/coin_zs/eval +fairseq: + common_eval: + path: runs/task/checkpoint_best.pt +predictor: COINZSPredictor diff --git a/data/fairseq/examples/MMPT/projects/task/test_crosstask.yaml b/data/fairseq/examples/MMPT/projects/task/test_crosstask.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6dd778e30be543a51997f16e326aa29f6e49e973 --- /dev/null +++ b/data/fairseq/examples/MMPT/projects/task/test_crosstask.yaml @@ -0,0 +1,32 @@ +includes: projects/task/test.yaml +dataset: + split: test + meta_processor: CrossTaskMetaProcessor + test_path: data/crosstask/crosstask_release/videos_val.csv + train_csv_path: data/crosstask/crosstask_release/videos.csv + val_path: data/crosstask/crosstask_release/videos_val.csv # dummy + val_csv_path: data/crosstask/crosstask_release/videos_val.csv + primary_path: data/crosstask/crosstask_release/tasks_primary.txt + related_path: data/crosstask/crosstask_release/tasks_related.txt + vfeat_dir: data/feat/feat_crosstask_s3d + annotation_path: data/crosstask/crosstask_release/annotations + n_train: 30 + video_processor: CrossTaskVideoProcessor + text_processor: CrossTaskTextProcessor + aligner: CrossTaskAligner + num_iso_layer: 12 + sliding_window: 16 + sliding_window_size: 32 +model: + model_cls: MMFusionActionLocalization + mm_encoder_cls: MMBertForJoint +eval: + save_path: runs/task/crosstask/eval +fairseq: + # read code and find what is the checkpoint arg. + dataset: + batch_size: 1 + common_eval: + path: runs/task/crosstask/checkpoint_best.pt +metric: CrossTaskMetric +predictor: CrossTaskPredictor diff --git a/data/fairseq/examples/MMPT/projects/task/test_crosstask_videoclip.yaml b/data/fairseq/examples/MMPT/projects/task/test_crosstask_videoclip.yaml new file mode 100644 index 0000000000000000000000000000000000000000..df12535d231ffe6d1759f77f161fe48d0833f4e4 --- /dev/null +++ b/data/fairseq/examples/MMPT/projects/task/test_crosstask_videoclip.yaml @@ -0,0 +1,7 @@ +includes: projects/task/test_crosstask.yaml +model: + model_cls: MMFusionSeparateActionLocalization + mm_encoder_cls: + video_encoder_cls: MMBertForEncoder + text_encoder_cls: BertModel # dummy, not used. + num_hidden_video_layers: 6 diff --git a/data/fairseq/examples/MMPT/projects/task/test_crosstask_zs.yaml b/data/fairseq/examples/MMPT/projects/task/test_crosstask_zs.yaml new file mode 100644 index 0000000000000000000000000000000000000000..19386e495b37ab2570cc1d763277193b28309814 --- /dev/null +++ b/data/fairseq/examples/MMPT/projects/task/test_crosstask_zs.yaml @@ -0,0 +1,32 @@ +includes: projects/task/test.yaml +dataset: + split: test + meta_processor: CrossTaskMetaProcessor + test_path: data/crosstask/crosstask_release/videos_val.csv + train_csv_path: data/crosstask/crosstask_release/videos.csv + val_path: data/crosstask/crosstask_release/videos_val.csv # dummy + val_csv_path: data/crosstask/crosstask_release/videos_val.csv + primary_path: data/crosstask/crosstask_release/tasks_primary.txt + related_path: data/crosstask/crosstask_release/tasks_related.txt + vfeat_dir: data/feat/feat_crosstask_s3d + annotation_path: data/crosstask/crosstask_release/annotations + n_train: 30 + video_processor: CrossTaskVideoProcessor + text_processor: CrossTaskTextProcessor + aligner: CrossTaskAligner + num_iso_layer: 12 + sliding_window: 16 + sliding_window_size: 32 +model: + model_cls: MMFusionActionLocalization + mm_encoder_cls: MMBertForJoint +eval: + save_path: runs/task/crosstask_zs/eval +fairseq: + # read code and find what is the checkpoint arg. + dataset: + batch_size: 1 + common_eval: + path: runs/task/checkpoint_best.pt # load the best from how2 on ACL submission: runs/task/checkpoint11.pt +metric: CrossTaskMetric +predictor: CrossTaskPredictor diff --git a/data/fairseq/examples/MMPT/projects/task/test_crosstask_zs_videoclip.yaml b/data/fairseq/examples/MMPT/projects/task/test_crosstask_zs_videoclip.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7f0198276f74f592413e99fb10672431543a8f67 --- /dev/null +++ b/data/fairseq/examples/MMPT/projects/task/test_crosstask_zs_videoclip.yaml @@ -0,0 +1,7 @@ +includes: projects/task/test_crosstask_zs.yaml +model: + model_cls: MMFusionSeparateActionLocalization + mm_encoder_cls: + video_encoder_cls: MMBertForEncoder + text_encoder_cls: BertModel # dummy, not used. + num_hidden_video_layers: 6 diff --git a/data/fairseq/examples/MMPT/projects/task/test_didemo_zs.yaml b/data/fairseq/examples/MMPT/projects/task/test_didemo_zs.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4b53dca71e4a0e77793483ee971379f0476366f3 --- /dev/null +++ b/data/fairseq/examples/MMPT/projects/task/test_didemo_zs.yaml @@ -0,0 +1,23 @@ +includes: projects/task/test.yaml +dataset: + meta_processor: DiDeMoMetaProcessor + test_path: data/didemo/test_data.json + video_processor: VideoProcessor + vfeat_dir: data/feat/feat_didemo_s3d + text_processor: DiDeMoTextProcessor + aligner: DiDeMoAligner + num_iso_layer: 12 +model: + model_cls: MMFusionSeparate + mm_encoder_cls: + video_encoder_cls: MMBertForEncoder + text_encoder_cls: BertModel + num_hidden_video_layers: 6 +eval: + save_path: runs/task/didemo_zs/eval +fairseq: + # read code and find what is the checkpoint arg. + common_eval: + path: runs/task/checkpoint_best.pt +metric: DiDeMoMetric +predictor: DiDeMoPredictor diff --git a/data/fairseq/examples/MMPT/projects/task/test_vtt.yaml b/data/fairseq/examples/MMPT/projects/task/test_vtt.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2f809b306d5d2b4d50d83ca3f9fa2db3f64530a5 --- /dev/null +++ b/data/fairseq/examples/MMPT/projects/task/test_vtt.yaml @@ -0,0 +1,19 @@ +includes: projects/task/test.yaml +dataset: + meta_processor: MSRVTTMetaProcessor + test_path: data/msrvtt/MSRVTT_JSFUSION_test.csv + video_processor: VideoProcessor + vfeat_dir: data/feat/feat_vtt_s3d + text_processor: MSRVTTTextProcessor + num_iso_layer: 12 +model: + model_cls: MMFusionJoint + mm_encoder_cls: MMBertForJoint +eval: + save_path: runs/task/vtt/eval +fairseq: + # read code and find what is the checkpoint arg. + common_eval: + path: runs/task/vtt/checkpoint_last.pt +metric: RetrievalMetric +predictor: RetrievalPredictor diff --git a/data/fairseq/examples/MMPT/projects/task/test_vtt_videoclip.yaml b/data/fairseq/examples/MMPT/projects/task/test_vtt_videoclip.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cb6564394c8de5edebe41d9a90e8233d2ef90edf --- /dev/null +++ b/data/fairseq/examples/MMPT/projects/task/test_vtt_videoclip.yaml @@ -0,0 +1,8 @@ +includes: projects/task/test_vtt.yaml +model: + model_cls: MMFusionSeparate + mm_encoder_cls: + video_encoder_cls: MMBertForEncoder + text_encoder_cls: BertModel + num_hidden_video_layers: 6 + diff --git a/data/fairseq/examples/MMPT/projects/task/test_vtt_zs.yaml b/data/fairseq/examples/MMPT/projects/task/test_vtt_zs.yaml new file mode 100644 index 0000000000000000000000000000000000000000..57340924b434338815743d0d2c326737ccfe3818 --- /dev/null +++ b/data/fairseq/examples/MMPT/projects/task/test_vtt_zs.yaml @@ -0,0 +1,13 @@ +includes: projects/task/test_vtt.yaml +model: + model_cls: MMFusionSeparate + mm_encoder_cls: + video_encoder_cls: MMBertForEncoder + text_encoder_cls: BertModel + num_hidden_video_layers: 6 +eval: + save_path: runs/task/vtt_zs/eval +fairseq: + # read code and find what is the checkpoint arg. + common_eval: + path: runs/task/checkpoint_best.pt diff --git a/data/fairseq/examples/MMPT/projects/task/test_vttqa.yaml b/data/fairseq/examples/MMPT/projects/task/test_vttqa.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ddf813c535c0ecd6e6fd81ba9013a88f8ec4296b --- /dev/null +++ b/data/fairseq/examples/MMPT/projects/task/test_vttqa.yaml @@ -0,0 +1,20 @@ +includes: projects/task/test.yaml +dataset: + meta_processor: MSRVTTQAMetaProcessor + test_path: data/msrvtt-qa/MSR_MC_test.csv + video_processor: VideoProcessor + vfeat_dir: data/feat/feat_vtt_s3d + text_processor: MSRVTTQATextProcessor + aligner: MSRVTTQAAligner + num_iso_layer: 12 +model: + model_cls: MMFusionJoint + mm_encoder_cls: MMBertForJoint +eval: + save_path: runs/task/vttqa/eval +fairseq: + # read code and find what is the checkpoint arg. + common_eval: + path: runs/task/vttqa/checkpoint_last.pt +metric: QAMetric +predictor: QAPredictor diff --git a/data/fairseq/examples/MMPT/projects/task/test_vttqa_videoclip.yaml b/data/fairseq/examples/MMPT/projects/task/test_vttqa_videoclip.yaml new file mode 100644 index 0000000000000000000000000000000000000000..32a41e861c2e71642d4e79f8cf9dc314a5a3f621 --- /dev/null +++ b/data/fairseq/examples/MMPT/projects/task/test_vttqa_videoclip.yaml @@ -0,0 +1,8 @@ +includes: projects/task/test_vttqa.yaml +model: + model_cls: MMFusionSeparate + mm_encoder_cls: + video_encoder_cls: MMBertForEncoder + text_encoder_cls: BertModel + num_hidden_video_layers: 6 + diff --git a/data/fairseq/examples/MMPT/projects/task/test_vttqa_zs.yaml b/data/fairseq/examples/MMPT/projects/task/test_vttqa_zs.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5e0e29d20729445258f5d07348fd1799f7feecf4 --- /dev/null +++ b/data/fairseq/examples/MMPT/projects/task/test_vttqa_zs.yaml @@ -0,0 +1,13 @@ +includes: projects/task/test_vttqa.yaml +model: + model_cls: MMFusionSeparate + mm_encoder_cls: + video_encoder_cls: MMBertForEncoder + text_encoder_cls: BertModel + num_hidden_video_layers: 6 +eval: + save_path: runs/task/vttqa_zs/eval +fairseq: + # read code and find what is the checkpoint arg. + common_eval: + path: runs/task/checkpoint_best.pt diff --git a/data/fairseq/examples/MMPT/projects/task/test_youcook.yaml b/data/fairseq/examples/MMPT/projects/task/test_youcook.yaml new file mode 100644 index 0000000000000000000000000000000000000000..092b680fa6400458c447abe4f5cc5e452ffdb54f --- /dev/null +++ b/data/fairseq/examples/MMPT/projects/task/test_youcook.yaml @@ -0,0 +1,22 @@ +includes: projects/task/test.yaml +dataset: + meta_processor: YoucookMetaProcessor + test_path: data/youcook/youcook_val.pkl + trainval_annotation: data/youcook/youcookii_annotations_trainval.json + use_annotation_text: True + video_processor: YoucookVideoProcessor + vfeat_dir: data/feat/feat_youcook_s3d # /checkpoint/huxu/feat/youcook_vmz # /checkpoint/prarora/berniehuang/feat_youcook_vmz + text_processor: TextProcessor + aligner: DSAligner + num_iso_layer: 12 +model: + model_cls: MMFusionJoint + mm_encoder_cls: MMBertForJoint +eval: + save_path: runs/task/youcook/eval +fairseq: + # read code and find what is the checkpoint arg. + common_eval: + path: runs/task/youcook/checkpoint_last.pt +metric: RetrievalMetric +predictor: RetrievalPredictor diff --git a/data/fairseq/examples/MMPT/projects/task/test_youcook_videoclip.yaml b/data/fairseq/examples/MMPT/projects/task/test_youcook_videoclip.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b85ea434749009f50eea057146403691bb48c4a9 --- /dev/null +++ b/data/fairseq/examples/MMPT/projects/task/test_youcook_videoclip.yaml @@ -0,0 +1,8 @@ +includes: projects/task/test_youcook.yaml +model: + model_cls: MMFusionSeparate + mm_encoder_cls: + video_encoder_cls: MMBertForEncoder + text_encoder_cls: BertModel + num_hidden_video_layers: 6 + diff --git a/data/fairseq/examples/MMPT/projects/task/test_youcook_zs.yaml b/data/fairseq/examples/MMPT/projects/task/test_youcook_zs.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0a5875bea4eaca1e46c65dfdc50b586550cc6f36 --- /dev/null +++ b/data/fairseq/examples/MMPT/projects/task/test_youcook_zs.yaml @@ -0,0 +1,13 @@ +includes: projects/task/test_youcook.yaml +model: + model_cls: MMFusionSeparate + mm_encoder_cls: + video_encoder_cls: MMBertForEncoder + text_encoder_cls: BertModel + num_hidden_video_layers: 6 +eval: + save_path: runs/task/youcook_zs/eval +fairseq: + # read code and find what is the checkpoint arg. + common_eval: + path: runs/task/checkpoint_best.pt diff --git a/data/fairseq/examples/MMPT/projects/task/test_youcookcap.yaml b/data/fairseq/examples/MMPT/projects/task/test_youcookcap.yaml new file mode 100644 index 0000000000000000000000000000000000000000..24f6518b7b4e210289f5200edafdb436d569adf7 --- /dev/null +++ b/data/fairseq/examples/MMPT/projects/task/test_youcookcap.yaml @@ -0,0 +1,23 @@ +includes: projects/task/test.yaml +dataset: + meta_processor: YoucookNLGMetaProcessor + test_path: data/youcook/val_list.txt + trainval_annotation: data/youcook/youcookii_annotations_trainval.json + video_processor: YoucookVideoProcessor + vfeat_dir: data/feat/feat_youcook_s3d + text_processor: NLGTextProcessor + aligner: DSNLGAligner +model: + model_cls: MMFusionNLG + mm_encoder_cls: MMBertForNLG + max_decode_length: 24 +eval: + save_path: runs/task/youcookcap/eval +fairseq: + # read code and find what is the checkpoint arg. + common_eval: + path: runs/task/youcookcap/checkpoint_best.pt +metric: NLGMetric +predictor: NLGPredictor +gen_param: + num_beams: 5 diff --git a/data/fairseq/examples/MMPT/projects/task/vtt.yaml b/data/fairseq/examples/MMPT/projects/task/vtt.yaml new file mode 100644 index 0000000000000000000000000000000000000000..395e2ee6fe96899f9c47e39ad33b558358431554 --- /dev/null +++ b/data/fairseq/examples/MMPT/projects/task/vtt.yaml @@ -0,0 +1,25 @@ +includes: projects/task/ft.yaml +dataset: + meta_processor: MSRVTTMetaProcessor + train_path: data/msrvtt/MSRVTT_train.csv + jsfusion_path: data/msrvtt/MSRVTT_JSFUSION_test.csv + full_test_path: data/msrvtt/MSRVTT_FULL_test.csv + dup: 20 + val_path: data/msrvtt/MSRVTT_JSFUSION_test.csv + vfeat_dir: data/feat/feat_vtt_s3d + text_processor: MSRVTTTextProcessor + json_path: data/msrvtt/MSRVTT_data.json + aligner: DSAligner + num_iso_layer: 12 +model: + model_cls: MMFusionJoint + mm_encoder_cls: MMBertForJoint +loss: + loss_cls: T2VContraLoss +fairseq: + dataset: + batch_size: 256 + optimization: + max_epoch: 10 + checkpoint: + save_dir: runs/task/vtt diff --git a/data/fairseq/examples/MMPT/projects/task/vtt_videoclip.yaml b/data/fairseq/examples/MMPT/projects/task/vtt_videoclip.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a9892cab01b810175dcef2ee91f0e64e96bd36d9 --- /dev/null +++ b/data/fairseq/examples/MMPT/projects/task/vtt_videoclip.yaml @@ -0,0 +1,12 @@ +includes: projects/task/vtt.yaml +model: + model_cls: MMFusionSeparate + mm_encoder_cls: + video_encoder_cls: MMBertForEncoder + text_encoder_cls: BertModel + num_hidden_video_layers: 6 +fairseq: + dataset: + batch_size: 224 +# model_cls: MMFusionShare +# mm_encoder_cls: MMBertForEncoder diff --git a/data/fairseq/examples/MMPT/projects/task/vttqa.yaml b/data/fairseq/examples/MMPT/projects/task/vttqa.yaml new file mode 100644 index 0000000000000000000000000000000000000000..56d578eff0104e5dcb4cea8720303100a9964db9 --- /dev/null +++ b/data/fairseq/examples/MMPT/projects/task/vttqa.yaml @@ -0,0 +1,23 @@ +includes: projects/task/ft.yaml +dataset: + meta_processor: MSRVTTMetaProcessor + train_path: data/msrvtt/MSRVTT_train.csv + dup: 20 + val_path: data/msrvtt/MSRVTT_JSFUSION_test.csv + vfeat_dir: data/feat/feat_vtt_s3d + text_processor: MSRVTTTextProcessor + json_path: data/msrvtt/MSRVTT_data.json + aligner: DSAligner + num_iso_layer: 12 +model: + model_cls: MMFusionJoint + mm_encoder_cls: MMBertForJoint +loss: + loss_cls: V2TContraLoss +fairseq: + dataset: + batch_size: 128 + optimization: + max_epoch: 5 + checkpoint: + save_dir: runs/task/vttqa diff --git a/data/fairseq/examples/MMPT/projects/task/vttqa_videoclip.yaml b/data/fairseq/examples/MMPT/projects/task/vttqa_videoclip.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2d484ca8a5de0e96172b49b746f93d052f50dea7 --- /dev/null +++ b/data/fairseq/examples/MMPT/projects/task/vttqa_videoclip.yaml @@ -0,0 +1,10 @@ +includes: projects/task/vttqa.yaml +model: + model_cls: MMFusionSeparate + mm_encoder_cls: + video_encoder_cls: MMBertForEncoder + text_encoder_cls: BertModel + num_hidden_video_layers: 6 + +# model_cls: MMFusionShare +# mm_encoder_cls: MMBertForEncoder diff --git a/data/fairseq/examples/MMPT/projects/task/youcook.yaml b/data/fairseq/examples/MMPT/projects/task/youcook.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e0cd841747a5cfa4c77a94d2478bfcb029517678 --- /dev/null +++ b/data/fairseq/examples/MMPT/projects/task/youcook.yaml @@ -0,0 +1,25 @@ +includes: projects/task/ft.yaml +dataset: + meta_processor: YoucookMetaProcessor + train_path: data/youcook/youcook_train.pkl + val_path: data/youcook/youcook_val.pkl + trainval_annotation: data/youcook/youcookii_annotations_trainval.json + use_annotation_text: True + video_processor: YoucookVideoProcessor + vfeat_dir: data/feat/feat_youcook_s3d # /checkpoint/huxu/feat/youcook_vmz # /checkpoint/prarora/berniehuang/feat_youcook_vmz + text_processor: TextProcessor + aligner: DSAligner + num_iso_layer: 12 +model: + model_cls: MMFusionJoint + mm_encoder_cls: MMBertForJoint +loss: + loss_cls: T2VContraLoss +fairseq: + dataset: + batch_size: 128 + optimization: + max_epoch: 10 + checkpoint: + save_dir: runs/task/youcook + diff --git a/data/fairseq/examples/MMPT/projects/task/youcook_videoclip.yaml b/data/fairseq/examples/MMPT/projects/task/youcook_videoclip.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e3e901c30ca8ef0e858b9fb0a317aea0b55f6e34 --- /dev/null +++ b/data/fairseq/examples/MMPT/projects/task/youcook_videoclip.yaml @@ -0,0 +1,9 @@ +includes: projects/task/youcook.yaml +model: + model_cls: MMFusionSeparate + mm_encoder_cls: + video_encoder_cls: MMBertForEncoder + text_encoder_cls: BertModel + num_hidden_video_layers: 6 + # model_cls: MMFusionShare + # mm_encoder_cls: MMBertForEncoder diff --git a/data/fairseq/examples/MMPT/projects/task/youcookcap.yaml b/data/fairseq/examples/MMPT/projects/task/youcookcap.yaml new file mode 100644 index 0000000000000000000000000000000000000000..047735f21716777354101ebaa230772244ac79e5 --- /dev/null +++ b/data/fairseq/examples/MMPT/projects/task/youcookcap.yaml @@ -0,0 +1,23 @@ +# finetuning for youcook captioning. +includes: projects/task/ft.yaml +dataset: + meta_processor: YoucookNLGMetaProcessor + train_path: data/youcook/train_list.txt + val_path: data/youcook/val_list.txt + trainval_annotation: data/youcook/youcookii_annotations_trainval.json + video_processor: YoucookVideoProcessor + vfeat_dir: data/feat/feat_youcook_s3d + text_processor: NLGTextProcessor + aligner: DSNLGAligner +model: + model_cls: MMFusionNLG + mm_encoder_cls: MMBertForNLG +loss: + loss_cls: NLGLoss +fairseq: + dataset: + batch_size: 128 + optimization: + max_epoch: 10 + checkpoint: + save_dir: runs/task/youcookcap diff --git a/data/fairseq/examples/MMPT/scripts/text_token_extractor/configs/bert-base-uncased.yaml b/data/fairseq/examples/MMPT/scripts/text_token_extractor/configs/bert-base-uncased.yaml new file mode 100644 index 0000000000000000000000000000000000000000..473dd9b45b78d6d4443921fb5f355356db4f5531 --- /dev/null +++ b/data/fairseq/examples/MMPT/scripts/text_token_extractor/configs/bert-base-uncased.yaml @@ -0,0 +1,5 @@ +dataset: + bert_name: bert-base-uncased + caption_pkl_path: data/how2/raw_caption_dedup.pkl + use_fast: true + target_dir: data/feat/feat_how2_s3d_shard_small diff --git a/data/fairseq/examples/MMPT/scripts/text_token_extractor/pretokenization.py b/data/fairseq/examples/MMPT/scripts/text_token_extractor/pretokenization.py new file mode 100644 index 0000000000000000000000000000000000000000..29ae5dc151085cc59de03e4e8bfc6e5341defb8b --- /dev/null +++ b/data/fairseq/examples/MMPT/scripts/text_token_extractor/pretokenization.py @@ -0,0 +1,106 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +# +# This source code is licensed under the MIT license found in the +# LICENSE file in the root directory of this source tree. + +import pickle +import os +import argparse +import numpy as np + +from torch.utils.data import Dataset, DataLoader +from mmpt.processors import PKLJSONStrTextProcessor +from mmpt.utils import ShardedTensor, recursive_config + + +class TokenizerDataset(Dataset): + def __init__(self, config): + self.text_processor = PKLJSONStrTextProcessor(config) + self.video_ids = list(self.text_processor.data.keys()) + + def __getitem__(self, idx): + video_id = self.video_ids[idx] + return video_id, self.text_processor(video_id) + + def __len__(self): + return len(self.video_ids) + + +def numpify(shard_idx, video_ids, captions, target_dir, split, prefix, max_cap_len=32): + startends = [] + caps_ids = [] + for video_id in video_ids: + caption = captions[video_id] + startend = [] + cap_ids = [] + for start, end, cap in zip( + caption["start"], caption["end"], caption["cap"]): + startend.append(np.array([start, end]).astype("float32")) + cap_id = np.full((max_cap_len,), -1, dtype=np.int32) + cap = cap[:max_cap_len] + cap_id[:len(cap)] = cap + cap_ids.append(cap_id) + startends.append(np.stack(startend)) + caps_ids.append(np.stack(cap_ids)) + + startends = ShardedTensor.from_list(startends) + target_path = os.path.join( + target_dir, + prefix + split + "_" + str(shard_idx) + ) + print("save to", target_path) + startends.save(target_path + ".startends") + caps_ids = ShardedTensor.from_list(caps_ids) + caps_ids.save(target_path + ".caps_ids") + + +def sharding(config, out_file): + with open(out_file, "rb") as fr: + captions = pickle.load(fr) + target_dir = config.target_dir + prefix = os.path.basename( + os.path.splitext(config.caption_pkl_path)[0] + ) + "." + config.bert_name + "." + for split in ["train", "val"]: + target_path = os.path.join(target_dir, split + "_meta") + with open(target_path + ".pkl", "rb") as fr: + meta = pickle.load(fr) + print("load meta", target_path, len(meta)) + for shard_id in meta: + numpify( + shard_id, meta[shard_id], captions, + target_dir, split, prefix + ) + + +def tokenize(config, out_file): + def collator(samples): + return samples + dataset = TokenizerDataset(config) + data = {} + for idx, batch in enumerate( + DataLoader(dataset, collate_fn=collator, num_workers=16)): + for video_id, caption in batch: + data[video_id] = caption + if idx % 5000 == 0: + print(idx) + with open(out_file, "wb") as fw: + pickle.dump(data, fw, pickle.HIGHEST_PROTOCOL) + + +def main(args): + config = recursive_config(args.config).dataset + + out_file = os.path.splitext(config.caption_pkl_path)[0] \ + + "." + config.bert_name + ".pkl" + if not os.path.isfile(out_file): + tokenize(config, out_file) + sharding(config, out_file) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description="pretokenize (raw_)caption.json into pkl.") + parser.add_argument('config', type=str) + args = parser.parse_args() + main(args) diff --git a/data/fairseq/examples/MMPT/scripts/video_feature_extractor/extract.py b/data/fairseq/examples/MMPT/scripts/video_feature_extractor/extract.py new file mode 100644 index 0000000000000000000000000000000000000000..b5ee7b778890994aa87d463b896f653792c073f2 --- /dev/null +++ b/data/fairseq/examples/MMPT/scripts/video_feature_extractor/extract.py @@ -0,0 +1,157 @@ +# Copyright Howto100M authors. +# Copyright (c) Facebook, Inc. All Rights Reserved + +import torch as th +import torch.nn.functional as F +import math +import numpy as np +import argparse + +from torch.utils.data import DataLoader +from model import get_model +from preprocessing import Preprocessing +from random_sequence_shuffler import RandomSequenceSampler + +from tqdm import tqdm +from pathbuilder import PathBuilder +from videoreader import VideoLoader + + +parser = argparse.ArgumentParser(description='Easy video feature extractor') + +parser.add_argument('--vdir', type=str) +parser.add_argument('--fdir', type=str) +parser.add_argument('--hflip', type=int, default=0) + +parser.add_argument('--batch_size', type=int, default=64, + help='batch size') +parser.add_argument('--type', type=str, default='2d', + help='CNN type') +parser.add_argument('--half_precision', type=int, default=0, + help='output half precision float') +parser.add_argument('--num_decoding_thread', type=int, default=4, + help='Num parallel thread for video decoding') +parser.add_argument('--l2_normalize', type=int, default=1, + help='l2 normalize feature') +parser.add_argument('--resnext101_model_path', type=str, default='model/resnext101.pth', + help='Resnext model path') +parser.add_argument('--vmz_model_path', type=str, default='model/r2plus1d_34_clip8_ig65m_from_scratch-9bae36ae.pth', + help='vmz model path') + +args = parser.parse_args() + + +# TODO: refactor all args into config. (current code is from different people.) +CONFIGS = { + "2d": { + "fps": 1, + "size": 224, + "centercrop": False, + "shards": 0, + }, + "3d": { + "fps": 24, + "size": 112, + "centercrop": True, + "shards": 0, + }, + "s3d": { + "fps": 30, + "size": 224, + "centercrop": True, + "shards": 0, + }, + "vmz": { + "fps": 24, + "size": 112, + "centercrop": True, + "shards": 0, + }, + "vae": { + "fps": 2, + "size": 256, + "centercrop": True, + "shards": 100, + } +} + +config = CONFIGS[args.type] + + +video_dirs = args.vdir +feature_dir = args.fdir + +video_dict = PathBuilder.build(video_dirs, feature_dir, ".npy", config["shards"]) + +dataset = VideoLoader( + video_dict=video_dict, + framerate=config["fps"], + size=config["size"], + centercrop=config["centercrop"], + hflip=args.hflip +) +n_dataset = len(dataset) +sampler = RandomSequenceSampler(n_dataset, 10) +loader = DataLoader( + dataset, + batch_size=1, + shuffle=False, + num_workers=args.num_decoding_thread, + sampler=sampler if n_dataset > 10 else None, +) +preprocess = Preprocessing(args.type) +model = get_model(args) + +with th.no_grad(): + for k, data in tqdm(enumerate(loader), total=loader.__len__(), ascii=True): + input_file = data['input'][0] + output_file = data['output'][0] + if len(data['video'].shape) > 3: + video = data['video'].squeeze() + if len(video.shape) == 4: + video = preprocess(video) + n_chunk = len(video) + if args.type == 'vmz': + n_chunk = math.ceil(n_chunk/float(3)) + features = th.cuda.FloatTensor(n_chunk, 512).fill_(0) + elif args.type == 's3d': + features = th.cuda.FloatTensor(n_chunk, 512).fill_(0) + elif args.type == "vae": + features = th.cuda.LongTensor(n_chunk, 1024).fill_(0) + else: + features = th.cuda.FloatTensor(n_chunk, 2048).fill_(0) + n_iter = int(math.ceil(n_chunk / float(args.batch_size))) + for i in range(n_iter): + factor = 1 + if args.type == 'vmz': + factor = 3 + min_ind = factor * i * args.batch_size + max_ind = factor * (i + 1) * args.batch_size + video_batch = video[min_ind:max_ind:factor].cuda() + if args.type == '2d': + batch_features = model(video_batch) # (51, 487), (51, 512) + elif args.type == 's3d': + batch_features = model(video_batch) + batch_features = batch_features['video_embedding'] + elif args.type == "vae": + # image_code. + batch_features = model(video_batch) + else: + batch_pred, batch_features = model(video_batch) # (51, 487), (51, 512) + if args.l2_normalize: + batch_features = F.normalize(batch_features, dim=1) + features[i*args.batch_size:(i+1)*args.batch_size] = batch_features + features = features.cpu().numpy() + if args.half_precision: + if args.type == "vae": + features = features.astype(np.int16) + else: + features = features.astype('float16') + else: + if args.type == "vae": + features = features.astype(np.int32) + else: + features = features.astype('float32') + np.save(output_file, features) + else: + print('Video {} error.'.format(input_file)) diff --git a/data/fairseq/examples/MMPT/scripts/video_feature_extractor/how2/s3d.sh b/data/fairseq/examples/MMPT/scripts/video_feature_extractor/how2/s3d.sh new file mode 100644 index 0000000000000000000000000000000000000000..90102c89fbb4fb413951dfd049b30b8dfe7cbe99 --- /dev/null +++ b/data/fairseq/examples/MMPT/scripts/video_feature_extractor/how2/s3d.sh @@ -0,0 +1,8 @@ +#!/bin/bash + + +python scripts/video_feature_extractor/extract.py \ + --vdir \ + --fdir data/feat/feat_how2_s3d \ + --type=s3d --num_decoding_thread=4 \ + --batch_size 32 --half_precision 1 diff --git a/data/fairseq/examples/MMPT/scripts/video_feature_extractor/model.py b/data/fairseq/examples/MMPT/scripts/video_feature_extractor/model.py new file mode 100644 index 0000000000000000000000000000000000000000..ac266e844c86246bbfce02b9e6a2999353661df9 --- /dev/null +++ b/data/fairseq/examples/MMPT/scripts/video_feature_extractor/model.py @@ -0,0 +1,58 @@ +# Copyright (c) Howto100M authors and Facebook, Inc. All Rights Reserved + +import torch as th + +from torch import nn + + +class GlobalAvgPool(nn.Module): + def __init__(self): + super(GlobalAvgPool, self).__init__() + + def forward(self, x): + return th.mean(x, dim=[-2, -1]) + + +def get_model(args): + assert args.type in ['2d', '3d', 'vmz', 's3d', 'vae'] + if args.type == '2d': + print('Loading 2D-ResNet-152 ...') + import torchvision.models as models + model = models.resnet152(pretrained=True) + model = nn.Sequential(*list(model.children())[:-2], GlobalAvgPool()) + model = model.cuda() + elif args.type == 'vmz': + print('Loading VMZ ...') + from vmz34 import r2plus1d_34 + model = r2plus1d_34(pretrained_path=args.vmz_model_path, pretrained_num_classes=487) + model = model.cuda() + elif args.type == 's3d': + # we use one copy of s3d instead of dup another one for feature extraction. + from mmpt.processors.models.s3dg import S3D + model = S3D('pretrained_models/s3d_dict.npy', 512) + model.load_state_dict(th.load('pretrained_models/s3d_howto100m.pth')) + model = model.cuda() + + elif args.type == '3d': + print('Loading 3D-ResneXt-101 ...') + from videocnn.models import resnext + model = resnext.resnet101( + num_classes=400, + shortcut_type='B', + cardinality=32, + sample_size=112, + sample_duration=16, + last_fc=False) + model = model.cuda() + model_data = th.load(args.resnext101_model_path) + model.load_state_dict(model_data) + elif args.type == 'vae': + from openaivae import OpenAIParallelDiscreteVAE + model = OpenAIParallelDiscreteVAE() + model = model.cuda() + else: + raise ValueError("model not supported yet.") + + model.eval() + print('loaded') + return model diff --git a/data/fairseq/examples/MMPT/scripts/video_feature_extractor/pathbuilder.py b/data/fairseq/examples/MMPT/scripts/video_feature_extractor/pathbuilder.py new file mode 100644 index 0000000000000000000000000000000000000000..2392d6d63bc293fb24068e33f707ee4ce0e74a6f --- /dev/null +++ b/data/fairseq/examples/MMPT/scripts/video_feature_extractor/pathbuilder.py @@ -0,0 +1,89 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +# +# This source code is licensed under the MIT license found in the +# LICENSE file in the root directory of this source tree. +import os +import urllib.parse +import json +import pandas as pd + +from tqdm import tqdm + + +# TODO: extending to other datasets. +supported_formats = {} + + +class PathBuilder(object): + @classmethod + def build(cls, video_dirs, feature_dir, ext, shards=0, split=None): + meta_fn = os.path.join(feature_dir, "meta_plan.json") + os.makedirs(feature_dir, exist_ok=True) + if os.path.isfile(meta_fn): + with open(meta_fn) as fr: + meta = json.load(fr) + return meta + print("searching videos...") + + video_id_to_path = {} + for video_dir in video_dirs.split(","): + # TODO: add supports of recursive listdir. + if video_dir in supported_formats: + supported_formats[video_dir].load(video_dir, video_id_to_path) + else: + for idx, fn in enumerate(tqdm(os.listdir(video_dir))): + video_fn = os.path.join(video_dir, fn) + if os.path.isfile(video_fn): + video_id = os.path.splitext(fn)[0] + video_id_to_path[video_id] = video_fn + elif os.path.isdir(video_fn): + # shards of folders. + shard_dir = video_fn + for idx, fn in enumerate(os.listdir(shard_dir)): + video_fn = os.path.join(shard_dir, fn) + if os.path.isfile(video_fn): + video_id = os.path.splitext(fn)[0] + video_id_to_path[video_id] = video_fn + + video_path, feature_path = [], [] + valid_ext = set() + for idx, video_id in enumerate(video_id_to_path): + video_path.append(video_id_to_path[video_id]) + if ext is None: + # use original file ext for format compatibility. + video_id_to_path[video_id] + path = urllib.parse.urlparse(video_id_to_path[video_id]).path + ext = os.path.splitext(path)[1] + if ext not in valid_ext: + valid_ext.add(ext) + print("adding", ext) + if shards: + shard_id = str(idx % shards) + feature_fn = os.path.join( + feature_dir, shard_id, video_id + ext) + else: + feature_fn = os.path.join( + feature_dir, video_id + ext) + feature_path.append(feature_fn) + + print("targeting", len(feature_path), "videos") + meta = { + "video_path": video_path, "feature_path": feature_path} + with open(meta_fn, "w") as fw: + json.dump(meta, fw) + + if split is not None: + splits = split.split("/") + assert len(splits) == 2 + cur, total = int(splits[0]), int(splits[1]) + assert cur < total + import math + chunk = math.ceil(len(meta["video_path"]) / total) + start = cur * chunk + end = (cur + 1) * chunk + meta = { + "video_path": meta["video_path"][start:end], + "feature_path": meta["feature_path"][start:end] + } + + return meta diff --git a/data/fairseq/examples/MMPT/scripts/video_feature_extractor/preprocessing.py b/data/fairseq/examples/MMPT/scripts/video_feature_extractor/preprocessing.py new file mode 100644 index 0000000000000000000000000000000000000000..fa0cec3a76b4665fae57d1feadd5a52469234226 --- /dev/null +++ b/data/fairseq/examples/MMPT/scripts/video_feature_extractor/preprocessing.py @@ -0,0 +1,57 @@ +# Copyright Howto100m authors. +# Copyright (c) Facebook, Inc. All Rights Reserved + +import torch as th + +class Normalize(object): + + def __init__(self, mean, std): + self.mean = th.FloatTensor(mean).view(1, 3, 1, 1) + self.std = th.FloatTensor(std).view(1, 3, 1, 1) + + def __call__(self, tensor): + tensor = (tensor - self.mean) / (self.std + 1e-8) + return tensor + +class Preprocessing(object): + + def __init__(self, type): + self.type = type + if type == '2d': + self.norm = Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) + elif type == '3d': + self.norm = Normalize(mean=[110.6, 103.2, 96.3], std=[1.0, 1.0, 1.0]) + elif type == 'vmz': + self.norm = Normalize(mean=[110.201, 100.64, 95.997], std=[58.1489, 56.4701, 55.3324]) + + def _zero_pad(self, tensor, size): + n = size - len(tensor) % size + if n == size: + return tensor + else: + z = th.zeros(n, tensor.shape[1], tensor.shape[2], tensor.shape[3]) + return th.cat((tensor, z), 0) + + def __call__(self, tensor): + if self.type == '2d': + tensor = tensor / 255.0 + tensor = self.norm(tensor) + elif self.type == 'vmz': + #tensor = self._zero_pad(tensor, 8) + tensor = self._zero_pad(tensor, 10) + tensor = self.norm(tensor) + #tensor = tensor.view(-1, 8, 3, 112, 112) + tensor = tensor.view(-1, 10, 3, 112, 112) + tensor = tensor.transpose(1, 2) + elif self.type == '3d': + tensor = self._zero_pad(tensor, 16) + tensor = self.norm(tensor) + tensor = tensor.view(-1, 16, 3, 112, 112) + tensor = tensor.transpose(1, 2) + elif self.type == 's3d': + tensor = tensor / 255.0 + tensor = self._zero_pad(tensor, 30) + tensor = tensor.view(-1, 30, 3, 224, 224) # N x 30 x 3 x H x W + tensor = tensor.transpose(1, 2) # N x 3 x 30 x H x W + # for vae do nothing + return tensor diff --git a/data/fairseq/examples/MMPT/scripts/video_feature_extractor/random_sequence_shuffler.py b/data/fairseq/examples/MMPT/scripts/video_feature_extractor/random_sequence_shuffler.py new file mode 100644 index 0000000000000000000000000000000000000000..1f3e4aceaa8f930c9eeac6283b15924e7eed5dc3 --- /dev/null +++ b/data/fairseq/examples/MMPT/scripts/video_feature_extractor/random_sequence_shuffler.py @@ -0,0 +1,29 @@ +# Copyright (c) Facebook, Inc. All Rights Reserved + +import numpy as np + +from torch.utils.data.sampler import Sampler + + +class RandomSequenceSampler(Sampler): + + def __init__(self, n_sample, seq_len): + self.n_sample = n_sample + self.seq_len = seq_len + + def _pad_ind(self, ind): + zeros = np.zeros(self.seq_len - self.n_sample % self.seq_len) + ind = np.concatenate((ind, zeros)) + return ind + + def __iter__(self): + idx = np.arange(self.n_sample) + if self.n_sample % self.seq_len != 0: + idx = self._pad_ind(idx) + idx = np.reshape(idx, (-1, self.seq_len)) + np.random.shuffle(idx) + idx = np.reshape(idx, (-1)) + return iter(idx.astype(int)) + + def __len__(self): + return self.n_sample + (self.seq_len - self.n_sample % self.seq_len) diff --git a/data/fairseq/examples/MMPT/scripts/video_feature_extractor/shard_feature.py b/data/fairseq/examples/MMPT/scripts/video_feature_extractor/shard_feature.py new file mode 100644 index 0000000000000000000000000000000000000000..f75e1dfae558fa366d15fa19e89d353003e728b3 --- /dev/null +++ b/data/fairseq/examples/MMPT/scripts/video_feature_extractor/shard_feature.py @@ -0,0 +1,64 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +# +# This source code is licensed under the MIT license found in the +# LICENSE file in the root directory of this source tree. +import numpy as np +import os +import pickle + +from mmpt.utils import ShardedTensor + + +class Shard(object): + def __init__( + self, + vfeat_dir, + tfeat_dir, + target_dir, + file_paths, + shard_size=4096 + ): + self.vfeat_dir = vfeat_dir + self.tfeat_dir = tfeat_dir + self.target_dir = target_dir + self.video_ids = {} + for split, file_path in zip(["train", "val"], file_paths): + with open(file_path) as fr: + self.video_ids[split] = [ + line.strip() for line in fr.readlines()] + self.shard_size = shard_size + + def __call__(self, split="train"): + for split in ["train", "val"]: + meta = {} + for shard_idx, shard_offset in enumerate( + range(0, len(self.video_ids[split]), self.shard_size) + ): + print(shard_idx) + meta_shard = [] + video_shard = [] + for video_id in self.video_ids[split][shard_offset:shard_offset+self.shard_size]: + meta_shard.append(video_id) + npy_file = os.path.join(self.vfeat_dir, video_id + ".npy") + video_shard.append(np.load(npy_file)) + + meta[shard_idx] = meta_shard + video_shard = ShardedTensor.from_list(video_shard) + target_path = os.path.join( + self.target_dir, split + "_" + str(shard_idx)) + video_shard.save(target_path) + + target_path = os.path.join(self.target_dir, split + "_meta") + with open(target_path + ".pkl", "wb") as fw: + pickle.dump(meta, fw, pickle.HIGHEST_PROTOCOL) + + +if __name__ == "__main__": + shard = Shard( + "data/feat/feat_how2_s3d", + "data/how2/raw_caption_dedup.bert-base-uncased", + "data/feat/feat_how2_s3d_shard_small", + ["data/how2/how2_s3d_train.lst", "data/how2/how2_s3d_val.lst"] + ) + + shard() diff --git a/data/fairseq/examples/MMPT/scripts/video_feature_extractor/videoreader.py b/data/fairseq/examples/MMPT/scripts/video_feature_extractor/videoreader.py new file mode 100644 index 0000000000000000000000000000000000000000..429e05f8bc8667408b8c2057578b1e8c0e98638c --- /dev/null +++ b/data/fairseq/examples/MMPT/scripts/video_feature_extractor/videoreader.py @@ -0,0 +1,242 @@ +# Copyright Howto100M authors. +# Copyright (c) Facebook, Inc. All Rights Reserved + +import torch as th +import pandas as pd +import os +import numpy as np +import ffmpeg +import random + +from torch.utils.data import Dataset + + +class VideoLoader(Dataset): + """modified from how2's video_feature_extractor.""" + def __init__( + self, + csv=None, + video_dict=None, + framerate=1, + size=112, + centercrop=False, + hflip=False, + **kwargs + ): + if csv is None and video_dict is None: + raise ValueError("csv and video_dict cannot be both None.") + if csv is not None: + self.csv = pd.read_csv(csv) + if video_dict is not None: + self.csv = pd.DataFrame.from_dict(video_dict) + + self.centercrop = centercrop + self.size = size + self.framerate = framerate + self.hflip = hflip + + def __len__(self): + return len(self.csv) + + def _get_video_dim(self, video_path): + probe = ffmpeg.probe(video_path) + video_stream = next((stream for stream in probe['streams'] + if stream['codec_type'] == 'video'), None) + width = int(video_stream['width']) + height = int(video_stream['height']) + return height, width + + def _get_video_info(self, video_path): + probe = ffmpeg.probe(video_path) + video_stream = next((stream for stream in probe['streams'] + if stream['codec_type'] == 'video'), None) + return video_stream + + def _get_output_dim(self, h, w): + if isinstance(self.size, tuple) and len(self.size) == 2: + return self.size + elif h >= w: + return int(h * self.size / w), self.size + else: + return self.size, int(w * self.size / h) + + def __getitem__(self, idx): + video_path = self.csv['video_path'].values[idx] + output_file = self.csv['feature_path'].values[idx] + return self._decode(output_file, video_path) + + def _decode(self, output_file, video_path): + if not(os.path.isfile(output_file)) and os.path.isfile(video_path): + try: + h, w = self._get_video_dim(video_path) + except Exception: + print('ffprobe failed at: {}'.format(video_path)) + return {'video': th.zeros(1), 'input': video_path, + 'output': output_file} + try: + os.makedirs(os.path.dirname(output_file), exist_ok=True) + height, width = self._get_output_dim(h, w) + + cmd = ( + ffmpeg + .input(video_path) + .filter('fps', fps=self.framerate) + .filter('scale', width, height) + ) + if self.hflip: + cmd = cmd.filter('hflip') + + if self.centercrop: + x = int((width - self.size) / 2.0) + y = int((height - self.size) / 2.0) + cmd = cmd.crop(x, y, self.size, self.size) + video = self._run(cmd, output_file) + except Exception: + video = th.zeros(1) + else: + video = th.zeros(1) + + return {'video': video, 'input': video_path, 'output': output_file} + + def _run(self, cmd, output_file): + out, _ = ( + cmd.output('pipe:', format='rawvideo', pix_fmt='rgb24') + .run(capture_stdout=True, quiet=True) + ) + if self.centercrop and isinstance(self.size, int): + height, width = self.size, self.size + video = np.frombuffer(out, np.uint8).reshape([-1, height, width, 3]) + video = th.from_numpy(video.astype('float32')) + return video.permute(0, 3, 1, 2) + + +class VideoVerifier(VideoLoader): + def __getitem__(self, idx): + video_path = self.csv['video_path'].values[idx] + try: + return self._get_video_info(video_path) + except Exception: + # print('ffprobe failed at: {}'.format(video_path)) + return None + + +class VideoCompressor(VideoLoader): + def __init__( + self, + csv=None, + video_dict=None, + framerate=1, + size=112, + centercrop=False, + hflip=False, + crf=32, + **kwargs + ): + super().__init__( + csv, + video_dict, + framerate, + size, + centercrop, + hflip + ) + self.crf = crf + + def _run(self, cmd, output_file): + out, _ = ( + cmd.output(filename=output_file, crf=self.crf) + .run(quiet=True) + ) + video = None + return video + + +class VideoDownloader(VideoCompressor): + """download""" + def __getitem__(self, idx): + video_path = self.csv['video_path'].values[idx] + output_file = self.csv['feature_path'].values[idx] + if not(os.path.isfile(output_file)): + os.makedirs(os.path.dirname(output_file), exist_ok=True) + cmd = "wget -O" + output_file + " " + video_path + # import subprocess + # subprocess.check_output( + # cmd, + # stderr=subprocess.STDOUT, shell=True) + os.system(cmd) + return {'video': None, 'input': video_path, 'output': output_file} + + +class AvKeyframeVideoCompressor(VideoLoader): + """extract keyframes from a video and save it as jpg. + TODO: consider to merge with `CodecProcessor`. + """ + def __init__( + self, + csv=None, + video_dict=None, + framerate=1, + size=112, + centercrop=False, + max_num_frames=5, + **kwargs + ): + super().__init__(csv, video_dict, framerate, size, centercrop) + self.max_num_frames = max_num_frames + + def _get_video_dim(self, video_fn): + """decord cannot probe the size of a video, we use pyav instead.""" + import av + with av.open(video_fn) as container: + height = container.streams.video[0].codec_context.height + width = container.streams.video[0].codec_context.width + return height, width + + def _get_output_dim(self, height, width): + """ + keep the shorter side be `self.size`, strech the other. + """ + if height >= width: + return int(height * self.size / width), self.size + else: + return self.size, int(width * self.size / height) + + def __getitem__(self, idx): + import av + video_path = self.csv['video_path'].values[idx] + output_file = self.csv['feature_path'].values[idx] + if not(os.path.isdir(output_file)) and os.path.isfile(video_path): + try: + h, w = self._get_video_dim(video_path) + except Exception: + print('probe failed at: {}'.format(video_path)) + return {'video': th.zeros(1), 'input': video_path, + 'output': output_file} + + try: + height, width = self._get_output_dim(h, w) + + # new for av. + with av.open(video_path) as container: + container.streams.video[0].thread_type = "AUTO" + container.streams.video[0].codec_context.height = height + container.streams.video[0].codec_context.width = width + if self.framerate == 0: # keyframe. + container.streams.video[0].codec_context.skip_frame = 'NONKEY' + frames = [] + for frame in container.decode(video=0): + frames.append(frame) + frames = random.sample(frames, self.max_num_frames) + + os.makedirs(output_file, exist_ok=True) + for frame in frames: + frame.to_image().save( + os.path.join( + output_file, + "%04d.jpg" % frame.index)) + except Exception: + print('extract failed at: {}'.format(video_path)) + return {'video': th.zeros(1), 'input': video_path, + 'output': output_file} + video = th.zeros(1) + return {'video': video, 'input': video_path, 'output': output_file} diff --git a/data/fairseq/examples/MMPT/setup.py b/data/fairseq/examples/MMPT/setup.py new file mode 100644 index 0000000000000000000000000000000000000000..a9a82296ea9c3a53f760c29c34020b5a90091a89 --- /dev/null +++ b/data/fairseq/examples/MMPT/setup.py @@ -0,0 +1,24 @@ +import setuptools + +with open("README.md", "r") as fh: + long_description = fh.read() + +setuptools.setup( + name="mmpt", + version="0.0.1", + author="Hu Xu, Po-yao Huang", + author_email="huxu@fb.com", + description="A package for multimodal pretraining.", + long_description=long_description, + long_description_content_type="text/markdown", + url="https://github.com/pytorch/fairseq/examples/MMPT", + packages=setuptools.find_packages(), + install_requires=[ + ], + classifiers=[ + "Programming Language :: Python :: 3", + "License :: CC-BY-NC", + "Operating System :: OS Independent", + ], + python_requires='>=3.6', +) diff --git a/data/fairseq/examples/byte_level_bpe/README.md b/data/fairseq/examples/byte_level_bpe/README.md new file mode 100644 index 0000000000000000000000000000000000000000..657092660eae42d20f67647417623b8b8cb7b66c --- /dev/null +++ b/data/fairseq/examples/byte_level_bpe/README.md @@ -0,0 +1,88 @@ +# Neural Machine Translation with Byte-Level Subwords + +https://arxiv.org/abs/1909.03341 + +We provide an implementation of byte-level byte-pair encoding (BBPE), taking IWSLT 2017 Fr-En translation as +example. + +## Data +Get data and generate fairseq binary dataset: +```bash +bash ./get_data.sh +``` + +## Model Training +Train Transformer model with Bi-GRU embedding contextualization (implemented in `gru_transformer.py`): +```bash +# VOCAB=bytes +# VOCAB=chars +VOCAB=bbpe2048 +# VOCAB=bpe2048 +# VOCAB=bbpe4096 +# VOCAB=bpe4096 +# VOCAB=bpe16384 +``` +```bash +fairseq-train "data/bin_${VOCAB}" --task translation --user-dir examples/byte_level_bpe/gru_transformer \ + --arch gru_transformer --encoder-layers 2 --decoder-layers 2 --dropout 0.3 --share-all-embeddings \ + --optimizer adam --adam-betas '(0.9, 0.98)' \ + --lr 5e-4 --lr-scheduler inverse_sqrt --warmup-updates 4000 \ + --criterion label_smoothed_cross_entropy --label-smoothing 0.1 \ + --log-format 'simple' --log-interval 100 --save-dir "checkpoints/${VOCAB}" \ + --batch-size 100 --max-update 100000 --update-freq 2 +``` + +## Generation +`fairseq-generate` requires bytes (BBPE) decoder to convert byte-level representation back to characters: +```bash +# BPE=--bpe bytes +# BPE=--bpe characters +BPE=--bpe byte_bpe --sentencepiece-model-path data/spm_bbpe2048.model +# BPE=--bpe sentencepiece --sentencepiece-model data/spm_bpe2048.model +# BPE=--bpe byte_bpe --sentencepiece-model-path data/spm_bbpe4096.model +# BPE=--bpe sentencepiece --sentencepiece-model data/spm_bpe4096.model +# BPE=--bpe sentencepiece --sentencepiece-model data/spm_bpe16384.model +``` + +```bash +fairseq-generate "data/bin_${VOCAB}" --task translation --user-dir examples/byte_level_bpe/gru_transformer \ + --source-lang fr --gen-subset test --sacrebleu --path "checkpoints/${VOCAB}/checkpoint_last.pt" \ + --tokenizer moses --moses-target-lang en ${BPE} +``` +When using `fairseq-interactive`, bytes (BBPE) encoder/decoder is required to tokenize input data and detokenize model predictions: +```bash +fairseq-interactive "data/bin_${VOCAB}" --task translation --user-dir examples/byte_level_bpe/gru_transformer \ + --path "checkpoints/${VOCAB}/checkpoint_last.pt" --input data/test.fr --tokenizer moses --moses-source-lang fr \ + --moses-target-lang en ${BPE} --buffer-size 1000 --max-tokens 10000 +``` + +## Results +| Vocabulary | Model | BLEU | +|:-------------:|:-------------:|:-------------:| +| Joint BPE 16k ([Kudo, 2018](https://arxiv.org/abs/1804.10959)) | 512d LSTM 2+2 | 33.81 | +| Joint BPE 16k | Transformer base 2+2 (w/ GRU) | 36.64 (36.72) | +| Joint BPE 4k | Transformer base 2+2 (w/ GRU) | 35.49 (36.10) | +| Joint BBPE 4k | Transformer base 2+2 (w/ GRU) | 35.61 (35.82) | +| Joint BPE 2k | Transformer base 2+2 (w/ GRU) | 34.87 (36.13) | +| Joint BBPE 2k | Transformer base 2+2 (w/ GRU) | 34.98 (35.43) | +| Characters | Transformer base 2+2 (w/ GRU) | 31.78 (33.30) | +| Bytes | Transformer base 2+2 (w/ GRU) | 31.57 (33.62) | + + +## Citation +``` +@misc{wang2019neural, + title={Neural Machine Translation with Byte-Level Subwords}, + author={Changhan Wang and Kyunghyun Cho and Jiatao Gu}, + year={2019}, + eprint={1909.03341}, + archivePrefix={arXiv}, + primaryClass={cs.CL} +} +``` + + +## Contact +Changhan Wang ([changhan@fb.com](mailto:changhan@fb.com)), +Kyunghyun Cho ([kyunghyuncho@fb.com](mailto:kyunghyuncho@fb.com)), +Jiatao Gu ([jgu@fb.com](mailto:jgu@fb.com)) diff --git a/data/fairseq/examples/byte_level_bpe/get_bitext.py b/data/fairseq/examples/byte_level_bpe/get_bitext.py new file mode 100644 index 0000000000000000000000000000000000000000..6ac1eeec1e6167ec6bafd76b37173ee6987cae7e --- /dev/null +++ b/data/fairseq/examples/byte_level_bpe/get_bitext.py @@ -0,0 +1,254 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +# +# This source code is licensed under the MIT license found in the +# LICENSE file in the root directory of this source tree. + + +import argparse +import os +import os.path as op +from collections import namedtuple +from multiprocessing import cpu_count +from typing import List, Optional + +import sentencepiece as sp +from fairseq.data.encoders.byte_bpe import ByteBPE +from fairseq.data.encoders.byte_utils import byte_encode +from fairseq.data.encoders.bytes import Bytes +from fairseq.data.encoders.characters import Characters +from fairseq.data.encoders.moses_tokenizer import MosesTokenizer +from fairseq.data.encoders.sentencepiece_bpe import SentencepieceBPE + + +SPLITS = ["train", "valid", "test"] + + +def _convert_xml(in_path: str, out_path: str): + with open(in_path) as f, open(out_path, "w") as f_o: + for s in f: + ss = s.strip() + if not ss.startswith("", "").split('">') + assert len(ss) == 2 + f_o.write(ss[1].strip() + "\n") + + +def _convert_train(in_path: str, out_path: str): + with open(in_path) as f, open(out_path, "w") as f_o: + for s in f: + ss = s.strip() + if ss.startswith("<"): + continue + f_o.write(ss.strip() + "\n") + + +def _get_bytes(in_path: str, out_path: str): + with open(in_path) as f, open(out_path, "w") as f_o: + for s in f: + f_o.write(Bytes.encode(s.strip()) + "\n") + + +def _get_chars(in_path: str, out_path: str): + with open(in_path) as f, open(out_path, "w") as f_o: + for s in f: + f_o.write(Characters.encode(s.strip()) + "\n") + + +def pretokenize(in_path: str, out_path: str, src: str, tgt: str): + Args = namedtuple( + "Args", + [ + "moses_source_lang", + "moses_target_lang", + "moses_no_dash_splits", + "moses_no_escape", + ], + ) + args = Args( + moses_source_lang=src, + moses_target_lang=tgt, + moses_no_dash_splits=False, + moses_no_escape=False, + ) + pretokenizer = MosesTokenizer(args) + with open(in_path) as f, open(out_path, "w") as f_o: + for s in f: + f_o.write(pretokenizer.encode(s.strip()) + "\n") + + +def _convert_to_bchar(in_path_prefix: str, src: str, tgt: str, out_path: str): + with open(out_path, "w") as f_o: + for lang in [src, tgt]: + with open(f"{in_path_prefix}.{lang}") as f: + for s in f: + f_o.write(byte_encode(s.strip()) + "\n") + + +def _get_bpe(in_path: str, model_prefix: str, vocab_size: int): + arguments = [ + f"--input={in_path}", + f"--model_prefix={model_prefix}", + f"--model_type=bpe", + f"--vocab_size={vocab_size}", + "--character_coverage=1.0", + "--normalization_rule_name=identity", + f"--num_threads={cpu_count()}", + ] + sp.SentencePieceTrainer.Train(" ".join(arguments)) + + +def _apply_bbpe(model_path: str, in_path: str, out_path: str): + Args = namedtuple("Args", ["sentencepiece_model_path"]) + args = Args(sentencepiece_model_path=model_path) + tokenizer = ByteBPE(args) + with open(in_path) as f, open(out_path, "w") as f_o: + for s in f: + f_o.write(tokenizer.encode(s.strip()) + "\n") + + +def _apply_bpe(model_path: str, in_path: str, out_path: str): + Args = namedtuple("Args", ["sentencepiece_model"]) + args = Args(sentencepiece_model=model_path) + tokenizer = SentencepieceBPE(args) + with open(in_path) as f, open(out_path, "w") as f_o: + for s in f: + f_o.write(tokenizer.encode(s.strip()) + "\n") + + +def _concat_files(in_paths: List[str], out_path: str): + with open(out_path, "w") as f_o: + for p in in_paths: + with open(p) as f: + for r in f: + f_o.write(r) + + +def preprocess_iwslt17( + root: str, + src: str, + tgt: str, + bpe_size: Optional[int], + need_chars: bool, + bbpe_size: Optional[int], + need_bytes: bool, +): + # extract bitext + in_root = op.join(root, f"{src}-{tgt}") + for lang in [src, tgt]: + _convert_train( + op.join(in_root, f"train.tags.{src}-{tgt}.{lang}"), + op.join(root, f"train.{lang}"), + ) + _convert_xml( + op.join(in_root, f"IWSLT17.TED.dev2010.{src}-{tgt}.{lang}.xml"), + op.join(root, f"valid.{lang}"), + ) + _convert_xml( + op.join(in_root, f"IWSLT17.TED.tst2015.{src}-{tgt}.{lang}.xml"), + op.join(root, f"test.{lang}"), + ) + # pre-tokenize + for lang in [src, tgt]: + for split in SPLITS: + pretokenize( + op.join(root, f"{split}.{lang}"), + op.join(root, f"{split}.moses.{lang}"), + src, + tgt, + ) + # tokenize with BPE vocabulary + if bpe_size is not None: + # learn vocabulary + concated_train_path = op.join(root, "train.all") + _concat_files( + [op.join(root, "train.moses.fr"), op.join(root, "train.moses.en")], + concated_train_path, + ) + bpe_model_prefix = op.join(root, f"spm_bpe{bpe_size}") + _get_bpe(concated_train_path, bpe_model_prefix, bpe_size) + os.remove(concated_train_path) + # apply + for lang in [src, tgt]: + for split in SPLITS: + _apply_bpe( + bpe_model_prefix + ".model", + op.join(root, f"{split}.moses.{lang}"), + op.join(root, f"{split}.moses.bpe{bpe_size}.{lang}"), + ) + # tokenize with bytes vocabulary + if need_bytes: + for lang in [src, tgt]: + for split in SPLITS: + _get_bytes( + op.join(root, f"{split}.moses.{lang}"), + op.join(root, f"{split}.moses.bytes.{lang}"), + ) + # tokenize with characters vocabulary + if need_chars: + for lang in [src, tgt]: + for split in SPLITS: + _get_chars( + op.join(root, f"{split}.moses.{lang}"), + op.join(root, f"{split}.moses.chars.{lang}"), + ) + # tokenize with byte-level BPE vocabulary + if bbpe_size is not None: + # learn vocabulary + bchar_path = op.join(root, "train.bchar") + _convert_to_bchar(op.join(root, "train.moses"), src, tgt, bchar_path) + bbpe_model_prefix = op.join(root, f"spm_bbpe{bbpe_size}") + _get_bpe(bchar_path, bbpe_model_prefix, bbpe_size) + os.remove(bchar_path) + # apply + for lang in [src, tgt]: + for split in SPLITS: + _apply_bbpe( + bbpe_model_prefix + ".model", + op.join(root, f"{split}.moses.{lang}"), + op.join(root, f"{split}.moses.bbpe{bbpe_size}.{lang}"), + ) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--root", type=str, default="data") + parser.add_argument( + "--bpe-vocab", + default=None, + type=int, + help="Generate tokenized bitext with BPE of size K." + "Default to None (disabled).", + ) + parser.add_argument( + "--bbpe-vocab", + default=None, + type=int, + help="Generate tokenized bitext with BBPE of size K." + "Default to None (disabled).", + ) + parser.add_argument( + "--byte-vocab", + action="store_true", + help="Generate tokenized bitext with bytes vocabulary", + ) + parser.add_argument( + "--char-vocab", + action="store_true", + help="Generate tokenized bitext with chars vocabulary", + ) + args = parser.parse_args() + + preprocess_iwslt17( + args.root, + "fr", + "en", + args.bpe_vocab, + args.char_vocab, + args.bbpe_vocab, + args.byte_vocab, + ) + + +if __name__ == "__main__": + main() diff --git a/data/fairseq/examples/byte_level_bpe/get_data.sh b/data/fairseq/examples/byte_level_bpe/get_data.sh new file mode 100644 index 0000000000000000000000000000000000000000..c3d55d4925a6e6e23d12d293f093c1ae14acf76e --- /dev/null +++ b/data/fairseq/examples/byte_level_bpe/get_data.sh @@ -0,0 +1,47 @@ +#!/bin/bash + +# Copyright (c) Facebook, Inc. and its affiliates. +# +# This source code is licensed under the MIT license found in the +# LICENSE file in the root directory of this source tree. + +PY_BIN_ROOT= + +# PyPI dependency +${PY_BIN_ROOT}pip install sentencepiece sacremoses + +# Get data +if [ ! -d "data" ]; then + mkdir data +fi + +if [ ! -f "data/fr-en.tgz" ]; then + wget https://wit3.fbk.eu/archive/2017-01-trnted/texts/fr/en/fr-en.tgz -P data + tar xvf data/fr-en.tgz -C data +fi +${PY_BIN_ROOT}python get_bitext.py --bpe-vocab 16384 --byte-vocab --char-vocab +for VOCAB_SIZE in 2048 4096; do + ${PY_BIN_ROOT}python get_bitext.py --bpe-vocab ${VOCAB_SIZE} --bbpe-vocab ${VOCAB_SIZE} +done +rm -r data/fr-en data/fr-en.tgz + +# Generate binary dataset +${PY_BIN_ROOT}/fairseq-preprocess --source-lang fr --target-lang en --destdir data/bin_bpe16384 --joined-dictionary \ + --workers "$(nproc)" --trainpref data/train.moses.bpe16384 --validpref data/valid.moses.bpe16384 \ + --testpref data/test.moses.bpe16384 + +${PY_BIN_ROOT}/fairseq-preprocess --source-lang fr --target-lang en --destdir data/bin_bytes --joined-dictionary \ + --workers "$(nproc)" --trainpref data/train.moses.bytes --validpref data/valid.moses.bytes \ + --testpref data/test.moses.bytes + +${PY_BIN_ROOT}/fairseq-preprocess --source-lang fr --target-lang en --destdir data/bin_chars --joined-dictionary \ + --workers "$(nproc)" --trainpref data/train.moses.chars --validpref data/valid.moses.chars \ + --testpref data/test.moses.chars + +for VOCAB_SIZE in 2048 4096; do + for TYPE in bbpe bpe; do + ${PY_BIN_ROOT}/fairseq-preprocess --source-lang fr --target-lang en --destdir "data/bin_${TYPE}${VOCAB_SIZE}" \ + --joined-dictionary --workers "$(nproc)" --trainpref "data/train.moses.${TYPE}${VOCAB_SIZE}" \ + --validpref "data/valid.moses.${TYPE}${VOCAB_SIZE}" --testpref "data/test.moses.${TYPE}${VOCAB_SIZE}" + done +done diff --git a/data/fairseq/examples/byte_level_bpe/gru_transformer.py b/data/fairseq/examples/byte_level_bpe/gru_transformer.py new file mode 100644 index 0000000000000000000000000000000000000000..d4efa93a4d75da71c78e786d7f62101ef3266af4 --- /dev/null +++ b/data/fairseq/examples/byte_level_bpe/gru_transformer.py @@ -0,0 +1,107 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +# +# This source code is licensed under the MIT license found in the +# LICENSE file in the root directory of this source tree. + +# Copyright (c) Facebook, Inc. and its affiliates. +# +# This source code is licensed under the MIT license found in the +# LICENSE file in the root directory of this source tree. + +import torch.nn as nn +import torch.nn.functional as F +from fairseq.models import register_model, register_model_architecture +from fairseq.models.transformer import TransformerEncoder, TransformerModel + + +@register_model("gru_transformer") +class GRUTransformerModel(TransformerModel): + @classmethod + def build_encoder(cls, args, src_dict, embed_tokens): + return GRUTransformerEncoder(args, src_dict, embed_tokens) + + +class GRUTransformerEncoder(TransformerEncoder): + def __init__(self, args, dictionary, embed_tokens): + super().__init__(args, dictionary, embed_tokens) + self.emb_ctx = nn.GRU( + input_size=embed_tokens.embedding_dim, + hidden_size=embed_tokens.embedding_dim // 2, + num_layers=1, + bidirectional=True, + ) + + def forward_embedding(self, src_tokens): + # embed tokens and positions + x = embed = self.embed_scale * self.embed_tokens(src_tokens) + if self.embed_positions is not None: + x = embed + self.embed_positions(src_tokens) + + # contextualize embeddings + x = x.transpose(0, 1) + x = self.dropout_module(x) + x, _ = self.emb_ctx.forward(x) + x = x.transpose(0, 1) + + if self.layernorm_embedding is not None: + x = self.layernorm_embedding(x) + x = self.dropout_module(x) + return x, embed + + +@register_model_architecture("gru_transformer", "gru_transformer") +def gru_transformer_base_architecture(args): + args.encoder_embed_path = getattr(args, "encoder_embed_path", None) + args.encoder_embed_dim = getattr(args, "encoder_embed_dim", 512) + args.encoder_ffn_embed_dim = getattr(args, "encoder_ffn_embed_dim", 2048) + args.encoder_layers = getattr(args, "encoder_layers", 6) + args.encoder_attention_heads = getattr(args, "encoder_attention_heads", 8) + args.encoder_normalize_before = getattr(args, "encoder_normalize_before", False) + args.encoder_learned_pos = getattr(args, "encoder_learned_pos", False) + args.decoder_embed_path = getattr(args, "decoder_embed_path", None) + args.decoder_embed_dim = getattr(args, "decoder_embed_dim", args.encoder_embed_dim) + args.decoder_ffn_embed_dim = getattr( + args, "decoder_ffn_embed_dim", args.encoder_ffn_embed_dim + ) + args.decoder_layers = getattr(args, "decoder_layers", 6) + args.decoder_attention_heads = getattr(args, "decoder_attention_heads", 8) + args.decoder_normalize_before = getattr(args, "decoder_normalize_before", False) + args.decoder_learned_pos = getattr(args, "decoder_learned_pos", False) + args.attention_dropout = getattr(args, "attention_dropout", 0.0) + args.activation_dropout = getattr(args, "activation_dropout", 0.0) + args.activation_fn = getattr(args, "activation_fn", "relu") + args.dropout = getattr(args, "dropout", 0.1) + args.adaptive_softmax_cutoff = getattr(args, "adaptive_softmax_cutoff", None) + args.adaptive_softmax_dropout = getattr(args, "adaptive_softmax_dropout", 0) + args.share_decoder_input_output_embed = getattr( + args, "share_decoder_input_output_embed", False + ) + args.share_all_embeddings = getattr(args, "share_all_embeddings", False) + args.no_token_positional_embeddings = getattr( + args, "no_token_positional_embeddings", False + ) + args.adaptive_input = getattr(args, "adaptive_input", False) + args.no_cross_attention = getattr(args, "no_cross_attention", False) + args.cross_self_attention = getattr(args, "cross_self_attention", False) + args.layer_wise_attention = getattr(args, "layer_wise_attention", False) + + args.decoder_output_dim = getattr( + args, "decoder_output_dim", args.decoder_embed_dim + ) + args.decoder_input_dim = getattr(args, "decoder_input_dim", args.decoder_embed_dim) + + args.no_scale_embedding = getattr(args, "no_scale_embedding", False) + args.layernorm_embedding = getattr(args, "layernorm_embedding", False) + + +@register_model_architecture("gru_transformer", "gru_transformer_big") +def gru_transformer_big(args): + args.encoder_embed_dim = getattr(args, "encoder_embed_dim", 1024) + args.encoder_ffn_embed_dim = getattr(args, "encoder_ffn_embed_dim", 4096) + args.encoder_attention_heads = getattr(args, "encoder_attention_heads", 16) + args.encoder_normalize_before = getattr(args, "encoder_normalize_before", False) + args.decoder_embed_dim = getattr(args, "decoder_embed_dim", 1024) + args.decoder_ffn_embed_dim = getattr(args, "decoder_ffn_embed_dim", 4096) + args.decoder_attention_heads = getattr(args, "decoder_attention_heads", 16) + args.dropout = getattr(args, "dropout", 0.3) + gru_transformer_base_architecture(args) diff --git a/data/fairseq/examples/constrained_decoding/README.md b/data/fairseq/examples/constrained_decoding/README.md new file mode 100644 index 0000000000000000000000000000000000000000..e04b8b6a018214c8233fa87fd91d46a6dd1519d4 --- /dev/null +++ b/data/fairseq/examples/constrained_decoding/README.md @@ -0,0 +1,123 @@ +# (Vectorized) Lexically constrained decoding with dynamic beam allocation + +This page provides instructions for how to use lexically constrained decoding in Fairseq. +Fairseq implements the code described in the following papers: + +* [Fast Lexically Constrained Decoding With Dynamic Beam Allocation](https://www.aclweb.org/anthology/N18-1119/) (Post & Vilar, 2018) +* [Improved Lexically Constrained Decoding for Translation and Monolingual Rewriting](https://www.aclweb.org/anthology/N19-1090/) (Hu et al., 2019) + +## Quick start + +Constrained search is enabled by adding the command-line argument `--constraints` to `fairseq-interactive`. +Constraints are appended to each line of input, separated by tabs. Each constraint (one or more tokens) +is a separate field. + +The following command, using [Fairseq's WMT19 German--English model](https://github.com/pytorch/fairseq/blob/main/examples/wmt19/README.md), +translates the sentence *Die maschinelle Übersetzung ist schwer zu kontrollieren.* with the constraints +"hard" and "to influence". + + echo -e "Die maschinelle Übersetzung ist schwer zu kontrollieren.\thard\ttoinfluence" \ + | normalize.py | tok.py \ + | fairseq-interactive /path/to/model \ + --path /path/to/model/model1.pt \ + --bpe fastbpe \ + --bpe-codes /path/to/model/bpecodes \ + --constraints \ + -s de -t en \ + --beam 10 + +(tok.py and normalize.py can be found in the same directory as this README; they are just shortcuts around Fairseq's WMT19 preprocessing). +This will generate the following output: + + [snip] + S-0 Die masch@@ in@@ elle Über@@ setzung ist schwer zu kontrollieren . + W-0 1.844 seconds + C-0 hard + C-0 influence + H-0 -1.5333266258239746 Mach@@ ine trans@@ lation is hard to influence . + D-0 -1.5333266258239746 Machine translation is hard to influence . + P-0 -0.5434 -0.1423 -0.1930 -0.1415 -0.2346 -1.8031 -0.1701 -11.7727 -0.1815 -0.1511 + +By default, constraints are generated in the order supplied, with any number (zero or more) of tokens generated +between constraints. If you wish for the decoder to order the constraints, then use `--constraints unordered`. +Note that you may want to use a larger beam. + +## Implementation details + +The heart of the implementation is in `fairseq/search.py`, which adds a `LexicallyConstrainedBeamSearch` instance. +This instance of beam search tracks the progress of each hypothesis in the beam through the set of constraints +provided for each input sentence. It does this using one of two classes, both found in `fairseq/token_generation_contstraints.py`: + +* OrderedConstraintState: assumes the `C` input constraints will be generated in the provided order +* UnorderedConstraintState: tries to apply `C` (phrasal) constraints in all `C!` orders + +## Differences from Sockeye + +There are a number of [differences from Sockeye's implementation](https://awslabs.github.io/sockeye/inference.html#lexical-constraints). + +* Generating constraints in the order supplied (the default option here) is not available in Sockeye. +* Due to an improved beam allocation method, there is no need to prune the beam. +* Again due to better allocation, beam sizes as low as 10 or even 5 are often sufficient. +* [The vector extensions described in Hu et al.](https://github.com/edwardjhu/sockeye/tree/trie_constraints) (NAACL 2019) were never merged + into the main Sockeye branch. + +## Citation + +The paper first describing lexical constraints for seq2seq decoding is: + +```bibtex +@inproceedings{hokamp-liu-2017-lexically, + title = "Lexically Constrained Decoding for Sequence Generation Using Grid Beam Search", + author = "Hokamp, Chris and + Liu, Qun", + booktitle = "Proceedings of the 55th Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)", + month = jul, + year = "2017", + address = "Vancouver, Canada", + publisher = "Association for Computational Linguistics", + url = "https://www.aclweb.org/anthology/P17-1141", + doi = "10.18653/v1/P17-1141", + pages = "1535--1546", +} +``` + +The fairseq implementation uses the extensions described in + +```bibtex +@inproceedings{post-vilar-2018-fast, + title = "Fast Lexically Constrained Decoding with Dynamic Beam Allocation for Neural Machine Translation", + author = "Post, Matt and + Vilar, David", + booktitle = "Proceedings of the 2018 Conference of the North {A}merican Chapter of the Association for Computational Linguistics: Human Language Technologies, Volume 1 (Long Papers)", + month = jun, + year = "2018", + address = "New Orleans, Louisiana", + publisher = "Association for Computational Linguistics", + url = "https://www.aclweb.org/anthology/N18-1119", + doi = "10.18653/v1/N18-1119", + pages = "1314--1324", +} +``` + +and + +```bibtex +@inproceedings{hu-etal-2019-improved, + title = "Improved Lexically Constrained Decoding for Translation and Monolingual Rewriting", + author = "Hu, J. Edward and + Khayrallah, Huda and + Culkin, Ryan and + Xia, Patrick and + Chen, Tongfei and + Post, Matt and + Van Durme, Benjamin", + booktitle = "Proceedings of the 2019 Conference of the North {A}merican Chapter of the Association for Computational Linguistics: Human Language Technologies, Volume 1 (Long and Short Papers)", + month = jun, + year = "2019", + address = "Minneapolis, Minnesota", + publisher = "Association for Computational Linguistics", + url = "https://www.aclweb.org/anthology/N19-1090", + doi = "10.18653/v1/N19-1090", + pages = "839--850", +} +``` diff --git a/data/fairseq/examples/constrained_decoding/normalize.py b/data/fairseq/examples/constrained_decoding/normalize.py new file mode 100644 index 0000000000000000000000000000000000000000..4ae2b5111ba025acb9e1613865c92fdc339a58d5 --- /dev/null +++ b/data/fairseq/examples/constrained_decoding/normalize.py @@ -0,0 +1,27 @@ +#!/usr/bin/env python3 +# +# Copyright (c) Facebook, Inc. and its affiliates. +# +# This source code is licensed under the MIT license found in the +# LICENSE file in the root directory of this source tree. + +import sys + +from sacremoses.normalize import MosesPunctNormalizer + + +def main(args): + normalizer = MosesPunctNormalizer(lang=args.lang, penn=args.penn) + for line in sys.stdin: + print(normalizer.normalize(line.rstrip()), flush=True) + + +if __name__ == "__main__": + import argparse + + parser = argparse.ArgumentParser() + parser.add_argument("--lang", "-l", default="en") + parser.add_argument("--penn", "-p", action="store_true") + args = parser.parse_args() + + main(args) diff --git a/data/fairseq/examples/constrained_decoding/tok.py b/data/fairseq/examples/constrained_decoding/tok.py new file mode 100644 index 0000000000000000000000000000000000000000..b1f888a8c0d1b8ec7174859476cc3222456e0d2c --- /dev/null +++ b/data/fairseq/examples/constrained_decoding/tok.py @@ -0,0 +1,34 @@ +#!/usr/bin/env python3 +# +# Copyright (c) Facebook, Inc. and its affiliates. +# +# This source code is licensed under the MIT license found in the +# LICENSE file in the root directory of this source tree. + +import sys + +import sacremoses + + +def main(args): + """Tokenizes, preserving tabs""" + mt = sacremoses.MosesTokenizer(lang=args.lang) + + def tok(s): + return mt.tokenize(s, return_str=True) + + for line in sys.stdin: + parts = list(map(tok, line.split("\t"))) + print(*parts, sep="\t", flush=True) + + +if __name__ == "__main__": + import argparse + + parser = argparse.ArgumentParser() + parser.add_argument("--lang", "-l", default="en") + parser.add_argument("--penn", "-p", action="store_true") + parser.add_argument("--fields", "-f", help="fields to tokenize") + args = parser.parse_args() + + main(args)