diff --git a/data/fairseq/examples/MMPT/.gitignore b/data/fairseq/examples/MMPT/.gitignore
new file mode 100644
index 0000000000000000000000000000000000000000..70a255dc918bf4a4242dd7ea5ded63e6848264db
--- /dev/null
+++ b/data/fairseq/examples/MMPT/.gitignore
@@ -0,0 +1,139 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+pip-wheel-metadata/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+# Usually these files are written by a python script from a template
+# before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+.python-version
+
+# pipenv
+# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+# However, in case of collaboration, if having platform-specific dependencies or dependencies
+# having no cross-platform support, pipenv may install dependencies that don't work, or not
+# install all needed dependencies.
+#Pipfile.lock
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+runs
+data
+pretrained_models
+projects/mmfusion_*
+log_test
+third-party
+python_log
+slurm_snapshot_code
+lightning_logs
+demos
diff --git a/data/fairseq/examples/MMPT/CONFIG.md b/data/fairseq/examples/MMPT/CONFIG.md
new file mode 100644
index 0000000000000000000000000000000000000000..bbd1403dfafc79ab581c691281ca49de28a4656f
--- /dev/null
+++ b/data/fairseq/examples/MMPT/CONFIG.md
@@ -0,0 +1,41 @@
+### Config Files Explained
+
+Taking `projects/mfmmlm.yaml` for example, which run pretraining using masked frame model (MFM) and masked language model (MLM) on a single BERT:
+
+```yaml
+project_dir: mfmmlm # specify the project dir for this baseline.
+run_task:
+ - how2.yaml # run pretraining on how2 when launching `projects/taskmfmmlm.yaml`
+ - [vtt.yaml, vttcap.yaml, vttqa.yaml, youcook.yaml, youcookcap.yaml, crosstask.yaml, coin.yaml] # run fine-tuning tasks.
+base_dir: task # a global template folder to specify each training task.
+task_group:
+ pretrain: # section for pretraining. Most baselines differs in this section.
+ task_list:
+ - how2.yaml # reconfig `projects/task/how2.yaml`
+ dataset:
+ aligner: MFMMLMAligner # overwrite the aligner for MFMMLM training task.
+ model:
+ model_cls: MMFusionMFMMLM # overwrite the model, which constructs negative examples for MFM on-the-fly.
+ loss:
+ loss_cls: MFMMLM # overwrite the loss as MFMMLM, which combines MFM and MLM together.
+ fairseq: # all fairseq args can be expecified under this name.
+ dataset:
+ batch_size: 128
+ finetune: # section for fine-tuning tasks, we don't need to change anything here mostly since we want to see how pretraining can contribute to finetuning.
+ task_list: # specify the list of downstream tasks, e.g., copy `projects/task/vtt.yaml` to `projects/mfmmlm`.
+ - vtt.yaml
+ - vttqa.yaml
+ - youcook.yaml
+ - youcookcap.yaml
+ - crosstask.yaml
+ - coin.yaml
+ test: # section for testing.
+ task_list:
+ - test_vtt.yaml
+ - test_vttqa.yaml
+ - test_youcook.yaml
+ - test_youcookcap.yaml
+ - test_crosstask.yaml
+ - test_crosstask_zs.yaml
+ - test_coin.yaml
+```
diff --git a/data/fairseq/examples/MMPT/DATASET.md b/data/fairseq/examples/MMPT/DATASET.md
new file mode 100644
index 0000000000000000000000000000000000000000..930403eb3651a14dfcd015d094383faeb052c8ac
--- /dev/null
+++ b/data/fairseq/examples/MMPT/DATASET.md
@@ -0,0 +1,34 @@
+# Dataset
+
+We understand video data are challenging to download and process. For videos, we provide our preprocessing scripts under `scripts/video_feature_extractor` (deeply adapted from `https://github.com/antoine77340/video_feature_extractor`); for text, we pre-tokenizing scripts under `scripts/text_token_extractor`.
+
+### S3D Feature Extraction
+We use pre-trained [S3D](https://github.com/antoine77340/S3D_HowTo100M) for video feature extraction. Please place the models as `pretrained_models/s3d_dict.npy` and `pretrained_models/s3d_howto100m.pth`.
+
+We implement a `PathBuilder` to automatically track video ids, source video paths to their feature locations (you may need `conda install -c anaconda pandas`). Decoding may need `pip install ffmpeg-python`.
+
+### Howto100M
+[Howto100M](https://www.di.ens.fr/willow/research/howto100m/) is a large-scale video pre-training datasets. You may download videos by yourself and run preprocessing of our scripts.
+
+Several key differences of our preprocessing from existing papers: (1) we use `raw_caption.json` instead of `caption.json` to have pure self-supervision on text (`caption.json` has manual removal of stop words); (2) we remove partially duplicated texts that are originally designed for real-time readability (see `mmpt/processors/dedupprocessor.py`); (3) then we shard video/text features using `SharedTensor` in `mmpt/utils/shardedtensor.py` for fast loading during training (faster than `h5py`).
+
+#### Steps
+##### video
+To extract video features: edit and run `bash scripts/video_feature_extractor/how2/s3d.sh`. (consider to run this on multiple machines; by default, we store features in fp16 to save space and also for faster training).
+
+Split available video ids as `data/how2/how2_s3d_train.lst` and `data/how2/how2_s3d_val.lst`.
+
+Lastly, pack video features into `ShardedTensor` using `python scripts/video_feature_extractor/shard_feature.py`.
+
+##### text
+Clean captions using `python -m mmpt.processors.dedupprocessor`.
+
+Tokenize dedupped captions `data/how2/raw_caption_dedup.pkl` into sharded numpy arrays:
+```
+python scripts/text_token_extractor/pretokenization.py scripts/text_token_extractor/configs/bert-base-uncased.yaml
+```
+
+### Youcook, MSRVTT etc.
+We use the version of Youcook and MSRVTT come with Howto100M and MILNCE. Please download the data to `data/youcook` and `data/msrvtt` accordingly, you can also check `projects/task/youcook.yaml` and `projects/task/vtt.yaml` etc. in details.
+We extract features for Youcook, MSRVTT similar to the first step of Howto100M but we read text from meta data directly and perform on-the-fly tokenization.
+
diff --git a/data/fairseq/examples/MMPT/README.md b/data/fairseq/examples/MMPT/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..4a84819d9ddc992170296958a9431ff0538a85be
--- /dev/null
+++ b/data/fairseq/examples/MMPT/README.md
@@ -0,0 +1,166 @@
+# VideoCLIP and VLM
+
+You just find this toolkit for multimodal video understanding! It contains implementation of two recent multi-modal video understanding papers [VideoCLIP](https://arxiv.org/pdf/2109.14084.pdf) (EMNLP, 2021) and [VLM](https://aclanthology.org/2021.findings-acl.370.pdf) (ACL Findings, 2021), along with high-performance toolkits that are typically lacking in existing codebase. The toolkit is desigend to contain generic performance-tuned components that can be potentially adapted to other frameworks (we initially use fairseq).
+
+VideoCLIP is a contrastive learning model for zero-shot transfer to retrieval/classification/sequence labeling style tasks.
+
+
+
+VLM is a masked language model style pre-training using only one encoder with masked modality model (MMM) for retrieval/generation/sequence labeling style tasks.
+
+
+
+### News
+[Oct. 2021] Initial release of implementation for the following papers:
+[VideoCLIP: Contrastive Pre-training for Zero-shot Video-Text Understanding](https://arxiv.org/pdf/2109.14084.pdf) (Xu et. al., EMNLP 2021)
+[VLM: Task-agnostic Video-Language Model Pre-training for Video Understanding](https://aclanthology.org/2021.findings-acl.370.pdf) (Xu et. al., ACL Findings 2021)
+
+
+### Installation
+We aim to minimize the dependency of this repo on other packages.
+We use fairseq as the main trainer (no models/datasets dependency on fairseq. We will support other trainer in future):
+```
+git clone https://github.com/pytorch/fairseq
+cd fairseq
+pip install -e . # also optionally follow fairseq README for apex installation for fp16 training.
+export MKL_THREADING_LAYER=GNU # fairseq may need this for numpy.
+```
+
+Then install this toolkit:
+```
+cd examples/MMPT # MMPT can be in any folder, not necessarily under fairseq/examples.
+pip install -e .
+```
+
+The code is developed under Python=3.8.8, Pytorch=1.8, cuda=11.0 with fairseq=1.0.0a0+af0389f and tested under Python=3.8.8 pytorch=1.9 cuda=11.0 fairseq=1.0.0a0+8e7bc73 during code release.
+Most models require `transformers==3.4` for API compatibility `pip install transformers==3.4`.
+In addition, some downstream tasks may need `conda install pandas`.
+
+
+### Usage
+#### Download Checkpoints
+We use pre-trained [S3D](https://github.com/antoine77340/S3D_HowTo100M) for video feature extraction. Please place the models as `pretrained_models/s3d_dict.npy` and `pretrained_models/s3d_howto100m.pth`.
+
+Download VideoCLIP checkpoint `https://dl.fbaipublicfiles.com/MMPT/retri/videoclip/checkpoint_best.pt` to `runs/retri/videoclip` or VLM checkpoint `https://dl.fbaipublicfiles.com/MMPT/mtm/vlm/checkpoint_best.pt` to `runs/mtm/vlm`.
+
+#### Demo of Inference
+run `python locallaunch.py projects/retri/videoclip.yaml --dryrun` to get all `.yaml`s for VideoCLIP.
+
+```python
+import torch
+
+from mmpt.models import MMPTModel
+
+
+model, tokenizer, aligner = MMPTModel.from_pretrained(
+ "projects/retri/videoclip/how2.yaml")
+
+model.eval()
+
+
+# B, T, FPS, H, W, C (VideoCLIP is trained on 30 fps of s3d)
+video_frames = torch.randn(1, 2, 30, 224, 224, 3)
+caps, cmasks = aligner._build_text_seq(
+ tokenizer("some text", add_special_tokens=False)["input_ids"]
+)
+
+caps, cmasks = caps[None, :], cmasks[None, :] # bsz=1
+
+with torch.no_grad():
+ output = model(video_frames, caps, cmasks, return_score=True)
+print(output["score"]) # dot-product
+```
+
+#### Data Preparation
+See [dataset](DATASET.md) for each dataset.
+
+#### Global Config for Training Pipeline
+We organize a global config file for a training/testing pipeline under projects (see a detailed [explanation](CONFIG.md)). For example, VideoCLIP in `projects/retri/videoclip.yaml` and VLM is in `projects/mtm/vlm.yaml`.
+
+We wrap all cmds into `locallaunch.py` and `mmpt_cli/localjob.py`. You can check concrete cmds by `--dryrun` and then drop it for actual run.
+
+First, run `python locallaunch.py projects/retri/videoclip.yaml --dryrun` will generate configs for all configs of pre-training, zero-shot evaluation, fine-tuning and testing, for VideoCLIP under `projects/retri/videoclip`.
+
+Then each (either training or evaluation) process will be configed by a concrete config file (we save all complex arguments into the concrete config file for reproducibility, including fairseq args). For example, run zero-shot evaluation on youcook,
+```
+python locallaunch.py projects/retri/videoclip/test_youcook_zs.yaml --jobtype local_predict # zero-shot evaluation.
+python locallaunch.py projects/retri/videoclip/youcook_videoclip.yaml --jobtype local_single --dryrun # fine-tuning: use --dryrun to check cmds and drop it to make an actual run; local_small will run on two gpus (as in paper).
+python locallaunch.py projects/retri/videoclip/test_youcook_videoclip.yaml --jobtype local_predict # testing on fine-tuned model.
+```
+
+Pretraining can be run as:
+```
+python locallaunch.py projects/retri/videoclip/how2.yaml --jobtype local_single --dryrun # check then drop dryrun; paper is ran on local_big as 8 gpus.
+```
+You may need to change `--jobtype`, check/extend `LocalJob` in `mmpt_cli/localjob.py` for multi-gpu/multi-node pre-training.
+
+The detailed instructions of pretraining and fine-tuning can be found at [pretraining instruction](pretraining.md) and [finetuning instruction](endtask.md).
+
+
+### Development
+Several components of this toolkit can be re-used for future research (and also our ongoing research).
+
+#### Framework Wrapper
+We currently only support fairseq, but most components can be easily fit into other frameworks like huggingface. This repo is a `--user-dir` of fairseq with fairseq wrapper. For example, `mmpt/tasks` includes a `FairseqMMTTask`, which manages `mmpt/datasets` with `FairseqDataset`, `mmpt/models` with `FairseqModel`, `mmpt/losses` with `FairseqCriterion`.
+
+#### Processors
+**Multi**modal research introduces the complexity on modality alignment from different input sources to losses. Inspired by [MMF](https://github.com/facebookresearch/mmf), this toolkit leverages `mmpt/processors` to handle various needs of data preprocessing and loading, **alleviating** the needs of multiple `torch.data.utils.Dataset` (that can be tricky for ablation study).
+Processors can also be decoupled from `torch.data.utils.Dataset` for offline preprocessing instead of on-the-fly data preprocessing.
+
+We decouple a `mmpt.MMDataset` as 3 types of processors: `MetaProcessor`, `VideoProcessor`, `TextProcessor` and `Aligner`. They can be configed in `dataset` field of a config file (e.g., see `projects/task/how2.yaml`).
+`MetaProcessor` is used to load the meta data about a dataset, aka, all video_ids of how2 dataset.
+`VideoProcessor` is used to load the video features about a dataset. For example, S3D features for each second of a video.
+`TextProcessor` is used to load the text (feature). For example, BERT pre-tokenized text clips for how2 dataset (with `start`s, `end`s of timestamps and `cap` for `token_ids`).
+`Aligner` is the core class for different baselines that prepares the training data. For example, sampling a clip, masking tokens for MLM, etc.
+
+#### Performance-tuned Components
+To speed up pre-training, this toolkit uses sharded features stored in mmaped numpy, backed by `ShardedTensor` in `mmpt/utils/shardedtensor.py` (adopted from MARGE paper). This reduces the loads of IO for multi-GPU training without loading all features for a video into the memory each time and `ShardedTensor` ensure features are stored in continuous disk space for near random access. This is used for both How2 video features and texts in `mmpt/processors/how2processor.py`.
+
+
+### Citation
+If this codebase is useful for your work, please cite the following papers:
+
+```BibTeX
+@inproceedings{xu-etal-2021-videoclip,
+ title = "{VideoCLIP}: Contrastive Pre-training for\\Zero-shot Video-Text Understanding",
+ author = "Xu, Hu and
+ Ghosh, Gargi and
+ Huang, Po-Yao and
+ Okhonko, Dmytro and
+ Aghajanyan, Armen and
+ Metze, Florian and
+ Zettlemoyer, Luke and
+ Feichtenhofer, Christoph",
+ booktitle = "Proceedings of the 2021 Conference on Empirical Methods in Natural Language Processing (EMNLP)",
+ month = nov,
+ year = "2021",
+ address = "Online",
+ publisher = "Association for Computational Linguistics",
+}
+
+@inproceedings{xu-etal-2021-vlm,
+ title = "{VLM}: Task-agnostic Video-Language Model Pre-training for Video Understanding",
+ author = "Xu, Hu and
+ Ghosh, Gargi and
+ Huang, Po-Yao and
+ Arora, Prahal and
+ Aminzadeh, Masoumeh and
+ Feichtenhofer, Christoph and
+ Metze, Florian and
+ Zettlemoyer, Luke",
+ booktitle = "Findings of the Association for Computational Linguistics: ACL-IJCNLP 2021",
+ month = aug,
+ year = "2021",
+ address = "Online",
+ publisher = "Association for Computational Linguistics",
+ url = "https://aclanthology.org/2021.findings-acl.370",
+ doi = "10.18653/v1/2021.findings-acl.370",
+ pages = "4227--4239",
+}
+```
+
+### Bug Reports
+This repo is in its initial stage, welcome bug reports to huxu@fb.com
+
+### Copyright
+The majority of Multimodal Pre-training (MMPT) is licensed under CC-BY-NC, however portions of the project are available under separate license terms: Evaluation Codes/Models: Howto100M and HuggingFace Transformers are licensed under the Apache2.0 license; COIN and NLG-eval are licensed under the MIT license; CrossTask is licensed under the BSD-3; DiDeMo is licensed under the BSD-2 license.
diff --git a/data/fairseq/examples/MMPT/endtask.md b/data/fairseq/examples/MMPT/endtask.md
new file mode 100644
index 0000000000000000000000000000000000000000..7690955327283ded4b37857a4a7b78463e0eb0f8
--- /dev/null
+++ b/data/fairseq/examples/MMPT/endtask.md
@@ -0,0 +1,41 @@
+# Zero-shot Transfer and Finetuning
+
+(If you are new to the ideas of `mmpt.processors`, see [README](README.md) first.)
+All finetuning datasets (specifically `processors`) are defined in `mmpt.processors.dsprocessor`.
+Given the complexity of different types of finetuning tasks, each task may have their own meta/video/text/aligner processors and `mmpt/evaluators/{Predictor,Metric}`.
+
+### Tasks
+
+Currently, we support 5 end datasets: `MSRVTT`, `Youcook`, `COIN`, `Crosstask` and `DiDeMo` with the following tasks:
+text-video retrieval: `MSRVTT`, `Youcook`, `DiDeMo`;
+video captioning: `Youcook`;
+Video Question and Answering: `MSRVTT-QA`.
+
+To add your own dataset, you can specify the corresponding processors and config them in the `dataset` field of a config file, such as `projects/task/vtt.yaml`.
+
+### Zero-shot Transfer (no Training)
+Zero-shot transfer will run the pre-trained model (e.g., VideoCLIP) directly on testing data. Configs with pattern: `projects/task/*_zs_*.yaml` are dedicated for zero-shot transfer.
+
+### Fine-tuning
+
+The training of a downstream task is similar to pretraining, execept you may need to specify the `restore_file` in `fairseq.checkpoint` and reset optimizers, see `projects/task/ft.yaml` that is included by `projects/task/vtt.yaml`.
+
+We typically do finetuning on 2 gpus (`local_small`).
+
+### Testing
+For each finetuning dataset, you may need to specify a testing config, similar to `projects/task/test_vtt.yaml`.
+
+We define `mmpt.evaluators.Predictor` for different types of prediction. For example, `MSRVTT` and `Youcook` are video-retrieval tasks and expecting to use `RetrievalPredictor`. You may need to define your new type of predictors and specify that in `predictor` field of a testing config.
+
+Each task may also have their own metric for evaluation. This can be created in `mmpt.evaluators.Metric` and specified in the `metric` field of a testing config.
+
+Launching a testing is as simple as training by specifying the path of a testing config:
+```python locallaunch.py projects/mfmmlm/test_vtt.yaml```
+Testing will be launched locally by default since prediction is computationally less expensive.
+
+### Third-party Libraries
+We list the following finetuning tasks that require third-party libraries.
+
+Youcook captioning: `https://github.com/Maluuba/nlg-eval`
+
+CrossTask: `https://github.com/DmZhukov/CrossTask`'s `dp` under `third-party/CrossTask` (`python setup.py build_ext --inplace`)
diff --git a/data/fairseq/examples/MMPT/locallaunch.py b/data/fairseq/examples/MMPT/locallaunch.py
new file mode 100644
index 0000000000000000000000000000000000000000..e20fd816fa3bed8b1af8f6a4d1a07ccb69a1fffa
--- /dev/null
+++ b/data/fairseq/examples/MMPT/locallaunch.py
@@ -0,0 +1,148 @@
+# Copyright (c) Facebook, Inc. and its affiliates.
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+import argparse
+import os
+
+from omegaconf import OmegaConf
+
+from mmpt.utils import recursive_config, overwrite_dir
+from mmpt_cli.localjob import LocalJob
+
+
+class JobLauncher(object):
+ JOB_CONFIG = {
+ "local": LocalJob,
+ }
+
+ def __init__(self, yaml_file):
+ self.yaml_file = yaml_file
+ job_key = "local"
+
+ if yaml_file.endswith(".yaml"):
+ config = recursive_config(yaml_file)
+ if config.task_type is not None:
+ job_key = config.task_type.split("_")[0]
+ else:
+ raise ValueError("unknown extension of job file:", yaml_file)
+ self.job_key = job_key
+
+ def __call__(self, job_type=None, dryrun=False):
+ if job_type is not None:
+ self.job_key = job_type.split("_")[0]
+ print("[JobLauncher] job_key", self.job_key)
+ job = JobLauncher.JOB_CONFIG[self.job_key](
+ self.yaml_file, job_type=job_type, dryrun=dryrun)
+ return job.submit()
+
+
+class Pipeline(object):
+ """a job that loads yaml config."""
+
+ def __init__(self, fn):
+ """
+ load a yaml config of a job and save generated configs as yaml for each task.
+ return: a list of files to run as specified by `run_task`.
+ """
+ if fn.endswith(".py"):
+ # a python command.
+ self.backend = "python"
+ self.run_yamls = [fn]
+ return
+
+ job_config = recursive_config(fn)
+ if job_config.base_dir is None: # single file job config.
+ self.run_yamls = [fn]
+ return
+
+ self.project_dir = os.path.join("projects", job_config.project_dir)
+ self.run_dir = os.path.join("runs", job_config.project_dir)
+
+ if job_config.run_task is not None:
+ run_yamls = []
+ for stage in job_config.run_task:
+ # each stage can have multiple tasks running in parallel.
+ if OmegaConf.is_list(stage):
+ stage_yamls = []
+ for task_file in stage:
+ stage_yamls.append(
+ os.path.join(self.project_dir, task_file))
+ run_yamls.append(stage_yamls)
+ else:
+ run_yamls.append(os.path.join(self.project_dir, stage))
+ self.run_yamls = run_yamls
+ configs_to_save = self._overwrite_task(job_config)
+ self._save_configs(configs_to_save)
+
+ def __getitem__(self, idx):
+ yaml_files = self.run_yamls[idx]
+ if isinstance(yaml_files, list):
+ return [JobLauncher(yaml_file) for yaml_file in yaml_files]
+ return [JobLauncher(yaml_files)]
+
+ def __len__(self):
+ return len(self.run_yamls)
+
+ def _save_configs(self, configs_to_save: dict):
+ # save
+ os.makedirs(self.project_dir, exist_ok=True)
+ for config_file in configs_to_save:
+ config = configs_to_save[config_file]
+ print("saving", config_file)
+ OmegaConf.save(config=config, f=config_file)
+
+ def _overwrite_task(self, job_config):
+ configs_to_save = {}
+ self.base_project_dir = os.path.join("projects", job_config.base_dir)
+ self.base_run_dir = os.path.join("runs", job_config.base_dir)
+
+ for config_sets in job_config.task_group:
+ overwrite_config = job_config.task_group[config_sets]
+ if (
+ overwrite_config.task_list is None
+ or len(overwrite_config.task_list) == 0
+ ):
+ print(
+ "[warning]",
+ job_config.task_group,
+ "has no task_list specified.")
+ # we don't want this added to a final config.
+ task_list = overwrite_config.pop("task_list", None)
+ for config_file in task_list:
+ config_file_path = os.path.join(
+ self.base_project_dir, config_file)
+ config = recursive_config(config_file_path)
+ # overwrite it.
+ if overwrite_config:
+ config = OmegaConf.merge(config, overwrite_config)
+ overwrite_dir(config, self.run_dir, basedir=self.base_run_dir)
+ save_file_path = os.path.join(self.project_dir, config_file)
+ configs_to_save[save_file_path] = config
+ return configs_to_save
+
+
+def main(args):
+ job_type = args.jobtype if args.jobtype else None
+ # parse multiple pipelines.
+ pipelines = [Pipeline(fn) for fn in args.yamls.split(",")]
+
+ for pipe_id, pipeline in enumerate(pipelines):
+ if not hasattr(pipeline, "project_dir"):
+ for job in pipeline[0]:
+ job(job_type=job_type, dryrun=args.dryrun)
+
+
+if __name__ == "__main__":
+ parser = argparse.ArgumentParser()
+ parser.add_argument("yamls", type=str)
+ parser.add_argument(
+ "--dryrun",
+ action="store_true",
+ help="run config and prepare to submit without launch the job.",
+ )
+ parser.add_argument(
+ "--jobtype", type=str, default="",
+ help="force to run jobs as specified.")
+ args = parser.parse_args()
+ main(args)
diff --git a/data/fairseq/examples/MMPT/mmpt/__init__.py b/data/fairseq/examples/MMPT/mmpt/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..6ff86ddd5ce0c454281e6568c628bd3c49ea5024
--- /dev/null
+++ b/data/fairseq/examples/MMPT/mmpt/__init__.py
@@ -0,0 +1,12 @@
+# Copyright (c) Facebook, Inc. and its affiliates.
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+try:
+ # fairseq user dir
+ from .datasets import FairseqMMDataset
+ from .losses import FairseqCriterion
+ from .models import FairseqMMModel
+ from .tasks import FairseqMMTask
+except ImportError:
+ pass
diff --git a/data/fairseq/examples/MMPT/mmpt/evaluators/__init__.py b/data/fairseq/examples/MMPT/mmpt/evaluators/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..2d06b9d7974db251143025124468f48cd230e89a
--- /dev/null
+++ b/data/fairseq/examples/MMPT/mmpt/evaluators/__init__.py
@@ -0,0 +1,13 @@
+# Copyright (c) Facebook, Inc. and its affiliates.
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+from .metric import *
+from .evaluator import *
+
+
+# experimental.
+try:
+ from .expmetric import *
+except ImportError:
+ pass
diff --git a/data/fairseq/examples/MMPT/mmpt/evaluators/predictor.py b/data/fairseq/examples/MMPT/mmpt/evaluators/predictor.py
new file mode 100644
index 0000000000000000000000000000000000000000..2ffef6ab474a7a275d90784250d04222bd6dc70f
--- /dev/null
+++ b/data/fairseq/examples/MMPT/mmpt/evaluators/predictor.py
@@ -0,0 +1,595 @@
+# Copyright (c) Facebook, Inc. and its affiliates.
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+import os
+import random
+import json
+import numpy as np
+import torch
+import pickle
+import math
+
+from tqdm import tqdm
+
+
+class Predictor(object):
+ """this base class is used to save predictions to disk
+ (and being called by a evaluator later).
+ Predictor has minimum support of single gpu prediction.
+ """
+ def __init__(self, config):
+ self.pred_dir = None # on-the-fly eval does not save the results.
+ if hasattr(config, "eval") and config.eval is not None:
+ self.pred_dir = config.eval.save_path
+ os.makedirs(self.pred_dir, exist_ok=True)
+
+ def __call__(self, outputs):
+ """extract the prediction and save it."""
+ raise NotImplementedError
+
+ def predict_loop(self, model, eval_dataloader, output_file=None):
+ """on-the-fly prediction on a single gpu."""
+ self.full_scores = []
+ model.eval()
+ model = model.to(0)
+ with torch.no_grad():
+ for data in eval_dataloader:
+ data = self.to_ctx(data)
+ outputs = model(**data)
+ outputs.update(data)
+ self(outputs)
+ return self.finalize(output_file)
+
+ def finalize(self, output_file):
+ pass
+
+ def to_ctx(self, data, ctx=0, dtype=None):
+ if isinstance(data, dict):
+ for key in data:
+ if torch.is_tensor(data[key]):
+ if dtype is not None and data[key].dtype == torch.float32:
+ data[key] = data[key].to(dtype)
+ data[key] = data[key].to(ctx)
+ return data
+ else:
+ raise ValueError("non-dict type of batch is not supported yet.")
+
+
+class NLGPredictor(Predictor):
+ """Predicting Text from MMFusion models."""
+ """TODO: make a context."""
+ def __init__(self, config):
+ super().__init__(config)
+ from transformers import AutoTokenizer
+
+ self.tokenizer = AutoTokenizer.from_pretrained(
+ config.dataset.bert_name,
+ bos_token="[CLS]", eos_token="[SEP]")
+ self.bos_token_id = self.tokenizer.bos_token_id
+ self.eos_token_id = self.tokenizer.eos_token_id
+
+ def predict_loop(self, model, eval_dataloader, output_file=None):
+ """TODO: refactor base classes."""
+ ctx = 0
+ outputs = {"outputs": [], "targets": [[]]}
+ model.eval()
+ model = model.to(ctx)
+ with torch.no_grad():
+ for data in tqdm(eval_dataloader):
+ data = self.to_ctx(data, ctx)
+ self(data, model, outputs)
+ return self.finalize(outputs, output_file)
+
+ def __call__(self, data, model, outputs):
+ data.update({
+ "bos_token_id": self.bos_token_id,
+ "eos_token_id": self.eos_token_id
+ })
+
+ output = model.generate(**data)
+ assert len(output) == len(data["ref"])
+ for idx, _output in enumerate(output):
+ generated_text = self.tokenizer.decode(
+ _output, skip_special_tokens=True)
+ if generated_text == "":
+ generated_text = "none"
+ outputs["outputs"].append(generated_text)
+ outputs["targets"][0].append(data["ref"][idx])
+ if random.random() < 0.001:
+ print("_output", _output)
+ print("generated_text", generated_text)
+ print("ref", data["ref"][idx])
+
+ def finalize(self, outputs, output_file=None):
+ if output_file is not None:
+ with open(os.path.join(
+ self.pred_dir, output_file + ".json"), "w") as fw:
+ json.dump(outputs, fw, indent=4)
+ return outputs
+
+
+class RetrievalPredictor(Predictor):
+ """generated `pooled_video` and `pooled_text`."""
+ def __init__(self, config):
+ super().__init__(config)
+ from transformers import AutoTokenizer
+ self.tokenizer = AutoTokenizer.from_pretrained(
+ config.dataset.bert_name)
+
+ def predict_loop(
+ self,
+ model,
+ eval_dataloader,
+ output_file="retrieval.npy"
+ ):
+ """on-the-fly prediction on a single gpu."""
+ full_scores = []
+ texts = []
+ model.eval()
+ model = model.cuda()
+ with torch.no_grad():
+ for data in eval_dataloader:
+ # convert to dict.
+ if not isinstance(data, dict):
+ data = {
+ "caps": data[0],
+ "cmasks": data[1],
+ "vfeats": data[2],
+ "vmasks": data[3],
+ "video_id": data[4]
+ }
+ data = self.to_ctx(data)
+ outputs = model(**data)
+ outputs.update(data)
+ self(outputs, full_scores)
+ for _cap in data["caps"]:
+ texts.append(
+ self.tokenizer.decode(_cap, skip_special_tokens=True)
+ )
+
+ return self.finalize(full_scores, texts, output_file)
+
+ def __call__(self, sample, full_scores):
+ scores = self._get_pooled_outputs(sample)
+ self._append_scores(scores, full_scores)
+
+ def finalize(self, full_scores, texts, output_file=None):
+ outputs = self._aggregate_scores(full_scores)
+ if output_file is not None:
+ np.save(os.path.join(self.pred_dir, output_file + ".npy"), outputs)
+ return {"outputs": outputs, "texts": texts}
+
+ def _get_pooled_outputs(self, outputs):
+ if "pooled_video" in outputs:
+ return outputs["pooled_video"], outputs["pooled_text"]
+ else:
+ raise ValueError("unknown format of outputs.")
+
+ def _append_scores(self, scores, full_scores):
+ assert len(scores) == 2
+ if len(full_scores) == 0:
+ full_scores.append([])
+ full_scores.append([])
+ full_scores[0].append(scores[0].cpu().detach().numpy())
+ full_scores[1].append(scores[1].cpu().detach().numpy())
+
+ def _aggregate_scores(self, scores):
+ assert len(scores) == 2
+ video_hidden = np.concatenate(scores[0], axis=0)
+ text_hidden = np.concatenate(scores[1], axis=0)
+ # clear up.
+ self.full_scores = []
+ return np.matmul(text_hidden, video_hidden.T)
+
+
+class QAPredictor(Predictor):
+ """generated `pooled_video` and `pooled_text`."""
+ def __init__(self, config):
+ super().__init__(config)
+ """predictor maintains scores and aggregate them."""
+
+ def predict_loop(self, model, eval_dataloader, output_file="qa.npy"):
+ """on-the-fly prediction on a single gpu."""
+ self.full_scores = []
+ model.eval()
+ model = model.cuda()
+ with torch.no_grad():
+ for data in eval_dataloader:
+ # reshape ans and dup video 5 times.
+ v_len = data["vfeats"].size(1)
+ hidden_size = data["vfeats"].size(2)
+ data["vfeats"] = data["vfeats"].unsqueeze(1).repeat(1, 5, 1, 1).view(-1, v_len, hidden_size)
+ data["vmasks"] = data["vmasks"].unsqueeze(1).repeat(1, 5, 1).view(-1, v_len)
+
+ t_len = data["caps"].size(-1)
+ data["caps"] = data["caps"].view(-1, t_len)
+ data["cmasks"] = data["cmasks"].view(-1, t_len)
+
+ data = self.to_ctx(data)
+ outputs = model(**data)
+ outputs.update(data)
+ self(outputs)
+ return self.finalize(output_file)
+
+ def __call__(self, sample):
+ hidden_size = sample["pooled_video"].size(-1)
+ pooled_video = sample["pooled_video"].view(-1, 5, hidden_size)
+ pooled_text = sample["pooled_text"].view(-1, 5, hidden_size)
+ scores = torch.bmm(pooled_video, pooled_text.transpose(2, 1))
+ scores = scores.argmax(-1)
+ self._append_scores(scores[:, 0], sample["answers"], self.full_scores)
+
+ def finalize(self, output_file=None):
+ outputs, targets = self._aggregate_scores(self.full_scores)
+ if output_file is not None:
+ np.save(os.path.join(self.pred_dir, output_file + ".npy"), outputs)
+ return {"outputs": outputs, "targets": targets}
+
+ def _append_scores(self, scores, answers, full_scores):
+ if len(full_scores) == 0:
+ full_scores.append([])
+ full_scores.append([])
+ full_scores[0].append(scores.cpu().detach().numpy())
+ full_scores[1].append(answers.cpu().detach().numpy())
+
+ def _aggregate_scores(self, scores):
+ assert len(scores) == 2
+ outputs = np.concatenate(scores[0], axis=0)
+ targets = np.concatenate(scores[1], axis=0)
+ # clear up.
+ self.full_scores = []
+ return outputs, targets
+
+
+class CrossTaskPredictor(Predictor):
+ """
+ CrossTaskPredictor needs to compute the average of logits
+ for overlapped sliding-window.
+ """
+ def __init__(self, config):
+ super().__init__(config)
+ self.lsm = torch.nn.LogSoftmax(dim=1)
+ self.max_video_len = config.dataset.max_video_len
+ self.sliding_window = config.dataset.sliding_window
+ self.sliding_window_size = config.dataset.sliding_window_size
+ self.annotation_path = config.dataset.annotation_path
+
+ def predict_loop(self, model, eval_dataloader, output_file="result.pkl"):
+ """refactored from line 144:
+ https://github.com/DmZhukov/CrossTask/blob/master/train.py
+ """
+ ctx = 0
+ model.eval()
+ model = model.to(ctx)
+ # this is not a loss but just compute neg_log_prob.
+ Y_pred = {}
+ Y_true = {}
+ with torch.no_grad():
+ for batch in eval_dataloader:
+ self(batch, model, Y_pred, Y_true)
+ return self.finalize(Y_pred, Y_true, output_file)
+
+ def __call__(self, sample, model, Y_pred, Y_true):
+ # please install dp from `https://github.com/DmZhukov/CrossTask`
+ from dp import dp
+ vid, task = sample['video_id'][0], sample['task'][0]
+ sample = self.to_ctx(sample)
+ # compute the average logits over sliding windows.
+ output = model(**sample)
+ batch_logits = output["logits"].cpu()
+
+ video_len = sample["video_len"][0]
+
+ # the following version is slow.
+ logits = torch.zeros((video_len, batch_logits.size(1)))
+ logits_counts = torch.zeros((video_len, 1), dtype=torch.long)
+ # use the same loop as aligner to recover.
+ batch_logit_idx = 0
+ for window_start in range(0, video_len, self.sliding_window):
+ video_end = min(video_len - window_start, self.sliding_window_size)
+ logits[window_start: window_start + video_end] += batch_logits[
+ batch_logit_idx: batch_logit_idx + video_end]
+ batch_logit_idx += video_end
+ logits_counts[window_start: window_start + video_end] += torch.ones((video_end, 1), dtype=torch.long)
+
+ if (video_len - window_start) <= self.sliding_window_size:
+ break
+
+ logits /= logits_counts
+ assert logits.size() == (video_len, batch_logits.size(1)), "{}, {}".format(logits.size(), video_len)
+
+ O = self.lsm(logits)
+ y = np.zeros(O.size(), dtype=np.float32)
+ dp(y, -O.detach().cpu().numpy())
+ if task not in Y_pred:
+ Y_pred[task] = {}
+ Y_pred[task][vid] = y
+ annot_path = os.path.join(
+ self.annotation_path, task+'_'+vid+'.csv')
+ if os.path.exists(annot_path):
+ if task not in Y_true:
+ Y_true[task] = {}
+ Y_true[task][vid] = self._read_assignment(
+ *y.shape, annot_path)
+
+ def finalize(self, Y_pred, Y_true, output_file=None):
+ if output_file is not None:
+ with open(
+ os.path.join(self.pred_dir, output_file + ".pkl"),
+ "wb") as fw:
+ pickle.dump(
+ {"Y_pred": Y_pred, "Y_true": Y_true}, fw,
+ protocol=pickle.HIGHEST_PROTOCOL)
+ return {"outputs": Y_pred, "targets": Y_true}
+
+ def _read_assignment(self, T, K, path):
+ """
+ refactored from https://github.com/DmZhukov/CrossTask/blob/master/data.py
+ Howto interpret contraints on loss that is going to be minimized:
+ lambd is a big number;
+ self.lambd * C is a big number for all valid position (csv stores invalids)
+
+ def forward(self, O, Y, C):
+ return (Y*(self.lambd * C - self.lsm(O))).mean(dim=0).sum()
+
+ This will load the csv file and fill-in the step col from start to end rows.
+ """
+
+ Y = np.zeros([T, K], dtype=np.uint8)
+ with open(path, 'r') as f:
+ for line in f:
+ step, start, end = line.strip().split(',')
+ start = int(math.floor(float(start)))
+ end = int(math.ceil(float(end)))
+ step = int(step) - 1
+ Y[start:end, step] = 1
+ return Y
+
+
+class COINPredictor(Predictor):
+ """
+ COINPredictor is similar to CrossTask on sliding windows.
+ """
+ def __init__(self, config):
+ super().__init__(config)
+ self.max_video_len = config.dataset.max_video_len
+ self.sliding_window = config.dataset.sliding_window
+ self.sliding_window_size = config.dataset.sliding_window_size
+
+ def predict_loop(self, model, eval_dataloader, output_file="result.pkl"):
+ """refactored from line 144:
+ https://github.com/DmZhukov/CrossTask/blob/master/train.py
+ """
+ ctx = 0
+ model.eval()
+ model = model.to(ctx)
+ # this is not a loss but just compute neg_log_prob.
+ Y_pred = []
+ Y_true = []
+ with torch.no_grad():
+ for batch in eval_dataloader:
+ self(batch, model, Y_pred, Y_true)
+ return self.finalize(Y_pred, Y_true, output_file)
+
+ def __call__(self, sample, model, Y_pred, Y_true):
+ sample = self.to_ctx(sample)
+ # compute the average logits over sliding windows.
+ output = model(**sample)
+ logits = self._merge_windows(sample, output)
+ Y_pred.append(logits.argmax(dim=1))
+ Y_true.append(sample["video_targets"].squeeze(0).cpu())
+
+ def _merge_windows(self, sample, output):
+ targets = sample["targets"].reshape(-1).cpu()
+ valid_mask = targets != -100
+ targets = targets[valid_mask]
+ batch_logits = output["logits"].cpu()
+ batch_logits = batch_logits.reshape(-1, batch_logits.size(-1))
+ batch_logits = batch_logits[valid_mask]
+
+ video_len = sample["video_len"][0]
+
+ # the following version is slow.
+ logits = torch.zeros((video_len, batch_logits.size(1)))
+ logits_counts = torch.zeros((video_len, 1), dtype=torch.long)
+ # use the same loop as aligner to recover.
+ batch_logit_idx = 0
+ for window_start in range(0, video_len, self.sliding_window):
+ video_end = min(video_len - window_start, self.sliding_window_size)
+ logits[window_start: window_start + video_end] += batch_logits[
+ batch_logit_idx: batch_logit_idx + video_end]
+ batch_logit_idx += video_end
+ logits_counts[window_start: window_start + video_end] += torch.ones((video_end, 1), dtype=torch.long)
+ if (video_len - window_start) <= self.sliding_window_size:
+ break
+ logits /= logits_counts
+ assert logits.size() == (video_len, batch_logits.size(1)), "{}, {}".format(logits.size(), video_len)
+ return logits
+
+ def finalize(self, Y_pred, Y_true, output_file=None):
+ Y_pred = torch.cat(Y_pred, dim=0).numpy()
+ Y_true = torch.cat(Y_true, dim=0).numpy()
+ assert len(Y_pred) == len(Y_true)
+
+ error_mask = Y_pred != Y_true
+ print("sample error", Y_pred[error_mask][:10], Y_true[error_mask][:10])
+ print("sample error", Y_pred[error_mask][10:20], Y_true[error_mask][10:20])
+
+ if output_file is not None:
+ with open(
+ os.path.join(self.pred_dir, output_file + ".pkl"),
+ "wb") as fw:
+ pickle.dump(
+ {"Y_pred": Y_pred, "Y_true": Y_true}, fw,
+ protocol=pickle.HIGHEST_PROTOCOL)
+ return {"outputs": Y_pred, "targets": Y_true}
+
+
+class COINZSPredictor(COINPredictor):
+ """
+ COINZSPredictor for COIN zero-shot prediction.
+ """
+
+ def __init__(self, config):
+ super().__init__(config)
+ self.dataset_config = config.dataset
+
+ def predict_loop(self, model, eval_dataloader, output_file="result.pkl"):
+ """refactored from line 144:
+ https://github.com/DmZhukov/CrossTask/blob/master/train.py
+ """
+ ctx = 0
+ model.eval()
+ model = model.to(ctx)
+
+ with torch.no_grad():
+ outputs = eval_dataloader.dataset.meta_processor.meta_text_labels(
+ self.dataset_config)
+ outputs = self.to_ctx(outputs, ctx)
+ label_hidden_states = model.forward_text(**outputs).cpu()
+ label_sim = label_hidden_states @ label_hidden_states.t()
+ num_labels = label_sim.size(0)
+ eye_mask = ~torch.eye(num_labels, dtype=torch.bool)
+ label_sim = label_sim.masked_select(eye_mask).view(num_labels, num_labels - 1)
+ lbd = label_sim.max()
+
+ # this is not a loss but just compute neg_log_prob.
+ Y_pred = []
+ Y_true = []
+ with torch.no_grad():
+ for batch in eval_dataloader:
+ self(batch, label_hidden_states, model, lbd, Y_pred, Y_true)
+ return self.finalize(Y_pred, Y_true, output_file)
+
+ def reshape_subsample(self, sample):
+ for key in sample:
+ if torch.is_tensor(sample[key]):
+ sample[key] = self.flat_subsample(sample[key])
+ return sample
+
+ def flat_subsample(self, tensor):
+ if len(tensor.size()) > 1 and tensor.size(0) == 1:
+ tensor = tensor.squeeze(0)
+ return tensor
+
+ def __call__(self, sample, label_hidden_states, model, lbd, Y_pred, Y_true):
+ sample = self.reshape_subsample(sample)
+ sample = self.to_ctx(sample)
+ # compute the average logits over sliding windows.
+ sample["output_hidden_states"] = True
+ video_outputs = model.forward_video(**sample).cpu()
+ output = {"logits": video_outputs[:, 1:sample["vmasks"].size(1)+1] @ label_hidden_states.t()}
+ logits = self._merge_windows(sample, output)
+ # logic of zero-shot for sequence labeling.
+ logits_argmax = logits.argmax(dim=1) + 1 # 0 is "O" label.
+ logits_max = logits.max(dim=1)[0]
+
+ pred = torch.zeros_like(logits_argmax)
+ label_select = logits_max > lbd # 73 or 74
+ pred[label_select] = logits_argmax[label_select]
+
+ Y_pred.append(pred)
+ Y_true.append(sample["video_targets"].squeeze(0).cpu())
+
+ def finalize(self, Y_pred, Y_true, output_file=None):
+ Y_pred = torch.cat(Y_pred, dim=0).numpy()
+ Y_true = torch.cat(Y_true, dim=0).numpy()
+ assert len(Y_pred) == len(Y_true)
+
+ error_mask = Y_pred != Y_true
+ print("sample error", Y_pred[error_mask][:10], Y_true[error_mask][:10])
+ print("sample error", Y_pred[error_mask][10:20], Y_true[error_mask][10:20])
+
+ if output_file is not None:
+ with open(
+ os.path.join(self.pred_dir, output_file + ".pkl"),
+ "wb") as fw:
+ pickle.dump(
+ {"Y_pred": Y_pred, "Y_true": Y_true}, fw,
+ protocol=pickle.HIGHEST_PROTOCOL)
+ return {"outputs": Y_pred, "targets": Y_true}
+
+
+class DiDeMoPredictor(Predictor):
+ """reference: https://github.com/LisaAnne/LocalizingMoments/blob/master/utils/eval.py
+ https://github.com/LisaAnne/LocalizingMoments/blob/master/utils/data_processing.py
+ """
+ def __init__(self, config):
+ super().__init__(config)
+ # load targets.
+ with open(config.dataset.test_path) as data_file:
+ self.test_data = json.load(data_file)
+
+ def predict_loop(self, model, eval_dataloader, output_file="didemo.npy"):
+ """
+ TODO: two solutions here.
+ """
+ import itertools
+ # 21 chunks.
+ self.possible_segments = [(0,0), (1,1), (2,2), (3,3), (4,4), (5,5)]
+ for i in itertools.combinations(range(6), 2):
+ self.possible_segments.append(i)
+ # pick segments from a video.
+
+ """on-the-fly prediction on a single gpu."""
+ self.full_scores = []
+ model.eval()
+ model = model.cuda()
+ with torch.no_grad():
+ for data in eval_dataloader:
+ # TODO special forwarding logic here.
+ data = self.to_ctx(data)
+ data["output_hidden_states"] = True
+ hidden_video = model.forward_video(**data)
+ data["output_hidden_states"] = False
+ pooled_text = model.forward_text(**data)
+ outputs = {
+ "hidden_video": hidden_video,
+ "pooled_text": pooled_text
+ }
+ outputs.update(data)
+ self(outputs)
+ return self.finalize(output_file)
+
+ def __call__(self, sample):
+ # TODO: make an index select from self.possible_segments.
+ hidden_video = sample["hidden_video"]
+ pooled_text = sample["pooled_text"]
+ vmasks = sample["vmasks"]
+ # probably maintain valid results here.
+
+ hidden_video = hidden_video[:, 1:-1, :]
+ # probably maintain valid results here.
+ pooled_video = []
+ for s, e in self.possible_segments:
+ pooled_video.append(
+ torch.mean(
+ hidden_video[:, int(s*5):int((e+1)*5), :],
+ dim=1, keepdim=True)
+ )
+ pooled_video = torch.cat(pooled_video, dim=1)
+ scores = torch.bmm(
+ pooled_video, pooled_text.unsqueeze(-1)).squeeze(-1).cpu()
+
+ ranks = scores.argsort(dim=-1, descending=True)
+
+ for batch_idx, rank in enumerate(ranks):
+ rank_of_moment = []
+ for m_idx, moment in enumerate(rank):
+ s, e = self.possible_segments[moment.item()]
+ if torch.any(
+ vmasks[batch_idx, int(s*5):int((e+1)*5)]
+ ):
+ rank_of_moment.append((s, e))
+ self.full_scores.append(rank_of_moment)
+
+ def finalize(self, output_file=None):
+ outputs = self._aggregate_scores(self.full_scores)
+ if output_file is not None:
+ np.save(os.path.join(self.pred_dir, output_file + ".npy"), outputs)
+ return {"outputs": outputs, "targets": self.test_data}
+
+ def _aggregate_scores(self, scores):
+ self.full_scores = []
+ return scores
diff --git a/data/fairseq/examples/MMPT/mmpt/utils/__init__.py b/data/fairseq/examples/MMPT/mmpt/utils/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..2429ee3757353e768f71b27d129eb3ca3bcbec73
--- /dev/null
+++ b/data/fairseq/examples/MMPT/mmpt/utils/__init__.py
@@ -0,0 +1,68 @@
+# Copyright (c) Facebook, Inc. and its affiliates.
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+import random
+import numpy as np
+import torch
+
+from .shardedtensor import *
+from .load_config import *
+
+
+def set_seed(seed=43211):
+ random.seed(seed)
+ np.random.seed(seed)
+ torch.manual_seed(seed)
+ torch.cuda.manual_seed_all(seed)
+ if torch.backends.cudnn.enabled:
+ torch.backends.cudnn.benchmark = False
+ torch.backends.cudnn.deterministic = True
+
+
+def get_world_size():
+ if torch.distributed.is_initialized():
+ world_size = torch.distributed.get_world_size()
+ else:
+ world_size = 1
+ return world_size
+
+
+def get_local_rank():
+ return torch.distributed.get_rank() \
+ if torch.distributed.is_initialized() else 0
+
+
+def print_on_rank0(func):
+ local_rank = get_local_rank()
+ if local_rank == 0:
+ print("[INFO]", func)
+
+
+class RetriMeter(object):
+ """
+ Statistics on whether retrieval yields a better pair.
+ """
+ def __init__(self, freq=1024):
+ self.freq = freq
+ self.total = 0
+ self.replace = 0
+ self.updates = 0
+
+ def __call__(self, data):
+ if isinstance(data, np.ndarray):
+ self.replace += data.shape[0] - int((data[:, 0] == -1).sum())
+ self.total += data.shape[0]
+ elif torch.is_tensor(data):
+ self.replace += int(data.sum())
+ self.total += data.size(0)
+ else:
+ raise ValueError("unsupported RetriMeter data type.", type(data))
+
+ self.updates += 1
+ if get_local_rank() == 0 and self.updates % self.freq == 0:
+ print("[INFO]", self)
+
+ def __repr__(self):
+ return "RetriMeter (" + str(self.replace / self.total) \
+ + "/" + str(self.replace) + "/" + str(self.total) + ")"
diff --git a/data/fairseq/examples/MMPT/mmpt/utils/load_config.py b/data/fairseq/examples/MMPT/mmpt/utils/load_config.py
new file mode 100644
index 0000000000000000000000000000000000000000..ede4f94117b197e5bdb551c5140604c6e47c91fb
--- /dev/null
+++ b/data/fairseq/examples/MMPT/mmpt/utils/load_config.py
@@ -0,0 +1,81 @@
+# Copyright (c) Facebook, Inc. and its affiliates.
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+import os
+import omegaconf
+from omegaconf import OmegaConf
+
+
+def load_config(args=None, config_file=None, overwrite_fairseq=False):
+ """TODO (huxu): move fairseq overwrite to another function."""
+ if args is not None:
+ config_file = args.taskconfig
+ config = recursive_config(config_file)
+
+ if config.dataset.subsampling is not None:
+ batch_size = config.fairseq.dataset.batch_size // config.dataset.subsampling
+ print(
+ "adjusting batch_size to {} due to subsampling {}.".format(
+ batch_size, config.dataset.subsampling
+ )
+ )
+ config.fairseq.dataset.batch_size = batch_size
+
+ is_test = config.dataset.split is not None and config.dataset.split == "test"
+ if not is_test:
+ if (
+ config.fairseq.checkpoint is None
+ or config.fairseq.checkpoint.save_dir is None
+ ):
+ raise ValueError("fairseq save_dir or save_path must be specified.")
+
+ save_dir = config.fairseq.checkpoint.save_dir
+ os.makedirs(save_dir, exist_ok=True)
+ if config.fairseq.common.tensorboard_logdir is not None:
+ tb_run_dir = suffix_rundir(
+ save_dir, config.fairseq.common.tensorboard_logdir
+ )
+ config.fairseq.common.tensorboard_logdir = tb_run_dir
+ print(
+ "update tensorboard_logdir as", config.fairseq.common.tensorboard_logdir
+ )
+ os.makedirs(save_dir, exist_ok=True)
+ OmegaConf.save(config=config, f=os.path.join(save_dir, "config.yaml"))
+
+ if overwrite_fairseq and config.fairseq is not None and args is not None:
+ # flatten fields.
+ for group in config.fairseq:
+ for field in config.fairseq[group]:
+ print("overwrite args." + field, "as", config.fairseq[group][field])
+ setattr(args, field, config.fairseq[group][field])
+ return config
+
+
+def recursive_config(config_path):
+ """allows for stacking of configs in any depth."""
+ config = OmegaConf.load(config_path)
+ if config.includes is not None:
+ includes = config.includes
+ config.pop("includes")
+ base_config = recursive_config(includes)
+ config = OmegaConf.merge(base_config, config)
+ return config
+
+
+def suffix_rundir(save_dir, run_dir):
+ max_id = -1
+ for search_dir in os.listdir(save_dir):
+ if search_dir.startswith(run_dir):
+ splits = search_dir.split("_")
+ cur_id = int(splits[1]) if len(splits) > 1 else 0
+ max_id = max(max_id, cur_id)
+ return os.path.join(save_dir, run_dir + "_" + str(max_id + 1))
+
+
+def overwrite_dir(config, replace, basedir):
+ for key in config:
+ if isinstance(config[key], str) and config[key].startswith(basedir):
+ config[key] = config[key].replace(basedir, replace)
+ if isinstance(config[key], omegaconf.dictconfig.DictConfig):
+ overwrite_dir(config[key], replace, basedir)
diff --git a/data/fairseq/examples/MMPT/mmpt/utils/shardedtensor.py b/data/fairseq/examples/MMPT/mmpt/utils/shardedtensor.py
new file mode 100644
index 0000000000000000000000000000000000000000..2424f360ef2a93f0f2783e38f0bd966086e92fd4
--- /dev/null
+++ b/data/fairseq/examples/MMPT/mmpt/utils/shardedtensor.py
@@ -0,0 +1,46 @@
+# Copyright (c) Facebook, Inc. and its affiliates.
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+import os
+import pickle
+import numpy as np
+
+
+class ShardedTensor(object):
+ def __init__(self, data, starts):
+ self.data = data
+ self.starts = starts
+ assert self.starts[0] == 0
+ assert self.starts[-1] == len(self.data)
+ assert (self.starts[1:] >= self.starts[:-1]).all()
+ assert (self.starts > -1).all()
+
+ @staticmethod
+ def from_list(xs):
+ starts = np.full((len(xs) + 1,), -1, dtype=np.long)
+ data = np.concatenate(xs, axis=0)
+ starts[0] = 0
+ for i, x in enumerate(xs):
+ starts[i + 1] = starts[i] + x.shape[0]
+ assert (starts > -1).all()
+ return ShardedTensor(data, starts)
+
+ def __getitem__(self, i):
+ return self.data[self.starts[i] : self.starts[i + 1]]
+
+ def __len__(self):
+ return len(self.starts) - 1
+
+ def lengths(self):
+ return self.starts[1:] - self.starts[:-1]
+
+ def save(self, path):
+ np.save(path + "_starts", self.starts)
+ np.save(path + "_data", self.data)
+
+ @staticmethod
+ def load(path, mmap_mode=None):
+ starts = np.load(path + "_starts.npy", mmap_mode)
+ data = np.load(path + "_data.npy", mmap_mode)
+ return ShardedTensor(data, starts)
diff --git a/data/fairseq/examples/MMPT/pretraining.md b/data/fairseq/examples/MMPT/pretraining.md
new file mode 100644
index 0000000000000000000000000000000000000000..8f8e6d0facaa47141342294d7eb26c4232a9677b
--- /dev/null
+++ b/data/fairseq/examples/MMPT/pretraining.md
@@ -0,0 +1,29 @@
+# Pretraining
+
+(If you are new to the ideas of `mmpt.processors`, see [README](README.md) first.)
+We mostly use [howto100M](https://github.com/antoine77340/howto100m) dataset for pretraining (other datasets are coming). So you are less likely to write a new `MetaProcessor`, `VideoProcessor` or `TextProcessor` but only working on a new `Aligner`, a new model and loss.
+
+### Data Sharding
+Pretraining on Howto100M is heavy on IO since we have millions of videos or captions on the hard disk that cannot be fit into the memory.
+It is desirable to have an optimized preprocessing step before the actual dataloading.
+
+We support data sharding to pack multiple videos into a shards of training data for both videos and captions. (see [dataset](DATASET.md) for preprocessing).
+These shards will be mapped into memory to reduce the frequency of IO access on millions of files. See (processors starting with `Sharded*`).
+This will be the default config for a how2 dataset `projects/task/how2.yaml`.
+
+Great thanks to Dmytro Okhonko for sharing the code from MARGE project.
+
+### Training
+Pretraining on Howto100m is expected on one or multiple nodes, where each node has 8 GPUS with 32 GB mem.
+launching a pretraing on MFM+MLM can be done, via:
+```python locallaunch.py projects/mfmmlm/how2.yaml```
+
+### Pre-training with a Retrieval Model (VideoCLIP)
+This projects now support alternatively run a retrieval model and pre-training.
+We implement a basic retrieval model that is built on the hidden states of a video and faiss.
+
+You may need to install faiss via `conda install faiss-cpu -c pytorch`.
+
+Right now, the hidden states of a video is computed as the average of 8 clips of their pooled visual/text hidden states.
+See `mmpt/tasks/retritask.py` for more details.
+The `.yaml` config for running pre-training with a retrieval model can be found at `projects/retri/videoretri.yaml`.
diff --git a/data/fairseq/examples/MMPT/projects/mfmmlm.yaml b/data/fairseq/examples/MMPT/projects/mfmmlm.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..0f3450a1e091517de19422b0c09cea6a01a02729
--- /dev/null
+++ b/data/fairseq/examples/MMPT/projects/mfmmlm.yaml
@@ -0,0 +1,59 @@
+project_dir: mfmmlm
+run_task:
+ - how2.yaml
+ - [vtt.yaml, vttcap.yaml, vttqa.yaml, youcook.yaml, youcookcap.yaml, crosstask.yaml, coin.yaml]
+base_dir: task
+task_group:
+ pretrain:
+ task_list:
+ - how2.yaml
+ dataset:
+ subsampling: 32
+ sampled_min_len: 10
+ sampled_max_len: 64
+ max_video_len: 32
+ max_len: 96
+ aligner: MFMMLMAligner
+ lazy_vfeat_mask: True
+ mfm_probability: 0.15
+ mlm_probability: 0.15
+ mm_prob: 0.5
+ model:
+ model_cls: MMFusionMFMMLM
+ mm_encoder_cls: MMFusionForMFMMLM
+ loss:
+ loss_cls: MFMMLM
+ fairseq:
+ common:
+ fp16: true
+ dataset:
+ batch_size: 256
+ optimization:
+ max_epoch: 15
+ finetune:
+ task_list:
+ - vtt.yaml
+ - vttqa.yaml
+ - youcook.yaml
+ - youcookcap.yaml
+ - crosstask.yaml
+ - coin.yaml
+ dataset:
+ max_video_len: 32
+ max_len: 96
+ fairseq:
+ common:
+ fp16: true
+ # do not write any model or loss here (they are expected to be fixed in mmfusion).
+ test:
+ task_list:
+ - test_vtt.yaml
+ - test_vttqa.yaml
+ - test_youcook.yaml
+ - test_youcookcap.yaml
+ - test_crosstask.yaml
+ - test_crosstask_zs.yaml
+ - test_coin.yaml
+ dataset:
+ max_video_len: 32
+ max_len: 96
diff --git a/data/fairseq/examples/MMPT/projects/mtm/mmfusionmtm.yaml b/data/fairseq/examples/MMPT/projects/mtm/mmfusionmtm.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..337d66a2aadcff86da033d6656bdc570a20c32e6
--- /dev/null
+++ b/data/fairseq/examples/MMPT/projects/mtm/mmfusionmtm.yaml
@@ -0,0 +1,19 @@
+includes: projects/mfmmlm.yaml
+project_dir: mtm/mmfusionmtm
+task_group:
+ pretrain:
+ task: VLMTask # reproducible
+ dataset:
+ aligner: MFMMLMAligner
+ model:
+ use_seg_emb: True # reproducible
+ model_cls: MMFusionMTM
+ mm_encoder_cls: MMBertForMFMMLM
+ loss:
+ loss_cls: MTM
+ finetune:
+ model:
+ use_seg_emb: True # reproducible
+ test:
+ model:
+ use_seg_emb: True # reproducible
diff --git a/data/fairseq/examples/MMPT/projects/mtm/vlm.yaml b/data/fairseq/examples/MMPT/projects/mtm/vlm.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..022a2623c5fdee1da51f211bef25a0dfebd7cbb0
--- /dev/null
+++ b/data/fairseq/examples/MMPT/projects/mtm/vlm.yaml
@@ -0,0 +1,8 @@
+includes: projects/mtm/mmfusionmtm.yaml
+project_dir: mtm/vlm
+task_group:
+ pretrain:
+ dataset:
+ sampled_min_len: 8
+ loss:
+ loss_cls: MTM
diff --git a/data/fairseq/examples/MMPT/projects/mtm/vlm/coin.yaml b/data/fairseq/examples/MMPT/projects/mtm/vlm/coin.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..48fd64a5f4996a03b412c418e83c4672b7252e8d
--- /dev/null
+++ b/data/fairseq/examples/MMPT/projects/mtm/vlm/coin.yaml
@@ -0,0 +1,47 @@
+dataset:
+ video_processor: VideoProcessor
+ bert_name: bert-base-uncased
+ meta_processor: COINActionSegmentationMetaProcessor
+ train_path: data/coin/COIN.json
+ val_path: data/coin/COIN.json
+ vfeat_dir: data/feat/feat_coin_s3d
+ text_processor: COINActionSegmentationTextProcessor
+ aligner: COINActionSegmentationAligner
+ num_iso_layer: 12
+ sliding_window: 8
+ sliding_window_size: 32
+ max_video_len: 32
+ max_len: 96
+fairseq:
+ common:
+ tensorboard_logdir: run
+ log_interval: 1000
+ fp16: true
+ dataset:
+ num_workers: 4
+ batch_size: 1
+ optimization:
+ lr:
+ - 5.0e-05
+ clip_norm: 2.0
+ optimizer: adam
+ adam_betas: (0.9, 0.98)
+ lr_scheduler: polynomial_decay
+ total_num_update: 1000000
+ warmup_updates: 122
+ weight_decay: 0.0
+ ddp_backend: no_c10d
+ max_epoch: 8
+ checkpoint:
+ restore_file: runs/mtm/vlm/checkpoint_best.pt
+ reset_optimizer: true
+ reset_dataloader: true
+ reset_meters: true
+ save_dir: runs/mtm/vlm/coin
+task_type: sweep_big
+model:
+ model_cls: MMFusionActionSegmentation
+ mm_encoder_cls: MMBertForTokenClassification
+ use_seg_emb: true
+loss:
+ loss_cls: CrossEntropy
diff --git a/data/fairseq/examples/MMPT/projects/mtm/vlm/crosstask.yaml b/data/fairseq/examples/MMPT/projects/mtm/vlm/crosstask.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..4e706b549e6085c9823049afaff3285aa2b88f4f
--- /dev/null
+++ b/data/fairseq/examples/MMPT/projects/mtm/vlm/crosstask.yaml
@@ -0,0 +1,53 @@
+dataset:
+ video_processor: CrossTaskVideoProcessor
+ bert_name: bert-base-uncased
+ meta_processor: CrossTaskMetaProcessor
+ train_path: data/crosstask/crosstask_release/videos.csv
+ train_csv_path: data/crosstask/crosstask_release/videos.csv
+ val_path: data/crosstask/crosstask_release/videos_val.csv
+ val_csv_path: data/crosstask/crosstask_release/videos_val.csv
+ primary_path: data/crosstask/crosstask_release/tasks_primary.txt
+ related_path: data/crosstask/crosstask_release/tasks_related.txt
+ vfeat_dir: data/feat/feat_crosstask_s3d
+ annotation_path: data/crosstask/crosstask_release/annotations
+ n_train: 30
+ text_processor: CrossTaskTextProcessor
+ aligner: CrossTaskAligner
+ num_iso_layer: 12
+ sliding_window: 16
+ sliding_window_size: 32
+ max_video_len: 32
+ max_len: 96
+fairseq:
+ common:
+ tensorboard_logdir: run
+ log_interval: 1000
+ fp16: true
+ dataset:
+ num_workers: 4
+ batch_size: 1
+ optimization:
+ lr:
+ - 5.0e-05
+ clip_norm: 2.0
+ optimizer: adam
+ adam_betas: (0.9, 0.98)
+ lr_scheduler: polynomial_decay
+ total_num_update: 1000000
+ warmup_updates: 122
+ weight_decay: 0.0
+ ddp_backend: no_c10d
+ max_epoch: 5
+ checkpoint:
+ restore_file: runs/mtm/vlm/checkpoint11.pt
+ reset_optimizer: true
+ reset_dataloader: true
+ reset_meters: true
+ save_dir: runs/mtm/vlm/crosstask
+task_type: sweep_small
+model:
+ model_cls: MMFusionActionLocalization
+ mm_encoder_cls: MMBertForJoint
+ use_seg_emb: true
+loss:
+ loss_cls: BCE
diff --git a/data/fairseq/examples/MMPT/projects/mtm/vlm/how2.yaml b/data/fairseq/examples/MMPT/projects/mtm/vlm/how2.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..7ca40ad815d9393bc9dc4d248cf7ec8810f1011f
--- /dev/null
+++ b/data/fairseq/examples/MMPT/projects/mtm/vlm/how2.yaml
@@ -0,0 +1,55 @@
+dataset:
+ video_processor: ShardedVideoProcessor
+ bert_name: bert-base-uncased
+ meta_processor: ShardedHow2MetaProcessor
+ train_path: data/how2/how2_s3d_train.lst
+ val_path: data/how2/how2_s3d_val.lst
+ vfeat_dir: data/feat/feat_how2_s3d_shard_small
+ text_processor: ShardedTextProcessor
+ tfeat_dir: data/feat/feat_how2_s3d_shard_small/raw_caption_dedup.bert-base-uncased.
+ aligner: MFMMLMAligner
+ subsampling: 32
+ sampled_min_len: 8
+ sampled_max_len: 64
+ max_video_len: 32
+ max_len: 96
+ lazy_vfeat_mask: true
+ mfm_probability: 0.15
+ mlm_probability: 0.15
+ mm_prob: 0.5
+fairseq:
+ common:
+ tensorboard_logdir: run
+ log_interval: 1000
+ fp16: true
+ dataset:
+ num_workers: 4
+ batch_size: 256
+ optimization:
+ lr:
+ - 5.0e-05
+ clip_norm: 2.0
+ optimizer: adam
+ adam_betas: (0.9, 0.98)
+ lr_scheduler: polynomial_decay
+ total_num_update: 1000000
+ warmup_updates: 1000
+ weight_decay: 0.0
+ ddp_backend: no_c10d
+ max_epoch: 15
+ checkpoint:
+ save_dir: runs/mtm/vlm
+ save_interval_updates: 1024
+ keep_interval_updates: 2
+ keep_last_epochs: 30
+task_type: sweep_big
+slurm_config: big
+eval:
+ save_path: runs/mtm/vlm
+model:
+ model_cls: MMFusionMTM
+ mm_encoder_cls: MMBertForMFMMLM
+ use_seg_emb: true
+loss:
+ loss_cls: MTM
+task: VLMTask
diff --git a/data/fairseq/examples/MMPT/projects/mtm/vlm/test_coin.yaml b/data/fairseq/examples/MMPT/projects/mtm/vlm/test_coin.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..8df2e66ad175529e0e86f69ade1e392849507036
--- /dev/null
+++ b/data/fairseq/examples/MMPT/projects/mtm/vlm/test_coin.yaml
@@ -0,0 +1,31 @@
+slurm_config: big
+task_type: local_predict
+dataset:
+ split: test
+ video_processor: VideoProcessor
+ aligner: COINActionSegmentationAligner
+ bert_name: bert-base-uncased
+ test_path: data/coin/COIN.json
+ meta_processor: COINActionSegmentationMetaProcessor
+ vfeat_dir: data/feat/feat_coin_s3d
+ text_processor: COINActionSegmentationTextProcessor
+ num_iso_layer: 12
+ sliding_window: 16
+ sliding_window_size: 32
+ max_video_len: 32
+ max_len: 96
+fairseq:
+ dataset:
+ batch_size: 1
+ valid_subset: test
+ num_workers: 2
+ common_eval:
+ path: runs/mtm/vlm/coin/checkpoint_best.pt
+model:
+ model_cls: MMFusionActionSegmentation
+ mm_encoder_cls: MMBertForTokenClassification
+ use_seg_emb: true
+eval:
+ save_path: runs/mtm/vlm/coin/eval
+metric: COINActionSegmentationMetric
+predictor: COINPredictor
diff --git a/data/fairseq/examples/MMPT/projects/mtm/vlm/test_crosstask.yaml b/data/fairseq/examples/MMPT/projects/mtm/vlm/test_crosstask.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..d159847875f74050fbae5f0c223fa6d56bbd3d38
--- /dev/null
+++ b/data/fairseq/examples/MMPT/projects/mtm/vlm/test_crosstask.yaml
@@ -0,0 +1,38 @@
+slurm_config: big
+task_type: local_predict
+dataset:
+ split: test
+ video_processor: CrossTaskVideoProcessor
+ aligner: CrossTaskAligner
+ bert_name: bert-base-uncased
+ meta_processor: CrossTaskMetaProcessor
+ test_path: data/crosstask/crosstask_release/videos_val.csv
+ train_csv_path: data/crosstask/crosstask_release/videos.csv
+ val_path: data/crosstask/crosstask_release/videos_val.csv
+ val_csv_path: data/crosstask/crosstask_release/videos_val.csv
+ primary_path: data/crosstask/crosstask_release/tasks_primary.txt
+ related_path: data/crosstask/crosstask_release/tasks_related.txt
+ vfeat_dir: data/feat/feat_crosstask_s3d
+ annotation_path: data/crosstask/crosstask_release/annotations
+ n_train: 30
+ text_processor: CrossTaskTextProcessor
+ num_iso_layer: 12
+ sliding_window: 16
+ sliding_window_size: 32
+ max_video_len: 32
+ max_len: 96
+fairseq:
+ dataset:
+ batch_size: 1
+ valid_subset: test
+ num_workers: 2
+ common_eval:
+ path: runs/mtm/vlm/crosstask/checkpoint_best.pt
+model:
+ model_cls: MMFusionActionLocalization
+ mm_encoder_cls: MMBertForJoint
+ use_seg_emb: true
+eval:
+ save_path: runs/mtm/vlm/crosstask/eval
+metric: CrossTaskMetric
+predictor: CrossTaskPredictor
diff --git a/data/fairseq/examples/MMPT/projects/mtm/vlm/test_crosstask_zs.yaml b/data/fairseq/examples/MMPT/projects/mtm/vlm/test_crosstask_zs.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..59833c55406ce53225d704b46eb0c141ad4d5e9d
--- /dev/null
+++ b/data/fairseq/examples/MMPT/projects/mtm/vlm/test_crosstask_zs.yaml
@@ -0,0 +1,38 @@
+slurm_config: big
+task_type: local_predict
+dataset:
+ split: test
+ video_processor: CrossTaskVideoProcessor
+ aligner: CrossTaskAligner
+ bert_name: bert-base-uncased
+ meta_processor: CrossTaskMetaProcessor
+ test_path: data/crosstask/crosstask_release/videos_val.csv
+ train_csv_path: data/crosstask/crosstask_release/videos.csv
+ val_path: data/crosstask/crosstask_release/videos_val.csv
+ val_csv_path: data/crosstask/crosstask_release/videos_val.csv
+ primary_path: data/crosstask/crosstask_release/tasks_primary.txt
+ related_path: data/crosstask/crosstask_release/tasks_related.txt
+ vfeat_dir: data/feat/feat_crosstask_s3d
+ annotation_path: data/crosstask/crosstask_release/annotations
+ n_train: 30
+ text_processor: CrossTaskTextProcessor
+ num_iso_layer: 12
+ sliding_window: 16
+ sliding_window_size: 32
+ max_video_len: 32
+ max_len: 96
+fairseq:
+ dataset:
+ batch_size: 1
+ valid_subset: test
+ num_workers: 2
+ common_eval:
+ path: runs/mtm/vlm/checkpoint_best.pt
+model:
+ model_cls: MMFusionActionLocalization
+ mm_encoder_cls: MMBertForJoint
+ use_seg_emb: true
+eval:
+ save_path: runs/mtm/vlm/crosstask_zs/eval
+metric: CrossTaskMetric
+predictor: CrossTaskPredictor
diff --git a/data/fairseq/examples/MMPT/projects/mtm/vlm/test_vtt.yaml b/data/fairseq/examples/MMPT/projects/mtm/vlm/test_vtt.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..a41557df6af496c755b0b32345a736b1cdc01ce4
--- /dev/null
+++ b/data/fairseq/examples/MMPT/projects/mtm/vlm/test_vtt.yaml
@@ -0,0 +1,29 @@
+slurm_config: big
+task_type: local_predict
+dataset:
+ split: test
+ video_processor: VideoProcessor
+ aligner: DSAligner
+ bert_name: bert-base-uncased
+ meta_processor: MSRVTTMetaProcessor
+ test_path: data/msrvtt/MSRVTT_JSFUSION_test.csv
+ vfeat_dir: data/feat/feat_vtt_s3d
+ text_processor: MSRVTTTextProcessor
+ num_iso_layer: 12
+ max_video_len: 32
+ max_len: 96
+fairseq:
+ dataset:
+ batch_size: 256
+ valid_subset: test
+ num_workers: 2
+ common_eval:
+ path: runs/mtm/vlm/vtt/checkpoint_last.pt
+model:
+ model_cls: MMFusionJoint
+ mm_encoder_cls: MMBertForJoint
+ use_seg_emb: true
+eval:
+ save_path: runs/mtm/vlm/vtt/eval
+metric: RetrievalMetric
+predictor: RetrievalPredictor
diff --git a/data/fairseq/examples/MMPT/projects/mtm/vlm/test_vttqa.yaml b/data/fairseq/examples/MMPT/projects/mtm/vlm/test_vttqa.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..abf3309f7072c88a6b7c72e2540d4950d0e55575
--- /dev/null
+++ b/data/fairseq/examples/MMPT/projects/mtm/vlm/test_vttqa.yaml
@@ -0,0 +1,29 @@
+slurm_config: big
+task_type: local_predict
+dataset:
+ split: test
+ video_processor: VideoProcessor
+ aligner: MSRVTTQAAligner
+ bert_name: bert-base-uncased
+ meta_processor: MSRVTTQAMetaProcessor
+ test_path: data/msrvtt-qa/MSR_MC_test.csv
+ vfeat_dir: data/feat/feat_vtt_s3d
+ text_processor: MSRVTTQATextProcessor
+ num_iso_layer: 12
+ max_video_len: 32
+ max_len: 96
+fairseq:
+ dataset:
+ batch_size: 256
+ valid_subset: test
+ num_workers: 2
+ common_eval:
+ path: runs/mtm/vlm/vttqa/checkpoint_last.pt
+model:
+ model_cls: MMFusionJoint
+ mm_encoder_cls: MMBertForJoint
+ use_seg_emb: true
+eval:
+ save_path: runs/mtm/vlm/vttqa/eval
+metric: QAMetric
+predictor: QAPredictor
diff --git a/data/fairseq/examples/MMPT/projects/mtm/vlm/test_youcook.yaml b/data/fairseq/examples/MMPT/projects/mtm/vlm/test_youcook.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..3a57d25c240758cf989ad0ec538562d046446232
--- /dev/null
+++ b/data/fairseq/examples/MMPT/projects/mtm/vlm/test_youcook.yaml
@@ -0,0 +1,31 @@
+slurm_config: big
+task_type: local_predict
+dataset:
+ split: test
+ video_processor: YoucookVideoProcessor
+ aligner: DSAligner
+ bert_name: bert-base-uncased
+ meta_processor: YoucookMetaProcessor
+ test_path: data/youcook/youcook_val.pkl
+ trainval_annotation: data/youcook/youcookii_annotations_trainval.json
+ use_annotation_text: true
+ vfeat_dir: data/feat/feat_youcook_s3d
+ text_processor: TextProcessor
+ num_iso_layer: 12
+ max_video_len: 32
+ max_len: 96
+fairseq:
+ dataset:
+ batch_size: 256
+ valid_subset: test
+ num_workers: 2
+ common_eval:
+ path: runs/mtm/vlm/youcook/checkpoint_last.pt
+model:
+ model_cls: MMFusionJoint
+ mm_encoder_cls: MMBertForJoint
+ use_seg_emb: true
+eval:
+ save_path: runs/mtm/vlm/youcook/eval
+metric: RetrievalMetric
+predictor: RetrievalPredictor
diff --git a/data/fairseq/examples/MMPT/projects/mtm/vlm/test_youcookcap.yaml b/data/fairseq/examples/MMPT/projects/mtm/vlm/test_youcookcap.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..b2595d7c3c633d0faac5e4891457067659f7fafc
--- /dev/null
+++ b/data/fairseq/examples/MMPT/projects/mtm/vlm/test_youcookcap.yaml
@@ -0,0 +1,32 @@
+slurm_config: big
+task_type: local_predict
+dataset:
+ split: test
+ video_processor: YoucookVideoProcessor
+ aligner: DSNLGAligner
+ bert_name: bert-base-uncased
+ meta_processor: YoucookNLGMetaProcessor
+ test_path: data/youcook/val_list.txt
+ trainval_annotation: data/youcook/youcookii_annotations_trainval.json
+ vfeat_dir: data/feat/feat_youcook_s3d
+ text_processor: NLGTextProcessor
+ max_video_len: 32
+ max_len: 96
+fairseq:
+ dataset:
+ batch_size: 256
+ valid_subset: test
+ num_workers: 2
+ common_eval:
+ path: runs/mtm/vlm/youcookcap/checkpoint_best.pt
+model:
+ model_cls: MMFusionNLG
+ mm_encoder_cls: MMBertForNLG
+ max_decode_length: 24
+ use_seg_emb: true
+eval:
+ save_path: runs/mtm/vlm/youcookcap/eval
+metric: NLGMetric
+predictor: NLGPredictor
+gen_param:
+ num_beams: 5
diff --git a/data/fairseq/examples/MMPT/projects/mtm/vlm/vtt.yaml b/data/fairseq/examples/MMPT/projects/mtm/vlm/vtt.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..c6c5b1ab40cbd2256919b42ea6b6a5fa52fe41a1
--- /dev/null
+++ b/data/fairseq/examples/MMPT/projects/mtm/vlm/vtt.yaml
@@ -0,0 +1,49 @@
+dataset:
+ video_processor: VideoProcessor
+ bert_name: bert-base-uncased
+ meta_processor: MSRVTTMetaProcessor
+ train_path: data/msrvtt/MSRVTT_train.csv
+ jsfusion_path: data/msrvtt/MSRVTT_JSFUSION_test.csv
+ full_test_path: data/msrvtt/MSRVTT_FULL_test.csv
+ dup: 20
+ val_path: data/msrvtt/MSRVTT_JSFUSION_test.csv
+ vfeat_dir: data/feat/feat_vtt_s3d
+ text_processor: MSRVTTTextProcessor
+ json_path: data/msrvtt/MSRVTT_data.json
+ aligner: DSAligner
+ num_iso_layer: 12
+ max_video_len: 32
+ max_len: 96
+fairseq:
+ common:
+ tensorboard_logdir: run
+ log_interval: 1000
+ fp16: true
+ dataset:
+ num_workers: 4
+ batch_size: 256
+ optimization:
+ lr:
+ - 5.0e-05
+ clip_norm: 2.0
+ optimizer: adam
+ adam_betas: (0.9, 0.98)
+ lr_scheduler: polynomial_decay
+ total_num_update: 1000000
+ warmup_updates: 122
+ weight_decay: 0.0
+ ddp_backend: no_c10d
+ max_epoch: 10
+ checkpoint:
+ restore_file: runs/mtm/vlm/checkpoint_best.pt
+ reset_optimizer: true
+ reset_dataloader: true
+ reset_meters: true
+ save_dir: runs/mtm/vlm/vtt
+task_type: sweep_small
+model:
+ model_cls: MMFusionJoint
+ mm_encoder_cls: MMBertForJoint
+ use_seg_emb: true
+loss:
+ loss_cls: T2VContraLoss
diff --git a/data/fairseq/examples/MMPT/projects/mtm/vlm/vttqa.yaml b/data/fairseq/examples/MMPT/projects/mtm/vlm/vttqa.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..0a440c7dd2f5b324d57367456d22ca5aab8f397f
--- /dev/null
+++ b/data/fairseq/examples/MMPT/projects/mtm/vlm/vttqa.yaml
@@ -0,0 +1,47 @@
+dataset:
+ video_processor: VideoProcessor
+ bert_name: bert-base-uncased
+ meta_processor: MSRVTTMetaProcessor
+ train_path: data/msrvtt/MSRVTT_train.csv
+ dup: 20
+ val_path: data/msrvtt/MSRVTT_JSFUSION_test.csv
+ vfeat_dir: data/feat/feat_vtt_s3d
+ text_processor: MSRVTTTextProcessor
+ json_path: data/msrvtt/MSRVTT_data.json
+ aligner: DSAligner
+ num_iso_layer: 12
+ max_video_len: 32
+ max_len: 96
+fairseq:
+ common:
+ tensorboard_logdir: run
+ log_interval: 1000
+ fp16: true
+ dataset:
+ num_workers: 4
+ batch_size: 128
+ optimization:
+ lr:
+ - 5.0e-05
+ clip_norm: 2.0
+ optimizer: adam
+ adam_betas: (0.9, 0.98)
+ lr_scheduler: polynomial_decay
+ total_num_update: 1000000
+ warmup_updates: 122
+ weight_decay: 0.0
+ ddp_backend: no_c10d
+ max_epoch: 5
+ checkpoint:
+ restore_file: runs/mtm/vlm/checkpoint_best.pt
+ reset_optimizer: true
+ reset_dataloader: true
+ reset_meters: true
+ save_dir: runs/mtm/vlm/vttqa
+task_type: sweep_small
+model:
+ model_cls: MMFusionJoint
+ mm_encoder_cls: MMBertForJoint
+ use_seg_emb: true
+loss:
+ loss_cls: V2TContraLoss
diff --git a/data/fairseq/examples/MMPT/projects/mtm/vlm/youcook.yaml b/data/fairseq/examples/MMPT/projects/mtm/vlm/youcook.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..9ee82b81b8b14a9f689e73bac22f21b780640984
--- /dev/null
+++ b/data/fairseq/examples/MMPT/projects/mtm/vlm/youcook.yaml
@@ -0,0 +1,47 @@
+dataset:
+ video_processor: YoucookVideoProcessor
+ bert_name: bert-base-uncased
+ meta_processor: YoucookMetaProcessor
+ train_path: data/youcook/youcook_train.pkl
+ val_path: data/youcook/youcook_val.pkl
+ trainval_annotation: data/youcook/youcookii_annotations_trainval.json
+ use_annotation_text: true
+ vfeat_dir: data/feat/feat_youcook_s3d
+ text_processor: TextProcessor
+ aligner: DSAligner
+ num_iso_layer: 12
+ max_video_len: 32
+ max_len: 96
+fairseq:
+ common:
+ tensorboard_logdir: run
+ log_interval: 1000
+ fp16: true
+ dataset:
+ num_workers: 4
+ batch_size: 128
+ optimization:
+ lr:
+ - 5.0e-05
+ clip_norm: 2.0
+ optimizer: adam
+ adam_betas: (0.9, 0.98)
+ lr_scheduler: polynomial_decay
+ total_num_update: 1000000
+ warmup_updates: 122
+ weight_decay: 0.0
+ ddp_backend: no_c10d
+ max_epoch: 10
+ checkpoint:
+ restore_file: runs/mtm/vlm/checkpoint_best.pt
+ reset_optimizer: true
+ reset_dataloader: true
+ reset_meters: true
+ save_dir: runs/mtm/vlm/youcook
+task_type: sweep_small
+model:
+ model_cls: MMFusionJoint
+ mm_encoder_cls: MMBertForJoint
+ use_seg_emb: true
+loss:
+ loss_cls: T2VContraLoss
diff --git a/data/fairseq/examples/MMPT/projects/mtm/vlm/youcookcap.yaml b/data/fairseq/examples/MMPT/projects/mtm/vlm/youcookcap.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..d29dfad5cd861cc1e8f587c730938ed491d60289
--- /dev/null
+++ b/data/fairseq/examples/MMPT/projects/mtm/vlm/youcookcap.yaml
@@ -0,0 +1,45 @@
+dataset:
+ video_processor: YoucookVideoProcessor
+ bert_name: bert-base-uncased
+ meta_processor: YoucookNLGMetaProcessor
+ train_path: data/youcook/train_list.txt
+ val_path: data/youcook/val_list.txt
+ trainval_annotation: data/youcook/youcookii_annotations_trainval.json
+ vfeat_dir: data/feat/feat_youcook_s3d
+ text_processor: NLGTextProcessor
+ aligner: DSNLGAligner
+ max_video_len: 32
+ max_len: 96
+fairseq:
+ common:
+ tensorboard_logdir: run
+ log_interval: 1000
+ fp16: true
+ dataset:
+ num_workers: 4
+ batch_size: 128
+ optimization:
+ lr:
+ - 5.0e-05
+ clip_norm: 2.0
+ optimizer: adam
+ adam_betas: (0.9, 0.98)
+ lr_scheduler: polynomial_decay
+ total_num_update: 1000000
+ warmup_updates: 122
+ weight_decay: 0.0
+ ddp_backend: no_c10d
+ max_epoch: 10
+ checkpoint:
+ restore_file: runs/mtm/vlm/checkpoint_best.pt
+ reset_optimizer: true
+ reset_dataloader: true
+ reset_meters: true
+ save_dir: runs/mtm/vlm/youcookcap
+task_type: sweep_small
+model:
+ model_cls: MMFusionNLG
+ mm_encoder_cls: MMBertForNLG
+ use_seg_emb: true
+loss:
+ loss_cls: NLGLoss
diff --git a/data/fairseq/examples/MMPT/projects/retri/videoclip.yaml b/data/fairseq/examples/MMPT/projects/retri/videoclip.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..afd040ab050554aa0ca9c65ab4cf36cf6f3155dd
--- /dev/null
+++ b/data/fairseq/examples/MMPT/projects/retri/videoclip.yaml
@@ -0,0 +1,10 @@
+includes: projects/retri/videoretri.yaml
+project_dir: retri/videoclip
+task_group:
+ pretrain:
+ model:
+ model_cls: MMFusionSeparate
+ mm_encoder_cls:
+ video_encoder_cls: MMBertForEncoder
+ text_encoder_cls: BertModel
+ num_hidden_video_layers: 6
diff --git a/data/fairseq/examples/MMPT/projects/retri/videoclip/coin_videoclip.yaml b/data/fairseq/examples/MMPT/projects/retri/videoclip/coin_videoclip.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..aaed5e47f62d421e5c434a2f539437831ae387db
--- /dev/null
+++ b/data/fairseq/examples/MMPT/projects/retri/videoclip/coin_videoclip.yaml
@@ -0,0 +1,49 @@
+dataset:
+ video_processor: VideoProcessor
+ bert_name: bert-base-uncased
+ meta_processor: COINActionSegmentationMetaProcessor
+ train_path: data/coin/COIN.json
+ val_path: data/coin/COIN.json
+ vfeat_dir: data/feat/feat_coin_s3d
+ text_processor: COINActionSegmentationTextProcessor
+ aligner: COINActionSegmentationAligner
+ num_iso_layer: 12
+ sliding_window: 8
+ sliding_window_size: 32
+ max_video_len: 32
+ max_len: 96
+fairseq:
+ common:
+ tensorboard_logdir: run
+ log_interval: 1000
+ fp16: true
+ dataset:
+ num_workers: 4
+ batch_size: 1
+ optimization:
+ lr:
+ - 5.0e-05
+ clip_norm: 2.0
+ optimizer: adam
+ adam_betas: (0.9, 0.98)
+ lr_scheduler: polynomial_decay
+ total_num_update: 1000000
+ warmup_updates: 122
+ weight_decay: 0.0
+ ddp_backend: no_c10d
+ max_epoch: 8
+ checkpoint:
+ restore_file: runs/retri/videoclip/checkpoint_best.pt
+ reset_optimizer: true
+ reset_dataloader: true
+ reset_meters: true
+ save_dir: runs/retri/videoclip/coin
+task_type: sweep_big
+model:
+ model_cls: MMFusionSeparateActionSegmentation
+ mm_encoder_cls: null
+ video_encoder_cls: MMBertForTokenClassification
+ text_encoder_cls: BertModel
+ num_hidden_video_layers: 6
+loss:
+ loss_cls: CrossEntropy
diff --git a/data/fairseq/examples/MMPT/projects/retri/videoclip/crosstask_videoclip.yaml b/data/fairseq/examples/MMPT/projects/retri/videoclip/crosstask_videoclip.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..758601e3593aa3960fee996dcfdfd71972f9f068
--- /dev/null
+++ b/data/fairseq/examples/MMPT/projects/retri/videoclip/crosstask_videoclip.yaml
@@ -0,0 +1,55 @@
+dataset:
+ video_processor: CrossTaskVideoProcessor
+ bert_name: bert-base-uncased
+ meta_processor: CrossTaskMetaProcessor
+ train_path: data/crosstask/crosstask_release/videos.csv
+ train_csv_path: data/crosstask/crosstask_release/videos.csv
+ val_path: data/crosstask/crosstask_release/videos_val.csv
+ val_csv_path: data/crosstask/crosstask_release/videos_val.csv
+ primary_path: data/crosstask/crosstask_release/tasks_primary.txt
+ related_path: data/crosstask/crosstask_release/tasks_related.txt
+ vfeat_dir: data/feat/feat_crosstask_s3d
+ annotation_path: data/crosstask/crosstask_release/annotations
+ n_train: 30
+ text_processor: CrossTaskTextProcessor
+ aligner: CrossTaskAligner
+ num_iso_layer: 12
+ sliding_window: 16
+ sliding_window_size: 32
+ max_video_len: 32
+ max_len: 96
+fairseq:
+ common:
+ tensorboard_logdir: run
+ log_interval: 1000
+ fp16: true
+ dataset:
+ num_workers: 4
+ batch_size: 1
+ optimization:
+ lr:
+ - 5.0e-05
+ clip_norm: 2.0
+ optimizer: adam
+ adam_betas: (0.9, 0.98)
+ lr_scheduler: polynomial_decay
+ total_num_update: 1000000
+ warmup_updates: 122
+ weight_decay: 0.0
+ ddp_backend: no_c10d
+ max_epoch: 5
+ checkpoint:
+ restore_file: runs/retri/videoclip/checkpoint_best.pt
+ reset_optimizer: true
+ reset_dataloader: true
+ reset_meters: true
+ save_dir: runs/retri/videoclip/crosstask
+task_type: sweep_small
+model:
+ model_cls: MMFusionSeparateActionLocalization
+ mm_encoder_cls: null
+ video_encoder_cls: MMBertForEncoder
+ text_encoder_cls: BertModel
+ num_hidden_video_layers: 6
+loss:
+ loss_cls: BCE
diff --git a/data/fairseq/examples/MMPT/projects/retri/videoclip/how2.yaml b/data/fairseq/examples/MMPT/projects/retri/videoclip/how2.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..b49581e878390cd8861926988a58a892b5a38606
--- /dev/null
+++ b/data/fairseq/examples/MMPT/projects/retri/videoclip/how2.yaml
@@ -0,0 +1,65 @@
+dataset:
+ video_processor: ShardedVideoRetriVideoProcessor
+ bert_name: bert-base-uncased
+ meta_processor: ShardedHow2VideoRetriMetaProcessor
+ train_path: data/how2/how2_s3d_train.lst
+ val_path: data/how2/how2_s3d_val.lst
+ vfeat_dir: data/feat/feat_how2_s3d_shard_small
+ text_processor: ShardedVideoRetriTextProcessor
+ tfeat_dir: data/feat/feat_how2_s3d_shard_small/raw_caption_dedup.bert-base-uncased.
+ aligner: VideoRetriOverlappedAligner
+ subsampling: 1
+ sampled_min_len: 8
+ sampled_max_len: 64
+ max_video_len: 32
+ max_len: 96
+ lazy_vfeat_mask: true
+ mfm_probability: 0.15
+ mlm_probability: 0.15
+ mm_prob: 0.5
+ sampled_video_min_len: 3
+ sampled_video_max_len: 32
+ num_video_per_batch: 32
+ clip_per_video: 16
+fairseq:
+ common:
+ tensorboard_logdir: run
+ log_interval: 1000
+ fp16: true
+ dataset:
+ num_workers: 4
+ batch_size: 1
+ optimization:
+ lr:
+ - 5.0e-05
+ clip_norm: 2.0
+ optimizer: adam
+ adam_betas: (0.9, 0.98)
+ lr_scheduler: polynomial_decay
+ total_num_update: 1000000
+ warmup_updates: 1000
+ weight_decay: 0.0
+ ddp_backend: no_c10d
+ max_epoch: 25
+ checkpoint:
+ save_dir: runs/retri/videoclip
+ save_interval_updates: 1024
+ keep_interval_updates: 2
+ keep_last_epochs: 30
+task_type: sweep_big
+slurm_config: big
+eval:
+ save_path: runs/retri/videoclip
+model:
+ model_cls: MMFusionSeparate
+ mm_encoder_cls: null
+ video_encoder_cls: MMBertForEncoder
+ text_encoder_cls: BertModel
+ num_hidden_video_layers: 6
+loss:
+ loss_cls: MMContraLoss
+task: VideoRetriTask
+retri_epoch: 1
+vectorpool_cls: VideoVectorPool
+retriever_cls: VectorRetriever
+num_cands: 64
diff --git a/data/fairseq/examples/MMPT/projects/retri/videoclip/test_coin_videoclip.yaml b/data/fairseq/examples/MMPT/projects/retri/videoclip/test_coin_videoclip.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..409906203c22d670c361d011d1035874961d4b5b
--- /dev/null
+++ b/data/fairseq/examples/MMPT/projects/retri/videoclip/test_coin_videoclip.yaml
@@ -0,0 +1,33 @@
+slurm_config: big
+task_type: local_predict
+dataset:
+ split: test
+ video_processor: VideoProcessor
+ aligner: COINActionSegmentationAligner
+ bert_name: bert-base-uncased
+ test_path: data/coin/COIN.json
+ meta_processor: COINActionSegmentationMetaProcessor
+ vfeat_dir: data/feat/feat_coin_s3d
+ text_processor: COINActionSegmentationTextProcessor
+ num_iso_layer: 12
+ sliding_window: 16
+ sliding_window_size: 32
+ max_video_len: 32
+ max_len: 96
+fairseq:
+ dataset:
+ batch_size: 1
+ valid_subset: test
+ num_workers: 2
+ common_eval:
+ path: runs/retri/videoclip/coin/checkpoint_best.pt
+model:
+ model_cls: MMFusionSeparateActionSegmentation
+ mm_encoder_cls: null
+ video_encoder_cls: MMBertForTokenClassification
+ text_encoder_cls: BertModel
+ num_hidden_video_layers: 6
+eval:
+ save_path: runs/retri/videoclip/coin/eval
+metric: COINActionSegmentationMetric
+predictor: COINPredictor
diff --git a/data/fairseq/examples/MMPT/projects/retri/videoclip/test_coin_zs.yaml b/data/fairseq/examples/MMPT/projects/retri/videoclip/test_coin_zs.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..b33739c7b6d7c4d935c0e4d78354a02f79e21d91
--- /dev/null
+++ b/data/fairseq/examples/MMPT/projects/retri/videoclip/test_coin_zs.yaml
@@ -0,0 +1,33 @@
+slurm_config: big
+task_type: local_predict
+dataset:
+ split: test
+ video_processor: VideoProcessor
+ aligner: COINActionSegmentationAligner
+ bert_name: bert-base-uncased
+ test_path: data/coin/COIN.json
+ meta_processor: COINActionSegmentationMetaProcessor
+ vfeat_dir: data/feat/feat_coin_s3d
+ text_processor: COINActionSegmentationTextProcessor
+ num_iso_layer: 12
+ sliding_window: 16
+ sliding_window_size: 32
+ max_video_len: 32
+ max_len: 96
+fairseq:
+ dataset:
+ batch_size: 1
+ valid_subset: test
+ num_workers: 2
+ common_eval:
+ path: runs/retri/videoclip/checkpoint_best.pt
+model:
+ model_cls: MMFusionSeparate
+ mm_encoder_cls: null
+ video_encoder_cls: MMBertForEncoder
+ text_encoder_cls: BertModel
+ num_hidden_video_layers: 6
+eval:
+ save_path: runs/retri/videoclip/coin_zs/eval
+metric: COINActionSegmentationMetric
+predictor: COINZSPredictor
diff --git a/data/fairseq/examples/MMPT/projects/retri/videoclip/test_crosstask_videoclip.yaml b/data/fairseq/examples/MMPT/projects/retri/videoclip/test_crosstask_videoclip.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..e82f54fbe52defa233f1b561415737924dd6ba9c
--- /dev/null
+++ b/data/fairseq/examples/MMPT/projects/retri/videoclip/test_crosstask_videoclip.yaml
@@ -0,0 +1,40 @@
+slurm_config: big
+task_type: local_predict
+dataset:
+ split: test
+ video_processor: CrossTaskVideoProcessor
+ aligner: CrossTaskAligner
+ bert_name: bert-base-uncased
+ meta_processor: CrossTaskMetaProcessor
+ test_path: data/crosstask/crosstask_release/videos_val.csv
+ train_csv_path: data/crosstask/crosstask_release/videos.csv
+ val_path: data/crosstask/crosstask_release/videos_val.csv
+ val_csv_path: data/crosstask/crosstask_release/videos_val.csv
+ primary_path: data/crosstask/crosstask_release/tasks_primary.txt
+ related_path: data/crosstask/crosstask_release/tasks_related.txt
+ vfeat_dir: data/feat/feat_crosstask_s3d
+ annotation_path: data/crosstask/crosstask_release/annotations
+ n_train: 30
+ text_processor: CrossTaskTextProcessor
+ num_iso_layer: 12
+ sliding_window: 16
+ sliding_window_size: 32
+ max_video_len: 32
+ max_len: 96
+fairseq:
+ dataset:
+ batch_size: 1
+ valid_subset: test
+ num_workers: 2
+ common_eval:
+ path: runs/retri/videoclip/crosstask/checkpoint_best.pt
+model:
+ model_cls: MMFusionSeparateActionLocalization
+ mm_encoder_cls: null
+ video_encoder_cls: MMBertForEncoder
+ text_encoder_cls: BertModel
+ num_hidden_video_layers: 6
+eval:
+ save_path: runs/retri/videoclip/crosstask/eval
+metric: CrossTaskMetric
+predictor: CrossTaskPredictor
diff --git a/data/fairseq/examples/MMPT/projects/retri/videoclip/test_crosstask_zs_videoclip.yaml b/data/fairseq/examples/MMPT/projects/retri/videoclip/test_crosstask_zs_videoclip.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..6fc357cc1f5285229a508a95dc31c2aae17c977d
--- /dev/null
+++ b/data/fairseq/examples/MMPT/projects/retri/videoclip/test_crosstask_zs_videoclip.yaml
@@ -0,0 +1,40 @@
+slurm_config: big
+task_type: local_predict
+dataset:
+ split: test
+ video_processor: CrossTaskVideoProcessor
+ aligner: CrossTaskAligner
+ bert_name: bert-base-uncased
+ meta_processor: CrossTaskMetaProcessor
+ test_path: data/crosstask/crosstask_release/videos_val.csv
+ train_csv_path: data/crosstask/crosstask_release/videos.csv
+ val_path: data/crosstask/crosstask_release/videos_val.csv
+ val_csv_path: data/crosstask/crosstask_release/videos_val.csv
+ primary_path: data/crosstask/crosstask_release/tasks_primary.txt
+ related_path: data/crosstask/crosstask_release/tasks_related.txt
+ vfeat_dir: data/feat/feat_crosstask_s3d
+ annotation_path: data/crosstask/crosstask_release/annotations
+ n_train: 30
+ text_processor: CrossTaskTextProcessor
+ num_iso_layer: 12
+ sliding_window: 16
+ sliding_window_size: 32
+ max_video_len: 32
+ max_len: 96
+fairseq:
+ dataset:
+ batch_size: 1
+ valid_subset: test
+ num_workers: 2
+ common_eval:
+ path: runs/retri/videoclip/checkpoint_best.pt
+model:
+ model_cls: MMFusionSeparateActionLocalization
+ mm_encoder_cls: null
+ video_encoder_cls: MMBertForEncoder
+ text_encoder_cls: BertModel
+ num_hidden_video_layers: 6
+eval:
+ save_path: runs/retri/videoclip/crosstask_zs/eval
+metric: CrossTaskMetric
+predictor: CrossTaskPredictor
diff --git a/data/fairseq/examples/MMPT/projects/retri/videoclip/test_didemo_zs.yaml b/data/fairseq/examples/MMPT/projects/retri/videoclip/test_didemo_zs.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..8dc716815d8d4812237c814fc17a90dd7ac3cf1e
--- /dev/null
+++ b/data/fairseq/examples/MMPT/projects/retri/videoclip/test_didemo_zs.yaml
@@ -0,0 +1,31 @@
+slurm_config: big
+task_type: local_predict
+dataset:
+ split: test
+ video_processor: VideoProcessor
+ aligner: DiDeMoAligner
+ bert_name: bert-base-uncased
+ meta_processor: DiDeMoMetaProcessor
+ test_path: data/didemo/test_data.json
+ vfeat_dir: data/feat/feat_didemo_s3d
+ text_processor: DiDeMoTextProcessor
+ num_iso_layer: 12
+ max_video_len: 32
+ max_len: 96
+fairseq:
+ dataset:
+ batch_size: 256
+ valid_subset: test
+ num_workers: 2
+ common_eval:
+ path: runs/retri/videoclip/checkpoint_best.pt
+model:
+ model_cls: MMFusionSeparate
+ mm_encoder_cls: null
+ video_encoder_cls: MMBertForEncoder
+ text_encoder_cls: BertModel
+ num_hidden_video_layers: 6
+eval:
+ save_path: runs/retri/videoclip/didemo_zs/eval
+metric: DiDeMoMetric
+predictor: DiDeMoPredictor
diff --git a/data/fairseq/examples/MMPT/projects/retri/videoclip/test_vtt_videoclip.yaml b/data/fairseq/examples/MMPT/projects/retri/videoclip/test_vtt_videoclip.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..19321ad5f489121381b1b78506b38ba3705b37e4
--- /dev/null
+++ b/data/fairseq/examples/MMPT/projects/retri/videoclip/test_vtt_videoclip.yaml
@@ -0,0 +1,31 @@
+slurm_config: big
+task_type: local_predict
+dataset:
+ split: test
+ video_processor: VideoProcessor
+ aligner: DSAligner
+ bert_name: bert-base-uncased
+ meta_processor: MSRVTTMetaProcessor
+ test_path: data/msrvtt/MSRVTT_JSFUSION_test.csv
+ vfeat_dir: data/feat/feat_vtt_s3d
+ text_processor: MSRVTTTextProcessor
+ num_iso_layer: 12
+ max_video_len: 32
+ max_len: 96
+fairseq:
+ dataset:
+ batch_size: 256
+ valid_subset: test
+ num_workers: 2
+ common_eval:
+ path: runs/retri/videoclip/vtt/checkpoint_last.pt
+model:
+ model_cls: MMFusionSeparate
+ mm_encoder_cls: null
+ video_encoder_cls: MMBertForEncoder
+ text_encoder_cls: BertModel
+ num_hidden_video_layers: 6
+eval:
+ save_path: runs/retri/videoclip/vtt/eval
+metric: RetrievalMetric
+predictor: RetrievalPredictor
diff --git a/data/fairseq/examples/MMPT/projects/retri/videoclip/test_vtt_zs.yaml b/data/fairseq/examples/MMPT/projects/retri/videoclip/test_vtt_zs.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..d149fa3960294f4d9dcac9c4f4f18631eb622435
--- /dev/null
+++ b/data/fairseq/examples/MMPT/projects/retri/videoclip/test_vtt_zs.yaml
@@ -0,0 +1,31 @@
+slurm_config: big
+task_type: local_predict
+dataset:
+ split: test
+ video_processor: VideoProcessor
+ aligner: DSAligner
+ bert_name: bert-base-uncased
+ meta_processor: MSRVTTMetaProcessor
+ test_path: data/msrvtt/MSRVTT_JSFUSION_test.csv
+ vfeat_dir: data/feat/feat_vtt_s3d
+ text_processor: MSRVTTTextProcessor
+ num_iso_layer: 12
+ max_video_len: 32
+ max_len: 96
+fairseq:
+ dataset:
+ batch_size: 256
+ valid_subset: test
+ num_workers: 2
+ common_eval:
+ path: runs/retri/videoclip/checkpoint_best.pt
+model:
+ model_cls: MMFusionSeparate
+ mm_encoder_cls: null
+ video_encoder_cls: MMBertForEncoder
+ text_encoder_cls: BertModel
+ num_hidden_video_layers: 6
+eval:
+ save_path: runs/retri/videoclip/vtt_zs/eval
+metric: RetrievalMetric
+predictor: RetrievalPredictor
diff --git a/data/fairseq/examples/MMPT/projects/retri/videoclip/test_vttqa_videoclip.yaml b/data/fairseq/examples/MMPT/projects/retri/videoclip/test_vttqa_videoclip.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..295aeedbb0f29a28ad9065da0a18e82ac98b6683
--- /dev/null
+++ b/data/fairseq/examples/MMPT/projects/retri/videoclip/test_vttqa_videoclip.yaml
@@ -0,0 +1,31 @@
+slurm_config: big
+task_type: local_predict
+dataset:
+ split: test
+ video_processor: VideoProcessor
+ aligner: MSRVTTQAAligner
+ bert_name: bert-base-uncased
+ meta_processor: MSRVTTQAMetaProcessor
+ test_path: data/msrvtt-qa/MSR_MC_test.csv
+ vfeat_dir: data/feat/feat_vtt_s3d
+ text_processor: MSRVTTQATextProcessor
+ num_iso_layer: 12
+ max_video_len: 32
+ max_len: 96
+fairseq:
+ dataset:
+ batch_size: 256
+ valid_subset: test
+ num_workers: 2
+ common_eval:
+ path: runs/retri/videoclip/vttqa/checkpoint_last.pt
+model:
+ model_cls: MMFusionSeparate
+ mm_encoder_cls: null
+ video_encoder_cls: MMBertForEncoder
+ text_encoder_cls: BertModel
+ num_hidden_video_layers: 6
+eval:
+ save_path: runs/retri/videoclip/vttqa/eval
+metric: QAMetric
+predictor: QAPredictor
diff --git a/data/fairseq/examples/MMPT/projects/retri/videoclip/test_vttqa_zs.yaml b/data/fairseq/examples/MMPT/projects/retri/videoclip/test_vttqa_zs.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..7a876c822ae38cadc03c69544e64025d516e8cca
--- /dev/null
+++ b/data/fairseq/examples/MMPT/projects/retri/videoclip/test_vttqa_zs.yaml
@@ -0,0 +1,31 @@
+slurm_config: big
+task_type: local_predict
+dataset:
+ split: test
+ video_processor: VideoProcessor
+ aligner: MSRVTTQAAligner
+ bert_name: bert-base-uncased
+ meta_processor: MSRVTTQAMetaProcessor
+ test_path: data/msrvtt-qa/MSR_MC_test.csv
+ vfeat_dir: data/feat/feat_vtt_s3d
+ text_processor: MSRVTTQATextProcessor
+ num_iso_layer: 12
+ max_video_len: 32
+ max_len: 96
+fairseq:
+ dataset:
+ batch_size: 256
+ valid_subset: test
+ num_workers: 2
+ common_eval:
+ path: runs/retri/videoclip/checkpoint_best.pt
+model:
+ model_cls: MMFusionSeparate
+ mm_encoder_cls: null
+ video_encoder_cls: MMBertForEncoder
+ text_encoder_cls: BertModel
+ num_hidden_video_layers: 6
+eval:
+ save_path: runs/retri/videoclip/vttqa_zs/eval
+metric: QAMetric
+predictor: QAPredictor
diff --git a/data/fairseq/examples/MMPT/projects/retri/videoclip/test_youcook_videoclip.yaml b/data/fairseq/examples/MMPT/projects/retri/videoclip/test_youcook_videoclip.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..86a4ab203e097ec156ce02d88ce9c1b2374cec9f
--- /dev/null
+++ b/data/fairseq/examples/MMPT/projects/retri/videoclip/test_youcook_videoclip.yaml
@@ -0,0 +1,33 @@
+slurm_config: big
+task_type: local_predict
+dataset:
+ split: test
+ video_processor: YoucookVideoProcessor
+ aligner: DSAligner
+ bert_name: bert-base-uncased
+ meta_processor: YoucookMetaProcessor
+ test_path: data/youcook/youcook_val.pkl
+ trainval_annotation: data/youcook/youcookii_annotations_trainval.json
+ use_annotation_text: true
+ vfeat_dir: data/feat/feat_youcook_s3d
+ text_processor: TextProcessor
+ num_iso_layer: 12
+ max_video_len: 32
+ max_len: 96
+fairseq:
+ dataset:
+ batch_size: 256
+ valid_subset: test
+ num_workers: 2
+ common_eval:
+ path: runs/retri/videoclip/youcook/checkpoint_last.pt
+model:
+ model_cls: MMFusionSeparate
+ mm_encoder_cls: null
+ video_encoder_cls: MMBertForEncoder
+ text_encoder_cls: BertModel
+ num_hidden_video_layers: 6
+eval:
+ save_path: runs/retri/videoclip/youcook/eval
+metric: RetrievalMetric
+predictor: RetrievalPredictor
diff --git a/data/fairseq/examples/MMPT/projects/retri/videoclip/test_youcook_zs.yaml b/data/fairseq/examples/MMPT/projects/retri/videoclip/test_youcook_zs.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..fd2941708be0baacf9e12ca757e0de40676e570d
--- /dev/null
+++ b/data/fairseq/examples/MMPT/projects/retri/videoclip/test_youcook_zs.yaml
@@ -0,0 +1,33 @@
+slurm_config: big
+task_type: local_predict
+dataset:
+ split: test
+ video_processor: YoucookVideoProcessor
+ aligner: DSAligner
+ bert_name: bert-base-uncased
+ meta_processor: YoucookMetaProcessor
+ test_path: data/youcook/youcook_val.pkl
+ trainval_annotation: data/youcook/youcookii_annotations_trainval.json
+ use_annotation_text: true
+ vfeat_dir: data/feat/feat_youcook_s3d
+ text_processor: TextProcessor
+ num_iso_layer: 12
+ max_video_len: 32
+ max_len: 96
+fairseq:
+ dataset:
+ batch_size: 256
+ valid_subset: test
+ num_workers: 2
+ common_eval:
+ path: runs/retri/videoclip/checkpoint_best.pt
+model:
+ model_cls: MMFusionSeparate
+ mm_encoder_cls: null
+ video_encoder_cls: MMBertForEncoder
+ text_encoder_cls: BertModel
+ num_hidden_video_layers: 6
+eval:
+ save_path: runs/retri/videoclip/youcook_zs/eval
+metric: RetrievalMetric
+predictor: RetrievalPredictor
diff --git a/data/fairseq/examples/MMPT/projects/retri/videoclip/vtt_videoclip.yaml b/data/fairseq/examples/MMPT/projects/retri/videoclip/vtt_videoclip.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..d8b4079ac233af17ee16cfed7f3e2316e0e236c5
--- /dev/null
+++ b/data/fairseq/examples/MMPT/projects/retri/videoclip/vtt_videoclip.yaml
@@ -0,0 +1,51 @@
+dataset:
+ video_processor: VideoProcessor
+ bert_name: bert-base-uncased
+ meta_processor: MSRVTTMetaProcessor
+ train_path: data/msrvtt/MSRVTT_train.csv
+ jsfusion_path: data/msrvtt/MSRVTT_JSFUSION_test.csv
+ full_test_path: data/msrvtt/MSRVTT_FULL_test.csv
+ dup: 20
+ val_path: data/msrvtt/MSRVTT_JSFUSION_test.csv
+ vfeat_dir: data/feat/feat_vtt_s3d
+ text_processor: MSRVTTTextProcessor
+ json_path: data/msrvtt/MSRVTT_data.json
+ aligner: DSAligner
+ num_iso_layer: 12
+ max_video_len: 32
+ max_len: 96
+fairseq:
+ common:
+ tensorboard_logdir: run
+ log_interval: 1000
+ fp16: true
+ dataset:
+ num_workers: 4
+ batch_size: 224
+ optimization:
+ lr:
+ - 5.0e-05
+ clip_norm: 2.0
+ optimizer: adam
+ adam_betas: (0.9, 0.98)
+ lr_scheduler: polynomial_decay
+ total_num_update: 1000000
+ warmup_updates: 122
+ weight_decay: 0.0
+ ddp_backend: no_c10d
+ max_epoch: 10
+ checkpoint:
+ restore_file: runs/retri/videoclip/checkpoint_best.pt
+ reset_optimizer: true
+ reset_dataloader: true
+ reset_meters: true
+ save_dir: runs/retri/videoclip/vtt
+task_type: sweep_small
+model:
+ model_cls: MMFusionSeparate
+ mm_encoder_cls: null
+ video_encoder_cls: MMBertForEncoder
+ text_encoder_cls: BertModel
+ num_hidden_video_layers: 6
+loss:
+ loss_cls: T2VContraLoss
diff --git a/data/fairseq/examples/MMPT/projects/retri/videoclip/vttqa_videoclip.yaml b/data/fairseq/examples/MMPT/projects/retri/videoclip/vttqa_videoclip.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..f0566d784ab53e0159a1db6b82222ad11ad12add
--- /dev/null
+++ b/data/fairseq/examples/MMPT/projects/retri/videoclip/vttqa_videoclip.yaml
@@ -0,0 +1,49 @@
+dataset:
+ video_processor: VideoProcessor
+ bert_name: bert-base-uncased
+ meta_processor: MSRVTTMetaProcessor
+ train_path: data/msrvtt/MSRVTT_train.csv
+ dup: 20
+ val_path: data/msrvtt/MSRVTT_JSFUSION_test.csv
+ vfeat_dir: data/feat/feat_vtt_s3d
+ text_processor: MSRVTTTextProcessor
+ json_path: data/msrvtt/MSRVTT_data.json
+ aligner: DSAligner
+ num_iso_layer: 12
+ max_video_len: 32
+ max_len: 96
+fairseq:
+ common:
+ tensorboard_logdir: run
+ log_interval: 1000
+ fp16: true
+ dataset:
+ num_workers: 4
+ batch_size: 128
+ optimization:
+ lr:
+ - 5.0e-05
+ clip_norm: 2.0
+ optimizer: adam
+ adam_betas: (0.9, 0.98)
+ lr_scheduler: polynomial_decay
+ total_num_update: 1000000
+ warmup_updates: 122
+ weight_decay: 0.0
+ ddp_backend: no_c10d
+ max_epoch: 5
+ checkpoint:
+ restore_file: runs/retri/videoclip/checkpoint_best.pt
+ reset_optimizer: true
+ reset_dataloader: true
+ reset_meters: true
+ save_dir: runs/retri/videoclip/vttqa
+task_type: sweep_small
+model:
+ model_cls: MMFusionSeparate
+ mm_encoder_cls: null
+ video_encoder_cls: MMBertForEncoder
+ text_encoder_cls: BertModel
+ num_hidden_video_layers: 6
+loss:
+ loss_cls: V2TContraLoss
diff --git a/data/fairseq/examples/MMPT/projects/retri/videoclip/youcook_videoclip.yaml b/data/fairseq/examples/MMPT/projects/retri/videoclip/youcook_videoclip.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..c2b13e5519fe0839c6b3bc681f7ee97b522077d8
--- /dev/null
+++ b/data/fairseq/examples/MMPT/projects/retri/videoclip/youcook_videoclip.yaml
@@ -0,0 +1,49 @@
+dataset:
+ video_processor: YoucookVideoProcessor
+ bert_name: bert-base-uncased
+ meta_processor: YoucookMetaProcessor
+ train_path: data/youcook/youcook_train.pkl
+ val_path: data/youcook/youcook_val.pkl
+ trainval_annotation: data/youcook/youcookii_annotations_trainval.json
+ use_annotation_text: true
+ vfeat_dir: data/feat/feat_youcook_s3d
+ text_processor: TextProcessor
+ aligner: DSAligner
+ num_iso_layer: 12
+ max_video_len: 32
+ max_len: 96
+fairseq:
+ common:
+ tensorboard_logdir: run
+ log_interval: 1000
+ fp16: true
+ dataset:
+ num_workers: 4
+ batch_size: 128
+ optimization:
+ lr:
+ - 5.0e-05
+ clip_norm: 2.0
+ optimizer: adam
+ adam_betas: (0.9, 0.98)
+ lr_scheduler: polynomial_decay
+ total_num_update: 1000000
+ warmup_updates: 122
+ weight_decay: 0.0
+ ddp_backend: no_c10d
+ max_epoch: 10
+ checkpoint:
+ restore_file: runs/retri/videoclip/checkpoint_best.pt
+ reset_optimizer: true
+ reset_dataloader: true
+ reset_meters: true
+ save_dir: runs/retri/videoclip/youcook
+task_type: sweep_small
+model:
+ model_cls: MMFusionSeparate
+ mm_encoder_cls: null
+ video_encoder_cls: MMBertForEncoder
+ text_encoder_cls: BertModel
+ num_hidden_video_layers: 6
+loss:
+ loss_cls: T2VContraLoss
diff --git a/data/fairseq/examples/MMPT/projects/retri/videoretri.yaml b/data/fairseq/examples/MMPT/projects/retri/videoretri.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..969e1fb2793d5381e52188c4ad78aea240698969
--- /dev/null
+++ b/data/fairseq/examples/MMPT/projects/retri/videoretri.yaml
@@ -0,0 +1,51 @@
+includes: projects/mfmmlm.yaml
+project_dir: retri/videoretri
+run_task:
+ - how2.yaml
+task_group:
+ pretrain:
+ task: VideoRetriTask
+ retri_epoch: 1
+ vectorpool_cls: VideoVectorPool
+ retriever_cls: VectorRetriever
+ num_cands: 64
+ dataset:
+ train_path: data/how2/how2_s3d_train.lst
+ meta_processor: ShardedHow2VideoRetriMetaProcessor
+ video_processor: ShardedVideoRetriVideoProcessor
+ text_processor: ShardedVideoRetriTextProcessor
+ aligner: VideoRetriOverlappedAligner
+ sampled_video_min_len: 3
+ sampled_video_max_len: 32
+ sampled_min_len: 8
+ sampled_max_len: 64
+ num_video_per_batch: 32
+ # do not use subsampling as it changes fairseq batch_size.
+ subsampling: 1 # disable subsampling
+ clip_per_video: 16
+ fairseq:
+ dataset:
+ batch_size: 1
+ optimization:
+ max_epoch: 25
+ model:
+ model_cls: MMFusionShare
+ mm_encoder_cls: MMBertForEncoder
+ loss:
+ loss_cls: MMContraLoss
+ finetune:
+ task_list: [vtt_videoclip.yaml, youcook_videoclip.yaml, vttqa_videoclip.yaml, crosstask_videoclip.yaml, coin_videoclip.yaml]
+ test:
+ task_list:
+ - test_youcook_zs.yaml
+ - test_vtt_zs.yaml
+ - test_vttqa_zs.yaml
+ - test_crosstask_zs_videoclip.yaml
+ - test_coin_zs.yaml
+ - test_didemo_zs.yaml
+ - test_youcook_videoclip.yaml
+ - test_vtt_videoclip.yaml
+ - test_vttqa_videoclip.yaml
+ - test_crosstask_videoclip.yaml
+ - test_coin_videoclip.yaml
+
diff --git a/data/fairseq/examples/MMPT/projects/task/coin.yaml b/data/fairseq/examples/MMPT/projects/task/coin.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..e7772486e166348deff1f7c9deceb68c7e1bb443
--- /dev/null
+++ b/data/fairseq/examples/MMPT/projects/task/coin.yaml
@@ -0,0 +1,25 @@
+includes: projects/task/ft.yaml
+task_type: sweep_big
+dataset:
+ meta_processor: COINActionSegmentationMetaProcessor
+ train_path: data/coin/COIN.json
+ val_path: data/coin/COIN.json
+ vfeat_dir: data/feat/feat_coin_s3d
+ video_processor: VideoProcessor
+ text_processor: COINActionSegmentationTextProcessor
+ aligner: COINActionSegmentationAligner
+ num_iso_layer: 12
+ sliding_window: 8
+ sliding_window_size: 32
+model:
+ model_cls: MMFusionActionSegmentation
+ mm_encoder_cls: MMBertForTokenClassification
+loss:
+ loss_cls: CrossEntropy
+fairseq:
+ dataset:
+ batch_size: 1
+ optimization:
+ max_epoch: 8
+ checkpoint:
+ save_dir: runs/task/coin
diff --git a/data/fairseq/examples/MMPT/projects/task/coin_videoclip.yaml b/data/fairseq/examples/MMPT/projects/task/coin_videoclip.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..69988bc18a03a8f9bf594266f3bf06aa54f11b6a
--- /dev/null
+++ b/data/fairseq/examples/MMPT/projects/task/coin_videoclip.yaml
@@ -0,0 +1,7 @@
+includes: projects/task/coin.yaml
+model:
+ model_cls: MMFusionSeparateActionSegmentation
+ mm_encoder_cls:
+ video_encoder_cls: MMBertForTokenClassification
+ text_encoder_cls: BertModel # dummy, not used.
+ num_hidden_video_layers: 6
diff --git a/data/fairseq/examples/MMPT/projects/task/crosstask.yaml b/data/fairseq/examples/MMPT/projects/task/crosstask.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..cb4dbb0cb4a07020d1fbba9a810eb30fd81ae96d
--- /dev/null
+++ b/data/fairseq/examples/MMPT/projects/task/crosstask.yaml
@@ -0,0 +1,31 @@
+includes: projects/task/ft.yaml
+dataset:
+ meta_processor: CrossTaskMetaProcessor
+ train_path: data/crosstask/crosstask_release/videos.csv # dummy
+ train_csv_path: data/crosstask/crosstask_release/videos.csv
+ val_path: data/crosstask/crosstask_release/videos_val.csv # dummy
+ val_csv_path: data/crosstask/crosstask_release/videos_val.csv
+ primary_path: data/crosstask/crosstask_release/tasks_primary.txt
+ related_path: data/crosstask/crosstask_release/tasks_related.txt
+ vfeat_dir: data/feat/feat_crosstask_s3d
+ annotation_path: data/crosstask/crosstask_release/annotations
+ n_train: 30
+ video_processor: CrossTaskVideoProcessor
+ text_processor: CrossTaskTextProcessor
+ aligner: CrossTaskAligner
+ num_iso_layer: 12
+ sliding_window: 16
+ sliding_window_size: 32
+model:
+ model_cls: MMFusionActionLocalization
+ mm_encoder_cls: MMBertForJoint
+loss:
+ loss_cls: BCE
+fairseq:
+ dataset:
+ batch_size: 1
+ optimization:
+ max_epoch: 5
+ checkpoint:
+ save_dir: runs/task/crosstask
+ restore_file: runs/task/checkpoint11.pt # for VLM
diff --git a/data/fairseq/examples/MMPT/projects/task/crosstask_videoclip.yaml b/data/fairseq/examples/MMPT/projects/task/crosstask_videoclip.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..6ec613c07fcacfa6e6d3c5fc78aaefcb2b33eff5
--- /dev/null
+++ b/data/fairseq/examples/MMPT/projects/task/crosstask_videoclip.yaml
@@ -0,0 +1,10 @@
+includes: projects/task/crosstask.yaml
+model:
+ model_cls: MMFusionSeparateActionLocalization
+ mm_encoder_cls:
+ video_encoder_cls: MMBertForEncoder
+ text_encoder_cls: BertModel # dummy, not used.
+ num_hidden_video_layers: 6
+fairseq:
+ checkpoint:
+ restore_file: runs/task/checkpoint_best.pt # overwrite the default of VLM.
diff --git a/data/fairseq/examples/MMPT/projects/task/default.yaml b/data/fairseq/examples/MMPT/projects/task/default.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..087fef71a4b94d81b74c5e9ef647db189626ff36
--- /dev/null
+++ b/data/fairseq/examples/MMPT/projects/task/default.yaml
@@ -0,0 +1,20 @@
+# this yaml cannot be run alone. you must use `how2.yaml`, `vtt.yaml` etc for training.
+dataset:
+ video_processor: VideoProcessor
+ bert_name: bert-base-uncased
+fairseq:
+ common:
+ tensorboard_logdir: run
+ log_interval: 1000
+ dataset:
+ num_workers: 4
+ optimization:
+ lr: [ 0.00005 ]
+ clip_norm: 2.0
+ optimizer: adam
+ adam_betas: (0.9, 0.98)
+ lr_scheduler: polynomial_decay
+ total_num_update: 1000000 # backward compatible on fairseq 1.0.0a0+af0389f for reproducibility.
+ warmup_updates: 1000
+ weight_decay: 0.0
+ ddp_backend: no_c10d
diff --git a/data/fairseq/examples/MMPT/projects/task/ft.yaml b/data/fairseq/examples/MMPT/projects/task/ft.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..c93b8a73ea93e00da8a0ee76751c9e1bd34ab8a2
--- /dev/null
+++ b/data/fairseq/examples/MMPT/projects/task/ft.yaml
@@ -0,0 +1,13 @@
+includes: projects/task/default.yaml
+# all derived config will be run by fairseq-train.
+task_type: sweep_small
+fairseq:
+ optimization:
+ warmup_updates: 122 # copied from roberta glue: https://github.com/pytorch/fairseq/blob/master/examples/roberta/README.glue.md
+ checkpoint:
+ # save_interval_updates: 512
+ # borrowed from Roberta script.
+ restore_file: runs/task/checkpoint_best.pt
+ reset_optimizer: True
+ reset_dataloader: True
+ reset_meters: True
diff --git a/data/fairseq/examples/MMPT/projects/task/how2.yaml b/data/fairseq/examples/MMPT/projects/task/how2.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..094dd04bfcfdb96699ac80066f874fca954dc56a
--- /dev/null
+++ b/data/fairseq/examples/MMPT/projects/task/how2.yaml
@@ -0,0 +1,22 @@
+includes: projects/task/default.yaml
+task_type: sweep_big
+slurm_config: big
+dataset:
+ meta_processor: ShardedHow2MetaProcessor
+ train_path: data/how2/how2_s3d_train.lst
+ val_path: data/how2/how2_s3d_val.lst
+ video_processor: ShardedVideoProcessor
+ vfeat_dir: data/feat/feat_how2_s3d_shard_small
+ text_processor: ShardedTextProcessor
+ tfeat_dir: data/feat/feat_how2_s3d_shard_small/raw_caption_dedup.bert-base-uncased.
+ aligner: FixedLenAligner
+# disable direct running of this yaml
+eval:
+ save_path: runs/task
+fairseq:
+ checkpoint:
+ save_dir: runs/task
+ save_interval_updates: 1024
+ keep_interval_updates: 2
+ keep_last_epochs: 30
+
diff --git a/data/fairseq/examples/MMPT/projects/task/test.yaml b/data/fairseq/examples/MMPT/projects/task/test.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..0a98445241a17bbfda84a3b3be0687d31e49d12f
--- /dev/null
+++ b/data/fairseq/examples/MMPT/projects/task/test.yaml
@@ -0,0 +1,13 @@
+# this yaml cannot be run alone: implement a test_${dataset}.yaml
+slurm_config: big
+task_type: local_predict
+dataset:
+ split: test
+ video_processor: VideoProcessor
+ aligner: DSAligner
+ bert_name: bert-base-uncased
+fairseq:
+ dataset:
+ batch_size: 256
+ valid_subset: test
+ num_workers: 2
diff --git a/data/fairseq/examples/MMPT/projects/task/test_coin.yaml b/data/fairseq/examples/MMPT/projects/task/test_coin.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..6d919df7c2a69de33debd03a6d9b1a513da39618
--- /dev/null
+++ b/data/fairseq/examples/MMPT/projects/task/test_coin.yaml
@@ -0,0 +1,24 @@
+includes: projects/task/test.yaml
+dataset:
+ split: test
+ test_path: data/coin/COIN.json
+ meta_processor: COINActionSegmentationMetaProcessor
+ vfeat_dir: data/feat/feat_coin_s3d
+ video_processor: VideoProcessor
+ text_processor: COINActionSegmentationTextProcessor
+ aligner: COINActionSegmentationAligner
+ num_iso_layer: 12
+ sliding_window: 16
+ sliding_window_size: 32
+model:
+ model_cls: MMFusionActionSegmentation
+ mm_encoder_cls: MMBertForTokenClassification
+eval:
+ save_path: runs/task/coin/eval
+fairseq:
+ dataset:
+ batch_size: 1
+ common_eval:
+ path: runs/task/coin/checkpoint_best.pt
+metric: COINActionSegmentationMetric
+predictor: COINPredictor
diff --git a/data/fairseq/examples/MMPT/projects/task/test_coin_videoclip.yaml b/data/fairseq/examples/MMPT/projects/task/test_coin_videoclip.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..b41f5bc4890405993217859c22653fa77a0dd4c3
--- /dev/null
+++ b/data/fairseq/examples/MMPT/projects/task/test_coin_videoclip.yaml
@@ -0,0 +1,7 @@
+includes: projects/task/test_coin.yaml
+model:
+ model_cls: MMFusionSeparateActionSegmentation
+ mm_encoder_cls:
+ video_encoder_cls: MMBertForTokenClassification
+ text_encoder_cls: BertModel # dummy, not used.
+ num_hidden_video_layers: 6
diff --git a/data/fairseq/examples/MMPT/projects/task/test_coin_zs.yaml b/data/fairseq/examples/MMPT/projects/task/test_coin_zs.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..5d19b09f1dee1922ffddf5bf030135cb46016dc1
--- /dev/null
+++ b/data/fairseq/examples/MMPT/projects/task/test_coin_zs.yaml
@@ -0,0 +1,13 @@
+includes: projects/task/test_coin.yaml
+model:
+ model_cls: MMFusionSeparate
+ mm_encoder_cls:
+ video_encoder_cls: MMBertForEncoder
+ text_encoder_cls: BertModel
+ num_hidden_video_layers: 6
+eval:
+ save_path: runs/task/coin_zs/eval
+fairseq:
+ common_eval:
+ path: runs/task/checkpoint_best.pt
+predictor: COINZSPredictor
diff --git a/data/fairseq/examples/MMPT/projects/task/test_crosstask.yaml b/data/fairseq/examples/MMPT/projects/task/test_crosstask.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..6dd778e30be543a51997f16e326aa29f6e49e973
--- /dev/null
+++ b/data/fairseq/examples/MMPT/projects/task/test_crosstask.yaml
@@ -0,0 +1,32 @@
+includes: projects/task/test.yaml
+dataset:
+ split: test
+ meta_processor: CrossTaskMetaProcessor
+ test_path: data/crosstask/crosstask_release/videos_val.csv
+ train_csv_path: data/crosstask/crosstask_release/videos.csv
+ val_path: data/crosstask/crosstask_release/videos_val.csv # dummy
+ val_csv_path: data/crosstask/crosstask_release/videos_val.csv
+ primary_path: data/crosstask/crosstask_release/tasks_primary.txt
+ related_path: data/crosstask/crosstask_release/tasks_related.txt
+ vfeat_dir: data/feat/feat_crosstask_s3d
+ annotation_path: data/crosstask/crosstask_release/annotations
+ n_train: 30
+ video_processor: CrossTaskVideoProcessor
+ text_processor: CrossTaskTextProcessor
+ aligner: CrossTaskAligner
+ num_iso_layer: 12
+ sliding_window: 16
+ sliding_window_size: 32
+model:
+ model_cls: MMFusionActionLocalization
+ mm_encoder_cls: MMBertForJoint
+eval:
+ save_path: runs/task/crosstask/eval
+fairseq:
+ # read code and find what is the checkpoint arg.
+ dataset:
+ batch_size: 1
+ common_eval:
+ path: runs/task/crosstask/checkpoint_best.pt
+metric: CrossTaskMetric
+predictor: CrossTaskPredictor
diff --git a/data/fairseq/examples/MMPT/projects/task/test_crosstask_videoclip.yaml b/data/fairseq/examples/MMPT/projects/task/test_crosstask_videoclip.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..df12535d231ffe6d1759f77f161fe48d0833f4e4
--- /dev/null
+++ b/data/fairseq/examples/MMPT/projects/task/test_crosstask_videoclip.yaml
@@ -0,0 +1,7 @@
+includes: projects/task/test_crosstask.yaml
+model:
+ model_cls: MMFusionSeparateActionLocalization
+ mm_encoder_cls:
+ video_encoder_cls: MMBertForEncoder
+ text_encoder_cls: BertModel # dummy, not used.
+ num_hidden_video_layers: 6
diff --git a/data/fairseq/examples/MMPT/projects/task/test_crosstask_zs.yaml b/data/fairseq/examples/MMPT/projects/task/test_crosstask_zs.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..19386e495b37ab2570cc1d763277193b28309814
--- /dev/null
+++ b/data/fairseq/examples/MMPT/projects/task/test_crosstask_zs.yaml
@@ -0,0 +1,32 @@
+includes: projects/task/test.yaml
+dataset:
+ split: test
+ meta_processor: CrossTaskMetaProcessor
+ test_path: data/crosstask/crosstask_release/videos_val.csv
+ train_csv_path: data/crosstask/crosstask_release/videos.csv
+ val_path: data/crosstask/crosstask_release/videos_val.csv # dummy
+ val_csv_path: data/crosstask/crosstask_release/videos_val.csv
+ primary_path: data/crosstask/crosstask_release/tasks_primary.txt
+ related_path: data/crosstask/crosstask_release/tasks_related.txt
+ vfeat_dir: data/feat/feat_crosstask_s3d
+ annotation_path: data/crosstask/crosstask_release/annotations
+ n_train: 30
+ video_processor: CrossTaskVideoProcessor
+ text_processor: CrossTaskTextProcessor
+ aligner: CrossTaskAligner
+ num_iso_layer: 12
+ sliding_window: 16
+ sliding_window_size: 32
+model:
+ model_cls: MMFusionActionLocalization
+ mm_encoder_cls: MMBertForJoint
+eval:
+ save_path: runs/task/crosstask_zs/eval
+fairseq:
+ # read code and find what is the checkpoint arg.
+ dataset:
+ batch_size: 1
+ common_eval:
+ path: runs/task/checkpoint_best.pt # load the best from how2 on ACL submission: runs/task/checkpoint11.pt
+metric: CrossTaskMetric
+predictor: CrossTaskPredictor
diff --git a/data/fairseq/examples/MMPT/projects/task/test_crosstask_zs_videoclip.yaml b/data/fairseq/examples/MMPT/projects/task/test_crosstask_zs_videoclip.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..7f0198276f74f592413e99fb10672431543a8f67
--- /dev/null
+++ b/data/fairseq/examples/MMPT/projects/task/test_crosstask_zs_videoclip.yaml
@@ -0,0 +1,7 @@
+includes: projects/task/test_crosstask_zs.yaml
+model:
+ model_cls: MMFusionSeparateActionLocalization
+ mm_encoder_cls:
+ video_encoder_cls: MMBertForEncoder
+ text_encoder_cls: BertModel # dummy, not used.
+ num_hidden_video_layers: 6
diff --git a/data/fairseq/examples/MMPT/projects/task/test_didemo_zs.yaml b/data/fairseq/examples/MMPT/projects/task/test_didemo_zs.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..4b53dca71e4a0e77793483ee971379f0476366f3
--- /dev/null
+++ b/data/fairseq/examples/MMPT/projects/task/test_didemo_zs.yaml
@@ -0,0 +1,23 @@
+includes: projects/task/test.yaml
+dataset:
+ meta_processor: DiDeMoMetaProcessor
+ test_path: data/didemo/test_data.json
+ video_processor: VideoProcessor
+ vfeat_dir: data/feat/feat_didemo_s3d
+ text_processor: DiDeMoTextProcessor
+ aligner: DiDeMoAligner
+ num_iso_layer: 12
+model:
+ model_cls: MMFusionSeparate
+ mm_encoder_cls:
+ video_encoder_cls: MMBertForEncoder
+ text_encoder_cls: BertModel
+ num_hidden_video_layers: 6
+eval:
+ save_path: runs/task/didemo_zs/eval
+fairseq:
+ # read code and find what is the checkpoint arg.
+ common_eval:
+ path: runs/task/checkpoint_best.pt
+metric: DiDeMoMetric
+predictor: DiDeMoPredictor
diff --git a/data/fairseq/examples/MMPT/projects/task/test_vtt.yaml b/data/fairseq/examples/MMPT/projects/task/test_vtt.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..2f809b306d5d2b4d50d83ca3f9fa2db3f64530a5
--- /dev/null
+++ b/data/fairseq/examples/MMPT/projects/task/test_vtt.yaml
@@ -0,0 +1,19 @@
+includes: projects/task/test.yaml
+dataset:
+ meta_processor: MSRVTTMetaProcessor
+ test_path: data/msrvtt/MSRVTT_JSFUSION_test.csv
+ video_processor: VideoProcessor
+ vfeat_dir: data/feat/feat_vtt_s3d
+ text_processor: MSRVTTTextProcessor
+ num_iso_layer: 12
+model:
+ model_cls: MMFusionJoint
+ mm_encoder_cls: MMBertForJoint
+eval:
+ save_path: runs/task/vtt/eval
+fairseq:
+ # read code and find what is the checkpoint arg.
+ common_eval:
+ path: runs/task/vtt/checkpoint_last.pt
+metric: RetrievalMetric
+predictor: RetrievalPredictor
diff --git a/data/fairseq/examples/MMPT/projects/task/test_vtt_videoclip.yaml b/data/fairseq/examples/MMPT/projects/task/test_vtt_videoclip.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..cb6564394c8de5edebe41d9a90e8233d2ef90edf
--- /dev/null
+++ b/data/fairseq/examples/MMPT/projects/task/test_vtt_videoclip.yaml
@@ -0,0 +1,8 @@
+includes: projects/task/test_vtt.yaml
+model:
+ model_cls: MMFusionSeparate
+ mm_encoder_cls:
+ video_encoder_cls: MMBertForEncoder
+ text_encoder_cls: BertModel
+ num_hidden_video_layers: 6
+
diff --git a/data/fairseq/examples/MMPT/projects/task/test_vtt_zs.yaml b/data/fairseq/examples/MMPT/projects/task/test_vtt_zs.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..57340924b434338815743d0d2c326737ccfe3818
--- /dev/null
+++ b/data/fairseq/examples/MMPT/projects/task/test_vtt_zs.yaml
@@ -0,0 +1,13 @@
+includes: projects/task/test_vtt.yaml
+model:
+ model_cls: MMFusionSeparate
+ mm_encoder_cls:
+ video_encoder_cls: MMBertForEncoder
+ text_encoder_cls: BertModel
+ num_hidden_video_layers: 6
+eval:
+ save_path: runs/task/vtt_zs/eval
+fairseq:
+ # read code and find what is the checkpoint arg.
+ common_eval:
+ path: runs/task/checkpoint_best.pt
diff --git a/data/fairseq/examples/MMPT/projects/task/test_vttqa.yaml b/data/fairseq/examples/MMPT/projects/task/test_vttqa.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..ddf813c535c0ecd6e6fd81ba9013a88f8ec4296b
--- /dev/null
+++ b/data/fairseq/examples/MMPT/projects/task/test_vttqa.yaml
@@ -0,0 +1,20 @@
+includes: projects/task/test.yaml
+dataset:
+ meta_processor: MSRVTTQAMetaProcessor
+ test_path: data/msrvtt-qa/MSR_MC_test.csv
+ video_processor: VideoProcessor
+ vfeat_dir: data/feat/feat_vtt_s3d
+ text_processor: MSRVTTQATextProcessor
+ aligner: MSRVTTQAAligner
+ num_iso_layer: 12
+model:
+ model_cls: MMFusionJoint
+ mm_encoder_cls: MMBertForJoint
+eval:
+ save_path: runs/task/vttqa/eval
+fairseq:
+ # read code and find what is the checkpoint arg.
+ common_eval:
+ path: runs/task/vttqa/checkpoint_last.pt
+metric: QAMetric
+predictor: QAPredictor
diff --git a/data/fairseq/examples/MMPT/projects/task/test_vttqa_videoclip.yaml b/data/fairseq/examples/MMPT/projects/task/test_vttqa_videoclip.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..32a41e861c2e71642d4e79f8cf9dc314a5a3f621
--- /dev/null
+++ b/data/fairseq/examples/MMPT/projects/task/test_vttqa_videoclip.yaml
@@ -0,0 +1,8 @@
+includes: projects/task/test_vttqa.yaml
+model:
+ model_cls: MMFusionSeparate
+ mm_encoder_cls:
+ video_encoder_cls: MMBertForEncoder
+ text_encoder_cls: BertModel
+ num_hidden_video_layers: 6
+
diff --git a/data/fairseq/examples/MMPT/projects/task/test_vttqa_zs.yaml b/data/fairseq/examples/MMPT/projects/task/test_vttqa_zs.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..5e0e29d20729445258f5d07348fd1799f7feecf4
--- /dev/null
+++ b/data/fairseq/examples/MMPT/projects/task/test_vttqa_zs.yaml
@@ -0,0 +1,13 @@
+includes: projects/task/test_vttqa.yaml
+model:
+ model_cls: MMFusionSeparate
+ mm_encoder_cls:
+ video_encoder_cls: MMBertForEncoder
+ text_encoder_cls: BertModel
+ num_hidden_video_layers: 6
+eval:
+ save_path: runs/task/vttqa_zs/eval
+fairseq:
+ # read code and find what is the checkpoint arg.
+ common_eval:
+ path: runs/task/checkpoint_best.pt
diff --git a/data/fairseq/examples/MMPT/projects/task/test_youcook.yaml b/data/fairseq/examples/MMPT/projects/task/test_youcook.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..092b680fa6400458c447abe4f5cc5e452ffdb54f
--- /dev/null
+++ b/data/fairseq/examples/MMPT/projects/task/test_youcook.yaml
@@ -0,0 +1,22 @@
+includes: projects/task/test.yaml
+dataset:
+ meta_processor: YoucookMetaProcessor
+ test_path: data/youcook/youcook_val.pkl
+ trainval_annotation: data/youcook/youcookii_annotations_trainval.json
+ use_annotation_text: True
+ video_processor: YoucookVideoProcessor
+ vfeat_dir: data/feat/feat_youcook_s3d # /checkpoint/huxu/feat/youcook_vmz # /checkpoint/prarora/berniehuang/feat_youcook_vmz
+ text_processor: TextProcessor
+ aligner: DSAligner
+ num_iso_layer: 12
+model:
+ model_cls: MMFusionJoint
+ mm_encoder_cls: MMBertForJoint
+eval:
+ save_path: runs/task/youcook/eval
+fairseq:
+ # read code and find what is the checkpoint arg.
+ common_eval:
+ path: runs/task/youcook/checkpoint_last.pt
+metric: RetrievalMetric
+predictor: RetrievalPredictor
diff --git a/data/fairseq/examples/MMPT/projects/task/test_youcook_videoclip.yaml b/data/fairseq/examples/MMPT/projects/task/test_youcook_videoclip.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..b85ea434749009f50eea057146403691bb48c4a9
--- /dev/null
+++ b/data/fairseq/examples/MMPT/projects/task/test_youcook_videoclip.yaml
@@ -0,0 +1,8 @@
+includes: projects/task/test_youcook.yaml
+model:
+ model_cls: MMFusionSeparate
+ mm_encoder_cls:
+ video_encoder_cls: MMBertForEncoder
+ text_encoder_cls: BertModel
+ num_hidden_video_layers: 6
+
diff --git a/data/fairseq/examples/MMPT/projects/task/test_youcook_zs.yaml b/data/fairseq/examples/MMPT/projects/task/test_youcook_zs.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..0a5875bea4eaca1e46c65dfdc50b586550cc6f36
--- /dev/null
+++ b/data/fairseq/examples/MMPT/projects/task/test_youcook_zs.yaml
@@ -0,0 +1,13 @@
+includes: projects/task/test_youcook.yaml
+model:
+ model_cls: MMFusionSeparate
+ mm_encoder_cls:
+ video_encoder_cls: MMBertForEncoder
+ text_encoder_cls: BertModel
+ num_hidden_video_layers: 6
+eval:
+ save_path: runs/task/youcook_zs/eval
+fairseq:
+ # read code and find what is the checkpoint arg.
+ common_eval:
+ path: runs/task/checkpoint_best.pt
diff --git a/data/fairseq/examples/MMPT/projects/task/test_youcookcap.yaml b/data/fairseq/examples/MMPT/projects/task/test_youcookcap.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..24f6518b7b4e210289f5200edafdb436d569adf7
--- /dev/null
+++ b/data/fairseq/examples/MMPT/projects/task/test_youcookcap.yaml
@@ -0,0 +1,23 @@
+includes: projects/task/test.yaml
+dataset:
+ meta_processor: YoucookNLGMetaProcessor
+ test_path: data/youcook/val_list.txt
+ trainval_annotation: data/youcook/youcookii_annotations_trainval.json
+ video_processor: YoucookVideoProcessor
+ vfeat_dir: data/feat/feat_youcook_s3d
+ text_processor: NLGTextProcessor
+ aligner: DSNLGAligner
+model:
+ model_cls: MMFusionNLG
+ mm_encoder_cls: MMBertForNLG
+ max_decode_length: 24
+eval:
+ save_path: runs/task/youcookcap/eval
+fairseq:
+ # read code and find what is the checkpoint arg.
+ common_eval:
+ path: runs/task/youcookcap/checkpoint_best.pt
+metric: NLGMetric
+predictor: NLGPredictor
+gen_param:
+ num_beams: 5
diff --git a/data/fairseq/examples/MMPT/projects/task/vtt.yaml b/data/fairseq/examples/MMPT/projects/task/vtt.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..395e2ee6fe96899f9c47e39ad33b558358431554
--- /dev/null
+++ b/data/fairseq/examples/MMPT/projects/task/vtt.yaml
@@ -0,0 +1,25 @@
+includes: projects/task/ft.yaml
+dataset:
+ meta_processor: MSRVTTMetaProcessor
+ train_path: data/msrvtt/MSRVTT_train.csv
+ jsfusion_path: data/msrvtt/MSRVTT_JSFUSION_test.csv
+ full_test_path: data/msrvtt/MSRVTT_FULL_test.csv
+ dup: 20
+ val_path: data/msrvtt/MSRVTT_JSFUSION_test.csv
+ vfeat_dir: data/feat/feat_vtt_s3d
+ text_processor: MSRVTTTextProcessor
+ json_path: data/msrvtt/MSRVTT_data.json
+ aligner: DSAligner
+ num_iso_layer: 12
+model:
+ model_cls: MMFusionJoint
+ mm_encoder_cls: MMBertForJoint
+loss:
+ loss_cls: T2VContraLoss
+fairseq:
+ dataset:
+ batch_size: 256
+ optimization:
+ max_epoch: 10
+ checkpoint:
+ save_dir: runs/task/vtt
diff --git a/data/fairseq/examples/MMPT/projects/task/vtt_videoclip.yaml b/data/fairseq/examples/MMPT/projects/task/vtt_videoclip.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..a9892cab01b810175dcef2ee91f0e64e96bd36d9
--- /dev/null
+++ b/data/fairseq/examples/MMPT/projects/task/vtt_videoclip.yaml
@@ -0,0 +1,12 @@
+includes: projects/task/vtt.yaml
+model:
+ model_cls: MMFusionSeparate
+ mm_encoder_cls:
+ video_encoder_cls: MMBertForEncoder
+ text_encoder_cls: BertModel
+ num_hidden_video_layers: 6
+fairseq:
+ dataset:
+ batch_size: 224
+# model_cls: MMFusionShare
+# mm_encoder_cls: MMBertForEncoder
diff --git a/data/fairseq/examples/MMPT/projects/task/vttqa.yaml b/data/fairseq/examples/MMPT/projects/task/vttqa.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..56d578eff0104e5dcb4cea8720303100a9964db9
--- /dev/null
+++ b/data/fairseq/examples/MMPT/projects/task/vttqa.yaml
@@ -0,0 +1,23 @@
+includes: projects/task/ft.yaml
+dataset:
+ meta_processor: MSRVTTMetaProcessor
+ train_path: data/msrvtt/MSRVTT_train.csv
+ dup: 20
+ val_path: data/msrvtt/MSRVTT_JSFUSION_test.csv
+ vfeat_dir: data/feat/feat_vtt_s3d
+ text_processor: MSRVTTTextProcessor
+ json_path: data/msrvtt/MSRVTT_data.json
+ aligner: DSAligner
+ num_iso_layer: 12
+model:
+ model_cls: MMFusionJoint
+ mm_encoder_cls: MMBertForJoint
+loss:
+ loss_cls: V2TContraLoss
+fairseq:
+ dataset:
+ batch_size: 128
+ optimization:
+ max_epoch: 5
+ checkpoint:
+ save_dir: runs/task/vttqa
diff --git a/data/fairseq/examples/MMPT/projects/task/vttqa_videoclip.yaml b/data/fairseq/examples/MMPT/projects/task/vttqa_videoclip.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..2d484ca8a5de0e96172b49b746f93d052f50dea7
--- /dev/null
+++ b/data/fairseq/examples/MMPT/projects/task/vttqa_videoclip.yaml
@@ -0,0 +1,10 @@
+includes: projects/task/vttqa.yaml
+model:
+ model_cls: MMFusionSeparate
+ mm_encoder_cls:
+ video_encoder_cls: MMBertForEncoder
+ text_encoder_cls: BertModel
+ num_hidden_video_layers: 6
+
+# model_cls: MMFusionShare
+# mm_encoder_cls: MMBertForEncoder
diff --git a/data/fairseq/examples/MMPT/projects/task/youcook.yaml b/data/fairseq/examples/MMPT/projects/task/youcook.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..e0cd841747a5cfa4c77a94d2478bfcb029517678
--- /dev/null
+++ b/data/fairseq/examples/MMPT/projects/task/youcook.yaml
@@ -0,0 +1,25 @@
+includes: projects/task/ft.yaml
+dataset:
+ meta_processor: YoucookMetaProcessor
+ train_path: data/youcook/youcook_train.pkl
+ val_path: data/youcook/youcook_val.pkl
+ trainval_annotation: data/youcook/youcookii_annotations_trainval.json
+ use_annotation_text: True
+ video_processor: YoucookVideoProcessor
+ vfeat_dir: data/feat/feat_youcook_s3d # /checkpoint/huxu/feat/youcook_vmz # /checkpoint/prarora/berniehuang/feat_youcook_vmz
+ text_processor: TextProcessor
+ aligner: DSAligner
+ num_iso_layer: 12
+model:
+ model_cls: MMFusionJoint
+ mm_encoder_cls: MMBertForJoint
+loss:
+ loss_cls: T2VContraLoss
+fairseq:
+ dataset:
+ batch_size: 128
+ optimization:
+ max_epoch: 10
+ checkpoint:
+ save_dir: runs/task/youcook
+
diff --git a/data/fairseq/examples/MMPT/projects/task/youcook_videoclip.yaml b/data/fairseq/examples/MMPT/projects/task/youcook_videoclip.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..e3e901c30ca8ef0e858b9fb0a317aea0b55f6e34
--- /dev/null
+++ b/data/fairseq/examples/MMPT/projects/task/youcook_videoclip.yaml
@@ -0,0 +1,9 @@
+includes: projects/task/youcook.yaml
+model:
+ model_cls: MMFusionSeparate
+ mm_encoder_cls:
+ video_encoder_cls: MMBertForEncoder
+ text_encoder_cls: BertModel
+ num_hidden_video_layers: 6
+ # model_cls: MMFusionShare
+ # mm_encoder_cls: MMBertForEncoder
diff --git a/data/fairseq/examples/MMPT/projects/task/youcookcap.yaml b/data/fairseq/examples/MMPT/projects/task/youcookcap.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..047735f21716777354101ebaa230772244ac79e5
--- /dev/null
+++ b/data/fairseq/examples/MMPT/projects/task/youcookcap.yaml
@@ -0,0 +1,23 @@
+# finetuning for youcook captioning.
+includes: projects/task/ft.yaml
+dataset:
+ meta_processor: YoucookNLGMetaProcessor
+ train_path: data/youcook/train_list.txt
+ val_path: data/youcook/val_list.txt
+ trainval_annotation: data/youcook/youcookii_annotations_trainval.json
+ video_processor: YoucookVideoProcessor
+ vfeat_dir: data/feat/feat_youcook_s3d
+ text_processor: NLGTextProcessor
+ aligner: DSNLGAligner
+model:
+ model_cls: MMFusionNLG
+ mm_encoder_cls: MMBertForNLG
+loss:
+ loss_cls: NLGLoss
+fairseq:
+ dataset:
+ batch_size: 128
+ optimization:
+ max_epoch: 10
+ checkpoint:
+ save_dir: runs/task/youcookcap
diff --git a/data/fairseq/examples/MMPT/scripts/text_token_extractor/configs/bert-base-uncased.yaml b/data/fairseq/examples/MMPT/scripts/text_token_extractor/configs/bert-base-uncased.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..473dd9b45b78d6d4443921fb5f355356db4f5531
--- /dev/null
+++ b/data/fairseq/examples/MMPT/scripts/text_token_extractor/configs/bert-base-uncased.yaml
@@ -0,0 +1,5 @@
+dataset:
+ bert_name: bert-base-uncased
+ caption_pkl_path: data/how2/raw_caption_dedup.pkl
+ use_fast: true
+ target_dir: data/feat/feat_how2_s3d_shard_small
diff --git a/data/fairseq/examples/MMPT/scripts/text_token_extractor/pretokenization.py b/data/fairseq/examples/MMPT/scripts/text_token_extractor/pretokenization.py
new file mode 100644
index 0000000000000000000000000000000000000000..29ae5dc151085cc59de03e4e8bfc6e5341defb8b
--- /dev/null
+++ b/data/fairseq/examples/MMPT/scripts/text_token_extractor/pretokenization.py
@@ -0,0 +1,106 @@
+# Copyright (c) Facebook, Inc. and its affiliates.
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+
+import pickle
+import os
+import argparse
+import numpy as np
+
+from torch.utils.data import Dataset, DataLoader
+from mmpt.processors import PKLJSONStrTextProcessor
+from mmpt.utils import ShardedTensor, recursive_config
+
+
+class TokenizerDataset(Dataset):
+ def __init__(self, config):
+ self.text_processor = PKLJSONStrTextProcessor(config)
+ self.video_ids = list(self.text_processor.data.keys())
+
+ def __getitem__(self, idx):
+ video_id = self.video_ids[idx]
+ return video_id, self.text_processor(video_id)
+
+ def __len__(self):
+ return len(self.video_ids)
+
+
+def numpify(shard_idx, video_ids, captions, target_dir, split, prefix, max_cap_len=32):
+ startends = []
+ caps_ids = []
+ for video_id in video_ids:
+ caption = captions[video_id]
+ startend = []
+ cap_ids = []
+ for start, end, cap in zip(
+ caption["start"], caption["end"], caption["cap"]):
+ startend.append(np.array([start, end]).astype("float32"))
+ cap_id = np.full((max_cap_len,), -1, dtype=np.int32)
+ cap = cap[:max_cap_len]
+ cap_id[:len(cap)] = cap
+ cap_ids.append(cap_id)
+ startends.append(np.stack(startend))
+ caps_ids.append(np.stack(cap_ids))
+
+ startends = ShardedTensor.from_list(startends)
+ target_path = os.path.join(
+ target_dir,
+ prefix + split + "_" + str(shard_idx)
+ )
+ print("save to", target_path)
+ startends.save(target_path + ".startends")
+ caps_ids = ShardedTensor.from_list(caps_ids)
+ caps_ids.save(target_path + ".caps_ids")
+
+
+def sharding(config, out_file):
+ with open(out_file, "rb") as fr:
+ captions = pickle.load(fr)
+ target_dir = config.target_dir
+ prefix = os.path.basename(
+ os.path.splitext(config.caption_pkl_path)[0]
+ ) + "." + config.bert_name + "."
+ for split in ["train", "val"]:
+ target_path = os.path.join(target_dir, split + "_meta")
+ with open(target_path + ".pkl", "rb") as fr:
+ meta = pickle.load(fr)
+ print("load meta", target_path, len(meta))
+ for shard_id in meta:
+ numpify(
+ shard_id, meta[shard_id], captions,
+ target_dir, split, prefix
+ )
+
+
+def tokenize(config, out_file):
+ def collator(samples):
+ return samples
+ dataset = TokenizerDataset(config)
+ data = {}
+ for idx, batch in enumerate(
+ DataLoader(dataset, collate_fn=collator, num_workers=16)):
+ for video_id, caption in batch:
+ data[video_id] = caption
+ if idx % 5000 == 0:
+ print(idx)
+ with open(out_file, "wb") as fw:
+ pickle.dump(data, fw, pickle.HIGHEST_PROTOCOL)
+
+
+def main(args):
+ config = recursive_config(args.config).dataset
+
+ out_file = os.path.splitext(config.caption_pkl_path)[0] \
+ + "." + config.bert_name + ".pkl"
+ if not os.path.isfile(out_file):
+ tokenize(config, out_file)
+ sharding(config, out_file)
+
+
+if __name__ == "__main__":
+ parser = argparse.ArgumentParser(
+ description="pretokenize (raw_)caption.json into pkl.")
+ parser.add_argument('config', type=str)
+ args = parser.parse_args()
+ main(args)
diff --git a/data/fairseq/examples/MMPT/scripts/video_feature_extractor/extract.py b/data/fairseq/examples/MMPT/scripts/video_feature_extractor/extract.py
new file mode 100644
index 0000000000000000000000000000000000000000..b5ee7b778890994aa87d463b896f653792c073f2
--- /dev/null
+++ b/data/fairseq/examples/MMPT/scripts/video_feature_extractor/extract.py
@@ -0,0 +1,157 @@
+# Copyright Howto100M authors.
+# Copyright (c) Facebook, Inc. All Rights Reserved
+
+import torch as th
+import torch.nn.functional as F
+import math
+import numpy as np
+import argparse
+
+from torch.utils.data import DataLoader
+from model import get_model
+from preprocessing import Preprocessing
+from random_sequence_shuffler import RandomSequenceSampler
+
+from tqdm import tqdm
+from pathbuilder import PathBuilder
+from videoreader import VideoLoader
+
+
+parser = argparse.ArgumentParser(description='Easy video feature extractor')
+
+parser.add_argument('--vdir', type=str)
+parser.add_argument('--fdir', type=str)
+parser.add_argument('--hflip', type=int, default=0)
+
+parser.add_argument('--batch_size', type=int, default=64,
+ help='batch size')
+parser.add_argument('--type', type=str, default='2d',
+ help='CNN type')
+parser.add_argument('--half_precision', type=int, default=0,
+ help='output half precision float')
+parser.add_argument('--num_decoding_thread', type=int, default=4,
+ help='Num parallel thread for video decoding')
+parser.add_argument('--l2_normalize', type=int, default=1,
+ help='l2 normalize feature')
+parser.add_argument('--resnext101_model_path', type=str, default='model/resnext101.pth',
+ help='Resnext model path')
+parser.add_argument('--vmz_model_path', type=str, default='model/r2plus1d_34_clip8_ig65m_from_scratch-9bae36ae.pth',
+ help='vmz model path')
+
+args = parser.parse_args()
+
+
+# TODO: refactor all args into config. (current code is from different people.)
+CONFIGS = {
+ "2d": {
+ "fps": 1,
+ "size": 224,
+ "centercrop": False,
+ "shards": 0,
+ },
+ "3d": {
+ "fps": 24,
+ "size": 112,
+ "centercrop": True,
+ "shards": 0,
+ },
+ "s3d": {
+ "fps": 30,
+ "size": 224,
+ "centercrop": True,
+ "shards": 0,
+ },
+ "vmz": {
+ "fps": 24,
+ "size": 112,
+ "centercrop": True,
+ "shards": 0,
+ },
+ "vae": {
+ "fps": 2,
+ "size": 256,
+ "centercrop": True,
+ "shards": 100,
+ }
+}
+
+config = CONFIGS[args.type]
+
+
+video_dirs = args.vdir
+feature_dir = args.fdir
+
+video_dict = PathBuilder.build(video_dirs, feature_dir, ".npy", config["shards"])
+
+dataset = VideoLoader(
+ video_dict=video_dict,
+ framerate=config["fps"],
+ size=config["size"],
+ centercrop=config["centercrop"],
+ hflip=args.hflip
+)
+n_dataset = len(dataset)
+sampler = RandomSequenceSampler(n_dataset, 10)
+loader = DataLoader(
+ dataset,
+ batch_size=1,
+ shuffle=False,
+ num_workers=args.num_decoding_thread,
+ sampler=sampler if n_dataset > 10 else None,
+)
+preprocess = Preprocessing(args.type)
+model = get_model(args)
+
+with th.no_grad():
+ for k, data in tqdm(enumerate(loader), total=loader.__len__(), ascii=True):
+ input_file = data['input'][0]
+ output_file = data['output'][0]
+ if len(data['video'].shape) > 3:
+ video = data['video'].squeeze()
+ if len(video.shape) == 4:
+ video = preprocess(video)
+ n_chunk = len(video)
+ if args.type == 'vmz':
+ n_chunk = math.ceil(n_chunk/float(3))
+ features = th.cuda.FloatTensor(n_chunk, 512).fill_(0)
+ elif args.type == 's3d':
+ features = th.cuda.FloatTensor(n_chunk, 512).fill_(0)
+ elif args.type == "vae":
+ features = th.cuda.LongTensor(n_chunk, 1024).fill_(0)
+ else:
+ features = th.cuda.FloatTensor(n_chunk, 2048).fill_(0)
+ n_iter = int(math.ceil(n_chunk / float(args.batch_size)))
+ for i in range(n_iter):
+ factor = 1
+ if args.type == 'vmz':
+ factor = 3
+ min_ind = factor * i * args.batch_size
+ max_ind = factor * (i + 1) * args.batch_size
+ video_batch = video[min_ind:max_ind:factor].cuda()
+ if args.type == '2d':
+ batch_features = model(video_batch) # (51, 487), (51, 512)
+ elif args.type == 's3d':
+ batch_features = model(video_batch)
+ batch_features = batch_features['video_embedding']
+ elif args.type == "vae":
+ # image_code.
+ batch_features = model(video_batch)
+ else:
+ batch_pred, batch_features = model(video_batch) # (51, 487), (51, 512)
+ if args.l2_normalize:
+ batch_features = F.normalize(batch_features, dim=1)
+ features[i*args.batch_size:(i+1)*args.batch_size] = batch_features
+ features = features.cpu().numpy()
+ if args.half_precision:
+ if args.type == "vae":
+ features = features.astype(np.int16)
+ else:
+ features = features.astype('float16')
+ else:
+ if args.type == "vae":
+ features = features.astype(np.int32)
+ else:
+ features = features.astype('float32')
+ np.save(output_file, features)
+ else:
+ print('Video {} error.'.format(input_file))
diff --git a/data/fairseq/examples/MMPT/scripts/video_feature_extractor/how2/s3d.sh b/data/fairseq/examples/MMPT/scripts/video_feature_extractor/how2/s3d.sh
new file mode 100644
index 0000000000000000000000000000000000000000..90102c89fbb4fb413951dfd049b30b8dfe7cbe99
--- /dev/null
+++ b/data/fairseq/examples/MMPT/scripts/video_feature_extractor/how2/s3d.sh
@@ -0,0 +1,8 @@
+#!/bin/bash
+
+
+python scripts/video_feature_extractor/extract.py \
+ --vdir \
+ --fdir data/feat/feat_how2_s3d \
+ --type=s3d --num_decoding_thread=4 \
+ --batch_size 32 --half_precision 1
diff --git a/data/fairseq/examples/MMPT/scripts/video_feature_extractor/model.py b/data/fairseq/examples/MMPT/scripts/video_feature_extractor/model.py
new file mode 100644
index 0000000000000000000000000000000000000000..ac266e844c86246bbfce02b9e6a2999353661df9
--- /dev/null
+++ b/data/fairseq/examples/MMPT/scripts/video_feature_extractor/model.py
@@ -0,0 +1,58 @@
+# Copyright (c) Howto100M authors and Facebook, Inc. All Rights Reserved
+
+import torch as th
+
+from torch import nn
+
+
+class GlobalAvgPool(nn.Module):
+ def __init__(self):
+ super(GlobalAvgPool, self).__init__()
+
+ def forward(self, x):
+ return th.mean(x, dim=[-2, -1])
+
+
+def get_model(args):
+ assert args.type in ['2d', '3d', 'vmz', 's3d', 'vae']
+ if args.type == '2d':
+ print('Loading 2D-ResNet-152 ...')
+ import torchvision.models as models
+ model = models.resnet152(pretrained=True)
+ model = nn.Sequential(*list(model.children())[:-2], GlobalAvgPool())
+ model = model.cuda()
+ elif args.type == 'vmz':
+ print('Loading VMZ ...')
+ from vmz34 import r2plus1d_34
+ model = r2plus1d_34(pretrained_path=args.vmz_model_path, pretrained_num_classes=487)
+ model = model.cuda()
+ elif args.type == 's3d':
+ # we use one copy of s3d instead of dup another one for feature extraction.
+ from mmpt.processors.models.s3dg import S3D
+ model = S3D('pretrained_models/s3d_dict.npy', 512)
+ model.load_state_dict(th.load('pretrained_models/s3d_howto100m.pth'))
+ model = model.cuda()
+
+ elif args.type == '3d':
+ print('Loading 3D-ResneXt-101 ...')
+ from videocnn.models import resnext
+ model = resnext.resnet101(
+ num_classes=400,
+ shortcut_type='B',
+ cardinality=32,
+ sample_size=112,
+ sample_duration=16,
+ last_fc=False)
+ model = model.cuda()
+ model_data = th.load(args.resnext101_model_path)
+ model.load_state_dict(model_data)
+ elif args.type == 'vae':
+ from openaivae import OpenAIParallelDiscreteVAE
+ model = OpenAIParallelDiscreteVAE()
+ model = model.cuda()
+ else:
+ raise ValueError("model not supported yet.")
+
+ model.eval()
+ print('loaded')
+ return model
diff --git a/data/fairseq/examples/MMPT/scripts/video_feature_extractor/pathbuilder.py b/data/fairseq/examples/MMPT/scripts/video_feature_extractor/pathbuilder.py
new file mode 100644
index 0000000000000000000000000000000000000000..2392d6d63bc293fb24068e33f707ee4ce0e74a6f
--- /dev/null
+++ b/data/fairseq/examples/MMPT/scripts/video_feature_extractor/pathbuilder.py
@@ -0,0 +1,89 @@
+# Copyright (c) Facebook, Inc. and its affiliates.
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+import os
+import urllib.parse
+import json
+import pandas as pd
+
+from tqdm import tqdm
+
+
+# TODO: extending to other datasets.
+supported_formats = {}
+
+
+class PathBuilder(object):
+ @classmethod
+ def build(cls, video_dirs, feature_dir, ext, shards=0, split=None):
+ meta_fn = os.path.join(feature_dir, "meta_plan.json")
+ os.makedirs(feature_dir, exist_ok=True)
+ if os.path.isfile(meta_fn):
+ with open(meta_fn) as fr:
+ meta = json.load(fr)
+ return meta
+ print("searching videos...")
+
+ video_id_to_path = {}
+ for video_dir in video_dirs.split(","):
+ # TODO: add supports of recursive listdir.
+ if video_dir in supported_formats:
+ supported_formats[video_dir].load(video_dir, video_id_to_path)
+ else:
+ for idx, fn in enumerate(tqdm(os.listdir(video_dir))):
+ video_fn = os.path.join(video_dir, fn)
+ if os.path.isfile(video_fn):
+ video_id = os.path.splitext(fn)[0]
+ video_id_to_path[video_id] = video_fn
+ elif os.path.isdir(video_fn):
+ # shards of folders.
+ shard_dir = video_fn
+ for idx, fn in enumerate(os.listdir(shard_dir)):
+ video_fn = os.path.join(shard_dir, fn)
+ if os.path.isfile(video_fn):
+ video_id = os.path.splitext(fn)[0]
+ video_id_to_path[video_id] = video_fn
+
+ video_path, feature_path = [], []
+ valid_ext = set()
+ for idx, video_id in enumerate(video_id_to_path):
+ video_path.append(video_id_to_path[video_id])
+ if ext is None:
+ # use original file ext for format compatibility.
+ video_id_to_path[video_id]
+ path = urllib.parse.urlparse(video_id_to_path[video_id]).path
+ ext = os.path.splitext(path)[1]
+ if ext not in valid_ext:
+ valid_ext.add(ext)
+ print("adding", ext)
+ if shards:
+ shard_id = str(idx % shards)
+ feature_fn = os.path.join(
+ feature_dir, shard_id, video_id + ext)
+ else:
+ feature_fn = os.path.join(
+ feature_dir, video_id + ext)
+ feature_path.append(feature_fn)
+
+ print("targeting", len(feature_path), "videos")
+ meta = {
+ "video_path": video_path, "feature_path": feature_path}
+ with open(meta_fn, "w") as fw:
+ json.dump(meta, fw)
+
+ if split is not None:
+ splits = split.split("/")
+ assert len(splits) == 2
+ cur, total = int(splits[0]), int(splits[1])
+ assert cur < total
+ import math
+ chunk = math.ceil(len(meta["video_path"]) / total)
+ start = cur * chunk
+ end = (cur + 1) * chunk
+ meta = {
+ "video_path": meta["video_path"][start:end],
+ "feature_path": meta["feature_path"][start:end]
+ }
+
+ return meta
diff --git a/data/fairseq/examples/MMPT/scripts/video_feature_extractor/preprocessing.py b/data/fairseq/examples/MMPT/scripts/video_feature_extractor/preprocessing.py
new file mode 100644
index 0000000000000000000000000000000000000000..fa0cec3a76b4665fae57d1feadd5a52469234226
--- /dev/null
+++ b/data/fairseq/examples/MMPT/scripts/video_feature_extractor/preprocessing.py
@@ -0,0 +1,57 @@
+# Copyright Howto100m authors.
+# Copyright (c) Facebook, Inc. All Rights Reserved
+
+import torch as th
+
+class Normalize(object):
+
+ def __init__(self, mean, std):
+ self.mean = th.FloatTensor(mean).view(1, 3, 1, 1)
+ self.std = th.FloatTensor(std).view(1, 3, 1, 1)
+
+ def __call__(self, tensor):
+ tensor = (tensor - self.mean) / (self.std + 1e-8)
+ return tensor
+
+class Preprocessing(object):
+
+ def __init__(self, type):
+ self.type = type
+ if type == '2d':
+ self.norm = Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])
+ elif type == '3d':
+ self.norm = Normalize(mean=[110.6, 103.2, 96.3], std=[1.0, 1.0, 1.0])
+ elif type == 'vmz':
+ self.norm = Normalize(mean=[110.201, 100.64, 95.997], std=[58.1489, 56.4701, 55.3324])
+
+ def _zero_pad(self, tensor, size):
+ n = size - len(tensor) % size
+ if n == size:
+ return tensor
+ else:
+ z = th.zeros(n, tensor.shape[1], tensor.shape[2], tensor.shape[3])
+ return th.cat((tensor, z), 0)
+
+ def __call__(self, tensor):
+ if self.type == '2d':
+ tensor = tensor / 255.0
+ tensor = self.norm(tensor)
+ elif self.type == 'vmz':
+ #tensor = self._zero_pad(tensor, 8)
+ tensor = self._zero_pad(tensor, 10)
+ tensor = self.norm(tensor)
+ #tensor = tensor.view(-1, 8, 3, 112, 112)
+ tensor = tensor.view(-1, 10, 3, 112, 112)
+ tensor = tensor.transpose(1, 2)
+ elif self.type == '3d':
+ tensor = self._zero_pad(tensor, 16)
+ tensor = self.norm(tensor)
+ tensor = tensor.view(-1, 16, 3, 112, 112)
+ tensor = tensor.transpose(1, 2)
+ elif self.type == 's3d':
+ tensor = tensor / 255.0
+ tensor = self._zero_pad(tensor, 30)
+ tensor = tensor.view(-1, 30, 3, 224, 224) # N x 30 x 3 x H x W
+ tensor = tensor.transpose(1, 2) # N x 3 x 30 x H x W
+ # for vae do nothing
+ return tensor
diff --git a/data/fairseq/examples/MMPT/scripts/video_feature_extractor/random_sequence_shuffler.py b/data/fairseq/examples/MMPT/scripts/video_feature_extractor/random_sequence_shuffler.py
new file mode 100644
index 0000000000000000000000000000000000000000..1f3e4aceaa8f930c9eeac6283b15924e7eed5dc3
--- /dev/null
+++ b/data/fairseq/examples/MMPT/scripts/video_feature_extractor/random_sequence_shuffler.py
@@ -0,0 +1,29 @@
+# Copyright (c) Facebook, Inc. All Rights Reserved
+
+import numpy as np
+
+from torch.utils.data.sampler import Sampler
+
+
+class RandomSequenceSampler(Sampler):
+
+ def __init__(self, n_sample, seq_len):
+ self.n_sample = n_sample
+ self.seq_len = seq_len
+
+ def _pad_ind(self, ind):
+ zeros = np.zeros(self.seq_len - self.n_sample % self.seq_len)
+ ind = np.concatenate((ind, zeros))
+ return ind
+
+ def __iter__(self):
+ idx = np.arange(self.n_sample)
+ if self.n_sample % self.seq_len != 0:
+ idx = self._pad_ind(idx)
+ idx = np.reshape(idx, (-1, self.seq_len))
+ np.random.shuffle(idx)
+ idx = np.reshape(idx, (-1))
+ return iter(idx.astype(int))
+
+ def __len__(self):
+ return self.n_sample + (self.seq_len - self.n_sample % self.seq_len)
diff --git a/data/fairseq/examples/MMPT/scripts/video_feature_extractor/shard_feature.py b/data/fairseq/examples/MMPT/scripts/video_feature_extractor/shard_feature.py
new file mode 100644
index 0000000000000000000000000000000000000000..f75e1dfae558fa366d15fa19e89d353003e728b3
--- /dev/null
+++ b/data/fairseq/examples/MMPT/scripts/video_feature_extractor/shard_feature.py
@@ -0,0 +1,64 @@
+# Copyright (c) Facebook, Inc. and its affiliates.
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+import numpy as np
+import os
+import pickle
+
+from mmpt.utils import ShardedTensor
+
+
+class Shard(object):
+ def __init__(
+ self,
+ vfeat_dir,
+ tfeat_dir,
+ target_dir,
+ file_paths,
+ shard_size=4096
+ ):
+ self.vfeat_dir = vfeat_dir
+ self.tfeat_dir = tfeat_dir
+ self.target_dir = target_dir
+ self.video_ids = {}
+ for split, file_path in zip(["train", "val"], file_paths):
+ with open(file_path) as fr:
+ self.video_ids[split] = [
+ line.strip() for line in fr.readlines()]
+ self.shard_size = shard_size
+
+ def __call__(self, split="train"):
+ for split in ["train", "val"]:
+ meta = {}
+ for shard_idx, shard_offset in enumerate(
+ range(0, len(self.video_ids[split]), self.shard_size)
+ ):
+ print(shard_idx)
+ meta_shard = []
+ video_shard = []
+ for video_id in self.video_ids[split][shard_offset:shard_offset+self.shard_size]:
+ meta_shard.append(video_id)
+ npy_file = os.path.join(self.vfeat_dir, video_id + ".npy")
+ video_shard.append(np.load(npy_file))
+
+ meta[shard_idx] = meta_shard
+ video_shard = ShardedTensor.from_list(video_shard)
+ target_path = os.path.join(
+ self.target_dir, split + "_" + str(shard_idx))
+ video_shard.save(target_path)
+
+ target_path = os.path.join(self.target_dir, split + "_meta")
+ with open(target_path + ".pkl", "wb") as fw:
+ pickle.dump(meta, fw, pickle.HIGHEST_PROTOCOL)
+
+
+if __name__ == "__main__":
+ shard = Shard(
+ "data/feat/feat_how2_s3d",
+ "data/how2/raw_caption_dedup.bert-base-uncased",
+ "data/feat/feat_how2_s3d_shard_small",
+ ["data/how2/how2_s3d_train.lst", "data/how2/how2_s3d_val.lst"]
+ )
+
+ shard()
diff --git a/data/fairseq/examples/MMPT/scripts/video_feature_extractor/videoreader.py b/data/fairseq/examples/MMPT/scripts/video_feature_extractor/videoreader.py
new file mode 100644
index 0000000000000000000000000000000000000000..429e05f8bc8667408b8c2057578b1e8c0e98638c
--- /dev/null
+++ b/data/fairseq/examples/MMPT/scripts/video_feature_extractor/videoreader.py
@@ -0,0 +1,242 @@
+# Copyright Howto100M authors.
+# Copyright (c) Facebook, Inc. All Rights Reserved
+
+import torch as th
+import pandas as pd
+import os
+import numpy as np
+import ffmpeg
+import random
+
+from torch.utils.data import Dataset
+
+
+class VideoLoader(Dataset):
+ """modified from how2's video_feature_extractor."""
+ def __init__(
+ self,
+ csv=None,
+ video_dict=None,
+ framerate=1,
+ size=112,
+ centercrop=False,
+ hflip=False,
+ **kwargs
+ ):
+ if csv is None and video_dict is None:
+ raise ValueError("csv and video_dict cannot be both None.")
+ if csv is not None:
+ self.csv = pd.read_csv(csv)
+ if video_dict is not None:
+ self.csv = pd.DataFrame.from_dict(video_dict)
+
+ self.centercrop = centercrop
+ self.size = size
+ self.framerate = framerate
+ self.hflip = hflip
+
+ def __len__(self):
+ return len(self.csv)
+
+ def _get_video_dim(self, video_path):
+ probe = ffmpeg.probe(video_path)
+ video_stream = next((stream for stream in probe['streams']
+ if stream['codec_type'] == 'video'), None)
+ width = int(video_stream['width'])
+ height = int(video_stream['height'])
+ return height, width
+
+ def _get_video_info(self, video_path):
+ probe = ffmpeg.probe(video_path)
+ video_stream = next((stream for stream in probe['streams']
+ if stream['codec_type'] == 'video'), None)
+ return video_stream
+
+ def _get_output_dim(self, h, w):
+ if isinstance(self.size, tuple) and len(self.size) == 2:
+ return self.size
+ elif h >= w:
+ return int(h * self.size / w), self.size
+ else:
+ return self.size, int(w * self.size / h)
+
+ def __getitem__(self, idx):
+ video_path = self.csv['video_path'].values[idx]
+ output_file = self.csv['feature_path'].values[idx]
+ return self._decode(output_file, video_path)
+
+ def _decode(self, output_file, video_path):
+ if not(os.path.isfile(output_file)) and os.path.isfile(video_path):
+ try:
+ h, w = self._get_video_dim(video_path)
+ except Exception:
+ print('ffprobe failed at: {}'.format(video_path))
+ return {'video': th.zeros(1), 'input': video_path,
+ 'output': output_file}
+ try:
+ os.makedirs(os.path.dirname(output_file), exist_ok=True)
+ height, width = self._get_output_dim(h, w)
+
+ cmd = (
+ ffmpeg
+ .input(video_path)
+ .filter('fps', fps=self.framerate)
+ .filter('scale', width, height)
+ )
+ if self.hflip:
+ cmd = cmd.filter('hflip')
+
+ if self.centercrop:
+ x = int((width - self.size) / 2.0)
+ y = int((height - self.size) / 2.0)
+ cmd = cmd.crop(x, y, self.size, self.size)
+ video = self._run(cmd, output_file)
+ except Exception:
+ video = th.zeros(1)
+ else:
+ video = th.zeros(1)
+
+ return {'video': video, 'input': video_path, 'output': output_file}
+
+ def _run(self, cmd, output_file):
+ out, _ = (
+ cmd.output('pipe:', format='rawvideo', pix_fmt='rgb24')
+ .run(capture_stdout=True, quiet=True)
+ )
+ if self.centercrop and isinstance(self.size, int):
+ height, width = self.size, self.size
+ video = np.frombuffer(out, np.uint8).reshape([-1, height, width, 3])
+ video = th.from_numpy(video.astype('float32'))
+ return video.permute(0, 3, 1, 2)
+
+
+class VideoVerifier(VideoLoader):
+ def __getitem__(self, idx):
+ video_path = self.csv['video_path'].values[idx]
+ try:
+ return self._get_video_info(video_path)
+ except Exception:
+ # print('ffprobe failed at: {}'.format(video_path))
+ return None
+
+
+class VideoCompressor(VideoLoader):
+ def __init__(
+ self,
+ csv=None,
+ video_dict=None,
+ framerate=1,
+ size=112,
+ centercrop=False,
+ hflip=False,
+ crf=32,
+ **kwargs
+ ):
+ super().__init__(
+ csv,
+ video_dict,
+ framerate,
+ size,
+ centercrop,
+ hflip
+ )
+ self.crf = crf
+
+ def _run(self, cmd, output_file):
+ out, _ = (
+ cmd.output(filename=output_file, crf=self.crf)
+ .run(quiet=True)
+ )
+ video = None
+ return video
+
+
+class VideoDownloader(VideoCompressor):
+ """download"""
+ def __getitem__(self, idx):
+ video_path = self.csv['video_path'].values[idx]
+ output_file = self.csv['feature_path'].values[idx]
+ if not(os.path.isfile(output_file)):
+ os.makedirs(os.path.dirname(output_file), exist_ok=True)
+ cmd = "wget -O" + output_file + " " + video_path
+ # import subprocess
+ # subprocess.check_output(
+ # cmd,
+ # stderr=subprocess.STDOUT, shell=True)
+ os.system(cmd)
+ return {'video': None, 'input': video_path, 'output': output_file}
+
+
+class AvKeyframeVideoCompressor(VideoLoader):
+ """extract keyframes from a video and save it as jpg.
+ TODO: consider to merge with `CodecProcessor`.
+ """
+ def __init__(
+ self,
+ csv=None,
+ video_dict=None,
+ framerate=1,
+ size=112,
+ centercrop=False,
+ max_num_frames=5,
+ **kwargs
+ ):
+ super().__init__(csv, video_dict, framerate, size, centercrop)
+ self.max_num_frames = max_num_frames
+
+ def _get_video_dim(self, video_fn):
+ """decord cannot probe the size of a video, we use pyav instead."""
+ import av
+ with av.open(video_fn) as container:
+ height = container.streams.video[0].codec_context.height
+ width = container.streams.video[0].codec_context.width
+ return height, width
+
+ def _get_output_dim(self, height, width):
+ """
+ keep the shorter side be `self.size`, strech the other.
+ """
+ if height >= width:
+ return int(height * self.size / width), self.size
+ else:
+ return self.size, int(width * self.size / height)
+
+ def __getitem__(self, idx):
+ import av
+ video_path = self.csv['video_path'].values[idx]
+ output_file = self.csv['feature_path'].values[idx]
+ if not(os.path.isdir(output_file)) and os.path.isfile(video_path):
+ try:
+ h, w = self._get_video_dim(video_path)
+ except Exception:
+ print('probe failed at: {}'.format(video_path))
+ return {'video': th.zeros(1), 'input': video_path,
+ 'output': output_file}
+
+ try:
+ height, width = self._get_output_dim(h, w)
+
+ # new for av.
+ with av.open(video_path) as container:
+ container.streams.video[0].thread_type = "AUTO"
+ container.streams.video[0].codec_context.height = height
+ container.streams.video[0].codec_context.width = width
+ if self.framerate == 0: # keyframe.
+ container.streams.video[0].codec_context.skip_frame = 'NONKEY'
+ frames = []
+ for frame in container.decode(video=0):
+ frames.append(frame)
+ frames = random.sample(frames, self.max_num_frames)
+
+ os.makedirs(output_file, exist_ok=True)
+ for frame in frames:
+ frame.to_image().save(
+ os.path.join(
+ output_file,
+ "%04d.jpg" % frame.index))
+ except Exception:
+ print('extract failed at: {}'.format(video_path))
+ return {'video': th.zeros(1), 'input': video_path,
+ 'output': output_file}
+ video = th.zeros(1)
+ return {'video': video, 'input': video_path, 'output': output_file}
diff --git a/data/fairseq/examples/MMPT/setup.py b/data/fairseq/examples/MMPT/setup.py
new file mode 100644
index 0000000000000000000000000000000000000000..a9a82296ea9c3a53f760c29c34020b5a90091a89
--- /dev/null
+++ b/data/fairseq/examples/MMPT/setup.py
@@ -0,0 +1,24 @@
+import setuptools
+
+with open("README.md", "r") as fh:
+ long_description = fh.read()
+
+setuptools.setup(
+ name="mmpt",
+ version="0.0.1",
+ author="Hu Xu, Po-yao Huang",
+ author_email="huxu@fb.com",
+ description="A package for multimodal pretraining.",
+ long_description=long_description,
+ long_description_content_type="text/markdown",
+ url="https://github.com/pytorch/fairseq/examples/MMPT",
+ packages=setuptools.find_packages(),
+ install_requires=[
+ ],
+ classifiers=[
+ "Programming Language :: Python :: 3",
+ "License :: CC-BY-NC",
+ "Operating System :: OS Independent",
+ ],
+ python_requires='>=3.6',
+)
diff --git a/data/fairseq/examples/byte_level_bpe/README.md b/data/fairseq/examples/byte_level_bpe/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..657092660eae42d20f67647417623b8b8cb7b66c
--- /dev/null
+++ b/data/fairseq/examples/byte_level_bpe/README.md
@@ -0,0 +1,88 @@
+# Neural Machine Translation with Byte-Level Subwords
+
+https://arxiv.org/abs/1909.03341
+
+We provide an implementation of byte-level byte-pair encoding (BBPE), taking IWSLT 2017 Fr-En translation as
+example.
+
+## Data
+Get data and generate fairseq binary dataset:
+```bash
+bash ./get_data.sh
+```
+
+## Model Training
+Train Transformer model with Bi-GRU embedding contextualization (implemented in `gru_transformer.py`):
+```bash
+# VOCAB=bytes
+# VOCAB=chars
+VOCAB=bbpe2048
+# VOCAB=bpe2048
+# VOCAB=bbpe4096
+# VOCAB=bpe4096
+# VOCAB=bpe16384
+```
+```bash
+fairseq-train "data/bin_${VOCAB}" --task translation --user-dir examples/byte_level_bpe/gru_transformer \
+ --arch gru_transformer --encoder-layers 2 --decoder-layers 2 --dropout 0.3 --share-all-embeddings \
+ --optimizer adam --adam-betas '(0.9, 0.98)' \
+ --lr 5e-4 --lr-scheduler inverse_sqrt --warmup-updates 4000 \
+ --criterion label_smoothed_cross_entropy --label-smoothing 0.1 \
+ --log-format 'simple' --log-interval 100 --save-dir "checkpoints/${VOCAB}" \
+ --batch-size 100 --max-update 100000 --update-freq 2
+```
+
+## Generation
+`fairseq-generate` requires bytes (BBPE) decoder to convert byte-level representation back to characters:
+```bash
+# BPE=--bpe bytes
+# BPE=--bpe characters
+BPE=--bpe byte_bpe --sentencepiece-model-path data/spm_bbpe2048.model
+# BPE=--bpe sentencepiece --sentencepiece-model data/spm_bpe2048.model
+# BPE=--bpe byte_bpe --sentencepiece-model-path data/spm_bbpe4096.model
+# BPE=--bpe sentencepiece --sentencepiece-model data/spm_bpe4096.model
+# BPE=--bpe sentencepiece --sentencepiece-model data/spm_bpe16384.model
+```
+
+```bash
+fairseq-generate "data/bin_${VOCAB}" --task translation --user-dir examples/byte_level_bpe/gru_transformer \
+ --source-lang fr --gen-subset test --sacrebleu --path "checkpoints/${VOCAB}/checkpoint_last.pt" \
+ --tokenizer moses --moses-target-lang en ${BPE}
+```
+When using `fairseq-interactive`, bytes (BBPE) encoder/decoder is required to tokenize input data and detokenize model predictions:
+```bash
+fairseq-interactive "data/bin_${VOCAB}" --task translation --user-dir examples/byte_level_bpe/gru_transformer \
+ --path "checkpoints/${VOCAB}/checkpoint_last.pt" --input data/test.fr --tokenizer moses --moses-source-lang fr \
+ --moses-target-lang en ${BPE} --buffer-size 1000 --max-tokens 10000
+```
+
+## Results
+| Vocabulary | Model | BLEU |
+|:-------------:|:-------------:|:-------------:|
+| Joint BPE 16k ([Kudo, 2018](https://arxiv.org/abs/1804.10959)) | 512d LSTM 2+2 | 33.81 |
+| Joint BPE 16k | Transformer base 2+2 (w/ GRU) | 36.64 (36.72) |
+| Joint BPE 4k | Transformer base 2+2 (w/ GRU) | 35.49 (36.10) |
+| Joint BBPE 4k | Transformer base 2+2 (w/ GRU) | 35.61 (35.82) |
+| Joint BPE 2k | Transformer base 2+2 (w/ GRU) | 34.87 (36.13) |
+| Joint BBPE 2k | Transformer base 2+2 (w/ GRU) | 34.98 (35.43) |
+| Characters | Transformer base 2+2 (w/ GRU) | 31.78 (33.30) |
+| Bytes | Transformer base 2+2 (w/ GRU) | 31.57 (33.62) |
+
+
+## Citation
+```
+@misc{wang2019neural,
+ title={Neural Machine Translation with Byte-Level Subwords},
+ author={Changhan Wang and Kyunghyun Cho and Jiatao Gu},
+ year={2019},
+ eprint={1909.03341},
+ archivePrefix={arXiv},
+ primaryClass={cs.CL}
+}
+```
+
+
+## Contact
+Changhan Wang ([changhan@fb.com](mailto:changhan@fb.com)),
+Kyunghyun Cho ([kyunghyuncho@fb.com](mailto:kyunghyuncho@fb.com)),
+Jiatao Gu ([jgu@fb.com](mailto:jgu@fb.com))
diff --git a/data/fairseq/examples/byte_level_bpe/get_bitext.py b/data/fairseq/examples/byte_level_bpe/get_bitext.py
new file mode 100644
index 0000000000000000000000000000000000000000..6ac1eeec1e6167ec6bafd76b37173ee6987cae7e
--- /dev/null
+++ b/data/fairseq/examples/byte_level_bpe/get_bitext.py
@@ -0,0 +1,254 @@
+# Copyright (c) Facebook, Inc. and its affiliates.
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+
+
+import argparse
+import os
+import os.path as op
+from collections import namedtuple
+from multiprocessing import cpu_count
+from typing import List, Optional
+
+import sentencepiece as sp
+from fairseq.data.encoders.byte_bpe import ByteBPE
+from fairseq.data.encoders.byte_utils import byte_encode
+from fairseq.data.encoders.bytes import Bytes
+from fairseq.data.encoders.characters import Characters
+from fairseq.data.encoders.moses_tokenizer import MosesTokenizer
+from fairseq.data.encoders.sentencepiece_bpe import SentencepieceBPE
+
+
+SPLITS = ["train", "valid", "test"]
+
+
+def _convert_xml(in_path: str, out_path: str):
+ with open(in_path) as f, open(out_path, "w") as f_o:
+ for s in f:
+ ss = s.strip()
+ if not ss.startswith("", "").split('">')
+ assert len(ss) == 2
+ f_o.write(ss[1].strip() + "\n")
+
+
+def _convert_train(in_path: str, out_path: str):
+ with open(in_path) as f, open(out_path, "w") as f_o:
+ for s in f:
+ ss = s.strip()
+ if ss.startswith("<"):
+ continue
+ f_o.write(ss.strip() + "\n")
+
+
+def _get_bytes(in_path: str, out_path: str):
+ with open(in_path) as f, open(out_path, "w") as f_o:
+ for s in f:
+ f_o.write(Bytes.encode(s.strip()) + "\n")
+
+
+def _get_chars(in_path: str, out_path: str):
+ with open(in_path) as f, open(out_path, "w") as f_o:
+ for s in f:
+ f_o.write(Characters.encode(s.strip()) + "\n")
+
+
+def pretokenize(in_path: str, out_path: str, src: str, tgt: str):
+ Args = namedtuple(
+ "Args",
+ [
+ "moses_source_lang",
+ "moses_target_lang",
+ "moses_no_dash_splits",
+ "moses_no_escape",
+ ],
+ )
+ args = Args(
+ moses_source_lang=src,
+ moses_target_lang=tgt,
+ moses_no_dash_splits=False,
+ moses_no_escape=False,
+ )
+ pretokenizer = MosesTokenizer(args)
+ with open(in_path) as f, open(out_path, "w") as f_o:
+ for s in f:
+ f_o.write(pretokenizer.encode(s.strip()) + "\n")
+
+
+def _convert_to_bchar(in_path_prefix: str, src: str, tgt: str, out_path: str):
+ with open(out_path, "w") as f_o:
+ for lang in [src, tgt]:
+ with open(f"{in_path_prefix}.{lang}") as f:
+ for s in f:
+ f_o.write(byte_encode(s.strip()) + "\n")
+
+
+def _get_bpe(in_path: str, model_prefix: str, vocab_size: int):
+ arguments = [
+ f"--input={in_path}",
+ f"--model_prefix={model_prefix}",
+ f"--model_type=bpe",
+ f"--vocab_size={vocab_size}",
+ "--character_coverage=1.0",
+ "--normalization_rule_name=identity",
+ f"--num_threads={cpu_count()}",
+ ]
+ sp.SentencePieceTrainer.Train(" ".join(arguments))
+
+
+def _apply_bbpe(model_path: str, in_path: str, out_path: str):
+ Args = namedtuple("Args", ["sentencepiece_model_path"])
+ args = Args(sentencepiece_model_path=model_path)
+ tokenizer = ByteBPE(args)
+ with open(in_path) as f, open(out_path, "w") as f_o:
+ for s in f:
+ f_o.write(tokenizer.encode(s.strip()) + "\n")
+
+
+def _apply_bpe(model_path: str, in_path: str, out_path: str):
+ Args = namedtuple("Args", ["sentencepiece_model"])
+ args = Args(sentencepiece_model=model_path)
+ tokenizer = SentencepieceBPE(args)
+ with open(in_path) as f, open(out_path, "w") as f_o:
+ for s in f:
+ f_o.write(tokenizer.encode(s.strip()) + "\n")
+
+
+def _concat_files(in_paths: List[str], out_path: str):
+ with open(out_path, "w") as f_o:
+ for p in in_paths:
+ with open(p) as f:
+ for r in f:
+ f_o.write(r)
+
+
+def preprocess_iwslt17(
+ root: str,
+ src: str,
+ tgt: str,
+ bpe_size: Optional[int],
+ need_chars: bool,
+ bbpe_size: Optional[int],
+ need_bytes: bool,
+):
+ # extract bitext
+ in_root = op.join(root, f"{src}-{tgt}")
+ for lang in [src, tgt]:
+ _convert_train(
+ op.join(in_root, f"train.tags.{src}-{tgt}.{lang}"),
+ op.join(root, f"train.{lang}"),
+ )
+ _convert_xml(
+ op.join(in_root, f"IWSLT17.TED.dev2010.{src}-{tgt}.{lang}.xml"),
+ op.join(root, f"valid.{lang}"),
+ )
+ _convert_xml(
+ op.join(in_root, f"IWSLT17.TED.tst2015.{src}-{tgt}.{lang}.xml"),
+ op.join(root, f"test.{lang}"),
+ )
+ # pre-tokenize
+ for lang in [src, tgt]:
+ for split in SPLITS:
+ pretokenize(
+ op.join(root, f"{split}.{lang}"),
+ op.join(root, f"{split}.moses.{lang}"),
+ src,
+ tgt,
+ )
+ # tokenize with BPE vocabulary
+ if bpe_size is not None:
+ # learn vocabulary
+ concated_train_path = op.join(root, "train.all")
+ _concat_files(
+ [op.join(root, "train.moses.fr"), op.join(root, "train.moses.en")],
+ concated_train_path,
+ )
+ bpe_model_prefix = op.join(root, f"spm_bpe{bpe_size}")
+ _get_bpe(concated_train_path, bpe_model_prefix, bpe_size)
+ os.remove(concated_train_path)
+ # apply
+ for lang in [src, tgt]:
+ for split in SPLITS:
+ _apply_bpe(
+ bpe_model_prefix + ".model",
+ op.join(root, f"{split}.moses.{lang}"),
+ op.join(root, f"{split}.moses.bpe{bpe_size}.{lang}"),
+ )
+ # tokenize with bytes vocabulary
+ if need_bytes:
+ for lang in [src, tgt]:
+ for split in SPLITS:
+ _get_bytes(
+ op.join(root, f"{split}.moses.{lang}"),
+ op.join(root, f"{split}.moses.bytes.{lang}"),
+ )
+ # tokenize with characters vocabulary
+ if need_chars:
+ for lang in [src, tgt]:
+ for split in SPLITS:
+ _get_chars(
+ op.join(root, f"{split}.moses.{lang}"),
+ op.join(root, f"{split}.moses.chars.{lang}"),
+ )
+ # tokenize with byte-level BPE vocabulary
+ if bbpe_size is not None:
+ # learn vocabulary
+ bchar_path = op.join(root, "train.bchar")
+ _convert_to_bchar(op.join(root, "train.moses"), src, tgt, bchar_path)
+ bbpe_model_prefix = op.join(root, f"spm_bbpe{bbpe_size}")
+ _get_bpe(bchar_path, bbpe_model_prefix, bbpe_size)
+ os.remove(bchar_path)
+ # apply
+ for lang in [src, tgt]:
+ for split in SPLITS:
+ _apply_bbpe(
+ bbpe_model_prefix + ".model",
+ op.join(root, f"{split}.moses.{lang}"),
+ op.join(root, f"{split}.moses.bbpe{bbpe_size}.{lang}"),
+ )
+
+
+def main():
+ parser = argparse.ArgumentParser()
+ parser.add_argument("--root", type=str, default="data")
+ parser.add_argument(
+ "--bpe-vocab",
+ default=None,
+ type=int,
+ help="Generate tokenized bitext with BPE of size K."
+ "Default to None (disabled).",
+ )
+ parser.add_argument(
+ "--bbpe-vocab",
+ default=None,
+ type=int,
+ help="Generate tokenized bitext with BBPE of size K."
+ "Default to None (disabled).",
+ )
+ parser.add_argument(
+ "--byte-vocab",
+ action="store_true",
+ help="Generate tokenized bitext with bytes vocabulary",
+ )
+ parser.add_argument(
+ "--char-vocab",
+ action="store_true",
+ help="Generate tokenized bitext with chars vocabulary",
+ )
+ args = parser.parse_args()
+
+ preprocess_iwslt17(
+ args.root,
+ "fr",
+ "en",
+ args.bpe_vocab,
+ args.char_vocab,
+ args.bbpe_vocab,
+ args.byte_vocab,
+ )
+
+
+if __name__ == "__main__":
+ main()
diff --git a/data/fairseq/examples/byte_level_bpe/get_data.sh b/data/fairseq/examples/byte_level_bpe/get_data.sh
new file mode 100644
index 0000000000000000000000000000000000000000..c3d55d4925a6e6e23d12d293f093c1ae14acf76e
--- /dev/null
+++ b/data/fairseq/examples/byte_level_bpe/get_data.sh
@@ -0,0 +1,47 @@
+#!/bin/bash
+
+# Copyright (c) Facebook, Inc. and its affiliates.
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+
+PY_BIN_ROOT=
+
+# PyPI dependency
+${PY_BIN_ROOT}pip install sentencepiece sacremoses
+
+# Get data
+if [ ! -d "data" ]; then
+ mkdir data
+fi
+
+if [ ! -f "data/fr-en.tgz" ]; then
+ wget https://wit3.fbk.eu/archive/2017-01-trnted/texts/fr/en/fr-en.tgz -P data
+ tar xvf data/fr-en.tgz -C data
+fi
+${PY_BIN_ROOT}python get_bitext.py --bpe-vocab 16384 --byte-vocab --char-vocab
+for VOCAB_SIZE in 2048 4096; do
+ ${PY_BIN_ROOT}python get_bitext.py --bpe-vocab ${VOCAB_SIZE} --bbpe-vocab ${VOCAB_SIZE}
+done
+rm -r data/fr-en data/fr-en.tgz
+
+# Generate binary dataset
+${PY_BIN_ROOT}/fairseq-preprocess --source-lang fr --target-lang en --destdir data/bin_bpe16384 --joined-dictionary \
+ --workers "$(nproc)" --trainpref data/train.moses.bpe16384 --validpref data/valid.moses.bpe16384 \
+ --testpref data/test.moses.bpe16384
+
+${PY_BIN_ROOT}/fairseq-preprocess --source-lang fr --target-lang en --destdir data/bin_bytes --joined-dictionary \
+ --workers "$(nproc)" --trainpref data/train.moses.bytes --validpref data/valid.moses.bytes \
+ --testpref data/test.moses.bytes
+
+${PY_BIN_ROOT}/fairseq-preprocess --source-lang fr --target-lang en --destdir data/bin_chars --joined-dictionary \
+ --workers "$(nproc)" --trainpref data/train.moses.chars --validpref data/valid.moses.chars \
+ --testpref data/test.moses.chars
+
+for VOCAB_SIZE in 2048 4096; do
+ for TYPE in bbpe bpe; do
+ ${PY_BIN_ROOT}/fairseq-preprocess --source-lang fr --target-lang en --destdir "data/bin_${TYPE}${VOCAB_SIZE}" \
+ --joined-dictionary --workers "$(nproc)" --trainpref "data/train.moses.${TYPE}${VOCAB_SIZE}" \
+ --validpref "data/valid.moses.${TYPE}${VOCAB_SIZE}" --testpref "data/test.moses.${TYPE}${VOCAB_SIZE}"
+ done
+done
diff --git a/data/fairseq/examples/byte_level_bpe/gru_transformer.py b/data/fairseq/examples/byte_level_bpe/gru_transformer.py
new file mode 100644
index 0000000000000000000000000000000000000000..d4efa93a4d75da71c78e786d7f62101ef3266af4
--- /dev/null
+++ b/data/fairseq/examples/byte_level_bpe/gru_transformer.py
@@ -0,0 +1,107 @@
+# Copyright (c) Facebook, Inc. and its affiliates.
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+
+# Copyright (c) Facebook, Inc. and its affiliates.
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+
+import torch.nn as nn
+import torch.nn.functional as F
+from fairseq.models import register_model, register_model_architecture
+from fairseq.models.transformer import TransformerEncoder, TransformerModel
+
+
+@register_model("gru_transformer")
+class GRUTransformerModel(TransformerModel):
+ @classmethod
+ def build_encoder(cls, args, src_dict, embed_tokens):
+ return GRUTransformerEncoder(args, src_dict, embed_tokens)
+
+
+class GRUTransformerEncoder(TransformerEncoder):
+ def __init__(self, args, dictionary, embed_tokens):
+ super().__init__(args, dictionary, embed_tokens)
+ self.emb_ctx = nn.GRU(
+ input_size=embed_tokens.embedding_dim,
+ hidden_size=embed_tokens.embedding_dim // 2,
+ num_layers=1,
+ bidirectional=True,
+ )
+
+ def forward_embedding(self, src_tokens):
+ # embed tokens and positions
+ x = embed = self.embed_scale * self.embed_tokens(src_tokens)
+ if self.embed_positions is not None:
+ x = embed + self.embed_positions(src_tokens)
+
+ # contextualize embeddings
+ x = x.transpose(0, 1)
+ x = self.dropout_module(x)
+ x, _ = self.emb_ctx.forward(x)
+ x = x.transpose(0, 1)
+
+ if self.layernorm_embedding is not None:
+ x = self.layernorm_embedding(x)
+ x = self.dropout_module(x)
+ return x, embed
+
+
+@register_model_architecture("gru_transformer", "gru_transformer")
+def gru_transformer_base_architecture(args):
+ args.encoder_embed_path = getattr(args, "encoder_embed_path", None)
+ args.encoder_embed_dim = getattr(args, "encoder_embed_dim", 512)
+ args.encoder_ffn_embed_dim = getattr(args, "encoder_ffn_embed_dim", 2048)
+ args.encoder_layers = getattr(args, "encoder_layers", 6)
+ args.encoder_attention_heads = getattr(args, "encoder_attention_heads", 8)
+ args.encoder_normalize_before = getattr(args, "encoder_normalize_before", False)
+ args.encoder_learned_pos = getattr(args, "encoder_learned_pos", False)
+ args.decoder_embed_path = getattr(args, "decoder_embed_path", None)
+ args.decoder_embed_dim = getattr(args, "decoder_embed_dim", args.encoder_embed_dim)
+ args.decoder_ffn_embed_dim = getattr(
+ args, "decoder_ffn_embed_dim", args.encoder_ffn_embed_dim
+ )
+ args.decoder_layers = getattr(args, "decoder_layers", 6)
+ args.decoder_attention_heads = getattr(args, "decoder_attention_heads", 8)
+ args.decoder_normalize_before = getattr(args, "decoder_normalize_before", False)
+ args.decoder_learned_pos = getattr(args, "decoder_learned_pos", False)
+ args.attention_dropout = getattr(args, "attention_dropout", 0.0)
+ args.activation_dropout = getattr(args, "activation_dropout", 0.0)
+ args.activation_fn = getattr(args, "activation_fn", "relu")
+ args.dropout = getattr(args, "dropout", 0.1)
+ args.adaptive_softmax_cutoff = getattr(args, "adaptive_softmax_cutoff", None)
+ args.adaptive_softmax_dropout = getattr(args, "adaptive_softmax_dropout", 0)
+ args.share_decoder_input_output_embed = getattr(
+ args, "share_decoder_input_output_embed", False
+ )
+ args.share_all_embeddings = getattr(args, "share_all_embeddings", False)
+ args.no_token_positional_embeddings = getattr(
+ args, "no_token_positional_embeddings", False
+ )
+ args.adaptive_input = getattr(args, "adaptive_input", False)
+ args.no_cross_attention = getattr(args, "no_cross_attention", False)
+ args.cross_self_attention = getattr(args, "cross_self_attention", False)
+ args.layer_wise_attention = getattr(args, "layer_wise_attention", False)
+
+ args.decoder_output_dim = getattr(
+ args, "decoder_output_dim", args.decoder_embed_dim
+ )
+ args.decoder_input_dim = getattr(args, "decoder_input_dim", args.decoder_embed_dim)
+
+ args.no_scale_embedding = getattr(args, "no_scale_embedding", False)
+ args.layernorm_embedding = getattr(args, "layernorm_embedding", False)
+
+
+@register_model_architecture("gru_transformer", "gru_transformer_big")
+def gru_transformer_big(args):
+ args.encoder_embed_dim = getattr(args, "encoder_embed_dim", 1024)
+ args.encoder_ffn_embed_dim = getattr(args, "encoder_ffn_embed_dim", 4096)
+ args.encoder_attention_heads = getattr(args, "encoder_attention_heads", 16)
+ args.encoder_normalize_before = getattr(args, "encoder_normalize_before", False)
+ args.decoder_embed_dim = getattr(args, "decoder_embed_dim", 1024)
+ args.decoder_ffn_embed_dim = getattr(args, "decoder_ffn_embed_dim", 4096)
+ args.decoder_attention_heads = getattr(args, "decoder_attention_heads", 16)
+ args.dropout = getattr(args, "dropout", 0.3)
+ gru_transformer_base_architecture(args)
diff --git a/data/fairseq/examples/constrained_decoding/README.md b/data/fairseq/examples/constrained_decoding/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..e04b8b6a018214c8233fa87fd91d46a6dd1519d4
--- /dev/null
+++ b/data/fairseq/examples/constrained_decoding/README.md
@@ -0,0 +1,123 @@
+# (Vectorized) Lexically constrained decoding with dynamic beam allocation
+
+This page provides instructions for how to use lexically constrained decoding in Fairseq.
+Fairseq implements the code described in the following papers:
+
+* [Fast Lexically Constrained Decoding With Dynamic Beam Allocation](https://www.aclweb.org/anthology/N18-1119/) (Post & Vilar, 2018)
+* [Improved Lexically Constrained Decoding for Translation and Monolingual Rewriting](https://www.aclweb.org/anthology/N19-1090/) (Hu et al., 2019)
+
+## Quick start
+
+Constrained search is enabled by adding the command-line argument `--constraints` to `fairseq-interactive`.
+Constraints are appended to each line of input, separated by tabs. Each constraint (one or more tokens)
+is a separate field.
+
+The following command, using [Fairseq's WMT19 German--English model](https://github.com/pytorch/fairseq/blob/main/examples/wmt19/README.md),
+translates the sentence *Die maschinelle Übersetzung ist schwer zu kontrollieren.* with the constraints
+"hard" and "to influence".
+
+ echo -e "Die maschinelle Übersetzung ist schwer zu kontrollieren.\thard\ttoinfluence" \
+ | normalize.py | tok.py \
+ | fairseq-interactive /path/to/model \
+ --path /path/to/model/model1.pt \
+ --bpe fastbpe \
+ --bpe-codes /path/to/model/bpecodes \
+ --constraints \
+ -s de -t en \
+ --beam 10
+
+(tok.py and normalize.py can be found in the same directory as this README; they are just shortcuts around Fairseq's WMT19 preprocessing).
+This will generate the following output:
+
+ [snip]
+ S-0 Die masch@@ in@@ elle Über@@ setzung ist schwer zu kontrollieren .
+ W-0 1.844 seconds
+ C-0 hard
+ C-0 influence
+ H-0 -1.5333266258239746 Mach@@ ine trans@@ lation is hard to influence .
+ D-0 -1.5333266258239746 Machine translation is hard to influence .
+ P-0 -0.5434 -0.1423 -0.1930 -0.1415 -0.2346 -1.8031 -0.1701 -11.7727 -0.1815 -0.1511
+
+By default, constraints are generated in the order supplied, with any number (zero or more) of tokens generated
+between constraints. If you wish for the decoder to order the constraints, then use `--constraints unordered`.
+Note that you may want to use a larger beam.
+
+## Implementation details
+
+The heart of the implementation is in `fairseq/search.py`, which adds a `LexicallyConstrainedBeamSearch` instance.
+This instance of beam search tracks the progress of each hypothesis in the beam through the set of constraints
+provided for each input sentence. It does this using one of two classes, both found in `fairseq/token_generation_contstraints.py`:
+
+* OrderedConstraintState: assumes the `C` input constraints will be generated in the provided order
+* UnorderedConstraintState: tries to apply `C` (phrasal) constraints in all `C!` orders
+
+## Differences from Sockeye
+
+There are a number of [differences from Sockeye's implementation](https://awslabs.github.io/sockeye/inference.html#lexical-constraints).
+
+* Generating constraints in the order supplied (the default option here) is not available in Sockeye.
+* Due to an improved beam allocation method, there is no need to prune the beam.
+* Again due to better allocation, beam sizes as low as 10 or even 5 are often sufficient.
+* [The vector extensions described in Hu et al.](https://github.com/edwardjhu/sockeye/tree/trie_constraints) (NAACL 2019) were never merged
+ into the main Sockeye branch.
+
+## Citation
+
+The paper first describing lexical constraints for seq2seq decoding is:
+
+```bibtex
+@inproceedings{hokamp-liu-2017-lexically,
+ title = "Lexically Constrained Decoding for Sequence Generation Using Grid Beam Search",
+ author = "Hokamp, Chris and
+ Liu, Qun",
+ booktitle = "Proceedings of the 55th Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)",
+ month = jul,
+ year = "2017",
+ address = "Vancouver, Canada",
+ publisher = "Association for Computational Linguistics",
+ url = "https://www.aclweb.org/anthology/P17-1141",
+ doi = "10.18653/v1/P17-1141",
+ pages = "1535--1546",
+}
+```
+
+The fairseq implementation uses the extensions described in
+
+```bibtex
+@inproceedings{post-vilar-2018-fast,
+ title = "Fast Lexically Constrained Decoding with Dynamic Beam Allocation for Neural Machine Translation",
+ author = "Post, Matt and
+ Vilar, David",
+ booktitle = "Proceedings of the 2018 Conference of the North {A}merican Chapter of the Association for Computational Linguistics: Human Language Technologies, Volume 1 (Long Papers)",
+ month = jun,
+ year = "2018",
+ address = "New Orleans, Louisiana",
+ publisher = "Association for Computational Linguistics",
+ url = "https://www.aclweb.org/anthology/N18-1119",
+ doi = "10.18653/v1/N18-1119",
+ pages = "1314--1324",
+}
+```
+
+and
+
+```bibtex
+@inproceedings{hu-etal-2019-improved,
+ title = "Improved Lexically Constrained Decoding for Translation and Monolingual Rewriting",
+ author = "Hu, J. Edward and
+ Khayrallah, Huda and
+ Culkin, Ryan and
+ Xia, Patrick and
+ Chen, Tongfei and
+ Post, Matt and
+ Van Durme, Benjamin",
+ booktitle = "Proceedings of the 2019 Conference of the North {A}merican Chapter of the Association for Computational Linguistics: Human Language Technologies, Volume 1 (Long and Short Papers)",
+ month = jun,
+ year = "2019",
+ address = "Minneapolis, Minnesota",
+ publisher = "Association for Computational Linguistics",
+ url = "https://www.aclweb.org/anthology/N19-1090",
+ doi = "10.18653/v1/N19-1090",
+ pages = "839--850",
+}
+```
diff --git a/data/fairseq/examples/constrained_decoding/normalize.py b/data/fairseq/examples/constrained_decoding/normalize.py
new file mode 100644
index 0000000000000000000000000000000000000000..4ae2b5111ba025acb9e1613865c92fdc339a58d5
--- /dev/null
+++ b/data/fairseq/examples/constrained_decoding/normalize.py
@@ -0,0 +1,27 @@
+#!/usr/bin/env python3
+#
+# Copyright (c) Facebook, Inc. and its affiliates.
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+
+import sys
+
+from sacremoses.normalize import MosesPunctNormalizer
+
+
+def main(args):
+ normalizer = MosesPunctNormalizer(lang=args.lang, penn=args.penn)
+ for line in sys.stdin:
+ print(normalizer.normalize(line.rstrip()), flush=True)
+
+
+if __name__ == "__main__":
+ import argparse
+
+ parser = argparse.ArgumentParser()
+ parser.add_argument("--lang", "-l", default="en")
+ parser.add_argument("--penn", "-p", action="store_true")
+ args = parser.parse_args()
+
+ main(args)
diff --git a/data/fairseq/examples/constrained_decoding/tok.py b/data/fairseq/examples/constrained_decoding/tok.py
new file mode 100644
index 0000000000000000000000000000000000000000..b1f888a8c0d1b8ec7174859476cc3222456e0d2c
--- /dev/null
+++ b/data/fairseq/examples/constrained_decoding/tok.py
@@ -0,0 +1,34 @@
+#!/usr/bin/env python3
+#
+# Copyright (c) Facebook, Inc. and its affiliates.
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+
+import sys
+
+import sacremoses
+
+
+def main(args):
+ """Tokenizes, preserving tabs"""
+ mt = sacremoses.MosesTokenizer(lang=args.lang)
+
+ def tok(s):
+ return mt.tokenize(s, return_str=True)
+
+ for line in sys.stdin:
+ parts = list(map(tok, line.split("\t")))
+ print(*parts, sep="\t", flush=True)
+
+
+if __name__ == "__main__":
+ import argparse
+
+ parser = argparse.ArgumentParser()
+ parser.add_argument("--lang", "-l", default="en")
+ parser.add_argument("--penn", "-p", action="store_true")
+ parser.add_argument("--fields", "-f", help="fields to tokenize")
+ args = parser.parse_args()
+
+ main(args)