Add files using upload-large-folder tool
Browse filesThis view is limited to 50 files because it contains too many changes. See raw diff
- data/fairseq/examples/attention_head_selection/README.md +161 -0
- data/fairseq/examples/attention_head_selection/src/__init__.py +0 -0
- data/fairseq/examples/attention_head_selection/src/data/__init__.py +0 -0
- data/fairseq/examples/attention_head_selection/src/data/speech_to_text_dataset_with_domain.py +242 -0
- data/fairseq/examples/attention_head_selection/src/loss/__init__.py +0 -0
- data/fairseq/examples/attention_head_selection/src/loss/attention_head_selection.py +27 -0
- data/fairseq/examples/attention_head_selection/src/models/__init__.py +0 -0
- data/fairseq/examples/attention_head_selection/src/models/head_selection_s2t_transformer.py +170 -0
- data/fairseq/examples/attention_head_selection/src/models/head_selection_transformer.py +215 -0
- data/fairseq/examples/attention_head_selection/src/modules/__init__.py +0 -0
- data/fairseq/examples/attention_head_selection/src/modules/attn_head_selector.py +81 -0
- data/fairseq/examples/attention_head_selection/src/modules/head_selection_transformer_layer.py +92 -0
- data/fairseq/examples/attention_head_selection/src/modules/multihead_attention_selection.py +355 -0
- data/fairseq/examples/attention_head_selection/src/modules/multihead_functional.py +278 -0
- data/fairseq/examples/attention_head_selection/src/speech_to_text_head_selection.py +180 -0
- data/fairseq/examples/camembert/README.md +75 -0
- data/fairseq/examples/conv_seq2seq/README.md +25 -0
- data/fairseq/examples/criss/README.md +61 -0
- data/fairseq/examples/criss/download_and_preprocess_flores_test.sh +64 -0
- data/fairseq/examples/criss/download_and_preprocess_tatoeba.sh +46 -0
- data/fairseq/examples/criss/mining/mine.py +240 -0
- data/fairseq/examples/criss/mining/mine_example.sh +103 -0
- data/fairseq/examples/criss/save_encoder.py +214 -0
- data/fairseq/examples/criss/sentence_retrieval/encoder_analysis.py +92 -0
- data/fairseq/examples/criss/sentence_retrieval/sentence_retrieval_tatoeba.sh +59 -0
- data/fairseq/examples/criss/unsupervised_mt/eval.sh +37 -0
- data/fairseq/examples/data2vec/README.md +261 -0
- data/fairseq/examples/data2vec/__init__.py +0 -0
- data/fairseq/examples/data2vec/config/audio/classification/base_classification.yaml +70 -0
- data/fairseq/examples/data2vec/config/audio/classification/run_config/slurm_1.yaml +35 -0
- data/fairseq/examples/data2vec/config/audio/classification/run_config/slurm_1g.yaml +35 -0
- data/fairseq/examples/data2vec/config/audio/classification/run_config/slurm_2.yaml +35 -0
- data/fairseq/examples/data2vec/config/audio/pretraining/audioset.yaml +91 -0
- data/fairseq/examples/data2vec/config/audio/pretraining/base_librispeech.yaml +83 -0
- data/fairseq/examples/data2vec/config/audio/pretraining/run_config/local.yaml +15 -0
- data/fairseq/examples/data2vec/config/audio/pretraining/run_config/slurm_1.yaml +37 -0
- data/fairseq/examples/data2vec/config/audio/pretraining/run_config/slurm_1_aws.yaml +36 -0
- data/fairseq/examples/data2vec/config/audio/pretraining/run_config/slurm_2.yaml +37 -0
- data/fairseq/examples/data2vec/config/audio/pretraining/run_config/slurm_2_aws.yaml +37 -0
- data/fairseq/examples/data2vec/config/audio/pretraining/run_config/slurm_3.yaml +36 -0
- data/fairseq/examples/data2vec/config/audio/pretraining/run_config/slurm_4.yaml +36 -0
- data/fairseq/examples/data2vec/config/audio/pretraining/run_config/slurm_4_aws.yaml +37 -0
- data/fairseq/examples/data2vec/config/audio/pretraining/run_config/slurm_6_aws.yaml +36 -0
- data/fairseq/examples/data2vec/config/audio/pretraining/run_config/slurm_8_aws.yaml +36 -0
- data/fairseq/examples/data2vec/config/text/pretraining/base.yaml +77 -0
- data/fairseq/examples/data2vec/config/text/pretraining/run_config/local.yaml +15 -0
- data/fairseq/examples/data2vec/config/text/pretraining/run_config/slurm_1_aws.yaml +37 -0
- data/fairseq/examples/data2vec/config/text/pretraining/run_config/slurm_2.yaml +37 -0
- data/fairseq/examples/data2vec/config/text/pretraining/run_config/slurm_2_aws.yaml +37 -0
- data/fairseq/examples/data2vec/config/text/pretraining/run_config/slurm_3.yaml +36 -0
data/fairseq/examples/attention_head_selection/README.md
ADDED
|
@@ -0,0 +1,161 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Pay Better Attention to Attention: Head Selection in Multilingual and Multi-Domain Sequence Modeling (Gong et al., 2021)
|
| 2 |
+
|
| 3 |
+
[https://arxiv.org/pdf/2106.10840.pdf](https://arxiv.org/pdf/2106.10840.pdf)
|
| 4 |
+
|
| 5 |
+
## Introduction
|
| 6 |
+
|
| 7 |
+
We present attention head selection strategies in multilingual and multi-domain sequence modeling including text translation, speech recognition and speech translation tasks.
|
| 8 |
+
|
| 9 |
+
Below is an example of training multilingual/multi-domain speech recognition models.
|
| 10 |
+
|
| 11 |
+
## Data Preparation
|
| 12 |
+
Prepare mTEDx data as in [mTEDx example](https://github.com/fairinternal/fairseq-py/blob/0d9c5851e6fac40f9e366b3633ccd615c2901788/examples/speech_to_text/docs/mtedx_example.md) and CoVoST data as in [CoVoST example](https://github.com/fairinternal/fairseq-py/blob/0d9c5851e6fac40f9e366b3633ccd615c2901788/examples/speech_to_text/docs/covost_example.md). Similarly prepare EuroParl data.
|
| 13 |
+
|
| 14 |
+
|
| 15 |
+
## Training a multilingual ASR model with attention head selection
|
| 16 |
+
|
| 17 |
+
```bash
|
| 18 |
+
data_dir=<path to mtedx data>
|
| 19 |
+
train_subset="train_ar_ar_tedx,train_de_de_tedx,train_el_el_tedx,train_es_es_tedx,train_fr_fr_tedx,train_it_it_tedx,train_pt_pt_tedx,train_ru_ru_tedx"
|
| 20 |
+
valid_subset="valid_ar_ar_tedx,valid_de_de_tedx,valid_el_el_tedx,valid_es_es_tedx,valid_fr_fr_tedx,valid_it_it_tedx,valid_pt_pt_tedx,valid_ru_ru_tedx"
|
| 21 |
+
strateg=<subset or group>
|
| 22 |
+
|
| 23 |
+
fairseq-train ${data_dir} \
|
| 24 |
+
--user-dir examples/attention_head_selection/src \
|
| 25 |
+
--train-subset "${train_subset}" \
|
| 26 |
+
--valid-subset "${valid_subset}" \
|
| 27 |
+
--config-yaml 'config_asr.yaml' \
|
| 28 |
+
--arch 'head_selection_s2t_transformer_s' \
|
| 29 |
+
--task 'speech_to_text_head_selection' \
|
| 30 |
+
--criterion label_smoothed_cross_entropy --label-smoothing 0.1 \
|
| 31 |
+
--lr-scheduler 'inverse_sqrt' --stop-min-lr -1.0 --warmup-updates 10000 \
|
| 32 |
+
--lr 5e-4 \
|
| 33 |
+
--clip-norm 10.0 \
|
| 34 |
+
--seed 1 \
|
| 35 |
+
--max-epoch 400 \
|
| 36 |
+
--max-tokens 32000 \
|
| 37 |
+
--ignore-prefix-size 1 \
|
| 38 |
+
--dropout 0.3 \
|
| 39 |
+
--optimizer adam --adam-eps 1e-06 --adam-betas '(0.9, 0.98)' \
|
| 40 |
+
--skip-invalid-size-inputs-valid-test \
|
| 41 |
+
--encoder-attn-head-select \
|
| 42 |
+
--total-encoder-attention-heads 8 \
|
| 43 |
+
--decoder-self-attn-head-select \
|
| 44 |
+
--total-decoder-attention-heads 8 \
|
| 45 |
+
--attn-head-select-strategy ${strategy} \
|
| 46 |
+
--task-type lang \
|
| 47 |
+
```
|
| 48 |
+
|
| 49 |
+
## Training a multi-domain ASR model with attention head selection
|
| 50 |
+
|
| 51 |
+
```bash
|
| 52 |
+
data_dir=<path to multi-domain data>
|
| 53 |
+
train_subset="train_es_es_tedx,train_fr_fr_tedx,train_pt_pt_tedx,train_it_it_tedx,train_ru_ru_tedx,train_el_el_tedx,train_ar_ar_tedx,train_de_de_tedx,train_ar_ar_cv,train_de_de_cv,train_es_es_cv,train_fr_fr_cv,train_it_it_cv,train_pt_pt_cv,train_ru_ru_cv,train_de_de_ep,train_es_es_ep,train_fr_fr_ep,train_it_it_ep,train_pt_pt_ep"
|
| 54 |
+
valid_subset="dev_es_es_tedx,dev_fr_fr_tedx,dev_pt_pt_tedx,dev_it_it_tedx,dev_ru_ru_tedx,dev_el_el_tedx,dev_ar_ar_tedx,dev_de_de_tedx,dev_ar_ar_cv,dev_de_de_cv,dev_es_es_cv,dev_fr_fr_cv,dev_it_it_cv,dev_pt_pt_cv,dev_ru_ru_cv,dev_de_de_ep,dev_es_es_ep,dev_fr_fr_ep,dev_it_it_ep,dev_pt_pt_ep"
|
| 55 |
+
strateg=<subset or group>
|
| 56 |
+
|
| 57 |
+
fairseq-train ${data_dir} \
|
| 58 |
+
--user-dir examples/attention_head_selection/src \
|
| 59 |
+
--train-subset "${train_subset}" \
|
| 60 |
+
--valid-subset "${valid_subset}" \
|
| 61 |
+
--config-yaml 'config_asr.yaml' \
|
| 62 |
+
--arch head_selection_s2t_transformer_s \
|
| 63 |
+
--task speech_to_text_head_selection \
|
| 64 |
+
--criterion label_smoothed_cross_entropy --label-smoothing 0.1 \
|
| 65 |
+
--lr-scheduler 'inverse_sqrt' --stop-min-lr -1.0 --warmup-updates 10000 \
|
| 66 |
+
--lr 5e-4 \
|
| 67 |
+
--clip-norm 10.0 \
|
| 68 |
+
--seed 1 \
|
| 69 |
+
--max-epoch 400 \
|
| 70 |
+
--max-tokens 32000 \
|
| 71 |
+
--ignore-prefix-size 1 \
|
| 72 |
+
--dropout 0.3 \
|
| 73 |
+
--optimizer adam --adam-eps 1e-06 --adam-betas '(0.9, 0.98)' \
|
| 74 |
+
--skip-invalid-size-inputs-valid-test \
|
| 75 |
+
--encoder-attn-head-select \
|
| 76 |
+
--total-encoder-attention-heads 8 \
|
| 77 |
+
--decoder-self-attn-head-select \
|
| 78 |
+
--total-decoder-attention-heads 8 \
|
| 79 |
+
--attn-head-select-strategy ${strategy} \
|
| 80 |
+
--task-type domain
|
| 81 |
+
```
|
| 82 |
+
|
| 83 |
+
## Inference in multilingual setting
|
| 84 |
+
|
| 85 |
+
```bash
|
| 86 |
+
MODEL_DIR=<checkpoint directory>
|
| 87 |
+
data_dir=<path to mtedx data>
|
| 88 |
+
gen_subset=<data to test, e.g., test_ar_ar_tedx>
|
| 89 |
+
train_subset="train_ar_ar_tedx,train_de_de_tedx,train_el_el_tedx,train_es_es_tedx,train_fr_fr_tedx,train_it_it_tedx,train_pt_pt_tedx,train_ru_ru_tedx"
|
| 90 |
+
last_n=10
|
| 91 |
+
CHECKPOINT_FILENAME="avg_last_${last_n}_checkpoint.pt"
|
| 92 |
+
CHECKPOINT="_avg"
|
| 93 |
+
RESULTS="${MODEL_DIR}/ckpt${CHECKPOINT}"
|
| 94 |
+
if [ ! -d $RESULTS ]; then
|
| 95 |
+
mkdir -p $RESULTS
|
| 96 |
+
fi;
|
| 97 |
+
|
| 98 |
+
python scripts/average_checkpoints.py \
|
| 99 |
+
--inputs ${MODEL_DIR} --num-epoch-checkpoints ${last_n} \
|
| 100 |
+
--output "${MODEL_DIR}/${CHECKPOINT_FILENAME}"
|
| 101 |
+
|
| 102 |
+
fairseq-generate ${data_dir} \
|
| 103 |
+
--user-dir examples/attention_head_selection/src \
|
| 104 |
+
--arch 'head_selection_s2t_transformer_s' \
|
| 105 |
+
--task 'speech_to_text_head_selection' \
|
| 106 |
+
--train-subset ${train_subset} \
|
| 107 |
+
--gen-subset ${gen_subset} \
|
| 108 |
+
--path "${MODEL_DIR}/${CHECKPOINT_FILENAME}" \
|
| 109 |
+
--config-yaml 'config_asr.yaml' \
|
| 110 |
+
--prefix-size 1 \
|
| 111 |
+
--max-tokens 40000 --beam 5 \
|
| 112 |
+
--skip-invalid-size-inputs-valid-test \
|
| 113 |
+
--results-path ${RESULTS} \
|
| 114 |
+
--scoring wer --wer-tokenizer 13a \
|
| 115 |
+
--wer-lowercase --wer-remove-punct --remove-bpe
|
| 116 |
+
```
|
| 117 |
+
|
| 118 |
+
## Inference in multi-domain setting
|
| 119 |
+
|
| 120 |
+
```bash
|
| 121 |
+
MODEL_DIR=<checkpoint directory>
|
| 122 |
+
data_dir=<path to multi-domain data>
|
| 123 |
+
gen_subset=<data to test, e.g., test_pt_pt_cv>
|
| 124 |
+
train_subset="train_es_es_tedx,train_fr_fr_tedx,train_pt_pt_tedx,train_it_it_tedx,train_ru_ru_tedx,train_el_el_tedx,train_ar_ar_tedx,train_de_de_tedx,train_ar_ar_cv,train_de_de_cv,train_es_es_cv,train_fr_fr_cv,train_it_it_cv,train_pt_pt_cv,train_ru_ru_cv,train_de_de_ep,train_es_es_ep,train_fr_fr_ep,train_it_it_ep,train_pt_pt_ep"
|
| 125 |
+
last_n=10
|
| 126 |
+
CHECKPOINT_FILENAME="avg_last_${last_n}_checkpoint.pt"
|
| 127 |
+
CHECKPOINT="_avg"
|
| 128 |
+
RESULTS="${MODEL_DIR}/ckpt${CHECKPOINT}"
|
| 129 |
+
if [ ! -d $RESULTS ]; then
|
| 130 |
+
mkdir -p $RESULTS
|
| 131 |
+
fi;
|
| 132 |
+
|
| 133 |
+
python scripts/average_checkpoints.py \
|
| 134 |
+
--inputs ${MODEL_DIR} --num-epoch-checkpoints ${last_n} \
|
| 135 |
+
--output "${MODEL_DIR}/${CHECKPOINT_FILENAME}"
|
| 136 |
+
|
| 137 |
+
fairseq-generate ${data_dir} \
|
| 138 |
+
--user-dir examples/attention_head_selection/src \
|
| 139 |
+
--arch 'head_selection_s2t_transformer_s' \
|
| 140 |
+
--task 'speech_to_text_head_selection' \
|
| 141 |
+
--train-subset ${train_subset} \
|
| 142 |
+
--gen-subset ${gen_subset} \
|
| 143 |
+
--path "${MODEL_DIR}/${CHECKPOINT_FILENAME}" \
|
| 144 |
+
--config-yaml 'config_asr.yaml' \
|
| 145 |
+
--prefix-size 1 \
|
| 146 |
+
--max-tokens 40000 --beam 5 \
|
| 147 |
+
--skip-invalid-size-inputs-valid-test \
|
| 148 |
+
--results-path ${RESULTS} \
|
| 149 |
+
--scoring wer --wer-tokenizer 13a \
|
| 150 |
+
--wer-lowercase --wer-remove-punct --remove-bpe
|
| 151 |
+
```
|
| 152 |
+
|
| 153 |
+
## Citation
|
| 154 |
+
```bibtex
|
| 155 |
+
@article{gong2021pay,
|
| 156 |
+
title={Pay Better Attention to Attention: Head Selection in Multilingual and Multi-Domain Sequence Modeling},
|
| 157 |
+
author={Gong, Hongyu and Tang, Yun and Pino, Juan and Li, Xian},
|
| 158 |
+
journal={arXiv preprint arXiv:2106.10840},
|
| 159 |
+
year={2021}
|
| 160 |
+
}
|
| 161 |
+
'''
|
data/fairseq/examples/attention_head_selection/src/__init__.py
ADDED
|
File without changes
|
data/fairseq/examples/attention_head_selection/src/data/__init__.py
ADDED
|
File without changes
|
data/fairseq/examples/attention_head_selection/src/data/speech_to_text_dataset_with_domain.py
ADDED
|
@@ -0,0 +1,242 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) Facebook, Inc. and its affiliates.
|
| 2 |
+
#
|
| 3 |
+
# This source code is licensed under the MIT license found in the
|
| 4 |
+
# LICENSE file in the root directory of this source tree.
|
| 5 |
+
|
| 6 |
+
import logging
|
| 7 |
+
from pathlib import Path
|
| 8 |
+
from typing import Dict, List, Optional
|
| 9 |
+
from dataclasses import dataclass
|
| 10 |
+
|
| 11 |
+
import torch
|
| 12 |
+
from fairseq.data import (
|
| 13 |
+
ConcatDataset,
|
| 14 |
+
Dictionary,
|
| 15 |
+
FairseqDataset,
|
| 16 |
+
ResamplingDataset
|
| 17 |
+
)
|
| 18 |
+
from fairseq.data.audio.data_cfg import S2TDataConfig
|
| 19 |
+
from fairseq.data.audio.speech_to_text_dataset import (
|
| 20 |
+
SpeechToTextDatasetItem,
|
| 21 |
+
SpeechToTextDataset,
|
| 22 |
+
SpeechToTextDatasetCreator
|
| 23 |
+
)
|
| 24 |
+
|
| 25 |
+
logger = logging.getLogger(__name__)
|
| 26 |
+
|
| 27 |
+
|
| 28 |
+
@dataclass
|
| 29 |
+
class SpeechToTextDatasetItemWithDomain(SpeechToTextDatasetItem):
|
| 30 |
+
src_lang_id: Optional[torch.Tensor] = None
|
| 31 |
+
tgt_lang_id: Optional[torch.Tensor] = None
|
| 32 |
+
domain_id: Optional[torch.Tensor] = None
|
| 33 |
+
|
| 34 |
+
|
| 35 |
+
class SpeechToTextDatasetWithDomain(SpeechToTextDataset):
|
| 36 |
+
|
| 37 |
+
def __init__(
|
| 38 |
+
self,
|
| 39 |
+
split: str,
|
| 40 |
+
is_train_split: bool,
|
| 41 |
+
cfg: S2TDataConfig,
|
| 42 |
+
audio_paths: List[str],
|
| 43 |
+
n_frames: List[int],
|
| 44 |
+
src_texts: Optional[List[str]] = None,
|
| 45 |
+
tgt_texts: Optional[List[str]] = None,
|
| 46 |
+
speakers: Optional[List[str]] = None,
|
| 47 |
+
src_langs: Optional[List[str]] = None,
|
| 48 |
+
tgt_langs: Optional[List[str]] = None,
|
| 49 |
+
ids: Optional[List[str]] = None,
|
| 50 |
+
tgt_dict: Optional[Dictionary] = None,
|
| 51 |
+
pre_tokenizer=None,
|
| 52 |
+
bpe_tokenizer=None,
|
| 53 |
+
n_frames_per_step=1,
|
| 54 |
+
speaker_to_id=None,
|
| 55 |
+
src_lang_ids: Optional[List[int]] = None,
|
| 56 |
+
tgt_lang_ids: Optional[List[int]] = None,
|
| 57 |
+
domain_ids: Optional[List[int]] = None
|
| 58 |
+
):
|
| 59 |
+
super().__init__(
|
| 60 |
+
split, is_train_split, cfg, audio_paths, n_frames,
|
| 61 |
+
src_texts, tgt_texts, speakers, src_langs, tgt_langs,
|
| 62 |
+
ids, tgt_dict, pre_tokenizer, bpe_tokenizer,
|
| 63 |
+
n_frames_per_step, speaker_to_id
|
| 64 |
+
)
|
| 65 |
+
assert src_lang_ids is None or len(src_lang_ids) == self.n_samples
|
| 66 |
+
assert tgt_lang_ids is None or len(tgt_lang_ids) == self.n_samples
|
| 67 |
+
assert domain_ids is None or len(domain_ids) == self.n_samples
|
| 68 |
+
|
| 69 |
+
self.src_lang_ids = src_lang_ids
|
| 70 |
+
self.tgt_lang_ids = tgt_lang_ids
|
| 71 |
+
self.domain_ids = domain_ids
|
| 72 |
+
|
| 73 |
+
def __getitem__(self, index: int) -> SpeechToTextDatasetItemWithDomain:
|
| 74 |
+
item = super().__getitem__(index)
|
| 75 |
+
src_lang_id = self.src_lang_ids[index]
|
| 76 |
+
tgt_lang_id = self.tgt_lang_ids[index]
|
| 77 |
+
domain_id = self.domain_ids[index]
|
| 78 |
+
return SpeechToTextDatasetItemWithDomain(
|
| 79 |
+
index=item.index, source=item.source,
|
| 80 |
+
target=item.target, speaker_id=item.speaker_id,
|
| 81 |
+
src_lang_id=src_lang_id,
|
| 82 |
+
tgt_lang_id=tgt_lang_id,
|
| 83 |
+
domain_id=domain_id
|
| 84 |
+
)
|
| 85 |
+
|
| 86 |
+
def collater(
|
| 87 |
+
self, samples: List[SpeechToTextDatasetItem], return_order: bool = False
|
| 88 |
+
) -> Dict:
|
| 89 |
+
if len(samples) == 0:
|
| 90 |
+
return {}
|
| 91 |
+
out = super().collater(samples, return_order=True)
|
| 92 |
+
order = out["order"]
|
| 93 |
+
src_lang_ids = torch.tensor([x.src_lang_id for x in samples], dtype=torch.long).index_select(0, order)
|
| 94 |
+
tgt_lang_ids = torch.tensor([x.tgt_lang_id for x in samples], dtype=torch.long).index_select(0, order)
|
| 95 |
+
domain_ids = torch.tensor([x.domain_id for x in samples], dtype=torch.long).index_select(0, order)
|
| 96 |
+
|
| 97 |
+
out["src_lang_ids"] = src_lang_ids
|
| 98 |
+
out["tgt_lang_ids"] = tgt_lang_ids
|
| 99 |
+
out["domain_ids"] = domain_ids
|
| 100 |
+
if not return_order:
|
| 101 |
+
del out["order"]
|
| 102 |
+
return out
|
| 103 |
+
|
| 104 |
+
|
| 105 |
+
class SpeechToTextDatasetCreatorWithDomain(SpeechToTextDatasetCreator):
|
| 106 |
+
KEY_SRC_LANG_ID, KEY_TGT_LANG_ID = "src_lang_id", "tgt_lang_id"
|
| 107 |
+
KEY_DOMAIN_ID = "domain_id"
|
| 108 |
+
# default values
|
| 109 |
+
DEFAULT_SRC_LANG_ID, DEFAULT_TGT_LANG_ID, DEFAULT_DOMAIN_ID = 0, 0, 0
|
| 110 |
+
|
| 111 |
+
@classmethod
|
| 112 |
+
def _from_list(
|
| 113 |
+
cls,
|
| 114 |
+
split_name: str,
|
| 115 |
+
is_train_split,
|
| 116 |
+
samples: List[Dict],
|
| 117 |
+
cfg: S2TDataConfig,
|
| 118 |
+
tgt_dict,
|
| 119 |
+
pre_tokenizer,
|
| 120 |
+
bpe_tokenizer,
|
| 121 |
+
n_frames_per_step,
|
| 122 |
+
speaker_to_id
|
| 123 |
+
) -> SpeechToTextDatasetWithDomain:
|
| 124 |
+
audio_root = Path(cfg.audio_root)
|
| 125 |
+
ids = [s[cls.KEY_ID] for s in samples]
|
| 126 |
+
audio_paths = [(audio_root / s[cls.KEY_AUDIO]).as_posix() for s in samples]
|
| 127 |
+
n_frames = [int(s[cls.KEY_N_FRAMES]) for s in samples]
|
| 128 |
+
tgt_texts = [s[cls.KEY_TGT_TEXT] for s in samples]
|
| 129 |
+
src_texts = [s.get(cls.KEY_SRC_TEXT, cls.DEFAULT_SRC_TEXT) for s in samples]
|
| 130 |
+
speakers = [s.get(cls.KEY_SPEAKER, cls.DEFAULT_SPEAKER) for s in samples]
|
| 131 |
+
src_langs = [s.get(cls.KEY_SRC_LANG, cls.DEFAULT_LANG) for s in samples]
|
| 132 |
+
tgt_langs = [s.get(cls.KEY_TGT_LANG, cls.DEFAULT_LANG) for s in samples]
|
| 133 |
+
src_lang_ids = [s.get(cls.KEY_SRC_LANG_ID, cls.DEFAULT_SRC_LANG_ID) for s in samples]
|
| 134 |
+
tgt_lang_ids = [s.get(cls.KEY_TGT_LANG_ID, cls.DEFAULT_TGT_LANG_ID) for s in samples]
|
| 135 |
+
domain_ids = [s.get(cls.KEY_DOMAIN_ID, cls.DEFAULT_DOMAIN_ID) for s in samples]
|
| 136 |
+
return SpeechToTextDatasetWithDomain(
|
| 137 |
+
split_name,
|
| 138 |
+
is_train_split,
|
| 139 |
+
cfg,
|
| 140 |
+
audio_paths,
|
| 141 |
+
n_frames,
|
| 142 |
+
src_texts=src_texts,
|
| 143 |
+
tgt_texts=tgt_texts,
|
| 144 |
+
speakers=speakers,
|
| 145 |
+
src_langs=src_langs,
|
| 146 |
+
tgt_langs=tgt_langs,
|
| 147 |
+
ids=ids,
|
| 148 |
+
tgt_dict=tgt_dict,
|
| 149 |
+
pre_tokenizer=pre_tokenizer,
|
| 150 |
+
bpe_tokenizer=bpe_tokenizer,
|
| 151 |
+
n_frames_per_step=n_frames_per_step,
|
| 152 |
+
speaker_to_id=speaker_to_id,
|
| 153 |
+
src_lang_ids=src_lang_ids,
|
| 154 |
+
tgt_lang_ids=tgt_lang_ids,
|
| 155 |
+
domain_ids=domain_ids
|
| 156 |
+
)
|
| 157 |
+
|
| 158 |
+
@classmethod
|
| 159 |
+
def _load_samples_from_tsv(
|
| 160 |
+
cls,
|
| 161 |
+
root: str,
|
| 162 |
+
split: str,
|
| 163 |
+
src_lang_map,
|
| 164 |
+
tgt_lang_map,
|
| 165 |
+
domain_map
|
| 166 |
+
):
|
| 167 |
+
# metadata from split
|
| 168 |
+
_, src_lang, tgt_lang, domain = split.split("_")
|
| 169 |
+
src_lang_id = src_lang_map[src_lang]
|
| 170 |
+
tgt_lang_id = tgt_lang_map[tgt_lang]
|
| 171 |
+
domain_id = domain_map[domain]
|
| 172 |
+
|
| 173 |
+
samples = SpeechToTextDatasetCreator._load_samples_from_tsv(root, split)
|
| 174 |
+
for s in samples:
|
| 175 |
+
s.update({
|
| 176 |
+
cls.KEY_SRC_LANG_ID: src_lang_id,
|
| 177 |
+
cls.KEY_TGT_LANG_ID: tgt_lang_id,
|
| 178 |
+
cls.KEY_DOMAIN_ID: domain_id
|
| 179 |
+
})
|
| 180 |
+
return samples
|
| 181 |
+
|
| 182 |
+
@classmethod
|
| 183 |
+
def _from_tsv(
|
| 184 |
+
cls,
|
| 185 |
+
root: str,
|
| 186 |
+
cfg: S2TDataConfig,
|
| 187 |
+
split: str,
|
| 188 |
+
tgt_dict,
|
| 189 |
+
is_train_split: bool,
|
| 190 |
+
pre_tokenizer,
|
| 191 |
+
bpe_tokenizer,
|
| 192 |
+
n_frames_per_step,
|
| 193 |
+
speaker_to_id,
|
| 194 |
+
src_lang_map: Dict[str, int],
|
| 195 |
+
tgt_lang_map: Dict[str, int],
|
| 196 |
+
domain_map: Dict[str, int]
|
| 197 |
+
) -> SpeechToTextDatasetItemWithDomain:
|
| 198 |
+
samples = cls._load_samples_from_tsv(
|
| 199 |
+
root, split, src_lang_map,
|
| 200 |
+
tgt_lang_map, domain_map
|
| 201 |
+
)
|
| 202 |
+
return cls._from_list(
|
| 203 |
+
split, is_train_split, samples, cfg, tgt_dict, pre_tokenizer,
|
| 204 |
+
bpe_tokenizer, n_frames_per_step, speaker_to_id
|
| 205 |
+
)
|
| 206 |
+
|
| 207 |
+
@classmethod
|
| 208 |
+
def from_tsv(
|
| 209 |
+
cls,
|
| 210 |
+
root: str,
|
| 211 |
+
cfg: S2TDataConfig,
|
| 212 |
+
splits: str,
|
| 213 |
+
tgt_dict,
|
| 214 |
+
pre_tokenizer,
|
| 215 |
+
bpe_tokenizer,
|
| 216 |
+
is_train_split: bool,
|
| 217 |
+
epoch: int,
|
| 218 |
+
seed: int,
|
| 219 |
+
src_lang_map: Dict[str, int],
|
| 220 |
+
tgt_lang_map: Dict[str, int],
|
| 221 |
+
domain_map: Dict[str, int],
|
| 222 |
+
n_frames_per_step: int = 1,
|
| 223 |
+
speaker_to_id=None
|
| 224 |
+
) -> SpeechToTextDatasetWithDomain:
|
| 225 |
+
datasets = [
|
| 226 |
+
cls._from_tsv(
|
| 227 |
+
root, cfg, split, tgt_dict, is_train_split, pre_tokenizer, bpe_tokenizer, n_frames_per_step, speaker_to_id, src_lang_map, tgt_lang_map, domain_map
|
| 228 |
+
)
|
| 229 |
+
for split in splits.split(",")
|
| 230 |
+
]
|
| 231 |
+
|
| 232 |
+
if is_train_split and len(datasets) > 1 and cfg.sampling_alpha != 1.0:
|
| 233 |
+
# temperature-based sampling
|
| 234 |
+
size_ratios = cls.get_size_ratios(datasets, alpha=cfg.sampling_alpha)
|
| 235 |
+
datasets = [
|
| 236 |
+
ResamplingDataset(
|
| 237 |
+
d, size_ratio=r, seed=seed, epoch=epoch, replace=(r >= 1.0)
|
| 238 |
+
)
|
| 239 |
+
for r, d in zip(size_ratios, datasets)
|
| 240 |
+
]
|
| 241 |
+
|
| 242 |
+
return ConcatDataset(datasets) if len(datasets) > 1 else datasets[0]
|
data/fairseq/examples/attention_head_selection/src/loss/__init__.py
ADDED
|
File without changes
|
data/fairseq/examples/attention_head_selection/src/loss/attention_head_selection.py
ADDED
|
@@ -0,0 +1,27 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) Facebook, Inc. and its affiliates.
|
| 2 |
+
#
|
| 3 |
+
# This source code is licensed under the MIT license found in the
|
| 4 |
+
# LICENSE file in the root directory of this source tree.
|
| 5 |
+
|
| 6 |
+
import math
|
| 7 |
+
|
| 8 |
+
import torch
|
| 9 |
+
from torch.nn.modules.loss import _Loss
|
| 10 |
+
|
| 11 |
+
|
| 12 |
+
class HeadSelectionLoss(_Loss):
|
| 13 |
+
|
| 14 |
+
def __init__(self, args):
|
| 15 |
+
super().__init__()
|
| 16 |
+
self.args = args
|
| 17 |
+
self.kl_weight = getattr(args, "kl_weight", 0.0)
|
| 18 |
+
|
| 19 |
+
def forward(self, head_samples, sample_sizes, prior=0.5, eps=1e-7):
|
| 20 |
+
"""
|
| 21 |
+
head_scores: (num_tasks, num_layers, num_heads)
|
| 22 |
+
sample_sizes: (num_tasks, )
|
| 23 |
+
"""
|
| 24 |
+
kl_loss = (head_samples * (torch.log(head_samples + eps) - math.log(prior))).sum(-1).sum(-1)
|
| 25 |
+
kl_loss /= (torch.numel(head_samples) / head_samples.size(0))
|
| 26 |
+
kl_loss = self.kl_weight * torch.matmul(kl_loss, sample_sizes)
|
| 27 |
+
return kl_loss
|
data/fairseq/examples/attention_head_selection/src/models/__init__.py
ADDED
|
File without changes
|
data/fairseq/examples/attention_head_selection/src/models/head_selection_s2t_transformer.py
ADDED
|
@@ -0,0 +1,170 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) Facebook, Inc. and its affiliates.
|
| 2 |
+
#
|
| 3 |
+
# This source code is licensed under the MIT license found in the
|
| 4 |
+
# LICENSE file in the root directory of this source tree.
|
| 5 |
+
|
| 6 |
+
import logging
|
| 7 |
+
from typing import Dict, List, Optional
|
| 8 |
+
from pathlib import Path
|
| 9 |
+
import torch.nn as nn
|
| 10 |
+
from torch import Tensor
|
| 11 |
+
from fairseq import checkpoint_utils
|
| 12 |
+
|
| 13 |
+
from fairseq.models import register_model, register_model_architecture
|
| 14 |
+
from fairseq.utils import safe_hasattr
|
| 15 |
+
from fairseq.models.speech_to_text.s2t_transformer import (
|
| 16 |
+
S2TTransformerModel,
|
| 17 |
+
S2TTransformerEncoder,
|
| 18 |
+
TransformerDecoderScriptable
|
| 19 |
+
)
|
| 20 |
+
from fairseq.models.speech_to_text.s2t_transformer import base_architecture as s2t_base_architecture
|
| 21 |
+
|
| 22 |
+
from ..modules.attn_head_selector import AttnHeadSelector
|
| 23 |
+
from ..modules.head_selection_transformer_layer import HeadSelectionTransformerEncoderLayer
|
| 24 |
+
from .head_selection_transformer import HeadSelectionTransformerDecoder
|
| 25 |
+
|
| 26 |
+
|
| 27 |
+
logger = logging.getLogger(__name__)
|
| 28 |
+
|
| 29 |
+
|
| 30 |
+
@register_model("head_selection_s2t_transformer")
|
| 31 |
+
class HeadSelectionS2TTransformerModel(S2TTransformerModel):
|
| 32 |
+
"""
|
| 33 |
+
Head selection implemented in S2TTransformer
|
| 34 |
+
"""
|
| 35 |
+
def __init__(self, encoder, decoder):
|
| 36 |
+
super().__init__(encoder, decoder)
|
| 37 |
+
|
| 38 |
+
@staticmethod
|
| 39 |
+
def add_args(parser):
|
| 40 |
+
S2TTransformerModel.add_args(parser)
|
| 41 |
+
# encoder head selection
|
| 42 |
+
parser.add_argument(
|
| 43 |
+
"--encoder-attn-head-select",
|
| 44 |
+
action="store_true",
|
| 45 |
+
default=False,
|
| 46 |
+
help="encoder head selection"
|
| 47 |
+
)
|
| 48 |
+
parser.add_argument(
|
| 49 |
+
"--total-encoder-attention-heads",
|
| 50 |
+
type=int,
|
| 51 |
+
help="total number of encoder attention heads"
|
| 52 |
+
)
|
| 53 |
+
# decoder self attention selection
|
| 54 |
+
parser.add_argument(
|
| 55 |
+
"--decoder-self-attn-head-select",
|
| 56 |
+
action="store_true",
|
| 57 |
+
default=False,
|
| 58 |
+
help="decoder self-attention head selection"
|
| 59 |
+
)
|
| 60 |
+
# decoder-encoder attention selection
|
| 61 |
+
parser.add_argument(
|
| 62 |
+
"--dec-enc-attn-head-select",
|
| 63 |
+
action="store_true",
|
| 64 |
+
default=False,
|
| 65 |
+
help="decoder-encoder attention head selection"
|
| 66 |
+
)
|
| 67 |
+
parser.add_argument(
|
| 68 |
+
"--total-decoder-attention-heads",
|
| 69 |
+
type=int,
|
| 70 |
+
help="total number of decoder attention heads"
|
| 71 |
+
)
|
| 72 |
+
# selection strategy
|
| 73 |
+
parser.add_argument(
|
| 74 |
+
"--attn-head-select-strategy",
|
| 75 |
+
type=str,
|
| 76 |
+
help="attention head selection strategy, subset or group"
|
| 77 |
+
)
|
| 78 |
+
|
| 79 |
+
@classmethod
|
| 80 |
+
def build_encoder(cls, args):
|
| 81 |
+
if safe_hasattr(args, "encoder_attn_head_select") and args.encoder_attn_head_select:
|
| 82 |
+
encoder = HeadSelectionS2TTransformerEncoder(args)
|
| 83 |
+
else:
|
| 84 |
+
encoder = S2TTransformerEncoder(args)
|
| 85 |
+
pretraining_path = getattr(args, "load_pretrained_encoder_from", None)
|
| 86 |
+
if pretraining_path is not None:
|
| 87 |
+
if not Path(pretraining_path).exists():
|
| 88 |
+
logger.warning(
|
| 89 |
+
f"skipped pretraining because {pretraining_path} does not exist"
|
| 90 |
+
)
|
| 91 |
+
else:
|
| 92 |
+
encoder = checkpoint_utils.load_pretrained_component_from_model(
|
| 93 |
+
component=encoder, checkpoint=pretraining_path
|
| 94 |
+
)
|
| 95 |
+
logger.info(f"loaded pretrained encoder from: {pretraining_path}")
|
| 96 |
+
return encoder
|
| 97 |
+
|
| 98 |
+
@classmethod
|
| 99 |
+
def build_decoder(cls, args, task, embed_tokens):
|
| 100 |
+
if (safe_hasattr(args, "decoder_self_attn_head_select") and args.decoder_self_attn_head_select) or (safe_hasattr(args, "dec_enc_attn_head_select") and args.dec_enc_attn_head_select):
|
| 101 |
+
return HeadSelectionTransformerDecoderScriptable(args, task.target_dictionary, embed_tokens)
|
| 102 |
+
else:
|
| 103 |
+
return TransformerDecoderScriptable(args, task.target_dictionary, embed_tokens)
|
| 104 |
+
|
| 105 |
+
|
| 106 |
+
class HeadSelectionS2TTransformerEncoder(S2TTransformerEncoder):
|
| 107 |
+
|
| 108 |
+
def __init__(self, args):
|
| 109 |
+
super().__init__(args)
|
| 110 |
+
self.attn_head_selector = AttnHeadSelector(
|
| 111 |
+
args.encoder_tasks,
|
| 112 |
+
args.encoder_layers,
|
| 113 |
+
args.total_encoder_attention_heads,
|
| 114 |
+
args.encoder_attention_heads,
|
| 115 |
+
args.attn_head_select_strategy,
|
| 116 |
+
)
|
| 117 |
+
self.task_ids = None
|
| 118 |
+
self.transformer_layers = nn.ModuleList([
|
| 119 |
+
HeadSelectionTransformerEncoderLayer(args, layer_idx, attn_head_selector=self.attn_head_selector) for layer_idx in range(args.encoder_layers)
|
| 120 |
+
])
|
| 121 |
+
|
| 122 |
+
def set_task_ids(self, task_ids):
|
| 123 |
+
self.task_ids = task_ids
|
| 124 |
+
|
| 125 |
+
def _forward(self, src_tokens, src_lengths, return_all_hiddens=False):
|
| 126 |
+
self.attn_head_selector.head_select(self.task_ids)
|
| 127 |
+
return super()._forward(src_tokens, src_lengths, return_all_hiddens)
|
| 128 |
+
|
| 129 |
+
|
| 130 |
+
class HeadSelectionTransformerDecoderScriptable(HeadSelectionTransformerDecoder):
|
| 131 |
+
def extract_features(
|
| 132 |
+
self,
|
| 133 |
+
prev_output_tokens,
|
| 134 |
+
encoder_out: Optional[Dict[str, List[Tensor]]] = None,
|
| 135 |
+
incremental_state: Optional[Dict[str, Dict[str, Optional[Tensor]]]] = None,
|
| 136 |
+
full_context_alignment: bool = False,
|
| 137 |
+
alignment_layer: Optional[int] = None,
|
| 138 |
+
alignment_heads: Optional[int] = None,
|
| 139 |
+
):
|
| 140 |
+
# call scriptable method from parent class
|
| 141 |
+
x, _ = self.extract_features_scriptable(
|
| 142 |
+
prev_output_tokens,
|
| 143 |
+
encoder_out,
|
| 144 |
+
incremental_state,
|
| 145 |
+
full_context_alignment,
|
| 146 |
+
alignment_layer,
|
| 147 |
+
alignment_heads,
|
| 148 |
+
)
|
| 149 |
+
return x, None
|
| 150 |
+
|
| 151 |
+
|
| 152 |
+
@register_model_architecture(model_name="head_selection_s2t_transformer", arch_name="head_selection_s2t_transformer")
|
| 153 |
+
def base_architecture(args):
|
| 154 |
+
s2t_base_architecture(args)
|
| 155 |
+
args.encoder_attn_head_select = getattr(args, "encoder_attn_head_select", False)
|
| 156 |
+
args.decoder_self_attn_head_select = getattr(args, "decoder_self_attn_head_select", False)
|
| 157 |
+
args.dec_enc_attn_head_select = getattr(args, "dec_enc_attn_head_select", False)
|
| 158 |
+
args.total_encoder_attention_heads = getattr(args, "total_encoder_attention_heads", 8)
|
| 159 |
+
args.total_decoder_attention_heads = getattr(args, "total_decoder_attention_heads", 8)
|
| 160 |
+
args.attn_head_select_strategy = getattr(args, "attn_head_select_strategy", "group")
|
| 161 |
+
|
| 162 |
+
|
| 163 |
+
@register_model_architecture("head_selection_s2t_transformer", "head_selection_s2t_transformer_s")
|
| 164 |
+
def head_selection_s2t_transformer_s(args):
|
| 165 |
+
args.encoder_embed_dim = getattr(args, "encoder_embed_dim", 256)
|
| 166 |
+
args.encoder_ffn_embed_dim = getattr(args, "encoder_ffn_embed_dim", 256 * 8)
|
| 167 |
+
args.encoder_attention_heads = getattr(args, "encoder_attention_heads", 4)
|
| 168 |
+
args.decoder_attention_heads = getattr(args, "decoder_attention_heads", 4)
|
| 169 |
+
args.dropout = getattr(args, "dropout", 0.1)
|
| 170 |
+
base_architecture(args)
|
data/fairseq/examples/attention_head_selection/src/models/head_selection_transformer.py
ADDED
|
@@ -0,0 +1,215 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) Facebook, Inc. and its affiliates.
|
| 2 |
+
#
|
| 3 |
+
# This source code is licensed under the MIT license found in the
|
| 4 |
+
# LICENSE file in the root directory of this source tree.
|
| 5 |
+
|
| 6 |
+
from typing import Any, List, Dict, Optional
|
| 7 |
+
import torch
|
| 8 |
+
import torch.nn as nn
|
| 9 |
+
from torch import Tensor
|
| 10 |
+
|
| 11 |
+
from fairseq.utils import safe_hasattr
|
| 12 |
+
from fairseq.models.transformer import (
|
| 13 |
+
TransformerModel,
|
| 14 |
+
TransformerEncoder,
|
| 15 |
+
TransformerDecoder
|
| 16 |
+
)
|
| 17 |
+
|
| 18 |
+
from ..modules.attn_head_selector import AttnHeadSelector
|
| 19 |
+
from ..modules.head_selection_transformer_layer import (
|
| 20 |
+
HeadSelectionTransformerEncoderLayer,
|
| 21 |
+
HeadSelectionTransformerDecoderLayer
|
| 22 |
+
)
|
| 23 |
+
|
| 24 |
+
|
| 25 |
+
class HeadSelectionTransformerModel(TransformerModel):
|
| 26 |
+
def __init__(self, args, encoder, decoder):
|
| 27 |
+
super().__init__(args, encoder, decoder)
|
| 28 |
+
|
| 29 |
+
@staticmethod
|
| 30 |
+
def add_args(parser):
|
| 31 |
+
TransformerModel.add_args(parser)
|
| 32 |
+
# encoder head selection
|
| 33 |
+
parser.add_argument(
|
| 34 |
+
"--encoder-attn-head-select",
|
| 35 |
+
action="store_true",
|
| 36 |
+
default=False,
|
| 37 |
+
help="encoder head selection"
|
| 38 |
+
)
|
| 39 |
+
parser.add_argument(
|
| 40 |
+
"--total-encoder-attention-heads",
|
| 41 |
+
type=int,
|
| 42 |
+
help="total number of encoder attention heads"
|
| 43 |
+
)
|
| 44 |
+
# decoder self attention
|
| 45 |
+
parser.add_argument(
|
| 46 |
+
"--decoder-self-attn-head-select",
|
| 47 |
+
action="store_true",
|
| 48 |
+
default=False,
|
| 49 |
+
help="decoder self-attention head selection"
|
| 50 |
+
)
|
| 51 |
+
# decoder-encoder attention
|
| 52 |
+
parser.add_argument(
|
| 53 |
+
"--dec-enc-attn-head-select",
|
| 54 |
+
action="store_true",
|
| 55 |
+
default=False,
|
| 56 |
+
help="decoder-encoder attention head selection"
|
| 57 |
+
)
|
| 58 |
+
parser.add_argument(
|
| 59 |
+
"--total-decoder-attention-heads",
|
| 60 |
+
type=int,
|
| 61 |
+
help="total number of decoder attention heads"
|
| 62 |
+
)
|
| 63 |
+
# selection strategy
|
| 64 |
+
parser.add_argument(
|
| 65 |
+
"--attn-head-select-strategy",
|
| 66 |
+
type=str,
|
| 67 |
+
help="attention head selection strategy, subset or group"
|
| 68 |
+
)
|
| 69 |
+
|
| 70 |
+
@classmethod
|
| 71 |
+
def build_encoder(cls, args, src_dict, embed_tokens):
|
| 72 |
+
if safe_hasattr(args, "encoder_attn_head_select") and args.encoder_attn_head_select:
|
| 73 |
+
return HeadSelectionTransformerEncoder(
|
| 74 |
+
args, src_dict, embed_tokens
|
| 75 |
+
)
|
| 76 |
+
else:
|
| 77 |
+
return TransformerEncoder(args, src_dict, embed_tokens)
|
| 78 |
+
|
| 79 |
+
@classmethod
|
| 80 |
+
def build_decoder(cls, args, tgt_dict, embed_tokens):
|
| 81 |
+
if (safe_hasattr(args, "decoder_self_attn_head_select") and args.decoder_self_attn_head_select) or (safe_hasattr(args, "dec_enc_attn_head_select") and args.dec_enc_attn_head_select):
|
| 82 |
+
return HeadSelectionTransformerDecoder(
|
| 83 |
+
args, tgt_dict, embed_tokens
|
| 84 |
+
)
|
| 85 |
+
else:
|
| 86 |
+
return TransformerDecoder(args, tgt_dict, embed_tokens)
|
| 87 |
+
|
| 88 |
+
|
| 89 |
+
class HeadSelectionTransformerEncoder(TransformerEncoder):
|
| 90 |
+
|
| 91 |
+
def __init__(self, args, dictionary, embed_tokens):
|
| 92 |
+
self.num_tasks = args.encoder_tasks
|
| 93 |
+
self.num_layers = args.encoder_layers
|
| 94 |
+
self.total_num_heads = args.total_encoder_attention_heads
|
| 95 |
+
self.num_heads = args.encoder_attention_heads
|
| 96 |
+
self.select_strategy = args.attn_head_select_strategy
|
| 97 |
+
|
| 98 |
+
super().__init__(args, dictionary, embed_tokens)
|
| 99 |
+
self.attn_head_selector = AttnHeadSelector(
|
| 100 |
+
self.num_tasks,
|
| 101 |
+
self.num_layers,
|
| 102 |
+
self.total_num_heads,
|
| 103 |
+
self.num_heads,
|
| 104 |
+
self.select_strategy
|
| 105 |
+
)
|
| 106 |
+
self.task_ids = None
|
| 107 |
+
self.layers = nn.ModuleList(
|
| 108 |
+
[self.build_encoder_layer(args, i) for i in range(args.encoder_layers)]
|
| 109 |
+
)
|
| 110 |
+
|
| 111 |
+
def set_task_ids(self, task_ids):
|
| 112 |
+
self.task_ids = task_ids
|
| 113 |
+
|
| 114 |
+
def build_encoder_layer(self, args, layer_idx=None):
|
| 115 |
+
return HeadSelectionTransformerEncoderLayer(
|
| 116 |
+
args,
|
| 117 |
+
layer_idx,
|
| 118 |
+
attn_head_selector=self.attn_head_selector
|
| 119 |
+
)
|
| 120 |
+
|
| 121 |
+
def forward(
|
| 122 |
+
self,
|
| 123 |
+
src_tokens,
|
| 124 |
+
src_lengths: Optional[torch.Tensor] = None,
|
| 125 |
+
return_all_hiddens: bool = False,
|
| 126 |
+
token_embeddings: Optional[torch.Tensor] = None,
|
| 127 |
+
):
|
| 128 |
+
self.attn_head_selector.head_select(self.task_ids)
|
| 129 |
+
return super().forward(src_tokens, src_lengths, return_all_hiddens, token_embeddings)
|
| 130 |
+
|
| 131 |
+
|
| 132 |
+
class HeadSelectionTransformerDecoder(TransformerDecoder):
|
| 133 |
+
|
| 134 |
+
def __init__(
|
| 135 |
+
self,
|
| 136 |
+
args,
|
| 137 |
+
dictionary,
|
| 138 |
+
embed_tokens,
|
| 139 |
+
no_encoder_attn=False,
|
| 140 |
+
output_projection=None,
|
| 141 |
+
):
|
| 142 |
+
self.num_tasks = args.decoder_tasks
|
| 143 |
+
self.num_layers = args.decoder_layers
|
| 144 |
+
self.total_num_heads = args.total_decoder_attention_heads
|
| 145 |
+
self.num_heads = args.decoder_attention_heads
|
| 146 |
+
self.select_strategy = args.attn_head_select_strategy
|
| 147 |
+
super().__init__(
|
| 148 |
+
args, dictionary, embed_tokens,
|
| 149 |
+
no_encoder_attn=no_encoder_attn,
|
| 150 |
+
output_projection=output_projection
|
| 151 |
+
)
|
| 152 |
+
self.self_attn_head_selector = None
|
| 153 |
+
self.enc_attn_head_selector = None
|
| 154 |
+
if safe_hasattr(args, "decoder_self_attn_head_select") and args.decoder_self_attn_head_select:
|
| 155 |
+
self.self_attn_head_selector = AttnHeadSelector(
|
| 156 |
+
self.num_tasks,
|
| 157 |
+
self.num_layers,
|
| 158 |
+
self.total_num_heads,
|
| 159 |
+
self.num_heads,
|
| 160 |
+
self.select_strategy
|
| 161 |
+
)
|
| 162 |
+
if safe_hasattr(args, "dec_enc_attn_head_select") and args.dec_enc_attn_head_select:
|
| 163 |
+
self.enc_attn_head_selector = AttnHeadSelector(
|
| 164 |
+
self.num_tasks,
|
| 165 |
+
self.num_layers,
|
| 166 |
+
self.total_num_heads,
|
| 167 |
+
self.num_heads,
|
| 168 |
+
self.select_strategy
|
| 169 |
+
)
|
| 170 |
+
self.task_ids = None
|
| 171 |
+
self.layers = nn.ModuleList(
|
| 172 |
+
[
|
| 173 |
+
self.build_head_selection_decoder_layer(args, no_encoder_attn, idx) for idx in range(args.decoder_layers)
|
| 174 |
+
]
|
| 175 |
+
)
|
| 176 |
+
|
| 177 |
+
def set_task_ids(self, task_ids):
|
| 178 |
+
self.task_ids = task_ids
|
| 179 |
+
|
| 180 |
+
def build_head_selection_decoder_layer(self, args, no_encoder_attn=False, layer_idx=None):
|
| 181 |
+
return HeadSelectionTransformerDecoderLayer(
|
| 182 |
+
args,
|
| 183 |
+
layer_idx,
|
| 184 |
+
self.self_attn_head_selector,
|
| 185 |
+
self.enc_attn_head_selector,
|
| 186 |
+
no_encoder_attn=no_encoder_attn
|
| 187 |
+
)
|
| 188 |
+
|
| 189 |
+
def forward(
|
| 190 |
+
self,
|
| 191 |
+
prev_output_tokens,
|
| 192 |
+
encoder_out: Optional[Dict[str, List[Tensor]]] = None,
|
| 193 |
+
incremental_state: Optional[Dict[str, Dict[str, Optional[Tensor]]]] = None,
|
| 194 |
+
features_only: bool = False,
|
| 195 |
+
full_context_alignment: bool = False,
|
| 196 |
+
alignment_layer: Optional[int] = None,
|
| 197 |
+
alignment_heads: Optional[int] = None,
|
| 198 |
+
src_lengths: Optional[Any] = None,
|
| 199 |
+
return_all_hiddens: bool = False,
|
| 200 |
+
):
|
| 201 |
+
if self.self_attn_head_selector is not None:
|
| 202 |
+
self.self_attn_head_selector.head_select(self.task_ids)
|
| 203 |
+
if self.enc_attn_head_selector is not None:
|
| 204 |
+
self.enc_attn_head_selector.head_select(self.task_ids)
|
| 205 |
+
return super().forward(
|
| 206 |
+
prev_output_tokens=prev_output_tokens,
|
| 207 |
+
encoder_out=encoder_out,
|
| 208 |
+
incremental_state=incremental_state,
|
| 209 |
+
features_only=features_only,
|
| 210 |
+
full_context_alignment=full_context_alignment,
|
| 211 |
+
alignment_layer=alignment_layer,
|
| 212 |
+
alignment_heads=alignment_heads,
|
| 213 |
+
src_lengths=src_lengths,
|
| 214 |
+
return_all_hiddens=return_all_hiddens
|
| 215 |
+
)
|
data/fairseq/examples/attention_head_selection/src/modules/__init__.py
ADDED
|
File without changes
|
data/fairseq/examples/attention_head_selection/src/modules/attn_head_selector.py
ADDED
|
@@ -0,0 +1,81 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# This source code is licensed under the MIT license found in the
|
| 2 |
+
# LICENSE file in the root directory of this source tree.
|
| 3 |
+
|
| 4 |
+
import torch
|
| 5 |
+
import torch.nn as nn
|
| 6 |
+
import math
|
| 7 |
+
|
| 8 |
+
|
| 9 |
+
class AttnHeadSelector(nn.Module):
|
| 10 |
+
"""
|
| 11 |
+
Latent variable modeling of attention head selection
|
| 12 |
+
"""
|
| 13 |
+
def __init__(
|
| 14 |
+
self, num_tasks, num_layers,
|
| 15 |
+
total_num_heads, num_heads,
|
| 16 |
+
select_strategy="group",
|
| 17 |
+
head_select_temp=5.0
|
| 18 |
+
):
|
| 19 |
+
super(AttnHeadSelector, self).__init__()
|
| 20 |
+
self.num_tasks = num_tasks
|
| 21 |
+
self.num_layers = num_layers
|
| 22 |
+
self.total_num_heads = total_num_heads
|
| 23 |
+
self.num_heads = num_heads
|
| 24 |
+
self.select_strategy = select_strategy
|
| 25 |
+
self.temp = head_select_temp
|
| 26 |
+
|
| 27 |
+
self.head_logits = torch.nn.Parameter(
|
| 28 |
+
torch.Tensor(self.num_tasks, self.num_layers, total_num_heads),
|
| 29 |
+
requires_grad=True
|
| 30 |
+
)
|
| 31 |
+
nn.init.uniform_(
|
| 32 |
+
self.head_logits, a=math.log(0.01),
|
| 33 |
+
b=math.log(1.0)
|
| 34 |
+
)
|
| 35 |
+
|
| 36 |
+
def gumbel_sample(self, logits, tau=1.0):
|
| 37 |
+
gumbels1 = -torch.empty_like(logits, memory_format=torch.legacy_contiguous_format).exponential_().log()
|
| 38 |
+
gumbels2 = -torch.empty_like(logits, memory_format=torch.legacy_contiguous_format).exponential_().log()
|
| 39 |
+
gumbels1 = (logits + gumbels1 - gumbels2) / tau
|
| 40 |
+
y_soft = gumbels1.sigmoid()
|
| 41 |
+
return y_soft
|
| 42 |
+
|
| 43 |
+
def subset_select(self, y_soft, topk, dim=-1):
|
| 44 |
+
top_values, top_inds = torch.topk(y_soft, k=topk, dim=dim)
|
| 45 |
+
top_ret = 1.0 - top_values.detach() + top_values
|
| 46 |
+
return top_inds.detach(), top_ret
|
| 47 |
+
|
| 48 |
+
def group_selet(self, y_soft, topk, dim=-1):
|
| 49 |
+
# top_values: (num_tasks, num_layers, topk)
|
| 50 |
+
top_values, top_inds = torch.max(
|
| 51 |
+
y_soft.view(self.num_tasks, self.num_layers, -1, topk), dim=2
|
| 52 |
+
)
|
| 53 |
+
top_inds = top_inds * topk + torch.arange(topk, device=top_inds.device).unsqueeze(0).unsqueeze(1)
|
| 54 |
+
top_ret = 1.0 - top_values.detach() + top_values
|
| 55 |
+
return top_inds.detach(), top_ret
|
| 56 |
+
|
| 57 |
+
def head_select(self, task_ids=None):
|
| 58 |
+
# gumbel_sample
|
| 59 |
+
self.head_samples = self.gumbel_sample(self.head_logits, tau=self.temp)
|
| 60 |
+
# head select
|
| 61 |
+
if self.select_strategy == "subset":
|
| 62 |
+
self.subset_heads, self.subset_weights = self.subset_select(
|
| 63 |
+
self.head_samples,
|
| 64 |
+
topk=self.num_heads,
|
| 65 |
+
)
|
| 66 |
+
elif self.select_strategy == "group":
|
| 67 |
+
self.subset_heads, self.subset_weights = self.group_selet(
|
| 68 |
+
self.head_samples,
|
| 69 |
+
topk=self.num_heads,
|
| 70 |
+
)
|
| 71 |
+
else:
|
| 72 |
+
raise ValueError("{} is not supported".format(self.select_strategy))
|
| 73 |
+
|
| 74 |
+
self.batch_subset = self.subset_heads[task_ids, :, :]
|
| 75 |
+
self.batch_weights = self.subset_weights[task_ids, :, :]
|
| 76 |
+
|
| 77 |
+
def forward(self, layer_idx):
|
| 78 |
+
assert layer_idx is not None
|
| 79 |
+
batch_subset = self.batch_subset[:, layer_idx, :]
|
| 80 |
+
batch_weights = self.batch_weights[:, layer_idx, :]
|
| 81 |
+
return batch_subset, batch_weights
|
data/fairseq/examples/attention_head_selection/src/modules/head_selection_transformer_layer.py
ADDED
|
@@ -0,0 +1,92 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) Facebook, Inc. and its affiliates.
|
| 2 |
+
#
|
| 3 |
+
# This source code is licensed under the MIT license found in the
|
| 4 |
+
# LICENSE file in the root directory of this source tree.
|
| 5 |
+
|
| 6 |
+
from fairseq.utils import safe_getattr
|
| 7 |
+
from fairseq.modules import TransformerEncoderLayer, TransformerDecoderLayer
|
| 8 |
+
from ..modules.multihead_attention_selection import MultiheadAttentionSelection
|
| 9 |
+
|
| 10 |
+
|
| 11 |
+
class HeadSelectionTransformerEncoderLayer(TransformerEncoderLayer):
|
| 12 |
+
|
| 13 |
+
def __init__(self, args, layer_idx, attn_head_selector=None):
|
| 14 |
+
super().__init__(args)
|
| 15 |
+
self.layer_idx = layer_idx
|
| 16 |
+
self.self_attn = self.build_self_attention_selection(
|
| 17 |
+
self.embed_dim, args, attn_head_selector
|
| 18 |
+
)
|
| 19 |
+
|
| 20 |
+
def build_self_attention_selection(self, embed_dim, args, attn_head_selector=None):
|
| 21 |
+
return MultiheadAttentionSelection(
|
| 22 |
+
embed_dim,
|
| 23 |
+
args.total_encoder_attention_heads,
|
| 24 |
+
args.encoder_attention_heads,
|
| 25 |
+
dropout=args.attention_dropout,
|
| 26 |
+
self_attention=True,
|
| 27 |
+
q_noise=self.quant_noise,
|
| 28 |
+
qn_block_size=self.quant_noise_block_size,
|
| 29 |
+
layer_idx=self.layer_idx,
|
| 30 |
+
attn_head_selector=attn_head_selector
|
| 31 |
+
)
|
| 32 |
+
|
| 33 |
+
|
| 34 |
+
class HeadSelectionTransformerDecoderLayer(TransformerDecoderLayer):
|
| 35 |
+
|
| 36 |
+
def __init__(
|
| 37 |
+
self,
|
| 38 |
+
args,
|
| 39 |
+
layer_idx,
|
| 40 |
+
self_attn_head_selector=None,
|
| 41 |
+
enc_attn_head_selector=None,
|
| 42 |
+
no_encoder_attn=False,
|
| 43 |
+
add_bias_kv=False,
|
| 44 |
+
add_zero_attn=False,
|
| 45 |
+
):
|
| 46 |
+
self.layer_idx = layer_idx
|
| 47 |
+
super().__init__(args, no_encoder_attn, add_bias_kv, add_zero_attn)
|
| 48 |
+
if self_attn_head_selector is not None:
|
| 49 |
+
self.self_attn = self.build_self_attention_selection(
|
| 50 |
+
self.embed_dim, args,
|
| 51 |
+
self_attn_head_selector=self_attn_head_selector,
|
| 52 |
+
add_bias_kv=add_bias_kv,
|
| 53 |
+
add_zero_attn=add_zero_attn
|
| 54 |
+
)
|
| 55 |
+
if enc_attn_head_selector is not None:
|
| 56 |
+
self.encoder_attn = self.build_encoder_attention_selection(
|
| 57 |
+
self.embed_dim, args,
|
| 58 |
+
enc_attn_head_selector=enc_attn_head_selector
|
| 59 |
+
)
|
| 60 |
+
|
| 61 |
+
def build_self_attention_selection(
|
| 62 |
+
self, embed_dim, args, self_attn_head_selector=None,
|
| 63 |
+
add_bias_kv=False, add_zero_attn=False
|
| 64 |
+
):
|
| 65 |
+
return MultiheadAttentionSelection(
|
| 66 |
+
embed_dim,
|
| 67 |
+
args.total_decoder_attention_heads,
|
| 68 |
+
args.decoder_attention_heads,
|
| 69 |
+
dropout=args.attention_dropout,
|
| 70 |
+
add_bias_kv=add_bias_kv,
|
| 71 |
+
add_zero_attn=add_zero_attn,
|
| 72 |
+
self_attention=not safe_getattr(args, "cross_self_attention"),
|
| 73 |
+
q_noise=self.quant_noise,
|
| 74 |
+
qn_block_size=self.quant_noise_block_size,
|
| 75 |
+
layer_idx=self.layer_idx,
|
| 76 |
+
attn_head_selector=self_attn_head_selector,
|
| 77 |
+
)
|
| 78 |
+
|
| 79 |
+
def build_encoder_attention_selection(self, embed_dim, args, enc_attn_head_selector=None):
|
| 80 |
+
return MultiheadAttentionSelection(
|
| 81 |
+
embed_dim,
|
| 82 |
+
args.total_decoder_attention_heads,
|
| 83 |
+
args.decoder_attention_heads,
|
| 84 |
+
kdim=args.encoder_embed_dim,
|
| 85 |
+
vdim=args.encoder_embed_dim,
|
| 86 |
+
dropout=args.attention_dropout,
|
| 87 |
+
encoder_decoder_attention=True,
|
| 88 |
+
q_noise=self.quant_noise,
|
| 89 |
+
qn_block_size=self.quant_noise_block_size,
|
| 90 |
+
layer_idx=self.layer_idx,
|
| 91 |
+
attn_head_selector=enc_attn_head_selector,
|
| 92 |
+
)
|
data/fairseq/examples/attention_head_selection/src/modules/multihead_attention_selection.py
ADDED
|
@@ -0,0 +1,355 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) Facebook, Inc. and its affiliates.
|
| 2 |
+
#
|
| 3 |
+
# This source code is licensed under the MIT license found in the
|
| 4 |
+
# LICENSE file in the root directory of this source tree.
|
| 5 |
+
|
| 6 |
+
from typing import Dict, Optional, Tuple
|
| 7 |
+
import torch
|
| 8 |
+
from fairseq import utils
|
| 9 |
+
from fairseq.modules.quant_noise import quant_noise
|
| 10 |
+
from torch import Tensor, nn
|
| 11 |
+
from torch.nn import Parameter
|
| 12 |
+
|
| 13 |
+
from fairseq.modules.multihead_attention import MultiheadAttention
|
| 14 |
+
from ..modules.multihead_functional import multi_head_attention_forward
|
| 15 |
+
|
| 16 |
+
|
| 17 |
+
class MultiheadAttentionSelection(MultiheadAttention):
|
| 18 |
+
|
| 19 |
+
def __init__(
|
| 20 |
+
self,
|
| 21 |
+
embed_dim,
|
| 22 |
+
total_num_heads,
|
| 23 |
+
num_heads,
|
| 24 |
+
kdim=None,
|
| 25 |
+
vdim=None,
|
| 26 |
+
dropout=0.0,
|
| 27 |
+
bias=True,
|
| 28 |
+
add_bias_kv=False,
|
| 29 |
+
add_zero_attn=False,
|
| 30 |
+
self_attention=False,
|
| 31 |
+
encoder_decoder_attention=False,
|
| 32 |
+
q_noise=0.0,
|
| 33 |
+
qn_block_size=8,
|
| 34 |
+
layer_idx=0,
|
| 35 |
+
attn_head_selector=None
|
| 36 |
+
):
|
| 37 |
+
super().__init__(
|
| 38 |
+
embed_dim,
|
| 39 |
+
num_heads,
|
| 40 |
+
kdim=kdim,
|
| 41 |
+
vdim=vdim,
|
| 42 |
+
dropout=dropout,
|
| 43 |
+
bias=bias,
|
| 44 |
+
add_bias_kv=add_bias_kv,
|
| 45 |
+
add_zero_attn=add_zero_attn,
|
| 46 |
+
self_attention=self_attention,
|
| 47 |
+
encoder_decoder_attention=encoder_decoder_attention,
|
| 48 |
+
q_noise=q_noise,
|
| 49 |
+
qn_block_size=qn_block_size,
|
| 50 |
+
)
|
| 51 |
+
self.layer_idx = layer_idx
|
| 52 |
+
self.attn_head_selector = attn_head_selector
|
| 53 |
+
self.total_num_heads = total_num_heads
|
| 54 |
+
self.total_embed_dim = self.head_dim * total_num_heads
|
| 55 |
+
self.k_proj = quant_noise(
|
| 56 |
+
nn.Linear(self.kdim, self.total_embed_dim, bias=bias), q_noise, qn_block_size
|
| 57 |
+
)
|
| 58 |
+
self.v_proj = quant_noise(
|
| 59 |
+
nn.Linear(self.vdim, self.total_embed_dim, bias=bias), q_noise, qn_block_size
|
| 60 |
+
)
|
| 61 |
+
self.q_proj = quant_noise(
|
| 62 |
+
nn.Linear(embed_dim, self.total_embed_dim, bias=bias), q_noise, qn_block_size
|
| 63 |
+
)
|
| 64 |
+
if add_bias_kv:
|
| 65 |
+
self.bias_k = Parameter(torch.Tensor(1, 1, self.total_embed_dim))
|
| 66 |
+
self.bias_v = Parameter(torch.Tensor(1, 1, self.total_embed_dim))
|
| 67 |
+
else:
|
| 68 |
+
self.bias_k = self.bias_v = None
|
| 69 |
+
self.reset_parameters()
|
| 70 |
+
|
| 71 |
+
def forward(
|
| 72 |
+
self,
|
| 73 |
+
query,
|
| 74 |
+
key: Optional[Tensor],
|
| 75 |
+
value: Optional[Tensor],
|
| 76 |
+
key_padding_mask: Optional[Tensor] = None,
|
| 77 |
+
incremental_state: Optional[Dict[str, Dict[str, Optional[Tensor]]]] = None,
|
| 78 |
+
need_weights: bool = True,
|
| 79 |
+
static_kv: bool = False,
|
| 80 |
+
attn_mask: Optional[Tensor] = None,
|
| 81 |
+
before_softmax: bool = False,
|
| 82 |
+
need_head_weights: bool = False,
|
| 83 |
+
# subset_heads: Optional[Tensor] = None,
|
| 84 |
+
# subset_weights: Optional[Tensor] = None
|
| 85 |
+
) -> Tuple[Tensor, Optional[Tensor]]:
|
| 86 |
+
if need_head_weights:
|
| 87 |
+
need_weights = True
|
| 88 |
+
|
| 89 |
+
is_tpu = query.device.type == "xla"
|
| 90 |
+
|
| 91 |
+
subset_heads, subset_weights = self.attn_head_selector(self.layer_idx)
|
| 92 |
+
|
| 93 |
+
tgt_len, bsz, embed_dim = query.size()
|
| 94 |
+
src_len = tgt_len
|
| 95 |
+
assert list(query.size()) == [tgt_len, bsz, self.embed_dim]
|
| 96 |
+
if key is not None:
|
| 97 |
+
src_len, key_bsz, _ = key.size()
|
| 98 |
+
if not torch.jit.is_scripting():
|
| 99 |
+
assert key_bsz == bsz
|
| 100 |
+
assert value is not None
|
| 101 |
+
assert src_len, bsz == value.shape[:2]
|
| 102 |
+
|
| 103 |
+
if (
|
| 104 |
+
not self.onnx_trace
|
| 105 |
+
and not is_tpu # don't use PyTorch version on TPUs
|
| 106 |
+
and incremental_state is None
|
| 107 |
+
and not static_kv
|
| 108 |
+
# A workaround for quantization to work. Otherwise JIT compilation
|
| 109 |
+
# treats bias in linear module as method.
|
| 110 |
+
and not torch.jit.is_scripting()
|
| 111 |
+
):
|
| 112 |
+
assert key is not None and value is not None
|
| 113 |
+
return multi_head_attention_forward(
|
| 114 |
+
query,
|
| 115 |
+
key,
|
| 116 |
+
value,
|
| 117 |
+
self.embed_dim,
|
| 118 |
+
self.total_num_heads,
|
| 119 |
+
self.num_heads,
|
| 120 |
+
torch.empty([0]),
|
| 121 |
+
torch.cat((self.q_proj.bias, self.k_proj.bias, self.v_proj.bias)),
|
| 122 |
+
self.bias_k,
|
| 123 |
+
self.bias_v,
|
| 124 |
+
self.add_zero_attn,
|
| 125 |
+
self.dropout_module.p,
|
| 126 |
+
self.out_proj.weight,
|
| 127 |
+
self.out_proj.bias,
|
| 128 |
+
self.training or self.dropout_module.apply_during_inference,
|
| 129 |
+
key_padding_mask,
|
| 130 |
+
need_weights,
|
| 131 |
+
attn_mask,
|
| 132 |
+
use_separate_proj_weight=True,
|
| 133 |
+
q_proj_weight=self.q_proj.weight,
|
| 134 |
+
k_proj_weight=self.k_proj.weight,
|
| 135 |
+
v_proj_weight=self.v_proj.weight,
|
| 136 |
+
subset_heads=subset_heads,
|
| 137 |
+
subset_weights=subset_weights
|
| 138 |
+
)
|
| 139 |
+
|
| 140 |
+
if incremental_state is not None:
|
| 141 |
+
saved_state = self._get_input_buffer(incremental_state)
|
| 142 |
+
if saved_state is not None and "prev_key" in saved_state:
|
| 143 |
+
# previous time steps are cached - no need to recompute
|
| 144 |
+
# key and value if they are static
|
| 145 |
+
if static_kv:
|
| 146 |
+
assert self.encoder_decoder_attention and not self.self_attention
|
| 147 |
+
key = value = None
|
| 148 |
+
else:
|
| 149 |
+
saved_state = None
|
| 150 |
+
|
| 151 |
+
if self.self_attention:
|
| 152 |
+
q = self.q_proj(query)
|
| 153 |
+
k = self.k_proj(query)
|
| 154 |
+
v = self.v_proj(query)
|
| 155 |
+
elif self.encoder_decoder_attention:
|
| 156 |
+
# encoder-decoder attention
|
| 157 |
+
q = self.q_proj(query)
|
| 158 |
+
if key is None:
|
| 159 |
+
assert value is None
|
| 160 |
+
k = v = None
|
| 161 |
+
else:
|
| 162 |
+
k = self.k_proj(key)
|
| 163 |
+
v = self.v_proj(key)
|
| 164 |
+
|
| 165 |
+
else:
|
| 166 |
+
assert key is not None and value is not None
|
| 167 |
+
q = self.q_proj(query)
|
| 168 |
+
k = self.k_proj(key)
|
| 169 |
+
v = self.v_proj(value)
|
| 170 |
+
q *= self.scaling
|
| 171 |
+
|
| 172 |
+
if self.bias_k is not None:
|
| 173 |
+
assert self.bias_v is not None
|
| 174 |
+
k = torch.cat([k, self.bias_k.repeat(1, bsz, 1)])
|
| 175 |
+
v = torch.cat([v, self.bias_v.repeat(1, bsz, 1)])
|
| 176 |
+
if attn_mask is not None:
|
| 177 |
+
attn_mask = torch.cat(
|
| 178 |
+
[attn_mask, attn_mask.new_zeros(attn_mask.size(0), 1)], dim=1
|
| 179 |
+
)
|
| 180 |
+
if key_padding_mask is not None:
|
| 181 |
+
key_padding_mask = torch.cat(
|
| 182 |
+
[
|
| 183 |
+
key_padding_mask,
|
| 184 |
+
key_padding_mask.new_zeros(key_padding_mask.size(0), 1),
|
| 185 |
+
],
|
| 186 |
+
dim=1,
|
| 187 |
+
)
|
| 188 |
+
|
| 189 |
+
q = (
|
| 190 |
+
q.contiguous()
|
| 191 |
+
.view(tgt_len, bsz * self.total_num_heads, self.head_dim)
|
| 192 |
+
.transpose(0, 1)
|
| 193 |
+
)
|
| 194 |
+
if k is not None:
|
| 195 |
+
k = (
|
| 196 |
+
k.contiguous()
|
| 197 |
+
.view(-1, bsz * self.total_num_heads, self.head_dim)
|
| 198 |
+
.transpose(0, 1)
|
| 199 |
+
)
|
| 200 |
+
if v is not None:
|
| 201 |
+
v = (
|
| 202 |
+
v.contiguous()
|
| 203 |
+
.view(-1, bsz * self.total_num_heads, self.head_dim)
|
| 204 |
+
.transpose(0, 1)
|
| 205 |
+
)
|
| 206 |
+
|
| 207 |
+
if saved_state is not None:
|
| 208 |
+
# saved states are stored with shape (bsz, num_heads, seq_len, head_dim)
|
| 209 |
+
if "prev_key" in saved_state:
|
| 210 |
+
_prev_key = saved_state["prev_key"]
|
| 211 |
+
assert _prev_key is not None
|
| 212 |
+
prev_key = _prev_key.view(bsz * self.total_num_heads, -1, self.head_dim)
|
| 213 |
+
if static_kv:
|
| 214 |
+
k = prev_key
|
| 215 |
+
else:
|
| 216 |
+
assert k is not None
|
| 217 |
+
k = torch.cat([prev_key, k], dim=1)
|
| 218 |
+
src_len = k.size(1)
|
| 219 |
+
if "prev_value" in saved_state:
|
| 220 |
+
_prev_value = saved_state["prev_value"]
|
| 221 |
+
assert _prev_value is not None
|
| 222 |
+
prev_value = _prev_value.view(bsz * self.total_num_heads, -1, self.head_dim)
|
| 223 |
+
if static_kv:
|
| 224 |
+
v = prev_value
|
| 225 |
+
else:
|
| 226 |
+
assert v is not None
|
| 227 |
+
v = torch.cat([prev_value, v], dim=1)
|
| 228 |
+
prev_key_padding_mask: Optional[Tensor] = None
|
| 229 |
+
if "prev_key_padding_mask" in saved_state:
|
| 230 |
+
prev_key_padding_mask = saved_state["prev_key_padding_mask"]
|
| 231 |
+
assert k is not None and v is not None
|
| 232 |
+
key_padding_mask = MultiheadAttention._append_prev_key_padding_mask(
|
| 233 |
+
key_padding_mask=key_padding_mask,
|
| 234 |
+
prev_key_padding_mask=prev_key_padding_mask,
|
| 235 |
+
batch_size=bsz,
|
| 236 |
+
src_len=k.size(1),
|
| 237 |
+
static_kv=static_kv,
|
| 238 |
+
)
|
| 239 |
+
|
| 240 |
+
saved_state["prev_key"] = k.view(bsz, self.total_num_heads, -1, self.head_dim)
|
| 241 |
+
saved_state["prev_value"] = v.view(bsz, self.total_num_heads, -1, self.head_dim)
|
| 242 |
+
saved_state["prev_key_padding_mask"] = key_padding_mask
|
| 243 |
+
# In this branch incremental_state is never None
|
| 244 |
+
assert incremental_state is not None
|
| 245 |
+
incremental_state = self._set_input_buffer(incremental_state, saved_state)
|
| 246 |
+
assert k is not None
|
| 247 |
+
assert k.size(1) == src_len
|
| 248 |
+
|
| 249 |
+
# This is part of a workaround to get around fork/join parallelism
|
| 250 |
+
# not supporting Optional types.
|
| 251 |
+
if key_padding_mask is not None and key_padding_mask.dim() == 0:
|
| 252 |
+
key_padding_mask = None
|
| 253 |
+
|
| 254 |
+
if key_padding_mask is not None:
|
| 255 |
+
assert key_padding_mask.size(0) == bsz
|
| 256 |
+
assert key_padding_mask.size(1) == src_len
|
| 257 |
+
|
| 258 |
+
if self.add_zero_attn:
|
| 259 |
+
assert v is not None
|
| 260 |
+
src_len += 1
|
| 261 |
+
k = torch.cat([k, k.new_zeros((k.size(0), 1) + k.size()[2:])], dim=1)
|
| 262 |
+
v = torch.cat([v, v.new_zeros((v.size(0), 1) + v.size()[2:])], dim=1)
|
| 263 |
+
if attn_mask is not None:
|
| 264 |
+
attn_mask = torch.cat(
|
| 265 |
+
[attn_mask, attn_mask.new_zeros(attn_mask.size(0), 1)], dim=1
|
| 266 |
+
)
|
| 267 |
+
if key_padding_mask is not None:
|
| 268 |
+
key_padding_mask = torch.cat(
|
| 269 |
+
[
|
| 270 |
+
key_padding_mask,
|
| 271 |
+
torch.zeros(key_padding_mask.size(0), 1).type_as(
|
| 272 |
+
key_padding_mask
|
| 273 |
+
),
|
| 274 |
+
],
|
| 275 |
+
dim=1,
|
| 276 |
+
)
|
| 277 |
+
|
| 278 |
+
attn_weights = torch.bmm(q, k.transpose(1, 2))
|
| 279 |
+
attn_weights = self.apply_sparse_mask(attn_weights, tgt_len, src_len, bsz)
|
| 280 |
+
|
| 281 |
+
assert list(attn_weights.size()) == [bsz * self.total_num_heads, tgt_len, src_len]
|
| 282 |
+
|
| 283 |
+
if attn_mask is not None:
|
| 284 |
+
attn_mask = attn_mask.unsqueeze(0)
|
| 285 |
+
if self.onnx_trace:
|
| 286 |
+
attn_mask = attn_mask.repeat(attn_weights.size(0), 1, 1)
|
| 287 |
+
attn_weights += attn_mask
|
| 288 |
+
|
| 289 |
+
if key_padding_mask is not None:
|
| 290 |
+
# don't attend to padding symbols
|
| 291 |
+
attn_weights = attn_weights.view(bsz, self.total_num_heads, tgt_len, src_len)
|
| 292 |
+
if not is_tpu:
|
| 293 |
+
attn_weights = attn_weights.masked_fill(
|
| 294 |
+
key_padding_mask.unsqueeze(1).unsqueeze(2).to(torch.bool),
|
| 295 |
+
float("-inf"),
|
| 296 |
+
)
|
| 297 |
+
else:
|
| 298 |
+
attn_weights = attn_weights.transpose(0, 2)
|
| 299 |
+
attn_weights = attn_weights.masked_fill(key_padding_mask, float("-inf"))
|
| 300 |
+
attn_weights = attn_weights.transpose(0, 2)
|
| 301 |
+
attn_weights = attn_weights.view(bsz * self.num_heads, tgt_len, src_len)
|
| 302 |
+
|
| 303 |
+
if before_softmax:
|
| 304 |
+
return attn_weights, v
|
| 305 |
+
|
| 306 |
+
attn_weights_float = utils.softmax(
|
| 307 |
+
attn_weights, dim=-1, onnx_trace=self.onnx_trace
|
| 308 |
+
)
|
| 309 |
+
attn_weights = attn_weights_float.type_as(attn_weights)
|
| 310 |
+
attn_probs = self.dropout_module(attn_weights)
|
| 311 |
+
|
| 312 |
+
assert v is not None
|
| 313 |
+
|
| 314 |
+
# evaluation
|
| 315 |
+
if subset_heads is not None and subset_heads.numel() == 1:
|
| 316 |
+
subset_heads = subset_heads.repeat(bsz)
|
| 317 |
+
subset_weights = subset_weights.repeat(bsz)
|
| 318 |
+
|
| 319 |
+
if subset_heads is None:
|
| 320 |
+
attn = torch.bmm(attn_probs, v)
|
| 321 |
+
else:
|
| 322 |
+
# training with head selection
|
| 323 |
+
mixed_attn = torch.bmm(attn_probs, v).contiguous().view(bsz, self.total_num_heads, tgt_len, self.head_dim)
|
| 324 |
+
attn = torch.stack(
|
| 325 |
+
[mixed_attn[torch.arange(bsz), subset_heads[:, col], :, :] for col in range(subset_heads.size(1))], dim=1
|
| 326 |
+
)
|
| 327 |
+
attn = attn * subset_weights.unsqueeze(2).unsqueeze(3)
|
| 328 |
+
attn = attn.contiguous().view(bsz * self.num_heads, tgt_len, self.head_dim)
|
| 329 |
+
|
| 330 |
+
assert list(attn.size()) == [bsz * self.num_heads, tgt_len, self.head_dim]
|
| 331 |
+
if self.onnx_trace and attn.size(1) == 1:
|
| 332 |
+
# when ONNX tracing a single decoder step (sequence length == 1)
|
| 333 |
+
# the transpose is a no-op copy before view, thus unnecessary
|
| 334 |
+
attn = attn.contiguous().view(tgt_len, bsz, embed_dim)
|
| 335 |
+
else:
|
| 336 |
+
attn = attn.transpose(0, 1).contiguous().view(tgt_len, bsz, embed_dim)
|
| 337 |
+
attn = self.out_proj(attn)
|
| 338 |
+
attn_weights: Optional[Tensor] = None
|
| 339 |
+
if need_weights:
|
| 340 |
+
if subset_heads is None:
|
| 341 |
+
attn_weights = attn_weights_float.view(
|
| 342 |
+
bsz, self.num_heads, tgt_len, src_len
|
| 343 |
+
).transpose(1, 0)
|
| 344 |
+
else:
|
| 345 |
+
mixed_attn_weights = attn_weights_float.view(
|
| 346 |
+
bsz, self.total_num_heads, tgt_len, src_len
|
| 347 |
+
)
|
| 348 |
+
attn_weights = torch.stack(
|
| 349 |
+
[mixed_attn_weights[torch.arange(bsz), subset_heads[:, col], :, :] for col in range(subset_heads.size(1))], dim=1
|
| 350 |
+
).transpose(1, 0)
|
| 351 |
+
if not need_head_weights:
|
| 352 |
+
# average attention weights over heads
|
| 353 |
+
attn_weights = attn_weights.mean(dim=0)
|
| 354 |
+
|
| 355 |
+
return attn, attn_weights
|
data/fairseq/examples/attention_head_selection/src/modules/multihead_functional.py
ADDED
|
@@ -0,0 +1,278 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) Facebook, Inc. and its affiliates.
|
| 2 |
+
#
|
| 3 |
+
# This source code is licensed under the MIT license found in the
|
| 4 |
+
# LICENSE file in the root directory of this source tree.
|
| 5 |
+
|
| 6 |
+
from typing import Optional, Tuple
|
| 7 |
+
import torch
|
| 8 |
+
from torch import Tensor
|
| 9 |
+
from torch.nn.functional import (
|
| 10 |
+
linear, softmax, dropout, pad,
|
| 11 |
+
has_torch_function,
|
| 12 |
+
handle_torch_function,
|
| 13 |
+
_in_projection_packed,
|
| 14 |
+
)
|
| 15 |
+
import math
|
| 16 |
+
import warnings
|
| 17 |
+
|
| 18 |
+
|
| 19 |
+
def _scaled_dot_product_attention(
|
| 20 |
+
q: Tensor,
|
| 21 |
+
k: Tensor,
|
| 22 |
+
v: Tensor,
|
| 23 |
+
attn_mask: Optional[Tensor] = None,
|
| 24 |
+
dropout_p: float = 0.0,
|
| 25 |
+
bsz: int = 1,
|
| 26 |
+
subset_heads: Optional[Tensor] = None,
|
| 27 |
+
subset_weights: Optional[Tensor] = None,
|
| 28 |
+
) -> Tuple[Tensor, Tensor]:
|
| 29 |
+
B, Nt, E = q.shape
|
| 30 |
+
q = q / math.sqrt(E)
|
| 31 |
+
# B: bsz * total_num_heads
|
| 32 |
+
# (B, Nt, E) x (B, E, Ns) -> (B, Nt, Ns)
|
| 33 |
+
attn = torch.bmm(q, k.transpose(-2, -1))
|
| 34 |
+
if attn_mask is not None:
|
| 35 |
+
attn += attn_mask
|
| 36 |
+
attn = softmax(attn, dim=-1)
|
| 37 |
+
if dropout_p > 0.0:
|
| 38 |
+
attn = dropout(attn, p=dropout_p)
|
| 39 |
+
if subset_heads is None:
|
| 40 |
+
# (B, Nt, Ns) x (B, Ns, E) -> (B, Nt, E)
|
| 41 |
+
output = torch.bmm(attn, v)
|
| 42 |
+
else:
|
| 43 |
+
mixed_output = torch.bmm(attn, v).contiguous().view(bsz, -1, Nt, E)
|
| 44 |
+
output = torch.stack(
|
| 45 |
+
[mixed_output[torch.arange(bsz), subset_heads[:, col], :, :] for col in range(subset_heads.size(1))],
|
| 46 |
+
dim=1
|
| 47 |
+
)
|
| 48 |
+
output = output * subset_weights.unsqueeze(2).unsqueeze(3)
|
| 49 |
+
output = output.contiguous().view(-1, Nt, E)
|
| 50 |
+
if subset_heads is not None:
|
| 51 |
+
_, Nt, Ns = attn.size()
|
| 52 |
+
mixed_attn = attn.view(bsz, -1, Nt, Ns)
|
| 53 |
+
attn = torch.stack(
|
| 54 |
+
[mixed_attn[torch.arange(bsz), subset_heads[:, col], :, :] for col in range(subset_heads.size(1))], dim=1
|
| 55 |
+
)
|
| 56 |
+
return output, attn
|
| 57 |
+
|
| 58 |
+
|
| 59 |
+
def _in_projection(
|
| 60 |
+
q: Tensor,
|
| 61 |
+
k: Tensor,
|
| 62 |
+
v: Tensor,
|
| 63 |
+
w_q: Tensor,
|
| 64 |
+
w_k: Tensor,
|
| 65 |
+
w_v: Tensor,
|
| 66 |
+
b_q: Optional[Tensor] = None,
|
| 67 |
+
b_k: Optional[Tensor] = None,
|
| 68 |
+
b_v: Optional[Tensor] = None,
|
| 69 |
+
) -> Tuple[Tensor, Tensor, Tensor]:
|
| 70 |
+
return linear(q, w_q, b_q), linear(k, w_k, b_k), linear(v, w_v, b_v)
|
| 71 |
+
|
| 72 |
+
|
| 73 |
+
def multi_head_attention_forward(
|
| 74 |
+
query: Tensor,
|
| 75 |
+
key: Tensor,
|
| 76 |
+
value: Tensor,
|
| 77 |
+
embed_dim_to_check: int,
|
| 78 |
+
total_num_heads: int,
|
| 79 |
+
num_heads: int,
|
| 80 |
+
in_proj_weight: Tensor,
|
| 81 |
+
in_proj_bias: Optional[Tensor],
|
| 82 |
+
bias_k: Optional[Tensor],
|
| 83 |
+
bias_v: Optional[Tensor],
|
| 84 |
+
add_zero_attn: bool,
|
| 85 |
+
dropout_p: float,
|
| 86 |
+
out_proj_weight: Tensor,
|
| 87 |
+
out_proj_bias: Optional[Tensor],
|
| 88 |
+
training: bool = True,
|
| 89 |
+
key_padding_mask: Optional[Tensor] = None,
|
| 90 |
+
need_weights: bool = True,
|
| 91 |
+
attn_mask: Optional[Tensor] = None,
|
| 92 |
+
use_separate_proj_weight: bool = False,
|
| 93 |
+
q_proj_weight: Optional[Tensor] = None,
|
| 94 |
+
k_proj_weight: Optional[Tensor] = None,
|
| 95 |
+
v_proj_weight: Optional[Tensor] = None,
|
| 96 |
+
static_k: Optional[Tensor] = None,
|
| 97 |
+
static_v: Optional[Tensor] = None,
|
| 98 |
+
subset_heads: Optional[Tensor] = None,
|
| 99 |
+
subset_weights: Optional[Tensor] = None,
|
| 100 |
+
):
|
| 101 |
+
tens_ops = (query, key, value, in_proj_weight, in_proj_bias, bias_k, bias_v, out_proj_weight, out_proj_bias)
|
| 102 |
+
if has_torch_function(tens_ops):
|
| 103 |
+
return handle_torch_function(
|
| 104 |
+
multi_head_attention_forward,
|
| 105 |
+
tens_ops,
|
| 106 |
+
query,
|
| 107 |
+
key,
|
| 108 |
+
value,
|
| 109 |
+
embed_dim_to_check,
|
| 110 |
+
total_num_heads,
|
| 111 |
+
num_heads,
|
| 112 |
+
in_proj_weight,
|
| 113 |
+
in_proj_bias,
|
| 114 |
+
bias_k,
|
| 115 |
+
bias_v,
|
| 116 |
+
add_zero_attn,
|
| 117 |
+
dropout_p,
|
| 118 |
+
out_proj_weight,
|
| 119 |
+
out_proj_bias,
|
| 120 |
+
training=training,
|
| 121 |
+
key_padding_mask=key_padding_mask,
|
| 122 |
+
need_weights=need_weights,
|
| 123 |
+
attn_mask=attn_mask,
|
| 124 |
+
use_separate_proj_weight=use_separate_proj_weight,
|
| 125 |
+
q_proj_weight=q_proj_weight,
|
| 126 |
+
k_proj_weight=k_proj_weight,
|
| 127 |
+
v_proj_weight=v_proj_weight,
|
| 128 |
+
static_k=static_k,
|
| 129 |
+
static_v=static_v,
|
| 130 |
+
subset_heads=subset_heads,
|
| 131 |
+
subset_weights=subset_weights
|
| 132 |
+
)
|
| 133 |
+
|
| 134 |
+
# set up shape vars
|
| 135 |
+
tgt_len, bsz, embed_dim = query.shape
|
| 136 |
+
src_len, _, _ = key.shape
|
| 137 |
+
assert embed_dim == embed_dim_to_check, \
|
| 138 |
+
f"was expecting embedding dimension of {embed_dim_to_check}, but got {embed_dim}"
|
| 139 |
+
if isinstance(embed_dim, torch.Tensor):
|
| 140 |
+
# embed_dim can be a tensor when JIT tracing
|
| 141 |
+
head_dim = embed_dim.div(num_heads, rounding_mode='trunc')
|
| 142 |
+
else:
|
| 143 |
+
head_dim = embed_dim // num_heads
|
| 144 |
+
assert head_dim * num_heads == embed_dim, f"embed_dim {embed_dim} not divisible by num_heads {num_heads}"
|
| 145 |
+
if use_separate_proj_weight:
|
| 146 |
+
# allow MHA to have different embedding dimensions when separate projection weights are used
|
| 147 |
+
assert key.shape[:2] == value.shape[:2], \
|
| 148 |
+
f"key's sequence and batch dims {key.shape[:2]} do not match value's {value.shape[:2]}"
|
| 149 |
+
else:
|
| 150 |
+
assert key.shape == value.shape, f"key shape {key.shape} does not match value shape {value.shape}"
|
| 151 |
+
|
| 152 |
+
#
|
| 153 |
+
# compute in-projection
|
| 154 |
+
#
|
| 155 |
+
if not use_separate_proj_weight:
|
| 156 |
+
q, k, v = _in_projection_packed(query, key, value, in_proj_weight, in_proj_bias)
|
| 157 |
+
else:
|
| 158 |
+
assert q_proj_weight is not None, "use_separate_proj_weight is True but q_proj_weight is None"
|
| 159 |
+
assert k_proj_weight is not None, "use_separate_proj_weight is True but k_proj_weight is None"
|
| 160 |
+
assert v_proj_weight is not None, "use_separate_proj_weight is True but v_proj_weight is None"
|
| 161 |
+
if in_proj_bias is None:
|
| 162 |
+
b_q = b_k = b_v = None
|
| 163 |
+
else:
|
| 164 |
+
b_q, b_k, b_v = in_proj_bias.chunk(3)
|
| 165 |
+
q, k, v = _in_projection(query, key, value, q_proj_weight, k_proj_weight, v_proj_weight, b_q, b_k, b_v)
|
| 166 |
+
|
| 167 |
+
# prep attention mask
|
| 168 |
+
if attn_mask is not None:
|
| 169 |
+
if attn_mask.dtype == torch.uint8:
|
| 170 |
+
warnings.warn("Byte tensor for attn_mask in nn.MultiheadAttention is deprecated. Use bool tensor instead.")
|
| 171 |
+
attn_mask = attn_mask.to(torch.bool)
|
| 172 |
+
else:
|
| 173 |
+
assert attn_mask.is_floating_point() or attn_mask.dtype == torch.bool, \
|
| 174 |
+
f"Only float, byte, and bool types are supported for attn_mask, not {attn_mask.dtype}"
|
| 175 |
+
# ensure attn_mask's dim is 3
|
| 176 |
+
if attn_mask.dim() == 2:
|
| 177 |
+
correct_2d_size = (tgt_len, src_len)
|
| 178 |
+
if attn_mask.shape != correct_2d_size:
|
| 179 |
+
raise RuntimeError(f"The shape of the 2D attn_mask is {attn_mask.shape}, but should be {correct_2d_size}.")
|
| 180 |
+
attn_mask = attn_mask.unsqueeze(0)
|
| 181 |
+
elif attn_mask.dim() == 3:
|
| 182 |
+
correct_3d_size = (bsz * total_num_heads, tgt_len, src_len)
|
| 183 |
+
if attn_mask.shape != correct_3d_size:
|
| 184 |
+
raise RuntimeError(f"The shape of the 3D attn_mask is {attn_mask.shape}, but should be {correct_3d_size}.")
|
| 185 |
+
else:
|
| 186 |
+
raise RuntimeError(f"attn_mask's dimension {attn_mask.dim()} is not supported")
|
| 187 |
+
|
| 188 |
+
# prep key padding mask
|
| 189 |
+
if key_padding_mask is not None and key_padding_mask.dtype == torch.uint8:
|
| 190 |
+
warnings.warn("Byte tensor for key_padding_mask in nn.MultiheadAttention is deprecated. Use bool tensor instead.")
|
| 191 |
+
key_padding_mask = key_padding_mask.to(torch.bool)
|
| 192 |
+
|
| 193 |
+
# add bias along batch dimension (currently second)
|
| 194 |
+
if bias_k is not None and bias_v is not None:
|
| 195 |
+
assert static_k is None, "bias cannot be added to static key."
|
| 196 |
+
assert static_v is None, "bias cannot be added to static value."
|
| 197 |
+
k = torch.cat([k, bias_k.repeat(1, bsz, 1)])
|
| 198 |
+
v = torch.cat([v, bias_v.repeat(1, bsz, 1)])
|
| 199 |
+
if attn_mask is not None:
|
| 200 |
+
attn_mask = pad(attn_mask, (0, 1))
|
| 201 |
+
if key_padding_mask is not None:
|
| 202 |
+
key_padding_mask = pad(key_padding_mask, (0, 1))
|
| 203 |
+
else:
|
| 204 |
+
assert bias_k is None
|
| 205 |
+
assert bias_v is None
|
| 206 |
+
|
| 207 |
+
#
|
| 208 |
+
# reshape q, k, v for multihead attention and make em batch first
|
| 209 |
+
#
|
| 210 |
+
q = q.contiguous().view(tgt_len, bsz * total_num_heads, head_dim).transpose(0, 1)
|
| 211 |
+
if static_k is None:
|
| 212 |
+
k = k.contiguous().view(k.shape[0], bsz * total_num_heads, head_dim).transpose(0, 1)
|
| 213 |
+
else:
|
| 214 |
+
# TODO finish disentangling control flow so we don't do in-projections when statics are passed
|
| 215 |
+
assert static_k.size(0) == bsz * total_num_heads, \
|
| 216 |
+
f"expecting static_k.size(0) of {bsz * total_num_heads}, but got {static_k.size(0)}"
|
| 217 |
+
assert static_k.size(2) == head_dim, \
|
| 218 |
+
f"expecting static_k.size(2) of {head_dim}, but got {static_k.size(2)}"
|
| 219 |
+
k = static_k
|
| 220 |
+
if static_v is None:
|
| 221 |
+
v = v.contiguous().view(v.shape[0], bsz * total_num_heads, head_dim).transpose(0, 1)
|
| 222 |
+
else:
|
| 223 |
+
# TODO finish disentangling control flow so we don't do in-projections when statics are passed
|
| 224 |
+
assert static_v.size(0) == bsz * total_num_heads, \
|
| 225 |
+
f"expecting static_v.size(0) of {bsz * total_num_heads}, but got {static_v.size(0)}"
|
| 226 |
+
assert static_v.size(2) == head_dim, \
|
| 227 |
+
f"expecting static_v.size(2) of {head_dim}, but got {static_v.size(2)}"
|
| 228 |
+
v = static_v
|
| 229 |
+
|
| 230 |
+
# add zero attention along batch dimension (now first)
|
| 231 |
+
if add_zero_attn:
|
| 232 |
+
zero_attn_shape = (bsz * total_num_heads, 1, head_dim)
|
| 233 |
+
k = torch.cat([k, torch.zeros(zero_attn_shape, dtype=k.dtype, device=k.device)], dim=1)
|
| 234 |
+
v = torch.cat([v, torch.zeros(zero_attn_shape, dtype=v.dtype, device=v.device)], dim=1)
|
| 235 |
+
if attn_mask is not None:
|
| 236 |
+
attn_mask = pad(attn_mask, (0, 1))
|
| 237 |
+
if key_padding_mask is not None:
|
| 238 |
+
key_padding_mask = pad(key_padding_mask, (0, 1))
|
| 239 |
+
|
| 240 |
+
# update source sequence length after adjustments
|
| 241 |
+
src_len = k.size(1)
|
| 242 |
+
|
| 243 |
+
# merge key padding and attention masks
|
| 244 |
+
if key_padding_mask is not None:
|
| 245 |
+
assert key_padding_mask.shape == (bsz, src_len), \
|
| 246 |
+
f"expecting key_padding_mask shape of {(bsz, src_len)}, but got {key_padding_mask.shape}"
|
| 247 |
+
key_padding_mask = key_padding_mask.view(bsz, 1, 1, src_len). \
|
| 248 |
+
expand(-1, total_num_heads, -1, -1).reshape(bsz * total_num_heads, 1, src_len)
|
| 249 |
+
if attn_mask is None:
|
| 250 |
+
attn_mask = key_padding_mask
|
| 251 |
+
elif attn_mask.dtype == torch.bool:
|
| 252 |
+
attn_mask = attn_mask.logical_or(key_padding_mask)
|
| 253 |
+
else:
|
| 254 |
+
attn_mask = attn_mask.masked_fill(key_padding_mask, float("-inf"))
|
| 255 |
+
|
| 256 |
+
# convert mask to float
|
| 257 |
+
if attn_mask is not None and attn_mask.dtype == torch.bool:
|
| 258 |
+
new_attn_mask = torch.zeros_like(attn_mask, dtype=torch.float)
|
| 259 |
+
new_attn_mask.masked_fill_(attn_mask, float("-inf"))
|
| 260 |
+
attn_mask = new_attn_mask
|
| 261 |
+
|
| 262 |
+
# adjust dropout probability
|
| 263 |
+
if not training:
|
| 264 |
+
dropout_p = 0.0
|
| 265 |
+
|
| 266 |
+
#
|
| 267 |
+
# (deep breath) calculate attention and out projection
|
| 268 |
+
#
|
| 269 |
+
attn_output, attn_output_weights = _scaled_dot_product_attention(q, k, v, attn_mask, dropout_p, bsz, subset_heads, subset_weights)
|
| 270 |
+
attn_output = attn_output.transpose(0, 1).contiguous().view(tgt_len, bsz, embed_dim)
|
| 271 |
+
attn_output = linear(attn_output, out_proj_weight, out_proj_bias)
|
| 272 |
+
|
| 273 |
+
if need_weights:
|
| 274 |
+
# average attention weights over heads
|
| 275 |
+
attn_output_weights = attn_output_weights.view(bsz, num_heads, tgt_len, src_len)
|
| 276 |
+
return attn_output, attn_output_weights.sum(dim=1) / num_heads
|
| 277 |
+
else:
|
| 278 |
+
return attn_output, None
|
data/fairseq/examples/attention_head_selection/src/speech_to_text_head_selection.py
ADDED
|
@@ -0,0 +1,180 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) Facebook, Inc. and its affiliates.
|
| 2 |
+
#
|
| 3 |
+
# This source code is licensed under the MIT license found in the
|
| 4 |
+
# LICENSE file in the root directory of this source tree.
|
| 5 |
+
|
| 6 |
+
import torch
|
| 7 |
+
from fairseq.optim.amp_optimizer import AMPOptimizer
|
| 8 |
+
from fairseq.tasks import register_task
|
| 9 |
+
from fairseq.tasks.speech_to_text import SpeechToTextTask
|
| 10 |
+
|
| 11 |
+
from .data.speech_to_text_dataset_with_domain import SpeechToTextDatasetCreatorWithDomain
|
| 12 |
+
from .loss.attention_head_selection import HeadSelectionLoss
|
| 13 |
+
|
| 14 |
+
|
| 15 |
+
@register_task("speech_to_text_head_selection")
|
| 16 |
+
class SpeechToTextHeadSelectionTask(SpeechToTextTask):
|
| 17 |
+
|
| 18 |
+
@classmethod
|
| 19 |
+
def add_args(cls, parser):
|
| 20 |
+
SpeechToTextTask.add_args(parser)
|
| 21 |
+
parser.add_argument(
|
| 22 |
+
"--task-type",
|
| 23 |
+
type=str,
|
| 24 |
+
default="lang",
|
| 25 |
+
help="task type for head selection, lang or domain"
|
| 26 |
+
)
|
| 27 |
+
parser.add_argument(
|
| 28 |
+
"--kl-weight",
|
| 29 |
+
type=float,
|
| 30 |
+
default=0.0,
|
| 31 |
+
help="the weight of KL loss"
|
| 32 |
+
)
|
| 33 |
+
|
| 34 |
+
def __init__(self, args, tgt_dict):
|
| 35 |
+
super().__init__(args, tgt_dict)
|
| 36 |
+
self.task_type = args.task_type
|
| 37 |
+
assert self.task_type in ["lang", "domain"], "invalid task_type: {}, should be either lang or domain".format(self.task_type)
|
| 38 |
+
self.map_task_to_id(args.train_subset)
|
| 39 |
+
self.encoder_head_prior = float(args.decoder_attention_heads) / args.total_decoder_attention_heads
|
| 40 |
+
self.decoder_head_prior = float(args.encoder_attention_heads) / args.total_encoder_attention_heads
|
| 41 |
+
self.kl_loss = HeadSelectionLoss(args)
|
| 42 |
+
|
| 43 |
+
def map_task_to_id(self, train_subset):
|
| 44 |
+
src_lang_set, tgt_lang_set, domain_set = set(), set(), set()
|
| 45 |
+
for split in train_subset.split(","):
|
| 46 |
+
seq = split.split("_")
|
| 47 |
+
assert len(seq) == 4, "subset {} should be in the format of train_src_tgt_domain".format(split)
|
| 48 |
+
_, src_lang, tgt_lang, domain = seq
|
| 49 |
+
src_lang_set.add(src_lang)
|
| 50 |
+
tgt_lang_set.add(tgt_lang)
|
| 51 |
+
domain_set.add(domain)
|
| 52 |
+
src_langs = sorted(src_lang_set)
|
| 53 |
+
tgt_langs = sorted(tgt_lang_set)
|
| 54 |
+
domains = sorted(domain_set)
|
| 55 |
+
self.src_lang_map = {src_lang: i for (i, src_lang) in enumerate(src_langs)}
|
| 56 |
+
self.tgt_lang_map = {tgt_lang: i for (i, tgt_lang) in enumerate(tgt_langs)}
|
| 57 |
+
self.domain_map = {domain: i for (i, domain) in enumerate(domains)}
|
| 58 |
+
if self.task_type == "lang":
|
| 59 |
+
self.encoder_tasks = len(self.src_lang_map)
|
| 60 |
+
self.decoder_tasks = len(self.tgt_lang_map)
|
| 61 |
+
elif self.task_type == "domain":
|
| 62 |
+
self.encoder_tasks = len(self.domain_map)
|
| 63 |
+
self.decoder_tasks = len(self.domain_map)
|
| 64 |
+
|
| 65 |
+
def load_dataset(self, split, epoch=1, combine=False, **kwargs):
|
| 66 |
+
is_train_split = split.startswith("train")
|
| 67 |
+
pre_tokenizer = self.build_tokenizer(self.args)
|
| 68 |
+
bpe_tokenizer = self.build_bpe(self.args)
|
| 69 |
+
self.datasets[split] = SpeechToTextDatasetCreatorWithDomain.from_tsv(
|
| 70 |
+
self.args.data,
|
| 71 |
+
self.data_cfg,
|
| 72 |
+
split,
|
| 73 |
+
self.tgt_dict,
|
| 74 |
+
pre_tokenizer,
|
| 75 |
+
bpe_tokenizer,
|
| 76 |
+
is_train_split=is_train_split,
|
| 77 |
+
epoch=epoch,
|
| 78 |
+
seed=self.args.seed,
|
| 79 |
+
src_lang_map=self.src_lang_map,
|
| 80 |
+
tgt_lang_map=self.tgt_lang_map,
|
| 81 |
+
domain_map=self.domain_map,
|
| 82 |
+
speaker_to_id=self.speaker_to_id
|
| 83 |
+
)
|
| 84 |
+
|
| 85 |
+
def build_model(self, args):
|
| 86 |
+
args.encoder_tasks = self.encoder_tasks
|
| 87 |
+
args.decoder_tasks = self.decoder_tasks
|
| 88 |
+
return super(SpeechToTextHeadSelectionTask, self).build_model(args)
|
| 89 |
+
|
| 90 |
+
def get_sample_sizes(self, sample, task_ids, num_tasks):
|
| 91 |
+
"""
|
| 92 |
+
task_ids: (bsz,)
|
| 93 |
+
get sample sizes for each task
|
| 94 |
+
"""
|
| 95 |
+
bsz = task_ids.size(0)
|
| 96 |
+
mat = torch.zeros((num_tasks, bsz), device=task_ids.device)
|
| 97 |
+
mat[task_ids, torch.arange(bsz)] = 1.0
|
| 98 |
+
ntokens = torch.sum(sample['target'] != 1, dim=-1)
|
| 99 |
+
sample_sizes = torch.matmul(mat, ntokens.float())
|
| 100 |
+
return sample_sizes
|
| 101 |
+
|
| 102 |
+
def train_step(
|
| 103 |
+
self, sample, model, criterion, optimizer, update_num, ignore_grad=False
|
| 104 |
+
):
|
| 105 |
+
model.train()
|
| 106 |
+
model.set_num_updates(update_num)
|
| 107 |
+
# task ids
|
| 108 |
+
if self.task_type == "lang":
|
| 109 |
+
encoder_task_ids = sample["src_lang_ids"]
|
| 110 |
+
decoder_task_ids = sample["tgt_lang_ids"]
|
| 111 |
+
elif self.task_type == "domain":
|
| 112 |
+
encoder_task_ids = sample["domain_ids"]
|
| 113 |
+
decoder_task_ids = sample["domain_ids"]
|
| 114 |
+
model.encoder.set_task_ids(encoder_task_ids)
|
| 115 |
+
model.decoder.set_task_ids(decoder_task_ids)
|
| 116 |
+
|
| 117 |
+
with torch.autograd.profiler.record_function("forward"):
|
| 118 |
+
with torch.cuda.amp.autocast(enabled=(isinstance(optimizer, AMPOptimizer))):
|
| 119 |
+
loss, sample_size, logging_output = criterion(model, sample)
|
| 120 |
+
# KL loss
|
| 121 |
+
if self.args.encoder_attn_head_select:
|
| 122 |
+
sample_sizes = self.get_sample_sizes(sample, encoder_task_ids, self.encoder_tasks)
|
| 123 |
+
loss += self.kl_loss(
|
| 124 |
+
model.encoder.attn_head_selector.head_samples,
|
| 125 |
+
sample_sizes,
|
| 126 |
+
self.encoder_head_prior
|
| 127 |
+
)
|
| 128 |
+
if self.args.decoder_self_attn_head_select:
|
| 129 |
+
sample_sizes = self.get_sample_sizes(sample, decoder_task_ids, self.decoder_tasks)
|
| 130 |
+
loss += self.kl_loss(
|
| 131 |
+
model.decoder.self_attn_head_selector.head_samples,
|
| 132 |
+
sample_sizes,
|
| 133 |
+
self.decoder_head_prior
|
| 134 |
+
)
|
| 135 |
+
if self.args.dec_enc_attn_head_select:
|
| 136 |
+
sample_sizes = self.get_sample_sizes(sample, decoder_task_ids, self.decoder_tasks)
|
| 137 |
+
loss += self.kl_loss(
|
| 138 |
+
model.decoder.enc_attn_head_selector.head_sampes,
|
| 139 |
+
sample_sizes,
|
| 140 |
+
self.decoder_head_prior
|
| 141 |
+
)
|
| 142 |
+
|
| 143 |
+
if ignore_grad:
|
| 144 |
+
loss *= 0
|
| 145 |
+
with torch.autograd.profiler.record_function("backward"):
|
| 146 |
+
optimizer.backward(loss)
|
| 147 |
+
return loss, sample_size, logging_output
|
| 148 |
+
|
| 149 |
+
def valid_step(self, sample, model, criterion):
|
| 150 |
+
model.eval()
|
| 151 |
+
# task ids
|
| 152 |
+
if self.task_type == "lang":
|
| 153 |
+
encoder_task_ids = sample["src_lang_ids"]
|
| 154 |
+
decoder_task_ids = sample["tgt_lang_ids"]
|
| 155 |
+
elif self.task_type == "domain":
|
| 156 |
+
encoder_task_ids = sample["domain_ids"]
|
| 157 |
+
decoder_task_ids = sample["domain_ids"]
|
| 158 |
+
model.encoder.set_task_ids(encoder_task_ids)
|
| 159 |
+
model.decoder.set_task_ids(decoder_task_ids)
|
| 160 |
+
with torch.no_grad():
|
| 161 |
+
loss, sample_size, logging_output = criterion(model, sample)
|
| 162 |
+
return loss, sample_size, logging_output
|
| 163 |
+
|
| 164 |
+
def inference_step(
|
| 165 |
+
self, generator, models, sample, prefix_tokens=None, constraints=None
|
| 166 |
+
):
|
| 167 |
+
with torch.no_grad():
|
| 168 |
+
# task ids
|
| 169 |
+
if self.task_type == "lang":
|
| 170 |
+
encoder_task_ids = sample["src_lang_ids"][:1]
|
| 171 |
+
decoder_task_ids = sample["tgt_lang_ids"][:1]
|
| 172 |
+
elif self.task_type == "domain":
|
| 173 |
+
encoder_task_ids = sample["domain_ids"][:1]
|
| 174 |
+
decoder_task_ids = sample["domain_ids"][:1]
|
| 175 |
+
for model in models:
|
| 176 |
+
model.encoder.set_task_ids(encoder_task_ids)
|
| 177 |
+
model.decoder.set_task_ids(decoder_task_ids)
|
| 178 |
+
return generator.generate(
|
| 179 |
+
models, sample, prefix_tokens=prefix_tokens, constraints=constraints
|
| 180 |
+
)
|
data/fairseq/examples/camembert/README.md
ADDED
|
@@ -0,0 +1,75 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# CamemBERT: a Tasty French Language Model
|
| 2 |
+
|
| 3 |
+
## Introduction
|
| 4 |
+
|
| 5 |
+
[CamemBERT](https://arxiv.org/abs/1911.03894) is a pretrained language model trained on 138GB of French text based on RoBERTa.
|
| 6 |
+
|
| 7 |
+
Also available in [github.com/huggingface/transformers](https://github.com/huggingface/transformers/).
|
| 8 |
+
|
| 9 |
+
## Pre-trained models
|
| 10 |
+
|
| 11 |
+
| Model | #params | Download | Arch. | Training data |
|
| 12 |
+
|--------------------------------|---------|--------------------------------------------------------------------------------------------------------------------------|-------|-----------------------------------|
|
| 13 |
+
| `camembert` / `camembert-base` | 110M | [camembert-base.tar.gz](https://dl.fbaipublicfiles.com/fairseq/models/camembert-base.tar.gz) | Base | OSCAR (138 GB of text) |
|
| 14 |
+
| `camembert-large` | 335M | [camembert-large.tar.gz](https://dl.fbaipublicfiles.com/fairseq/models/camembert-large.tar.gz) | Large | CCNet (135 GB of text) |
|
| 15 |
+
| `camembert-base-ccnet` | 110M | [camembert-base-ccnet.tar.gz](https://dl.fbaipublicfiles.com/fairseq/models/camembert-base-ccnet.tar.gz) | Base | CCNet (135 GB of text) |
|
| 16 |
+
| `camembert-base-wikipedia-4gb` | 110M | [camembert-base-wikipedia-4gb.tar.gz](https://dl.fbaipublicfiles.com/fairseq/models/camembert-base-wikipedia-4gb.tar.gz) | Base | Wikipedia (4 GB of text) |
|
| 17 |
+
| `camembert-base-oscar-4gb` | 110M | [camembert-base-oscar-4gb.tar.gz](https://dl.fbaipublicfiles.com/fairseq/models/camembert-base-oscar-4gb.tar.gz) | Base | Subsample of OSCAR (4 GB of text) |
|
| 18 |
+
| `camembert-base-ccnet-4gb` | 110M | [camembert-base-ccnet-4gb.tar.gz](https://dl.fbaipublicfiles.com/fairseq/models/camembert-base-ccnet-4gb.tar.gz) | Base | Subsample of CCNet (4 GB of text) |
|
| 19 |
+
|
| 20 |
+
## Example usage
|
| 21 |
+
|
| 22 |
+
### fairseq
|
| 23 |
+
##### Load CamemBERT from torch.hub (PyTorch >= 1.1):
|
| 24 |
+
```python
|
| 25 |
+
import torch
|
| 26 |
+
camembert = torch.hub.load('pytorch/fairseq', 'camembert')
|
| 27 |
+
camembert.eval() # disable dropout (or leave in train mode to finetune)
|
| 28 |
+
```
|
| 29 |
+
|
| 30 |
+
##### Load CamemBERT (for PyTorch 1.0 or custom models):
|
| 31 |
+
```python
|
| 32 |
+
# Download camembert model
|
| 33 |
+
wget https://dl.fbaipublicfiles.com/fairseq/models/camembert-base.tar.gz
|
| 34 |
+
tar -xzvf camembert.tar.gz
|
| 35 |
+
|
| 36 |
+
# Load the model in fairseq
|
| 37 |
+
from fairseq.models.roberta import CamembertModel
|
| 38 |
+
camembert = CamembertModel.from_pretrained('/path/to/camembert')
|
| 39 |
+
camembert.eval() # disable dropout (or leave in train mode to finetune)
|
| 40 |
+
```
|
| 41 |
+
|
| 42 |
+
##### Filling masks:
|
| 43 |
+
```python
|
| 44 |
+
masked_line = 'Le camembert est <mask> :)'
|
| 45 |
+
camembert.fill_mask(masked_line, topk=3)
|
| 46 |
+
# [('Le camembert est délicieux :)', 0.4909118115901947, ' délicieux'),
|
| 47 |
+
# ('Le camembert est excellent :)', 0.10556942224502563, ' excellent'),
|
| 48 |
+
# ('Le camembert est succulent :)', 0.03453322499990463, ' succulent')]
|
| 49 |
+
```
|
| 50 |
+
|
| 51 |
+
##### Extract features from Camembert:
|
| 52 |
+
```python
|
| 53 |
+
# Extract the last layer's features
|
| 54 |
+
line = "J'aime le camembert !"
|
| 55 |
+
tokens = camembert.encode(line)
|
| 56 |
+
last_layer_features = camembert.extract_features(tokens)
|
| 57 |
+
assert last_layer_features.size() == torch.Size([1, 10, 768])
|
| 58 |
+
|
| 59 |
+
# Extract all layer's features (layer 0 is the embedding layer)
|
| 60 |
+
all_layers = camembert.extract_features(tokens, return_all_hiddens=True)
|
| 61 |
+
assert len(all_layers) == 13
|
| 62 |
+
assert torch.all(all_layers[-1] == last_layer_features)
|
| 63 |
+
```
|
| 64 |
+
|
| 65 |
+
## Citation
|
| 66 |
+
If you use our work, please cite:
|
| 67 |
+
|
| 68 |
+
```bibtex
|
| 69 |
+
@inproceedings{martin2020camembert,
|
| 70 |
+
title={CamemBERT: a Tasty French Language Model},
|
| 71 |
+
author={Martin, Louis and Muller, Benjamin and Su{\'a}rez, Pedro Javier Ortiz and Dupont, Yoann and Romary, Laurent and de la Clergerie, {\'E}ric Villemonte and Seddah, Djam{\'e} and Sagot, Beno{\^\i}t},
|
| 72 |
+
booktitle={Proceedings of the 58th Annual Meeting of the Association for Computational Linguistics},
|
| 73 |
+
year={2020}
|
| 74 |
+
}
|
| 75 |
+
```
|
data/fairseq/examples/conv_seq2seq/README.md
ADDED
|
@@ -0,0 +1,25 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Convolutional Sequence to Sequence Learning (Gehring et al., 2017)
|
| 2 |
+
|
| 3 |
+
## Pre-trained models
|
| 4 |
+
|
| 5 |
+
Description | Dataset | Model | Test set(s)
|
| 6 |
+
---|---|---|---
|
| 7 |
+
Convolutional <br> ([Gehring et al., 2017](https://arxiv.org/abs/1705.03122)) | [WMT14 English-French](http://statmt.org/wmt14/translation-task.html#Download) | [download (.tar.bz2)](https://dl.fbaipublicfiles.com/fairseq/models/wmt14.v2.en-fr.fconv-py.tar.bz2) | newstest2014: <br> [download (.tar.bz2)](https://dl.fbaipublicfiles.com/fairseq/data/wmt14.v2.en-fr.newstest2014.tar.bz2) <br> newstest2012/2013: <br> [download (.tar.bz2)](https://dl.fbaipublicfiles.com/fairseq/data/wmt14.v2.en-fr.ntst1213.tar.bz2)
|
| 8 |
+
Convolutional <br> ([Gehring et al., 2017](https://arxiv.org/abs/1705.03122)) | [WMT14 English-German](http://statmt.org/wmt14/translation-task.html#Download) | [download (.tar.bz2)](https://dl.fbaipublicfiles.com/fairseq/models/wmt14.en-de.fconv-py.tar.bz2) | newstest2014: <br> [download (.tar.bz2)](https://dl.fbaipublicfiles.com/fairseq/data/wmt14.en-de.newstest2014.tar.bz2)
|
| 9 |
+
Convolutional <br> ([Gehring et al., 2017](https://arxiv.org/abs/1705.03122)) | [WMT17 English-German](http://statmt.org/wmt17/translation-task.html#Download) | [download (.tar.bz2)](https://dl.fbaipublicfiles.com/fairseq/models/wmt17.v2.en-de.fconv-py.tar.bz2) | newstest2014: <br> [download (.tar.bz2)](https://dl.fbaipublicfiles.com/fairseq/data/wmt17.v2.en-de.newstest2014.tar.bz2)
|
| 10 |
+
|
| 11 |
+
## Example usage
|
| 12 |
+
|
| 13 |
+
See the [translation README](../translation/README.md) for instructions on reproducing results for WMT'14 En-De and
|
| 14 |
+
WMT'14 En-Fr using the `fconv_wmt_en_de` and `fconv_wmt_en_fr` model architectures.
|
| 15 |
+
|
| 16 |
+
## Citation
|
| 17 |
+
|
| 18 |
+
```bibtex
|
| 19 |
+
@inproceedings{gehring2017convs2s,
|
| 20 |
+
title = {Convolutional Sequence to Sequence Learning},
|
| 21 |
+
author = {Gehring, Jonas, and Auli, Michael and Grangier, David and Yarats, Denis and Dauphin, Yann N},
|
| 22 |
+
booktitle = {Proc. of ICML},
|
| 23 |
+
year = 2017,
|
| 24 |
+
}
|
| 25 |
+
```
|
data/fairseq/examples/criss/README.md
ADDED
|
@@ -0,0 +1,61 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Cross-lingual Retrieval for Iterative Self-Supervised Training
|
| 2 |
+
|
| 3 |
+
https://arxiv.org/pdf/2006.09526.pdf
|
| 4 |
+
|
| 5 |
+
## Introduction
|
| 6 |
+
|
| 7 |
+
CRISS is a multilingual sequence-to-sequnce pretraining method where mining and training processes are applied iteratively, improving cross-lingual alignment and translation ability at the same time.
|
| 8 |
+
|
| 9 |
+
## Requirements:
|
| 10 |
+
|
| 11 |
+
* faiss: https://github.com/facebookresearch/faiss
|
| 12 |
+
* mosesdecoder: https://github.com/moses-smt/mosesdecoder
|
| 13 |
+
* flores: https://github.com/facebookresearch/flores
|
| 14 |
+
* LASER: https://github.com/facebookresearch/LASER
|
| 15 |
+
|
| 16 |
+
## Unsupervised Machine Translation
|
| 17 |
+
##### 1. Download and decompress CRISS checkpoints
|
| 18 |
+
```
|
| 19 |
+
cd examples/criss
|
| 20 |
+
wget https://dl.fbaipublicfiles.com/criss/criss_3rd_checkpoints.tar.gz
|
| 21 |
+
tar -xf criss_checkpoints.tar.gz
|
| 22 |
+
```
|
| 23 |
+
##### 2. Download and preprocess Flores test dataset
|
| 24 |
+
Make sure to run all scripts from examples/criss directory
|
| 25 |
+
```
|
| 26 |
+
bash download_and_preprocess_flores_test.sh
|
| 27 |
+
```
|
| 28 |
+
|
| 29 |
+
##### 3. Run Evaluation on Sinhala-English
|
| 30 |
+
```
|
| 31 |
+
bash unsupervised_mt/eval.sh
|
| 32 |
+
```
|
| 33 |
+
|
| 34 |
+
## Sentence Retrieval
|
| 35 |
+
##### 1. Download and preprocess Tatoeba dataset
|
| 36 |
+
```
|
| 37 |
+
bash download_and_preprocess_tatoeba.sh
|
| 38 |
+
```
|
| 39 |
+
|
| 40 |
+
##### 2. Run Sentence Retrieval on Tatoeba Kazakh-English
|
| 41 |
+
```
|
| 42 |
+
bash sentence_retrieval/sentence_retrieval_tatoeba.sh
|
| 43 |
+
```
|
| 44 |
+
|
| 45 |
+
## Mining
|
| 46 |
+
##### 1. Install faiss
|
| 47 |
+
Follow instructions on https://github.com/facebookresearch/faiss/blob/master/INSTALL.md
|
| 48 |
+
##### 2. Mine pseudo-parallel data between Kazakh and English
|
| 49 |
+
```
|
| 50 |
+
bash mining/mine_example.sh
|
| 51 |
+
```
|
| 52 |
+
|
| 53 |
+
## Citation
|
| 54 |
+
```bibtex
|
| 55 |
+
@article{tran2020cross,
|
| 56 |
+
title={Cross-lingual retrieval for iterative self-supervised training},
|
| 57 |
+
author={Tran, Chau and Tang, Yuqing and Li, Xian and Gu, Jiatao},
|
| 58 |
+
journal={arXiv preprint arXiv:2006.09526},
|
| 59 |
+
year={2020}
|
| 60 |
+
}
|
| 61 |
+
```
|
data/fairseq/examples/criss/download_and_preprocess_flores_test.sh
ADDED
|
@@ -0,0 +1,64 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/bin/bash
|
| 2 |
+
# Copyright (c) Facebook, Inc. and its affiliates.
|
| 3 |
+
# All rights reserved.
|
| 4 |
+
#
|
| 5 |
+
# This source code is licensed under the license found in the
|
| 6 |
+
# LICENSE file in the root directory of this source tree.
|
| 7 |
+
|
| 8 |
+
SPM_ENCODE=flores/scripts/spm_encode.py
|
| 9 |
+
DATA=data_tmp
|
| 10 |
+
SPM_MODEL=criss_checkpoints/sentence.bpe.model
|
| 11 |
+
DICT=criss_checkpoints/dict.txt
|
| 12 |
+
|
| 13 |
+
download_data() {
|
| 14 |
+
CORPORA=$1
|
| 15 |
+
URL=$2
|
| 16 |
+
|
| 17 |
+
if [ -f $CORPORA ]; then
|
| 18 |
+
echo "$CORPORA already exists, skipping download"
|
| 19 |
+
else
|
| 20 |
+
echo "Downloading $URL"
|
| 21 |
+
wget $URL -O $CORPORA --no-check-certificate || rm -f $CORPORA
|
| 22 |
+
if [ -f $CORPORA ]; then
|
| 23 |
+
echo "$URL successfully downloaded."
|
| 24 |
+
else
|
| 25 |
+
echo "$URL not successfully downloaded."
|
| 26 |
+
rm -f $CORPORA
|
| 27 |
+
fi
|
| 28 |
+
fi
|
| 29 |
+
}
|
| 30 |
+
|
| 31 |
+
if [[ -f flores ]]; then
|
| 32 |
+
echo "flores already cloned"
|
| 33 |
+
else
|
| 34 |
+
git clone https://github.com/facebookresearch/flores
|
| 35 |
+
fi
|
| 36 |
+
|
| 37 |
+
mkdir -p $DATA
|
| 38 |
+
download_data $DATA/wikipedia_en_ne_si_test_sets.tgz "https://github.com/facebookresearch/flores/raw/master/data/wikipedia_en_ne_si_test_sets.tgz"
|
| 39 |
+
pushd $DATA
|
| 40 |
+
pwd
|
| 41 |
+
tar -vxf wikipedia_en_ne_si_test_sets.tgz
|
| 42 |
+
popd
|
| 43 |
+
|
| 44 |
+
|
| 45 |
+
for lang in ne_NP si_LK; do
|
| 46 |
+
datadir=$DATA/${lang}-en_XX-flores
|
| 47 |
+
rm -rf $datadir
|
| 48 |
+
mkdir -p $datadir
|
| 49 |
+
TEST_PREFIX=$DATA/wikipedia_en_ne_si_test_sets/wikipedia.test
|
| 50 |
+
python $SPM_ENCODE \
|
| 51 |
+
--model ${SPM_MODEL} \
|
| 52 |
+
--output_format=piece \
|
| 53 |
+
--inputs ${TEST_PREFIX}.${lang:0:2}-en.${lang:0:2} ${TEST_PREFIX}.${lang:0:2}-en.en \
|
| 54 |
+
--outputs $datadir/test.bpe.${lang}-en_XX.${lang} $datadir/test.bpe.${lang}-en_XX.en_XX
|
| 55 |
+
|
| 56 |
+
# binarize data
|
| 57 |
+
fairseq-preprocess \
|
| 58 |
+
--source-lang ${lang} --target-lang en_XX \
|
| 59 |
+
--testpref $datadir/test.bpe.${lang}-en_XX \
|
| 60 |
+
--destdir $datadir \
|
| 61 |
+
--srcdict ${DICT} \
|
| 62 |
+
--joined-dictionary \
|
| 63 |
+
--workers 4
|
| 64 |
+
done
|
data/fairseq/examples/criss/download_and_preprocess_tatoeba.sh
ADDED
|
@@ -0,0 +1,46 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/bin/bash
|
| 2 |
+
# Copyright (c) Facebook, Inc. and its affiliates.
|
| 3 |
+
# All rights reserved.
|
| 4 |
+
#
|
| 5 |
+
# This source code is licensed under the license found in the
|
| 6 |
+
# LICENSE file in the root directory of this source tree.
|
| 7 |
+
|
| 8 |
+
SPM_ENCODE=flores/scripts/spm_encode.py
|
| 9 |
+
DATA=data_tmp
|
| 10 |
+
SPM_MODEL=criss_checkpoints/sentence.bpe.model
|
| 11 |
+
DICT=criss_checkpoints/dict.txt
|
| 12 |
+
|
| 13 |
+
if [[ -f flores ]]; then
|
| 14 |
+
echo "flores already cloned"
|
| 15 |
+
else
|
| 16 |
+
git clone https://github.com/facebookresearch/flores
|
| 17 |
+
fi
|
| 18 |
+
if [[ -f LASER ]]; then
|
| 19 |
+
echo "LASER already cloned"
|
| 20 |
+
else
|
| 21 |
+
git clone https://github.com/facebookresearch/LASER
|
| 22 |
+
fi
|
| 23 |
+
mkdir -p data_tmp
|
| 24 |
+
declare -A lang_tatoeba_map=( ["ar_AR"]="ara" ["de_DE"]="deu" ["es_XX"]="spa" ["et_EE"]="est" ["fi_FI"]="fin" ["fr_XX"]="fra" ["hi_IN"]="hin" ["it_IT"]="ita" ["ja_XX"]="jpn" ["ko_KR"]="kor" ["kk_KZ"]="kaz" ["nl_XX"]="nld" ["ru_RU"]="rus" ["tr_TR"]="tur" ["vi_VN"]="vie" ["zh_CN"]="cmn")
|
| 25 |
+
for lang in ar_AR de_DE es_XX et_EE fi_FI fr_XX hi_IN it_IT ja_XX kk_KZ ko_KR nl_XX ru_RU tr_TR vi_VN zh_CN; do
|
| 26 |
+
lang_tatoeba=${lang_tatoeba_map[$lang]}
|
| 27 |
+
echo $lang_tatoeba
|
| 28 |
+
datadir=$DATA/${lang}-en_XX-tatoeba
|
| 29 |
+
rm -rf $datadir
|
| 30 |
+
mkdir -p $datadir
|
| 31 |
+
TEST_PREFIX=LASER/data/tatoeba/v1/tatoeba
|
| 32 |
+
python $SPM_ENCODE \
|
| 33 |
+
--model ${SPM_MODEL} \
|
| 34 |
+
--output_format=piece \
|
| 35 |
+
--inputs ${TEST_PREFIX}.${lang_tatoeba}-eng.${lang_tatoeba} ${TEST_PREFIX}.${lang_tatoeba}-eng.eng \
|
| 36 |
+
--outputs $datadir/test.bpe.${lang}-en_XX.${lang} $datadir/test.bpe.${lang}-en_XX.en_XX
|
| 37 |
+
|
| 38 |
+
# binarize data
|
| 39 |
+
fairseq-preprocess \
|
| 40 |
+
--source-lang ${lang} --target-lang en_XX \
|
| 41 |
+
--testpref $datadir/test.bpe.${lang}-en_XX \
|
| 42 |
+
--destdir $datadir \
|
| 43 |
+
--srcdict ${DICT} \
|
| 44 |
+
--joined-dictionary \
|
| 45 |
+
--workers 4
|
| 46 |
+
done
|
data/fairseq/examples/criss/mining/mine.py
ADDED
|
@@ -0,0 +1,240 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python3 -u
|
| 2 |
+
# Copyright (c) Facebook, Inc. and its affiliates.
|
| 3 |
+
#
|
| 4 |
+
# This source code is licensed under the MIT license found in the
|
| 5 |
+
# LICENSE file in the root directory of this source tree.
|
| 6 |
+
import argparse
|
| 7 |
+
import glob
|
| 8 |
+
from subprocess import check_call
|
| 9 |
+
|
| 10 |
+
try:
|
| 11 |
+
import faiss
|
| 12 |
+
|
| 13 |
+
has_faiss = True
|
| 14 |
+
except ImportError:
|
| 15 |
+
has_faiss = False
|
| 16 |
+
import numpy as np
|
| 17 |
+
|
| 18 |
+
|
| 19 |
+
GB = 1024 * 1024 * 1024
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
def call(cmd):
|
| 23 |
+
print(cmd)
|
| 24 |
+
check_call(cmd, shell=True)
|
| 25 |
+
|
| 26 |
+
|
| 27 |
+
def get_batches(directory, lang, prefix="all_avg_pool"):
|
| 28 |
+
print(f"Finding in {directory}/{prefix}.{lang}*")
|
| 29 |
+
files = glob.glob(f"{directory}/{prefix}.{lang}*")
|
| 30 |
+
emb_files = []
|
| 31 |
+
txt_files = []
|
| 32 |
+
for emb_fi in files:
|
| 33 |
+
emb_files.append(emb_fi)
|
| 34 |
+
txt_fi = emb_fi.replace(prefix, "sentences")
|
| 35 |
+
txt_files.append(txt_fi)
|
| 36 |
+
return emb_files, txt_files
|
| 37 |
+
|
| 38 |
+
|
| 39 |
+
def load_batch(emb_file, dim):
|
| 40 |
+
embeddings = np.fromfile(emb_file, dtype=np.float32)
|
| 41 |
+
num_rows = int(embeddings.shape[0] / dim)
|
| 42 |
+
embeddings = embeddings.reshape((num_rows, dim))
|
| 43 |
+
faiss.normalize_L2(embeddings)
|
| 44 |
+
return embeddings
|
| 45 |
+
|
| 46 |
+
|
| 47 |
+
def knnGPU_sharded(x_batches_f, y_batches_f, dim, k, direction="x2y"):
|
| 48 |
+
if not has_faiss:
|
| 49 |
+
raise ImportError("Please install Faiss")
|
| 50 |
+
sims = []
|
| 51 |
+
inds = []
|
| 52 |
+
xfrom = 0
|
| 53 |
+
xto = 0
|
| 54 |
+
for x_batch_f in x_batches_f:
|
| 55 |
+
yfrom = 0
|
| 56 |
+
yto = 0
|
| 57 |
+
x_batch = load_batch(x_batch_f, dim)
|
| 58 |
+
xto = xfrom + x_batch.shape[0]
|
| 59 |
+
bsims, binds = [], []
|
| 60 |
+
for y_batch_f in y_batches_f:
|
| 61 |
+
y_batch = load_batch(y_batch_f, dim)
|
| 62 |
+
neighbor_size = min(k, y_batch.shape[0])
|
| 63 |
+
yto = yfrom + y_batch.shape[0]
|
| 64 |
+
print("{}-{} -> {}-{}".format(xfrom, xto, yfrom, yto))
|
| 65 |
+
idx = faiss.IndexFlatIP(dim)
|
| 66 |
+
idx = faiss.index_cpu_to_all_gpus(idx)
|
| 67 |
+
idx.add(y_batch)
|
| 68 |
+
bsim, bind = idx.search(x_batch, neighbor_size)
|
| 69 |
+
|
| 70 |
+
bsims.append(bsim)
|
| 71 |
+
binds.append(bind + yfrom)
|
| 72 |
+
yfrom += y_batch.shape[0]
|
| 73 |
+
del idx
|
| 74 |
+
del y_batch
|
| 75 |
+
bsims = np.concatenate(bsims, axis=1)
|
| 76 |
+
binds = np.concatenate(binds, axis=1)
|
| 77 |
+
aux = np.argsort(-bsims, axis=1)
|
| 78 |
+
sim_batch = np.zeros((x_batch.shape[0], k), dtype=np.float32)
|
| 79 |
+
ind_batch = np.zeros((x_batch.shape[0], k), dtype=np.int64)
|
| 80 |
+
for i in range(x_batch.shape[0]):
|
| 81 |
+
for j in range(k):
|
| 82 |
+
sim_batch[i, j] = bsims[i, aux[i, j]]
|
| 83 |
+
ind_batch[i, j] = binds[i, aux[i, j]]
|
| 84 |
+
sims.append(sim_batch)
|
| 85 |
+
inds.append(ind_batch)
|
| 86 |
+
xfrom += x_batch.shape[0]
|
| 87 |
+
del x_batch
|
| 88 |
+
sim = np.concatenate(sims, axis=0)
|
| 89 |
+
ind = np.concatenate(inds, axis=0)
|
| 90 |
+
return sim, ind
|
| 91 |
+
|
| 92 |
+
|
| 93 |
+
def score(sim, fwd_mean, bwd_mean, margin):
|
| 94 |
+
return margin(sim, (fwd_mean + bwd_mean) / 2)
|
| 95 |
+
|
| 96 |
+
|
| 97 |
+
def score_candidates(
|
| 98 |
+
sim_mat, candidate_inds, fwd_mean, bwd_mean, margin, verbose=False
|
| 99 |
+
):
|
| 100 |
+
print(" - scoring {:d} candidates".format(sim_mat.shape[0]))
|
| 101 |
+
scores = np.zeros(candidate_inds.shape)
|
| 102 |
+
for i in range(scores.shape[0]):
|
| 103 |
+
for j in range(scores.shape[1]):
|
| 104 |
+
k = int(candidate_inds[i, j])
|
| 105 |
+
scores[i, j] = score(sim_mat[i, j], fwd_mean[i], bwd_mean[k], margin)
|
| 106 |
+
return scores
|
| 107 |
+
|
| 108 |
+
|
| 109 |
+
def load_text(files):
|
| 110 |
+
all_sentences = []
|
| 111 |
+
for fi in files:
|
| 112 |
+
with open(fi) as sentence_fi:
|
| 113 |
+
for line in sentence_fi:
|
| 114 |
+
all_sentences.append(line.strip())
|
| 115 |
+
print(f"Read {len(all_sentences)} sentences")
|
| 116 |
+
return all_sentences
|
| 117 |
+
|
| 118 |
+
|
| 119 |
+
if __name__ == "__main__":
|
| 120 |
+
parser = argparse.ArgumentParser(description="Mine bitext")
|
| 121 |
+
parser.add_argument("--src-lang", help="Source language")
|
| 122 |
+
parser.add_argument("--tgt-lang", help="Target language")
|
| 123 |
+
parser.add_argument(
|
| 124 |
+
"--dict-path", help="Path to dictionary file", default="dict.txt"
|
| 125 |
+
)
|
| 126 |
+
parser.add_argument(
|
| 127 |
+
"--spm-path", help="Path to SPM model file", default="sentence.bpe.model"
|
| 128 |
+
)
|
| 129 |
+
parser.add_argument("--dim", type=int, default=1024, help="Embedding dimension")
|
| 130 |
+
parser.add_argument("--mem", type=int, default=5, help="Memory in GB")
|
| 131 |
+
parser.add_argument("--src-dir", help="Source directory")
|
| 132 |
+
parser.add_argument("--tgt-dir", help="Target directory")
|
| 133 |
+
parser.add_argument("--output", help="Output path")
|
| 134 |
+
parser.add_argument(
|
| 135 |
+
"--neighborhood", type=int, default=4, help="Embedding dimension"
|
| 136 |
+
)
|
| 137 |
+
parser.add_argument(
|
| 138 |
+
"--threshold", type=float, default=1.06, help="Threshold on mined bitext"
|
| 139 |
+
)
|
| 140 |
+
parser.add_argument(
|
| 141 |
+
"--valid-size",
|
| 142 |
+
type=int,
|
| 143 |
+
default=2000,
|
| 144 |
+
help="Number of sentences used for validation set",
|
| 145 |
+
)
|
| 146 |
+
parser.add_argument(
|
| 147 |
+
"--min-count",
|
| 148 |
+
type=int,
|
| 149 |
+
default=50000,
|
| 150 |
+
help="Min num sentences used for each language",
|
| 151 |
+
)
|
| 152 |
+
args = parser.parse_args()
|
| 153 |
+
|
| 154 |
+
x_batches_f, x_sents_f = get_batches(args.src_dir, args.src_lang)
|
| 155 |
+
y_batches_f, y_sents_f = get_batches(args.tgt_dir, args.tgt_lang)
|
| 156 |
+
margin = lambda a, b: a / b
|
| 157 |
+
y2x_sim, y2x_ind = knnGPU_sharded(
|
| 158 |
+
y_batches_f, x_batches_f, args.dim, args.neighborhood, direction="y2x"
|
| 159 |
+
)
|
| 160 |
+
x2y_sim, x2y_ind = knnGPU_sharded(
|
| 161 |
+
x_batches_f, y_batches_f, args.dim, args.neighborhood, direction="x2y"
|
| 162 |
+
)
|
| 163 |
+
|
| 164 |
+
x2y_mean = x2y_sim.mean(axis=1)
|
| 165 |
+
y2x_mean = y2x_sim.mean(axis=1)
|
| 166 |
+
fwd_scores = score_candidates(x2y_sim, x2y_ind, x2y_mean, y2x_mean, margin)
|
| 167 |
+
bwd_scores = score_candidates(y2x_sim, y2x_ind, y2x_mean, x2y_mean, margin)
|
| 168 |
+
fwd_best = x2y_ind[np.arange(x2y_sim.shape[0]), fwd_scores.argmax(axis=1)]
|
| 169 |
+
bwd_best = y2x_ind[np.arange(y2x_sim.shape[0]), bwd_scores.argmax(axis=1)]
|
| 170 |
+
indices = np.stack(
|
| 171 |
+
(
|
| 172 |
+
np.concatenate((np.arange(x2y_ind.shape[0]), bwd_best)),
|
| 173 |
+
np.concatenate((fwd_best, np.arange(y2x_ind.shape[0]))),
|
| 174 |
+
),
|
| 175 |
+
axis=1,
|
| 176 |
+
)
|
| 177 |
+
scores = np.concatenate((fwd_scores.max(axis=1), bwd_scores.max(axis=1)))
|
| 178 |
+
|
| 179 |
+
x_sentences = load_text(x_sents_f)
|
| 180 |
+
y_sentences = load_text(y_sents_f)
|
| 181 |
+
|
| 182 |
+
threshold = args.threshold
|
| 183 |
+
min_count = args.min_count
|
| 184 |
+
seen_src, seen_trg = set(), set()
|
| 185 |
+
directory = args.output
|
| 186 |
+
call(f"mkdir -p {directory}")
|
| 187 |
+
src_out = open(
|
| 188 |
+
f"{directory}/all.{args.src_lang}",
|
| 189 |
+
mode="w",
|
| 190 |
+
encoding="utf-8",
|
| 191 |
+
errors="surrogateescape",
|
| 192 |
+
)
|
| 193 |
+
tgt_out = open(
|
| 194 |
+
f"{directory}/all.{args.tgt_lang}",
|
| 195 |
+
mode="w",
|
| 196 |
+
encoding="utf-8",
|
| 197 |
+
errors="surrogateescape",
|
| 198 |
+
)
|
| 199 |
+
scores_out = open(
|
| 200 |
+
f"{directory}/all.scores", mode="w", encoding="utf-8", errors="surrogateescape"
|
| 201 |
+
)
|
| 202 |
+
count = 0
|
| 203 |
+
for i in np.argsort(-scores):
|
| 204 |
+
src_ind, trg_ind = indices[i]
|
| 205 |
+
if src_ind not in seen_src and trg_ind not in seen_trg:
|
| 206 |
+
seen_src.add(src_ind)
|
| 207 |
+
seen_trg.add(trg_ind)
|
| 208 |
+
if scores[i] > threshold or count < min_count:
|
| 209 |
+
if x_sentences[src_ind]:
|
| 210 |
+
print(scores[i], file=scores_out)
|
| 211 |
+
print(x_sentences[src_ind], file=src_out)
|
| 212 |
+
print(y_sentences[trg_ind], file=tgt_out)
|
| 213 |
+
count += 1
|
| 214 |
+
else:
|
| 215 |
+
print(f"Ignoring sentence: {x_sentences[src_ind]}")
|
| 216 |
+
src_out.close()
|
| 217 |
+
tgt_out.close()
|
| 218 |
+
scores_out.close()
|
| 219 |
+
|
| 220 |
+
print(f"Found {count} pairs for threshold={threshold}")
|
| 221 |
+
with open(f"{directory}/all.{args.src_lang}") as all_s, open(
|
| 222 |
+
f"{directory}/all.{args.tgt_lang}"
|
| 223 |
+
) as all_t, open(f"{directory}/valid.{args.src_lang}", "w") as valid_s, open(
|
| 224 |
+
f"{directory}/valid.{args.tgt_lang}", "w"
|
| 225 |
+
) as valid_t, open(
|
| 226 |
+
f"{directory}/train.{args.src_lang}", "w"
|
| 227 |
+
) as train_s, open(
|
| 228 |
+
f"{directory}/train.{args.tgt_lang}", "w"
|
| 229 |
+
) as train_t:
|
| 230 |
+
count = 0
|
| 231 |
+
for s_line, t_line in zip(all_s, all_t):
|
| 232 |
+
s_line = s_line.split("\t")[1]
|
| 233 |
+
t_line = t_line.split("\t")[1]
|
| 234 |
+
if count >= args.valid_size:
|
| 235 |
+
train_s.write(s_line)
|
| 236 |
+
train_t.write(t_line)
|
| 237 |
+
else:
|
| 238 |
+
valid_s.write(s_line)
|
| 239 |
+
valid_t.write(t_line)
|
| 240 |
+
count += 1
|
data/fairseq/examples/criss/mining/mine_example.sh
ADDED
|
@@ -0,0 +1,103 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/bin/bash
|
| 2 |
+
# Copyright (c) Facebook, Inc. and its affiliates.
|
| 3 |
+
# All rights reserved.
|
| 4 |
+
#
|
| 5 |
+
# This source code is licensed under the license found in the
|
| 6 |
+
# LICENSE file in the root directory of this source tree.
|
| 7 |
+
#
|
| 8 |
+
source_lang=kk_KZ
|
| 9 |
+
target_lang=en_XX
|
| 10 |
+
MODEL=criss_checkpoints/criss.3rd.pt
|
| 11 |
+
SPM=criss_checkpoints/sentence.bpe.model
|
| 12 |
+
SPLIT=test
|
| 13 |
+
LANG_DICT=criss_checkpoints/lang_dict.txt
|
| 14 |
+
SPM_ENCODE=flores/scripts/spm_encode.py
|
| 15 |
+
SAVE_ENCODER=save_encoder.py
|
| 16 |
+
ENCODER_SAVE_ROOT=sentence_embeddings/$MODEL
|
| 17 |
+
DICT=criss_checkpoints/dict.txt
|
| 18 |
+
THRESHOLD=1.02
|
| 19 |
+
MIN_COUNT=500
|
| 20 |
+
|
| 21 |
+
DATA_DIR=data_tmp
|
| 22 |
+
SAVE_DIR=mining/${source_lang}_${target_lang}_mined
|
| 23 |
+
ENCODER_SAVE_DIR=${ENCODER_SAVE_ROOT}/${source_lang}-${target_lang}
|
| 24 |
+
INPUT_DIR=$DATA_DIR/${source_lang}-${target_lang}-tatoeba
|
| 25 |
+
|
| 26 |
+
mkdir -p $ENCODER_SAVE_DIR/${target_lang}
|
| 27 |
+
mkdir -p $ENCODER_SAVE_DIR/${source_lang}
|
| 28 |
+
mkdir -p $SAVE_DIR
|
| 29 |
+
|
| 30 |
+
## Save encoder outputs
|
| 31 |
+
|
| 32 |
+
# Save encoder outputs for source sentences
|
| 33 |
+
python $SAVE_ENCODER \
|
| 34 |
+
${INPUT_DIR} \
|
| 35 |
+
--path ${MODEL} \
|
| 36 |
+
--task translation_multi_simple_epoch \
|
| 37 |
+
--lang-pairs ${source_lang}-${target_lang} \
|
| 38 |
+
--lang-dict ${LANG_DICT} \
|
| 39 |
+
--gen-subset ${SPLIT} \
|
| 40 |
+
--bpe 'sentencepiece' \
|
| 41 |
+
-s ${source_lang} -t ${target_lang} \
|
| 42 |
+
--sentencepiece-model ${SPM} \
|
| 43 |
+
--remove-bpe 'sentencepiece' \
|
| 44 |
+
--beam 1 \
|
| 45 |
+
--lang-tok-style mbart \
|
| 46 |
+
--encoder-save-dir ${ENCODER_SAVE_DIR}/${source_lang}
|
| 47 |
+
|
| 48 |
+
## Save encoder outputs for target sentences
|
| 49 |
+
python $SAVE_ENCODER \
|
| 50 |
+
${INPUT_DIR} \
|
| 51 |
+
--path ${MODEL} \
|
| 52 |
+
--lang-pairs ${source_lang}-${target_lang} \
|
| 53 |
+
--lang-dict ${LANG_DICT} \
|
| 54 |
+
--task translation_multi_simple_epoch \
|
| 55 |
+
--gen-subset ${SPLIT} \
|
| 56 |
+
--bpe 'sentencepiece' \
|
| 57 |
+
-t ${source_lang} -s ${target_lang} \
|
| 58 |
+
--sentencepiece-model ${SPM} \
|
| 59 |
+
--remove-bpe 'sentencepiece' \
|
| 60 |
+
--beam 1 \
|
| 61 |
+
--lang-tok-style mbart \
|
| 62 |
+
--encoder-save-dir ${ENCODER_SAVE_DIR}/${target_lang}
|
| 63 |
+
|
| 64 |
+
## Mining
|
| 65 |
+
python mining/mine.py \
|
| 66 |
+
--src-lang ${source_lang} \
|
| 67 |
+
--tgt-lang ${target_lang} \
|
| 68 |
+
--dim 1024 \
|
| 69 |
+
--mem 10 \
|
| 70 |
+
--neighborhood 4 \
|
| 71 |
+
--src-dir ${ENCODER_SAVE_DIR}/${source_lang} \
|
| 72 |
+
--tgt-dir ${ENCODER_SAVE_DIR}/${target_lang} \
|
| 73 |
+
--output $SAVE_DIR \
|
| 74 |
+
--threshold ${THRESHOLD} \
|
| 75 |
+
--min-count ${MIN_COUNT} \
|
| 76 |
+
--valid-size 100 \
|
| 77 |
+
--dict-path ${DICT} \
|
| 78 |
+
--spm-path ${SPM} \
|
| 79 |
+
|
| 80 |
+
|
| 81 |
+
## Process and binarize mined data
|
| 82 |
+
python $SPM_ENCODE \
|
| 83 |
+
--model ${SPM} \
|
| 84 |
+
--output_format=piece \
|
| 85 |
+
--inputs mining/${source_lang}_${target_lang}_mined/train.${source_lang} mining/${source_lang}_${target_lang}_mined/train.${target_lang} \
|
| 86 |
+
--outputs mining/${source_lang}_${target_lang}_mined/train.bpe.${source_lang} mining/${source_lang}_${target_lang}_mined/train.bpe.${target_lang}
|
| 87 |
+
|
| 88 |
+
python $SPM_ENCODE \
|
| 89 |
+
--model ${SPM} \
|
| 90 |
+
--output_format=piece \
|
| 91 |
+
--inputs mining/${source_lang}_${target_lang}_mined/valid.${source_lang} mining/${source_lang}_${target_lang}_mined/valid.${target_lang} \
|
| 92 |
+
--outputs mining/${source_lang}_${target_lang}_mined/valid.bpe.${source_lang} mining/${source_lang}_${target_lang}_mined/valid.bpe.${target_lang}
|
| 93 |
+
|
| 94 |
+
|
| 95 |
+
fairseq-preprocess \
|
| 96 |
+
--source-lang ${source_lang} \
|
| 97 |
+
--target-lang ${target_lang} \
|
| 98 |
+
--trainpref mining/${source_lang}_${target_lang}_mined/train.bpe \
|
| 99 |
+
--validpref mining/${source_lang}_${target_lang}_mined/valid.bpe \
|
| 100 |
+
--destdir mining/${source_lang}_${target_lang}_mined \
|
| 101 |
+
--srcdict ${DICT} \
|
| 102 |
+
--joined-dictionary \
|
| 103 |
+
--workers 8
|
data/fairseq/examples/criss/save_encoder.py
ADDED
|
@@ -0,0 +1,214 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python3 -u
|
| 2 |
+
# Copyright (c) Facebook, Inc. and its affiliates.
|
| 3 |
+
#
|
| 4 |
+
# This source code is licensed under the MIT license found in the
|
| 5 |
+
# LICENSE file in the root directory of this source tree.
|
| 6 |
+
"""
|
| 7 |
+
Translate pre-processed data with a trained model.
|
| 8 |
+
"""
|
| 9 |
+
|
| 10 |
+
import numpy as np
|
| 11 |
+
import torch
|
| 12 |
+
from fairseq import checkpoint_utils, options, progress_bar, tasks, utils
|
| 13 |
+
from fairseq.sequence_generator import EnsembleModel
|
| 14 |
+
from fairseq.utils import safe_hasattr
|
| 15 |
+
|
| 16 |
+
|
| 17 |
+
def get_avg_pool(
|
| 18 |
+
models, sample, prefix_tokens, src_dict, remove_bpe, has_langtok=False
|
| 19 |
+
):
|
| 20 |
+
model = EnsembleModel(models)
|
| 21 |
+
|
| 22 |
+
# model.forward normally channels prev_output_tokens into the decoder
|
| 23 |
+
# separately, but SequenceGenerator directly calls model.encoder
|
| 24 |
+
encoder_input = {
|
| 25 |
+
k: v for k, v in sample["net_input"].items() if k != "prev_output_tokens"
|
| 26 |
+
}
|
| 27 |
+
|
| 28 |
+
# compute the encoder output for each beam
|
| 29 |
+
encoder_outs = model.forward_encoder(encoder_input)
|
| 30 |
+
np_encoder_outs = encoder_outs[0].encoder_out.cpu().numpy().astype(np.float32)
|
| 31 |
+
encoder_mask = 1 - encoder_outs[0].encoder_padding_mask.cpu().numpy().astype(
|
| 32 |
+
np.float32
|
| 33 |
+
)
|
| 34 |
+
encoder_mask = np.expand_dims(encoder_mask.T, axis=2)
|
| 35 |
+
if has_langtok:
|
| 36 |
+
encoder_mask = encoder_mask[1:, :, :]
|
| 37 |
+
np_encoder_outs = np_encoder_outs[1, :, :]
|
| 38 |
+
masked_encoder_outs = encoder_mask * np_encoder_outs
|
| 39 |
+
avg_pool = (masked_encoder_outs / encoder_mask.sum(axis=0)).sum(axis=0)
|
| 40 |
+
return avg_pool
|
| 41 |
+
|
| 42 |
+
|
| 43 |
+
def main(args):
|
| 44 |
+
assert args.path is not None, "--path required for generation!"
|
| 45 |
+
assert (
|
| 46 |
+
not args.sampling or args.nbest == args.beam
|
| 47 |
+
), "--sampling requires --nbest to be equal to --beam"
|
| 48 |
+
assert (
|
| 49 |
+
args.replace_unk is None or args.raw_text
|
| 50 |
+
), "--replace-unk requires a raw text dataset (--raw-text)"
|
| 51 |
+
|
| 52 |
+
args.beam = 1
|
| 53 |
+
utils.import_user_module(args)
|
| 54 |
+
|
| 55 |
+
if args.max_tokens is None:
|
| 56 |
+
args.max_tokens = 12000
|
| 57 |
+
print(args)
|
| 58 |
+
use_cuda = torch.cuda.is_available() and not args.cpu
|
| 59 |
+
|
| 60 |
+
# Load dataset splits
|
| 61 |
+
task = tasks.setup_task(args)
|
| 62 |
+
task.load_dataset(args.gen_subset)
|
| 63 |
+
|
| 64 |
+
# Set dictionaries
|
| 65 |
+
try:
|
| 66 |
+
src_dict = getattr(task, "source_dictionary", None)
|
| 67 |
+
except NotImplementedError:
|
| 68 |
+
src_dict = None
|
| 69 |
+
tgt_dict = task.target_dictionary
|
| 70 |
+
|
| 71 |
+
# Load ensemble
|
| 72 |
+
print("| loading model(s) from {}".format(args.path))
|
| 73 |
+
models, _model_args = checkpoint_utils.load_model_ensemble(
|
| 74 |
+
args.path.split(":"),
|
| 75 |
+
arg_overrides=eval(args.model_overrides),
|
| 76 |
+
task=task,
|
| 77 |
+
)
|
| 78 |
+
|
| 79 |
+
# Optimize ensemble for generation
|
| 80 |
+
for model in models:
|
| 81 |
+
model.make_generation_fast_(
|
| 82 |
+
beamable_mm_beam_size=None if args.no_beamable_mm else args.beam,
|
| 83 |
+
need_attn=args.print_alignment,
|
| 84 |
+
)
|
| 85 |
+
if args.fp16:
|
| 86 |
+
model.half()
|
| 87 |
+
if use_cuda:
|
| 88 |
+
model.cuda()
|
| 89 |
+
|
| 90 |
+
# Load alignment dictionary for unknown word replacement
|
| 91 |
+
# (None if no unknown word replacement, empty if no path to align dictionary)
|
| 92 |
+
align_dict = utils.load_align_dict(args.replace_unk)
|
| 93 |
+
|
| 94 |
+
# Load dataset (possibly sharded)
|
| 95 |
+
itr = task.get_batch_iterator(
|
| 96 |
+
dataset=task.dataset(args.gen_subset),
|
| 97 |
+
max_tokens=args.max_tokens,
|
| 98 |
+
max_positions=utils.resolve_max_positions(
|
| 99 |
+
task.max_positions(),
|
| 100 |
+
),
|
| 101 |
+
ignore_invalid_inputs=args.skip_invalid_size_inputs_valid_test,
|
| 102 |
+
required_batch_size_multiple=args.required_batch_size_multiple,
|
| 103 |
+
num_shards=args.num_shards,
|
| 104 |
+
shard_id=args.shard_id,
|
| 105 |
+
num_workers=args.num_workers,
|
| 106 |
+
).next_epoch_itr(shuffle=False)
|
| 107 |
+
|
| 108 |
+
num_sentences = 0
|
| 109 |
+
source_sentences = []
|
| 110 |
+
shard_id = 0
|
| 111 |
+
all_avg_pool = None
|
| 112 |
+
encoder_has_langtok = (
|
| 113 |
+
safe_hasattr(task.args, "encoder_langtok")
|
| 114 |
+
and task.args.encoder_langtok is not None
|
| 115 |
+
and safe_hasattr(task.args, "lang_tok_replacing_bos_eos")
|
| 116 |
+
and not task.args.lang_tok_replacing_bos_eos
|
| 117 |
+
)
|
| 118 |
+
with progress_bar.build_progress_bar(args, itr) as t:
|
| 119 |
+
for sample in t:
|
| 120 |
+
if sample is None:
|
| 121 |
+
print("Skipping None")
|
| 122 |
+
continue
|
| 123 |
+
sample = utils.move_to_cuda(sample) if use_cuda else sample
|
| 124 |
+
if "net_input" not in sample:
|
| 125 |
+
continue
|
| 126 |
+
|
| 127 |
+
prefix_tokens = None
|
| 128 |
+
if args.prefix_size > 0:
|
| 129 |
+
prefix_tokens = sample["target"][:, : args.prefix_size]
|
| 130 |
+
|
| 131 |
+
with torch.no_grad():
|
| 132 |
+
avg_pool = get_avg_pool(
|
| 133 |
+
models,
|
| 134 |
+
sample,
|
| 135 |
+
prefix_tokens,
|
| 136 |
+
src_dict,
|
| 137 |
+
args.post_process,
|
| 138 |
+
has_langtok=encoder_has_langtok,
|
| 139 |
+
)
|
| 140 |
+
if all_avg_pool is not None:
|
| 141 |
+
all_avg_pool = np.concatenate((all_avg_pool, avg_pool))
|
| 142 |
+
else:
|
| 143 |
+
all_avg_pool = avg_pool
|
| 144 |
+
|
| 145 |
+
if not isinstance(sample["id"], list):
|
| 146 |
+
sample_ids = sample["id"].tolist()
|
| 147 |
+
else:
|
| 148 |
+
sample_ids = sample["id"]
|
| 149 |
+
for i, sample_id in enumerate(sample_ids):
|
| 150 |
+
# Remove padding
|
| 151 |
+
src_tokens = utils.strip_pad(
|
| 152 |
+
sample["net_input"]["src_tokens"][i, :], tgt_dict.pad()
|
| 153 |
+
)
|
| 154 |
+
|
| 155 |
+
# Either retrieve the original sentences or regenerate them from tokens.
|
| 156 |
+
if align_dict is not None:
|
| 157 |
+
src_str = task.dataset(args.gen_subset).src.get_original_text(
|
| 158 |
+
sample_id
|
| 159 |
+
)
|
| 160 |
+
else:
|
| 161 |
+
if src_dict is not None:
|
| 162 |
+
src_str = src_dict.string(src_tokens, args.post_process)
|
| 163 |
+
else:
|
| 164 |
+
src_str = ""
|
| 165 |
+
|
| 166 |
+
if not args.quiet:
|
| 167 |
+
if src_dict is not None:
|
| 168 |
+
print("S-{}\t{}".format(sample_id, src_str))
|
| 169 |
+
|
| 170 |
+
source_sentences.append(f"{sample_id}\t{src_str}")
|
| 171 |
+
|
| 172 |
+
num_sentences += sample["nsentences"]
|
| 173 |
+
if all_avg_pool.shape[0] >= 1000000:
|
| 174 |
+
with open(
|
| 175 |
+
f"{args.encoder_save_dir}/all_avg_pool.{args.source_lang}.{shard_id}",
|
| 176 |
+
"w",
|
| 177 |
+
) as avg_pool_file:
|
| 178 |
+
all_avg_pool.tofile(avg_pool_file)
|
| 179 |
+
with open(
|
| 180 |
+
f"{args.encoder_save_dir}/sentences.{args.source_lang}.{shard_id}",
|
| 181 |
+
"w",
|
| 182 |
+
) as sentence_file:
|
| 183 |
+
sentence_file.writelines(f"{line}\n" for line in source_sentences)
|
| 184 |
+
all_avg_pool = None
|
| 185 |
+
source_sentences = []
|
| 186 |
+
shard_id += 1
|
| 187 |
+
|
| 188 |
+
if all_avg_pool is not None:
|
| 189 |
+
with open(
|
| 190 |
+
f"{args.encoder_save_dir}/all_avg_pool.{args.source_lang}.{shard_id}", "w"
|
| 191 |
+
) as avg_pool_file:
|
| 192 |
+
all_avg_pool.tofile(avg_pool_file)
|
| 193 |
+
with open(
|
| 194 |
+
f"{args.encoder_save_dir}/sentences.{args.source_lang}.{shard_id}", "w"
|
| 195 |
+
) as sentence_file:
|
| 196 |
+
sentence_file.writelines(f"{line}\n" for line in source_sentences)
|
| 197 |
+
return None
|
| 198 |
+
|
| 199 |
+
|
| 200 |
+
def cli_main():
|
| 201 |
+
parser = options.get_generation_parser()
|
| 202 |
+
parser.add_argument(
|
| 203 |
+
"--encoder-save-dir",
|
| 204 |
+
default="",
|
| 205 |
+
type=str,
|
| 206 |
+
metavar="N",
|
| 207 |
+
help="directory to save encoder outputs",
|
| 208 |
+
)
|
| 209 |
+
args = options.parse_args_and_arch(parser)
|
| 210 |
+
main(args)
|
| 211 |
+
|
| 212 |
+
|
| 213 |
+
if __name__ == "__main__":
|
| 214 |
+
cli_main()
|
data/fairseq/examples/criss/sentence_retrieval/encoder_analysis.py
ADDED
|
@@ -0,0 +1,92 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python3 -u
|
| 2 |
+
# Copyright (c) Facebook, Inc. and its affiliates.
|
| 3 |
+
#
|
| 4 |
+
# This source code is licensed under the MIT license found in the
|
| 5 |
+
# LICENSE file in the root directory of this source tree.
|
| 6 |
+
import argparse
|
| 7 |
+
import glob
|
| 8 |
+
|
| 9 |
+
import numpy as np
|
| 10 |
+
|
| 11 |
+
|
| 12 |
+
DIM = 1024
|
| 13 |
+
|
| 14 |
+
|
| 15 |
+
def compute_dist(source_embs, target_embs, k=5, return_sim_mat=False):
|
| 16 |
+
target_ids = [tid for tid in target_embs]
|
| 17 |
+
source_mat = np.stack(source_embs.values(), axis=0)
|
| 18 |
+
normalized_source_mat = source_mat / np.linalg.norm(
|
| 19 |
+
source_mat, axis=1, keepdims=True
|
| 20 |
+
)
|
| 21 |
+
target_mat = np.stack(target_embs.values(), axis=0)
|
| 22 |
+
normalized_target_mat = target_mat / np.linalg.norm(
|
| 23 |
+
target_mat, axis=1, keepdims=True
|
| 24 |
+
)
|
| 25 |
+
sim_mat = normalized_source_mat.dot(normalized_target_mat.T)
|
| 26 |
+
if return_sim_mat:
|
| 27 |
+
return sim_mat
|
| 28 |
+
neighbors_map = {}
|
| 29 |
+
for i, sentence_id in enumerate(source_embs):
|
| 30 |
+
idx = np.argsort(sim_mat[i, :])[::-1][:k]
|
| 31 |
+
neighbors_map[sentence_id] = [target_ids[tid] for tid in idx]
|
| 32 |
+
return neighbors_map
|
| 33 |
+
|
| 34 |
+
|
| 35 |
+
def load_embeddings(directory, LANGS):
|
| 36 |
+
sentence_embeddings = {}
|
| 37 |
+
sentence_texts = {}
|
| 38 |
+
for lang in LANGS:
|
| 39 |
+
sentence_embeddings[lang] = {}
|
| 40 |
+
sentence_texts[lang] = {}
|
| 41 |
+
lang_dir = f"{directory}/{lang}"
|
| 42 |
+
embedding_files = glob.glob(f"{lang_dir}/all_avg_pool.{lang}.*")
|
| 43 |
+
for embed_file in embedding_files:
|
| 44 |
+
shard_id = embed_file.split(".")[-1]
|
| 45 |
+
embeddings = np.fromfile(embed_file, dtype=np.float32)
|
| 46 |
+
num_rows = embeddings.shape[0] // DIM
|
| 47 |
+
embeddings = embeddings.reshape((num_rows, DIM))
|
| 48 |
+
|
| 49 |
+
with open(f"{lang_dir}/sentences.{lang}.{shard_id}") as sentence_file:
|
| 50 |
+
for idx, line in enumerate(sentence_file):
|
| 51 |
+
sentence_id, sentence = line.strip().split("\t")
|
| 52 |
+
sentence_texts[lang][sentence_id] = sentence
|
| 53 |
+
sentence_embeddings[lang][sentence_id] = embeddings[idx, :]
|
| 54 |
+
|
| 55 |
+
return sentence_embeddings, sentence_texts
|
| 56 |
+
|
| 57 |
+
|
| 58 |
+
def compute_accuracy(directory, LANGS):
|
| 59 |
+
sentence_embeddings, sentence_texts = load_embeddings(directory, LANGS)
|
| 60 |
+
|
| 61 |
+
top_1_accuracy = {}
|
| 62 |
+
|
| 63 |
+
top1_str = " ".join(LANGS) + "\n"
|
| 64 |
+
for source_lang in LANGS:
|
| 65 |
+
top_1_accuracy[source_lang] = {}
|
| 66 |
+
top1_str += f"{source_lang} "
|
| 67 |
+
for target_lang in LANGS:
|
| 68 |
+
top1 = 0
|
| 69 |
+
top5 = 0
|
| 70 |
+
neighbors_map = compute_dist(
|
| 71 |
+
sentence_embeddings[source_lang], sentence_embeddings[target_lang]
|
| 72 |
+
)
|
| 73 |
+
for sentence_id, neighbors in neighbors_map.items():
|
| 74 |
+
if sentence_id == neighbors[0]:
|
| 75 |
+
top1 += 1
|
| 76 |
+
if sentence_id in neighbors[:5]:
|
| 77 |
+
top5 += 1
|
| 78 |
+
n = len(sentence_embeddings[target_lang])
|
| 79 |
+
top1_str += f"{top1/n} "
|
| 80 |
+
top1_str += "\n"
|
| 81 |
+
|
| 82 |
+
print(top1_str)
|
| 83 |
+
print(top1_str, file=open(f"{directory}/accuracy", "w"))
|
| 84 |
+
|
| 85 |
+
|
| 86 |
+
if __name__ == "__main__":
|
| 87 |
+
parser = argparse.ArgumentParser(description="Analyze encoder outputs")
|
| 88 |
+
parser.add_argument("directory", help="Source language corpus")
|
| 89 |
+
parser.add_argument("--langs", help="List of langs")
|
| 90 |
+
args = parser.parse_args()
|
| 91 |
+
langs = args.langs.split(",")
|
| 92 |
+
compute_accuracy(args.directory, langs)
|
data/fairseq/examples/criss/sentence_retrieval/sentence_retrieval_tatoeba.sh
ADDED
|
@@ -0,0 +1,59 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/bin/bash
|
| 2 |
+
# Copyright (c) Facebook, Inc. and its affiliates.
|
| 3 |
+
# All rights reserved.
|
| 4 |
+
#
|
| 5 |
+
# This source code is licensed under the license found in the
|
| 6 |
+
# LICENSE file in the root directory of this source tree.
|
| 7 |
+
#
|
| 8 |
+
source_lang=kk_KZ
|
| 9 |
+
target_lang=en_XX
|
| 10 |
+
MODEL=criss_checkpoints/criss.3rd.pt
|
| 11 |
+
SPM=criss_checkpoints/sentence.bpe.model
|
| 12 |
+
SPLIT=test
|
| 13 |
+
LANG_DICT=criss_checkpoints/lang_dict.txt
|
| 14 |
+
ENCODER_ANALYSIS=sentence_retrieval/encoder_analysis.py
|
| 15 |
+
SAVE_ENCODER=save_encoder.py
|
| 16 |
+
ENCODER_SAVE_ROOT=sentence_embeddings/$MODEL
|
| 17 |
+
|
| 18 |
+
|
| 19 |
+
|
| 20 |
+
DATA_DIR=data_tmp
|
| 21 |
+
INPUT_DIR=$DATA_DIR/${source_lang}-${target_lang}-tatoeba
|
| 22 |
+
ENCODER_SAVE_DIR=${ENCODER_SAVE_ROOT}/${source_lang}-${target_lang}
|
| 23 |
+
mkdir -p $ENCODER_SAVE_DIR/${target_lang}
|
| 24 |
+
mkdir -p $ENCODER_SAVE_DIR/${source_lang}
|
| 25 |
+
|
| 26 |
+
# Save encoder outputs for source sentences
|
| 27 |
+
python $SAVE_ENCODER \
|
| 28 |
+
${INPUT_DIR} \
|
| 29 |
+
--path ${MODEL} \
|
| 30 |
+
--task translation_multi_simple_epoch \
|
| 31 |
+
--lang-dict ${LANG_DICT} \
|
| 32 |
+
--gen-subset ${SPLIT} \
|
| 33 |
+
--bpe 'sentencepiece' \
|
| 34 |
+
--lang-pairs ${source_lang}-${target_lang} \
|
| 35 |
+
-s ${source_lang} -t ${target_lang} \
|
| 36 |
+
--sentencepiece-model ${SPM} \
|
| 37 |
+
--remove-bpe 'sentencepiece' \
|
| 38 |
+
--beam 1 \
|
| 39 |
+
--lang-tok-style mbart \
|
| 40 |
+
--encoder-save-dir ${ENCODER_SAVE_DIR}/${source_lang}
|
| 41 |
+
|
| 42 |
+
# Save encoder outputs for target sentences
|
| 43 |
+
python $SAVE_ENCODER \
|
| 44 |
+
${INPUT_DIR} \
|
| 45 |
+
--path ${MODEL} \
|
| 46 |
+
--lang-dict ${LANG_DICT} \
|
| 47 |
+
--task translation_multi_simple_epoch \
|
| 48 |
+
--gen-subset ${SPLIT} \
|
| 49 |
+
--bpe 'sentencepiece' \
|
| 50 |
+
--lang-pairs ${target_lang}-${source_lang} \
|
| 51 |
+
-t ${source_lang} -s ${target_lang} \
|
| 52 |
+
--sentencepiece-model ${SPM} \
|
| 53 |
+
--remove-bpe 'sentencepiece' \
|
| 54 |
+
--beam 1 \
|
| 55 |
+
--lang-tok-style mbart \
|
| 56 |
+
--encoder-save-dir ${ENCODER_SAVE_DIR}/${target_lang}
|
| 57 |
+
|
| 58 |
+
# Analyze sentence retrieval accuracy
|
| 59 |
+
python $ENCODER_ANALYSIS --langs "${source_lang},${target_lang}" ${ENCODER_SAVE_DIR}
|
data/fairseq/examples/criss/unsupervised_mt/eval.sh
ADDED
|
@@ -0,0 +1,37 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/bin/bash
|
| 2 |
+
# Copyright (c) Facebook, Inc. and its affiliates.
|
| 3 |
+
# All rights reserved.
|
| 4 |
+
#
|
| 5 |
+
# This source code is licensed under the license found in the
|
| 6 |
+
# LICENSE file in the root directory of this source tree.
|
| 7 |
+
#
|
| 8 |
+
SRC=si_LK
|
| 9 |
+
TGT=en_XX
|
| 10 |
+
MODEL=criss_checkpoints/criss.3rd.pt
|
| 11 |
+
|
| 12 |
+
MULTIBLEU=mosesdecoder/scripts/generic/multi-bleu.perl
|
| 13 |
+
MOSES=mosesdecoder
|
| 14 |
+
REPLACE_UNICODE_PUNCT=$MOSES/scripts/tokenizer/replace-unicode-punctuation.perl
|
| 15 |
+
NORM_PUNC=$MOSES/scripts/tokenizer/normalize-punctuation.perl
|
| 16 |
+
REM_NON_PRINT_CHAR=$MOSES/scripts/tokenizer/remove-non-printing-char.perl
|
| 17 |
+
TOKENIZER=$MOSES/scripts/tokenizer/tokenizer.perl
|
| 18 |
+
GEN_TMP_DIR=gen_tmp
|
| 19 |
+
LANG_DICT=criss_checkpoints/lang_dict.txt
|
| 20 |
+
|
| 21 |
+
if [ ! -d "mosesdecoder" ]; then
|
| 22 |
+
git clone https://github.com/moses-smt/mosesdecoder
|
| 23 |
+
fi
|
| 24 |
+
mkdir -p $GEN_TMP_DIR
|
| 25 |
+
fairseq-generate data_tmp/${SRC}-${TGT}-flores \
|
| 26 |
+
--task translation_multi_simple_epoch \
|
| 27 |
+
--max-tokens 2000 \
|
| 28 |
+
--path ${MODEL} \
|
| 29 |
+
--skip-invalid-size-inputs-valid-test \
|
| 30 |
+
--beam 5 --lenpen 1.0 --gen-subset test \
|
| 31 |
+
--remove-bpe=sentencepiece \
|
| 32 |
+
--source-lang ${SRC} --target-lang ${TGT} \
|
| 33 |
+
--decoder-langtok --lang-pairs 'en_XX-ar_AR,en_XX-de_DE,en_XX-es_XX,en_XX-fr_XX,en_XX-hi_IN,en_XX-it_IT,en_XX-ja_XX,en_XX-ko_KR,en_XX-nl_XX,en_XX-ru_RU,en_XX-zh_CN,en_XX-tr_TR,en_XX-vi_VN,en_XX-ro_RO,en_XX-my_MM,en_XX-ne_NP,en_XX-si_LK,en_XX-cs_CZ,en_XX-lt_LT,en_XX-kk_KZ,en_XX-gu_IN,en_XX-fi_FI,en_XX-et_EE,en_XX-lv_LV,ar_AR-en_XX,cs_CZ-en_XX,de_DE-en_XX,es_XX-en_XX,et_EE-en_XX,fi_FI-en_XX,fr_XX-en_XX,gu_IN-en_XX,hi_IN-en_XX,it_IT-en_XX,ja_XX-en_XX,kk_KZ-en_XX,ko_KR-en_XX,lt_LT-en_XX,lv_LV-en_XX,my_MM-en_XX,ne_NP-en_XX,nl_XX-en_XX,ro_RO-en_XX,ru_RU-en_XX,si_LK-en_XX,tr_TR-en_XX,vi_VN-en_XX,zh_CN-en_XX,ar_AR-es_XX,es_XX-ar_AR,ar_AR-hi_IN,hi_IN-ar_AR,ar_AR-zh_CN,zh_CN-ar_AR,cs_CZ-es_XX,es_XX-cs_CZ,cs_CZ-hi_IN,hi_IN-cs_CZ,cs_CZ-zh_CN,zh_CN-cs_CZ,de_DE-es_XX,es_XX-de_DE,de_DE-hi_IN,hi_IN-de_DE,de_DE-zh_CN,zh_CN-de_DE,es_XX-hi_IN,hi_IN-es_XX,es_XX-zh_CN,zh_CN-es_XX,et_EE-es_XX,es_XX-et_EE,et_EE-hi_IN,hi_IN-et_EE,et_EE-zh_CN,zh_CN-et_EE,fi_FI-es_XX,es_XX-fi_FI,fi_FI-hi_IN,hi_IN-fi_FI,fi_FI-zh_CN,zh_CN-fi_FI,fr_XX-es_XX,es_XX-fr_XX,fr_XX-hi_IN,hi_IN-fr_XX,fr_XX-zh_CN,zh_CN-fr_XX,gu_IN-es_XX,es_XX-gu_IN,gu_IN-hi_IN,hi_IN-gu_IN,gu_IN-zh_CN,zh_CN-gu_IN,hi_IN-zh_CN,zh_CN-hi_IN,it_IT-es_XX,es_XX-it_IT,it_IT-hi_IN,hi_IN-it_IT,it_IT-zh_CN,zh_CN-it_IT,ja_XX-es_XX,es_XX-ja_XX,ja_XX-hi_IN,hi_IN-ja_XX,ja_XX-zh_CN,zh_CN-ja_XX,kk_KZ-es_XX,es_XX-kk_KZ,kk_KZ-hi_IN,hi_IN-kk_KZ,kk_KZ-zh_CN,zh_CN-kk_KZ,ko_KR-es_XX,es_XX-ko_KR,ko_KR-hi_IN,hi_IN-ko_KR,ko_KR-zh_CN,zh_CN-ko_KR,lt_LT-es_XX,es_XX-lt_LT,lt_LT-hi_IN,hi_IN-lt_LT,lt_LT-zh_CN,zh_CN-lt_LT,lv_LV-es_XX,es_XX-lv_LV,lv_LV-hi_IN,hi_IN-lv_LV,lv_LV-zh_CN,zh_CN-lv_LV,my_MM-es_XX,es_XX-my_MM,my_MM-hi_IN,hi_IN-my_MM,my_MM-zh_CN,zh_CN-my_MM,ne_NP-es_XX,es_XX-ne_NP,ne_NP-hi_IN,hi_IN-ne_NP,ne_NP-zh_CN,zh_CN-ne_NP,nl_XX-es_XX,es_XX-nl_XX,nl_XX-hi_IN,hi_IN-nl_XX,nl_XX-zh_CN,zh_CN-nl_XX,ro_RO-es_XX,es_XX-ro_RO,ro_RO-hi_IN,hi_IN-ro_RO,ro_RO-zh_CN,zh_CN-ro_RO,ru_RU-es_XX,es_XX-ru_RU,ru_RU-hi_IN,hi_IN-ru_RU,ru_RU-zh_CN,zh_CN-ru_RU,si_LK-es_XX,es_XX-si_LK,si_LK-hi_IN,hi_IN-si_LK,si_LK-zh_CN,zh_CN-si_LK,tr_TR-es_XX,es_XX-tr_TR,tr_TR-hi_IN,hi_IN-tr_TR,tr_TR-zh_CN,zh_CN-tr_TR,vi_VN-es_XX,es_XX-vi_VN,vi_VN-hi_IN,hi_IN-vi_VN,vi_VN-zh_CN,zh_CN-vi_VN' \
|
| 34 |
+
--lang-dict ${LANG_DICT} --lang-tok-style 'mbart' --sampling-method 'temperature' --sampling-temperature '1.0' > $GEN_TMP_DIR/${SRC}_${TGT}.gen
|
| 35 |
+
cat $GEN_TMP_DIR/${SRC}_${TGT}.gen | grep -P "^T-" | cut -f2 | $REPLACE_UNICODE_PUNCT | $NORM_PUNC -l ${TGT:0:2} | $REM_NON_PRINT_CHAR | $TOKENIZER -no-escape ${TGT:0:2} > $GEN_TMP_DIR/${SRC}_${TGT}.hyp
|
| 36 |
+
cat $GEN_TMP_DIR/${SRC}_${TGT}.gen | grep -P "^H-" | cut -f3 | $REPLACE_UNICODE_PUNCT | $NORM_PUNC -l ${TGT:0:2} | $REM_NON_PRINT_CHAR | $TOKENIZER -no-escape ${TGT:0:2} > $GEN_TMP_DIR/${SRC}_${TGT}.ref
|
| 37 |
+
${MULTIBLEU} $GEN_TMP_DIR/${SRC}_${TGT}.ref < $GEN_TMP_DIR/${SRC}_${TGT}.hyp
|
data/fairseq/examples/data2vec/README.md
ADDED
|
@@ -0,0 +1,261 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# data2vec 2.0
|
| 2 |
+
|
| 3 |
+
data2vec 2.0 improves the training efficiency of the original data2vec algorithm. We make the following improvements for efficiency considerations - we forward only the unmasked timesteps through the encoder, we use convolutional decoder and we use multimasking to amortize the compute overhead of the teacher model. You can find details in the paper [Efficient Self-supervised Learning with Contextualized Target Representations for Vision, Speech and Language](https://arxiv.org/abs/2212.07525) and our [blog post](https://ai.facebook.com/blog/ai-self-supervised-learning-data2vec/).
|
| 4 |
+
|
| 5 |
+
## Pretrained and finetuned models
|
| 6 |
+
### Vision
|
| 7 |
+
| Model | Finetuning split | Link
|
| 8 |
+
|---|---|---
|
| 9 |
+
data2vec ViT-B | No fine-tuning | [download](https://dl.fbaipublicfiles.com/fairseq/data2vec2/base_imagenet.pt)
|
| 10 |
+
data2vec ViT-B | Imagenet-1K | [download](https://dl.fbaipublicfiles.com/fairseq/data2vec2/base_imagenet_ft.pt)
|
| 11 |
+
data2vec ViT-L | No fine-tuning | [download](https://dl.fbaipublicfiles.com/fairseq/data2vec2/large_imagenet.pt)
|
| 12 |
+
data2vec ViT-L | Imagenet-1K | [download](https://dl.fbaipublicfiles.com/fairseq/data2vec2/large_imagenet_ft.pt)
|
| 13 |
+
data2vec ViT-H | No fine-tuning | [download](https://dl.fbaipublicfiles.com/fairseq/data2vec2/huge_imagenet.pt)
|
| 14 |
+
data2vec ViT-H | Imagenet-1K | [download](https://dl.fbaipublicfiles.com/fairseq/data2vec2/huge_imagenet_ft.pt)
|
| 15 |
+
|
| 16 |
+
Vision models only are license under CC-BY-NC.
|
| 17 |
+
### Speech
|
| 18 |
+
|
| 19 |
+
| Model | Finetuning split | Dataset | Link
|
| 20 |
+
|---|---|---|---
|
| 21 |
+
data2vec Base | No fine-tuning | [Librispeech](http://www.openslr.org/12) | [download](https://dl.fbaipublicfiles.com/fairseq/data2vec2/base_libri.pt)
|
| 22 |
+
data2vec Base | 960 hours | [Librispeech](http://www.openslr.org/12) | [download](https://dl.fbaipublicfiles.com/fairseq/data2vec2/base_libri_960h.pt)
|
| 23 |
+
data2vec Large | No fine-tuning | [Libri-light](https://github.com/facebookresearch/libri-light) | [download](https://dl.fbaipublicfiles.com/fairseq/data2vec2/large_vox.pt)
|
| 24 |
+
data2vec Large | 960 hours | [Libri-light](https://github.com/facebookresearch/libri-light) | [download](https://dl.fbaipublicfiles.com/fairseq/data2vec2/large_vox_960h.pt)
|
| 25 |
+
|
| 26 |
+
### NLP
|
| 27 |
+
|
| 28 |
+
| Model | Fine-tuning data | Dataset | Link | Dict | BPE
|
| 29 |
+
|---|---|---|---|---|---
|
| 30 |
+
data2vec Base | No fine-tuning | Books + Wiki | [download](https://dl.fbaipublicfiles.com/fairseq/data2vec2/nlp_base.pt) | [dict](https://dl.fbaipublicfiles.com/fairseq/data2vec2/dict.txt) | [encoder](https://dl.fbaipublicfiles.com/fairseq/data2vec2/encoder.json) / [vocab](https://dl.fbaipublicfiles.com/fairseq/data2vec2/vocab.bpe)
|
| 31 |
+
|
| 32 |
+
[//]: # (## Data Preparation)
|
| 33 |
+
|
| 34 |
+
[//]: # ()
|
| 35 |
+
[//]: # (### Vision)
|
| 36 |
+
|
| 37 |
+
[//]: # (add details)
|
| 38 |
+
|
| 39 |
+
[//]: # (### Speech)
|
| 40 |
+
|
| 41 |
+
[//]: # (add details)
|
| 42 |
+
|
| 43 |
+
[//]: # ()
|
| 44 |
+
[//]: # (### NLP)
|
| 45 |
+
|
| 46 |
+
[//]: # (add details)
|
| 47 |
+
|
| 48 |
+
|
| 49 |
+
## Commands to train different models using data2vec 2.0
|
| 50 |
+
|
| 51 |
+
### Vision
|
| 52 |
+
|
| 53 |
+
Commands to pretrain different model configurations
|
| 54 |
+
```shell script
|
| 55 |
+
$ python fairseq_cli/hydra_train.py -m --config-dir examples/data2vec/config/v2 \
|
| 56 |
+
--config-name base_images_only_task task.data=/path/to/dir
|
| 57 |
+
```
|
| 58 |
+
|
| 59 |
+
```shell script
|
| 60 |
+
$ python fairseq_cli/hydra_train.py -m --config-dir examples/data2vec/config/v2 \
|
| 61 |
+
--config-name large_images_only_task task.data=/path/to/dir
|
| 62 |
+
```
|
| 63 |
+
|
| 64 |
+
```shell script
|
| 65 |
+
$ python fairseq_cli/hydra_train.py -m --config-dir examples/data2vec/config/v2 \
|
| 66 |
+
--config-name huge_images14_only_task task.data=/path/to/dir
|
| 67 |
+
```
|
| 68 |
+
|
| 69 |
+
Commands to finetune different model configurations
|
| 70 |
+
|
| 71 |
+
```shell script
|
| 72 |
+
$ python fairseq_cli/hydra_train.py -m --config-dir examples/data2vec/config/vision/finetuning \
|
| 73 |
+
--config-name mae_imagenet_clean task.data=/path/to/dir model.model_path=/path/to/pretrained/model
|
| 74 |
+
```
|
| 75 |
+
|
| 76 |
+
```shell script
|
| 77 |
+
$ python fairseq_cli/hydra_train.py -m --config-dir examples/data2vec/config/vision/finetuning \
|
| 78 |
+
--config-name mae_imagenet_large_clean task.data=/path/to/dir model.model_path=/path/to/pretrained/model
|
| 79 |
+
```
|
| 80 |
+
|
| 81 |
+
```shell script
|
| 82 |
+
$ python fairseq_cli/hydra_train.py -m --config-dir examples/data2vec/config/vision/finetuning \
|
| 83 |
+
--config-name mae_imagenet_huge_clean task.data=/path/to/dir model.model_path=/path/to/pretrained/model
|
| 84 |
+
```
|
| 85 |
+
|
| 86 |
+
### Speech
|
| 87 |
+
|
| 88 |
+
```shell script
|
| 89 |
+
$ python fairseq_cli/hydra_train.py -m --config-dir examples/data2vec/config/v2 \
|
| 90 |
+
--config-name base_audio_only_task task.data=/path/to/manifests
|
| 91 |
+
```
|
| 92 |
+
|
| 93 |
+
```shell script
|
| 94 |
+
$ python fairseq_cli/hydra_train.py -m --config-dir examples/data2vec/config/v2 \
|
| 95 |
+
--config-name large_audio_only_task task.data=/path/to/manifests
|
| 96 |
+
```
|
| 97 |
+
|
| 98 |
+
Finetuning:
|
| 99 |
+
|
| 100 |
+
```shell script
|
| 101 |
+
$ python fairseq_cli/hydra_train.py -m --config-dir examples/wav2vec/config/finetuning --config-name vox_10h \
|
| 102 |
+
task.data=/path/to/manifests model.w2v_path=/path/to/pretrained/model common.user_dir=examples/data2vec
|
| 103 |
+
```
|
| 104 |
+
|
| 105 |
+
Replace vox_10h with the right config depending on your model and fine-tuning split.
|
| 106 |
+
See examples/wav2vec/config/finetuning for all available configs.
|
| 107 |
+
|
| 108 |
+
### NLP
|
| 109 |
+
|
| 110 |
+
Commands to pretrain
|
| 111 |
+
```shell script
|
| 112 |
+
$ python fairseq_cli/hydra_train.py -m --config-dir examples/data2vec/config/v2 \
|
| 113 |
+
--config-name base_text_only_task task.data=/path/to/file
|
| 114 |
+
```
|
| 115 |
+
|
| 116 |
+
Commands to fine-tune all GLUE tasks
|
| 117 |
+
```shell script
|
| 118 |
+
$ task=cola # choose from [cola|qnli|mrpc|rte|sst_2|mnli|qqp|sts_b]
|
| 119 |
+
$ lr=1e-5 # sweep [1e-5|2e-5|4e-5|6e-5] for each task
|
| 120 |
+
$ python fairseq_cli/hydra_train.py -m --config-dir examples/data2vec/config/v2/text_finetuning \
|
| 121 |
+
--config-name $task task.data=/path/to/file model.model_path=/path/to/pretrained/model "optimization.lr=[${lr}]"
|
| 122 |
+
```
|
| 123 |
+
|
| 124 |
+
# data2vec
|
| 125 |
+
|
| 126 |
+
data2vec is a framework for self-supervised representation learning for images, speech, and text as described in [data2vec: A General Framework for Self-supervised Learning in Speech, Vision and Language (Baevski et al., 2022)](https://ai.facebook.com/research/data2vec-a-general-framework-for-self-supervised-learning-in-speech-vision-and-language). The algorithm uses the same learning mechanism for different modalities.
|
| 127 |
+
|
| 128 |
+
|
| 129 |
+
## Pre-trained models
|
| 130 |
+
|
| 131 |
+
### Vision
|
| 132 |
+
|
| 133 |
+
Code and pre-trained models for data2vec visions can be found [here](https://github.com/facebookresearch/data2vec_vision/tree/main/beit).
|
| 134 |
+
|
| 135 |
+
### Speech
|
| 136 |
+
|
| 137 |
+
| Model | Finetuning split | Dataset | Link
|
| 138 |
+
|---|---|---|---
|
| 139 |
+
data2vec Base | No fine-tuning | [Librispeech](http://www.openslr.org/12) | [download](https://dl.fbaipublicfiles.com/fairseq/data2vec/audio_base_ls.pt)
|
| 140 |
+
data2vec Base | 10 minutes | [Librispeech](http://www.openslr.org/12) | [download](https://dl.fbaipublicfiles.com/fairseq/data2vec/audio_base_ls_10m.pt)
|
| 141 |
+
data2vec Base | 100 hours | [Librispeech](http://www.openslr.org/12) | [download](https://dl.fbaipublicfiles.com/fairseq/data2vec/audio_base_ls_100h.pt)
|
| 142 |
+
data2vec Base | 960 hours | [Librispeech](http://www.openslr.org/12) | [download](https://dl.fbaipublicfiles.com/fairseq/data2vec/audio_base_ls_960h.pt)
|
| 143 |
+
data2vec Large | No fine-tuning | [Libri-light](https://github.com/facebookresearch/libri-light) | [download](https://dl.fbaipublicfiles.com/fairseq/data2vec/vox_pretrained.pt)
|
| 144 |
+
data2vec Large | 10 minutes | [Libri-light](https://github.com/facebookresearch/libri-light) | [download](https://dl.fbaipublicfiles.com/fairseq/data2vec/vox_10m.pt)
|
| 145 |
+
data2vec Large | 100 hours | [Libri-light](https://github.com/facebookresearch/libri-light) | [download](https://dl.fbaipublicfiles.com/fairseq/data2vec/vox_100h.pt)
|
| 146 |
+
data2vec Large | 960 hours | [Libri-light](https://github.com/facebookresearch/libri-light) | [download](https://dl.fbaipublicfiles.com/fairseq/data2vec/vox_960h.pt)
|
| 147 |
+
---
|
| 148 |
+
|
| 149 |
+
### NLP
|
| 150 |
+
|
| 151 |
+
Model | Fine-tuning data | Dataset | Link
|
| 152 |
+
|---|---|---|---|
|
| 153 |
+
data2vec Base | No fine-tuning | Books + Wiki | [download](https://dl.fbaipublicfiles.com/fairseq/data2vec/nlp_base.pt)
|
| 154 |
+
|
| 155 |
+
## Training a new speech model with the CLI tools
|
| 156 |
+
|
| 157 |
+
Given a directory containing wav files to be used for pretraining (we recommend splitting each file into separate file 10 to 30 seconds in length)
|
| 158 |
+
|
| 159 |
+
### Prepare training data manifest:
|
| 160 |
+
|
| 161 |
+
First, install the `soundfile` library:
|
| 162 |
+
```shell script
|
| 163 |
+
pip install soundfile
|
| 164 |
+
```
|
| 165 |
+
|
| 166 |
+
Next, run:
|
| 167 |
+
|
| 168 |
+
```shell script
|
| 169 |
+
$ python examples/wav2vec/wav2vec_manifest.py /path/to/waves --dest /manifest/path --ext $ext --valid-percent $valid
|
| 170 |
+
```
|
| 171 |
+
|
| 172 |
+
$ext should be set to flac, wav, or whatever format your dataset happens to use that soundfile can read.
|
| 173 |
+
|
| 174 |
+
$valid should be set to some reasonable percentage (like 0.01) of training data to use for validation.
|
| 175 |
+
To use a pre-defined validation set (like dev-other from librispeech), set to it 0 and then overwrite valid.tsv with a
|
| 176 |
+
separately pre-processed manifest file.
|
| 177 |
+
|
| 178 |
+
### Train a data2vec Base model:
|
| 179 |
+
|
| 180 |
+
This configuration was used for the base model trained on the Librispeech dataset in the data2vec paper
|
| 181 |
+
|
| 182 |
+
Note that the input is expected to be single channel, sampled at 16 kHz
|
| 183 |
+
|
| 184 |
+
```shell script
|
| 185 |
+
$ python fairseq_cli/hydra_train.py -m --config-dir examples/data2vec/config/audio/pretraining \
|
| 186 |
+
--config-name base_librispeech task.data=/path/to/manifests common.user_dir=examples/data2vec
|
| 187 |
+
```
|
| 188 |
+
|
| 189 |
+
Note: you can simulate 16 GPUs by using k GPUs and adding command line parameters
|
| 190 |
+
`distributed_training.distributed_world_size=k` `+optimization.update_freq='[x]'` where x = 16/k
|
| 191 |
+
|
| 192 |
+
### Fine-tune a pre-trained model with CTC:
|
| 193 |
+
|
| 194 |
+
Fine-tuning a model requires parallel audio and labels file, as well as a vocabulary file in fairseq format.
|
| 195 |
+
A letter vocabulary can be downloaded [here](https://dl.fbaipublicfiles.com/fairseq/wav2vec/dict.ltr.txt).
|
| 196 |
+
An example [script](../wav2vec/libri_labels.py) that generates labels for the Librispeech dataset from the tsv file produced by wav2vec_manifest.py can be used as follows:
|
| 197 |
+
|
| 198 |
+
```shell script
|
| 199 |
+
split=train
|
| 200 |
+
$ python libri_labels.py /path/to/tsv --output-dir /output/dir --output-name $split
|
| 201 |
+
```
|
| 202 |
+
|
| 203 |
+
Fine-tuning on 100h of Librispeech with letter targets:
|
| 204 |
+
```shell script
|
| 205 |
+
$ fairseq-hydra-train \
|
| 206 |
+
distributed_training.distributed_port=$PORT \
|
| 207 |
+
task.data=/path/to/data \
|
| 208 |
+
model.w2v_path=/path/to/model.pt \
|
| 209 |
+
--config-dir /path/to/fairseq-py/examples/wav2vec/config/finetuning \
|
| 210 |
+
--config-name base_100h common.user_dir=examples/data2vec
|
| 211 |
+
```
|
| 212 |
+
|
| 213 |
+
There are other config files in the config/finetuning directory that can be used to fine-tune on other splits.
|
| 214 |
+
You can specify the right config via the `--config-name` parameter.
|
| 215 |
+
|
| 216 |
+
Decoding with a language model during training requires flashlight [python bindings](https://github.com/facebookresearch/flashlight/tree/master/bindings/python) (previously called [wav2letter](https://github.com/facebookresearch/wav2letter).
|
| 217 |
+
If you want to use a language model, add `+criterion.wer_args='[/path/to/kenlm, /path/to/lexicon, 2, -1]'` to the command line.
|
| 218 |
+
|
| 219 |
+
### Evaluating a CTC model:
|
| 220 |
+
|
| 221 |
+
Evaluating a CTC model with a language model requires [flashlight python bindings](https://github.com/facebookresearch/flashlight/tree/master/bindings/python) (previously called [wav2letter](https://github.com/facebookresearch/wav2letter) to be installed.
|
| 222 |
+
|
| 223 |
+
Fairseq transformer language model used in the wav2vec 2.0 paper can be obtained from the [wav2letter model repository](https://github.com/facebookresearch/wav2letter/tree/master/recipes/sota/2019).
|
| 224 |
+
Be sure to upper-case the language model vocab after downloading it.
|
| 225 |
+
|
| 226 |
+
Letter dictionary for pre-trained models can be found [here](https://dl.fbaipublicfiles.com/fairseq/wav2vec/dict.ltr.txt).
|
| 227 |
+
|
| 228 |
+
Next, run the evaluation command:
|
| 229 |
+
|
| 230 |
+
```shell script
|
| 231 |
+
python examples/speech_recognition/new/infer.py --config-dir examples/speech_recognition/new/conf \
|
| 232 |
+
--config-name infer task=audio_finetuning task.data=/path/to/manifests common.user_dir=examples/data2vec \
|
| 233 |
+
task.labels=ltr decoding.type=kenlm \
|
| 234 |
+
decoding.lmweight=${lmweight} decoding.wordscore=${wordscore} decoding.silweight=${silscore} \
|
| 235 |
+
decoding.lexicon=/path/to/lexicon \
|
| 236 |
+
decoding.lmpath=/path/to/lm decoding.unique_wer_file=True \
|
| 237 |
+
dataset.gen_subset=dev_clean,dev_other,test_clean,test_other \
|
| 238 |
+
common_eval.path=/path/to/checkpoint.pt decoding.beam=1500 distributed_training.distributed_world_size=${num_gpus}
|
| 239 |
+
```
|
| 240 |
+
|
| 241 |
+
To get raw numbers, use decoding.type=viterbi and omit the lexicon. To use the transformer language model, use decoding.type=fairseqlm.
|
| 242 |
+
|
| 243 |
+
## Training a new NLP model with the CLI tools
|
| 244 |
+
|
| 245 |
+
Please follow the [RoBERTa](../roberta/README.md) instructions to preprocess your data. To train a data2vec model on run:
|
| 246 |
+
|
| 247 |
+
```shell script
|
| 248 |
+
$ python fairseq_cli/hydra_train.py -m --config-dir examples/data2vec/config/text/pretraining \
|
| 249 |
+
--config-name base task.data=/path/to/data common.user_dir=examples/data2vec
|
| 250 |
+
```
|
| 251 |
+
|
| 252 |
+
As for speech models, you can simulate 16 gpus by using the update_freq parameter.
|
| 253 |
+
|
| 254 |
+
### Finetuning data2vec-text on GLUE
|
| 255 |
+
|
| 256 |
+
Please use a command similar to this:
|
| 257 |
+
|
| 258 |
+
```shell
|
| 259 |
+
$ python fairseq_cli/hydra_train.py -m --config-dir examples/roberta/config/finetuning \
|
| 260 |
+
--config-name $task task.data=$data_path checkpoint.restore_file="${/path/to/pretrained/model.pt}"
|
| 261 |
+
```
|
data/fairseq/examples/data2vec/__init__.py
ADDED
|
File without changes
|
data/fairseq/examples/data2vec/config/audio/classification/base_classification.yaml
ADDED
|
@@ -0,0 +1,70 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# @package _group_
|
| 2 |
+
|
| 3 |
+
common:
|
| 4 |
+
fp16: true
|
| 5 |
+
log_format: json
|
| 6 |
+
log_interval: 200
|
| 7 |
+
all_gather_list_size: 70000
|
| 8 |
+
tensorboard_logdir: tb
|
| 9 |
+
min_loss_scale: 1e-6
|
| 10 |
+
|
| 11 |
+
checkpoint:
|
| 12 |
+
save_interval: 1
|
| 13 |
+
no_epoch_checkpoints: true
|
| 14 |
+
best_checkpoint_metric: mAP
|
| 15 |
+
maximize_best_checkpoint_metric: true
|
| 16 |
+
|
| 17 |
+
task:
|
| 18 |
+
_name: audio_classification
|
| 19 |
+
data: ???
|
| 20 |
+
normalize: true
|
| 21 |
+
labels: lbl
|
| 22 |
+
|
| 23 |
+
dataset:
|
| 24 |
+
num_workers: 6
|
| 25 |
+
max_tokens: 2560000
|
| 26 |
+
skip_invalid_size_inputs_valid_test: true
|
| 27 |
+
valid_subset: eval
|
| 28 |
+
validate_interval: 5
|
| 29 |
+
|
| 30 |
+
distributed_training:
|
| 31 |
+
ddp_backend: legacy_ddp
|
| 32 |
+
distributed_world_size: 8
|
| 33 |
+
|
| 34 |
+
criterion:
|
| 35 |
+
_name: model
|
| 36 |
+
can_sum: false
|
| 37 |
+
log_keys:
|
| 38 |
+
- _predictions
|
| 39 |
+
- _targets
|
| 40 |
+
|
| 41 |
+
optimization:
|
| 42 |
+
max_update: 30000
|
| 43 |
+
lr: [0.00006] # scratch 53-5
|
| 44 |
+
|
| 45 |
+
optimizer:
|
| 46 |
+
_name: adam
|
| 47 |
+
adam_betas: (0.9,0.98)
|
| 48 |
+
adam_eps: 1e-08
|
| 49 |
+
|
| 50 |
+
lr_scheduler:
|
| 51 |
+
_name: cosine
|
| 52 |
+
warmup_updates: 5000
|
| 53 |
+
|
| 54 |
+
model:
|
| 55 |
+
_name: audio_classification
|
| 56 |
+
model_path: ???
|
| 57 |
+
apply_mask: true
|
| 58 |
+
mask_prob: 0.6
|
| 59 |
+
mask_length: 5 # scratch 1
|
| 60 |
+
mask_channel_prob: 0
|
| 61 |
+
mask_channel_length: 64
|
| 62 |
+
layerdrop: 0.1
|
| 63 |
+
dropout: 0.1
|
| 64 |
+
activation_dropout: 0.1
|
| 65 |
+
attention_dropout: 0.2
|
| 66 |
+
feature_grad_mult: 0 # scratch 1
|
| 67 |
+
label_mixup: true
|
| 68 |
+
source_mixup: 0.5
|
| 69 |
+
prediction_mode: lin_softmax # scratch average_sigmoid
|
| 70 |
+
|
data/fairseq/examples/data2vec/config/audio/classification/run_config/slurm_1.yaml
ADDED
|
@@ -0,0 +1,35 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# @package _global_
|
| 2 |
+
|
| 3 |
+
hydra:
|
| 4 |
+
job:
|
| 5 |
+
config:
|
| 6 |
+
override_dirname:
|
| 7 |
+
kv_sep: ':'
|
| 8 |
+
item_sep: '/'
|
| 9 |
+
exclude_keys:
|
| 10 |
+
- run_config
|
| 11 |
+
- distributed_training.distributed_port
|
| 12 |
+
- distributed_training.distributed_world_size
|
| 13 |
+
- model.pretrained_model_path
|
| 14 |
+
- model.target_network_path
|
| 15 |
+
- next_script
|
| 16 |
+
- task.cache_in_scratch
|
| 17 |
+
- task.data
|
| 18 |
+
- checkpoint.save_interval_updates
|
| 19 |
+
- checkpoint.keep_interval_updates
|
| 20 |
+
- checkpoint.save_on_overflow
|
| 21 |
+
sweep:
|
| 22 |
+
dir: /checkpoint/${env:USER}/${env:PREFIX}/${hydra.job.config_name}_${hydra.launcher.gpus_per_node}/${hydra.job.override_dirname}
|
| 23 |
+
subdir: ''
|
| 24 |
+
launcher:
|
| 25 |
+
submitit_folder: ${hydra.sweep.dir}
|
| 26 |
+
timeout_min: 4320
|
| 27 |
+
cpus_per_task: 10
|
| 28 |
+
gpus_per_node: 8
|
| 29 |
+
tasks_per_node: 8
|
| 30 |
+
mem_gb: 450
|
| 31 |
+
nodes: 1
|
| 32 |
+
name: ${env:PREFIX}_${hydra.job.config_name}
|
| 33 |
+
partition: devlab,learnlab,learnfair,scavenge
|
| 34 |
+
constraint: volta32gb,ib4
|
| 35 |
+
max_num_timeout: 30
|
data/fairseq/examples/data2vec/config/audio/classification/run_config/slurm_1g.yaml
ADDED
|
@@ -0,0 +1,35 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# @package _global_
|
| 2 |
+
|
| 3 |
+
hydra:
|
| 4 |
+
job:
|
| 5 |
+
config:
|
| 6 |
+
override_dirname:
|
| 7 |
+
kv_sep: ':'
|
| 8 |
+
item_sep: '/'
|
| 9 |
+
exclude_keys:
|
| 10 |
+
- run_config
|
| 11 |
+
- distributed_training.distributed_port
|
| 12 |
+
- distributed_training.distributed_world_size
|
| 13 |
+
- model.pretrained_model_path
|
| 14 |
+
- model.target_network_path
|
| 15 |
+
- next_script
|
| 16 |
+
- task.cache_in_scratch
|
| 17 |
+
- task.data
|
| 18 |
+
- checkpoint.save_interval_updates
|
| 19 |
+
- checkpoint.keep_interval_updates
|
| 20 |
+
- checkpoint.save_on_overflow
|
| 21 |
+
sweep:
|
| 22 |
+
dir: /checkpoint/${env:USER}/${env:PREFIX}/${hydra.job.config_name}_${hydra.launcher.gpus_per_node}/${hydra.job.override_dirname}
|
| 23 |
+
subdir: ''
|
| 24 |
+
launcher:
|
| 25 |
+
submitit_folder: ${hydra.sweep.dir}
|
| 26 |
+
timeout_min: 4320
|
| 27 |
+
cpus_per_task: 10
|
| 28 |
+
gpus_per_node: 1
|
| 29 |
+
tasks_per_node: 1
|
| 30 |
+
mem_gb: 100
|
| 31 |
+
nodes: 1
|
| 32 |
+
name: ${env:PREFIX}_${hydra.job.config_name}
|
| 33 |
+
partition: devlab,learnlab,learnfair,scavenge
|
| 34 |
+
constraint: volta32gb
|
| 35 |
+
max_num_timeout: 30
|
data/fairseq/examples/data2vec/config/audio/classification/run_config/slurm_2.yaml
ADDED
|
@@ -0,0 +1,35 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# @package _global_
|
| 2 |
+
|
| 3 |
+
hydra:
|
| 4 |
+
job:
|
| 5 |
+
config:
|
| 6 |
+
override_dirname:
|
| 7 |
+
kv_sep: ':'
|
| 8 |
+
item_sep: '/'
|
| 9 |
+
exclude_keys:
|
| 10 |
+
- run_config
|
| 11 |
+
- distributed_training.distributed_port
|
| 12 |
+
- distributed_training.distributed_world_size
|
| 13 |
+
- model.pretrained_model_path
|
| 14 |
+
- model.target_network_path
|
| 15 |
+
- next_script
|
| 16 |
+
- task.cache_in_scratch
|
| 17 |
+
- task.data
|
| 18 |
+
- checkpoint.save_interval_updates
|
| 19 |
+
- checkpoint.keep_interval_updates
|
| 20 |
+
- checkpoint.save_on_overflow
|
| 21 |
+
sweep:
|
| 22 |
+
dir: /checkpoint/${env:USER}/${env:PREFIX}/${hydra.job.config_name}_${hydra.launcher.gpus_per_node}/${hydra.job.override_dirname}
|
| 23 |
+
subdir: ''
|
| 24 |
+
launcher:
|
| 25 |
+
submitit_folder: ${hydra.sweep.dir}
|
| 26 |
+
timeout_min: 4320
|
| 27 |
+
cpus_per_task: 10
|
| 28 |
+
gpus_per_node: 8
|
| 29 |
+
tasks_per_node: 8
|
| 30 |
+
mem_gb: 450
|
| 31 |
+
nodes: 2
|
| 32 |
+
name: ${env:PREFIX}_${hydra.job.config_name}
|
| 33 |
+
partition: devlab,learnlab,learnfair,scavenge
|
| 34 |
+
constraint: volta32gb,ib4
|
| 35 |
+
max_num_timeout: 30
|
data/fairseq/examples/data2vec/config/audio/pretraining/audioset.yaml
ADDED
|
@@ -0,0 +1,91 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# @package _group_
|
| 2 |
+
|
| 3 |
+
common:
|
| 4 |
+
fp16: true
|
| 5 |
+
log_format: json
|
| 6 |
+
log_interval: 200
|
| 7 |
+
tensorboard_logdir: tb
|
| 8 |
+
min_loss_scale: 1e-6
|
| 9 |
+
user_dir: /private/home/abaevski/fairseq-py/examples/data2vec
|
| 10 |
+
|
| 11 |
+
checkpoint:
|
| 12 |
+
save_interval: 1
|
| 13 |
+
save_interval_updates: 25000
|
| 14 |
+
keep_interval_updates: 1
|
| 15 |
+
no_epoch_checkpoints: true
|
| 16 |
+
|
| 17 |
+
task:
|
| 18 |
+
_name: audio_pretraining
|
| 19 |
+
data: /private/home/abaevski/data/audioset
|
| 20 |
+
max_sample_size: 320000
|
| 21 |
+
min_sample_size: 32000
|
| 22 |
+
normalize: true
|
| 23 |
+
|
| 24 |
+
dataset:
|
| 25 |
+
num_workers: 6
|
| 26 |
+
max_tokens: 3400000
|
| 27 |
+
skip_invalid_size_inputs_valid_test: true
|
| 28 |
+
validate_interval: 5
|
| 29 |
+
required_batch_size_multiple: 1
|
| 30 |
+
disable_validation: true
|
| 31 |
+
|
| 32 |
+
distributed_training:
|
| 33 |
+
distributed_world_size: 24
|
| 34 |
+
ddp_backend: legacy_ddp
|
| 35 |
+
|
| 36 |
+
criterion:
|
| 37 |
+
_name: model
|
| 38 |
+
log_keys:
|
| 39 |
+
- ema_decay
|
| 40 |
+
- target_var
|
| 41 |
+
- pred_var
|
| 42 |
+
# - avg_self_attn
|
| 43 |
+
# - weights
|
| 44 |
+
|
| 45 |
+
optimization:
|
| 46 |
+
max_update: 200000
|
| 47 |
+
lr: [0.0005]
|
| 48 |
+
|
| 49 |
+
optimizer:
|
| 50 |
+
_name: adam
|
| 51 |
+
adam_betas: (0.9,0.98)
|
| 52 |
+
adam_eps: 1e-06
|
| 53 |
+
weight_decay: 0.01
|
| 54 |
+
|
| 55 |
+
lr_scheduler:
|
| 56 |
+
_name: cosine
|
| 57 |
+
warmup_updates: 10000
|
| 58 |
+
|
| 59 |
+
model:
|
| 60 |
+
_name: data2vec_audio
|
| 61 |
+
extractor_mode: layer_norm
|
| 62 |
+
encoder_layerdrop: 0.05
|
| 63 |
+
dropout_input: 0.0
|
| 64 |
+
dropout_features: 0.0
|
| 65 |
+
feature_grad_mult: 1.0
|
| 66 |
+
encoder_embed_dim: 768
|
| 67 |
+
|
| 68 |
+
mask_prob: 0.65
|
| 69 |
+
mask_length: 10
|
| 70 |
+
|
| 71 |
+
loss_beta: 0
|
| 72 |
+
loss_scale: null
|
| 73 |
+
|
| 74 |
+
instance_norm_target_layer: true
|
| 75 |
+
layer_norm_targets: true
|
| 76 |
+
average_top_k_layers: 12
|
| 77 |
+
|
| 78 |
+
self_attn_norm_type: deepnorm
|
| 79 |
+
final_norm_type: deepnorm
|
| 80 |
+
|
| 81 |
+
pos_conv_depth: 5
|
| 82 |
+
conv_pos: 95
|
| 83 |
+
|
| 84 |
+
ema_decay: 0.999
|
| 85 |
+
ema_end_decay: 0.9999
|
| 86 |
+
ema_anneal_end_step: 30000
|
| 87 |
+
ema_transformer_only: true
|
| 88 |
+
ema_layers_only: false
|
| 89 |
+
|
| 90 |
+
require_same_masks: true
|
| 91 |
+
mask_dropout: 0
|
data/fairseq/examples/data2vec/config/audio/pretraining/base_librispeech.yaml
ADDED
|
@@ -0,0 +1,83 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# @package _group_
|
| 2 |
+
|
| 3 |
+
common:
|
| 4 |
+
fp16: true
|
| 5 |
+
log_format: json
|
| 6 |
+
log_interval: 200
|
| 7 |
+
tensorboard_logdir: tb
|
| 8 |
+
|
| 9 |
+
checkpoint:
|
| 10 |
+
save_interval: 5
|
| 11 |
+
save_interval_updates: 25000
|
| 12 |
+
keep_interval_updates: 1
|
| 13 |
+
no_epoch_checkpoints: true
|
| 14 |
+
|
| 15 |
+
task:
|
| 16 |
+
_name: audio_pretraining
|
| 17 |
+
data: ???
|
| 18 |
+
max_sample_size: 320000
|
| 19 |
+
min_sample_size: 32000
|
| 20 |
+
normalize: true
|
| 21 |
+
|
| 22 |
+
dataset:
|
| 23 |
+
num_workers: 6
|
| 24 |
+
max_tokens: 3800000
|
| 25 |
+
skip_invalid_size_inputs_valid_test: true
|
| 26 |
+
validate_interval: 5
|
| 27 |
+
required_batch_size_multiple: 1
|
| 28 |
+
disable_validation: true
|
| 29 |
+
|
| 30 |
+
distributed_training:
|
| 31 |
+
distributed_world_size: 16
|
| 32 |
+
ddp_backend: legacy_ddp
|
| 33 |
+
|
| 34 |
+
criterion:
|
| 35 |
+
_name: model
|
| 36 |
+
log_keys:
|
| 37 |
+
- ema_decay
|
| 38 |
+
- target_var
|
| 39 |
+
- pred_var
|
| 40 |
+
|
| 41 |
+
optimization:
|
| 42 |
+
max_update: 400000
|
| 43 |
+
lr: [0.0005]
|
| 44 |
+
|
| 45 |
+
optimizer:
|
| 46 |
+
_name: adam
|
| 47 |
+
adam_betas: (0.9,0.98)
|
| 48 |
+
adam_eps: 1e-06
|
| 49 |
+
weight_decay: 0.01
|
| 50 |
+
|
| 51 |
+
lr_scheduler:
|
| 52 |
+
_name: tri_stage
|
| 53 |
+
phase_ratio: [0.03,0.9,0.07]
|
| 54 |
+
|
| 55 |
+
model:
|
| 56 |
+
_name: data2vec_audio
|
| 57 |
+
extractor_mode: layer_norm
|
| 58 |
+
encoder_layerdrop: 0.05
|
| 59 |
+
dropout_input: 0.0
|
| 60 |
+
dropout_features: 0.0
|
| 61 |
+
feature_grad_mult: 1.0
|
| 62 |
+
encoder_embed_dim: 768
|
| 63 |
+
|
| 64 |
+
mask_prob: 0.65
|
| 65 |
+
mask_length: 10
|
| 66 |
+
|
| 67 |
+
loss_beta: 0
|
| 68 |
+
loss_scale: null
|
| 69 |
+
|
| 70 |
+
instance_norm_target_layer: true
|
| 71 |
+
average_top_k_layers: 8
|
| 72 |
+
|
| 73 |
+
pos_conv_depth: 5
|
| 74 |
+
conv_pos: 95
|
| 75 |
+
|
| 76 |
+
ema_decay: 0.999
|
| 77 |
+
ema_end_decay: 0.9999
|
| 78 |
+
ema_anneal_end_step: 30000
|
| 79 |
+
ema_transformer_only: true
|
| 80 |
+
ema_layers_only: true
|
| 81 |
+
|
| 82 |
+
require_same_masks: true
|
| 83 |
+
mask_dropout: 0
|
data/fairseq/examples/data2vec/config/audio/pretraining/run_config/local.yaml
ADDED
|
@@ -0,0 +1,15 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# @package _global_
|
| 2 |
+
hydra:
|
| 3 |
+
sweep:
|
| 4 |
+
dir: ${env:PWD}/tmp_dbg/${now:%H-%M-%S}
|
| 5 |
+
|
| 6 |
+
distributed_training:
|
| 7 |
+
distributed_world_size: 1
|
| 8 |
+
nprocs_per_node: 1
|
| 9 |
+
distributed_port: -1
|
| 10 |
+
|
| 11 |
+
common:
|
| 12 |
+
log_interval: 1
|
| 13 |
+
|
| 14 |
+
dataset:
|
| 15 |
+
num_workers: 0
|
data/fairseq/examples/data2vec/config/audio/pretraining/run_config/slurm_1.yaml
ADDED
|
@@ -0,0 +1,37 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# @package _global_
|
| 2 |
+
|
| 3 |
+
hydra:
|
| 4 |
+
job:
|
| 5 |
+
config:
|
| 6 |
+
override_dirname:
|
| 7 |
+
kv_sep: ':'
|
| 8 |
+
item_sep: '/'
|
| 9 |
+
exclude_keys:
|
| 10 |
+
- run_config
|
| 11 |
+
- distributed_training.distributed_port
|
| 12 |
+
- distributed_training.distributed_world_size
|
| 13 |
+
- model.pretrained_model_path
|
| 14 |
+
- model.target_network_path
|
| 15 |
+
- next_script
|
| 16 |
+
- task.cache_in_scratch
|
| 17 |
+
- task.data
|
| 18 |
+
- checkpoint.save_interval_updates
|
| 19 |
+
- checkpoint.keep_interval_updates
|
| 20 |
+
- checkpoint.save_on_overflow
|
| 21 |
+
- common.log_interval
|
| 22 |
+
- common.user_dir
|
| 23 |
+
sweep:
|
| 24 |
+
dir: /checkpoint/${env:USER}/${env:PREFIX}/${hydra.job.config_name}_${hydra.launcher.gpus_per_node}/${hydra.job.override_dirname}
|
| 25 |
+
subdir: ''
|
| 26 |
+
launcher:
|
| 27 |
+
submitit_folder: ${hydra.sweep.dir}
|
| 28 |
+
timeout_min: 4320
|
| 29 |
+
cpus_per_task: 80
|
| 30 |
+
gpus_per_node: 8
|
| 31 |
+
tasks_per_node: 1
|
| 32 |
+
mem_gb: 450
|
| 33 |
+
nodes: 1
|
| 34 |
+
name: ${env:PREFIX}_${hydra.job.config_name}
|
| 35 |
+
partition: devlab,learnlab,learnfair,scavenge
|
| 36 |
+
constraint: volta32gb,ib4
|
| 37 |
+
max_num_timeout: 30
|
data/fairseq/examples/data2vec/config/audio/pretraining/run_config/slurm_1_aws.yaml
ADDED
|
@@ -0,0 +1,36 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# @package _global_
|
| 2 |
+
|
| 3 |
+
hydra:
|
| 4 |
+
job:
|
| 5 |
+
config:
|
| 6 |
+
override_dirname:
|
| 7 |
+
kv_sep: ':'
|
| 8 |
+
item_sep: '/'
|
| 9 |
+
exclude_keys:
|
| 10 |
+
- run_config
|
| 11 |
+
- distributed_training.distributed_port
|
| 12 |
+
- distributed_training.distributed_world_size
|
| 13 |
+
- model.pretrained_model_path
|
| 14 |
+
- model.target_network_path
|
| 15 |
+
- next_script
|
| 16 |
+
- task.cache_in_scratch
|
| 17 |
+
- task.data
|
| 18 |
+
- checkpoint.save_interval_updates
|
| 19 |
+
- checkpoint.keep_interval_updates
|
| 20 |
+
- checkpoint.save_on_overflow
|
| 21 |
+
- common.log_interval
|
| 22 |
+
- common.user_dir
|
| 23 |
+
sweep:
|
| 24 |
+
dir: /checkpoint/${env:USER}/${env:PREFIX}/${hydra.job.config_name}_${hydra.launcher.gpus_per_node}/${hydra.job.override_dirname}
|
| 25 |
+
subdir: ''
|
| 26 |
+
launcher:
|
| 27 |
+
submitit_folder: ${hydra.sweep.dir}
|
| 28 |
+
timeout_min: 4320
|
| 29 |
+
cpus_per_task: 80
|
| 30 |
+
gpus_per_node: 8
|
| 31 |
+
tasks_per_node: 1
|
| 32 |
+
mem_gb: 0
|
| 33 |
+
nodes: 1
|
| 34 |
+
name: ${env:PREFIX}_${hydra.job.config_name}
|
| 35 |
+
partition: wav2vec,learnlab,learnfair
|
| 36 |
+
max_num_timeout: 30
|
data/fairseq/examples/data2vec/config/audio/pretraining/run_config/slurm_2.yaml
ADDED
|
@@ -0,0 +1,37 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# @package _global_
|
| 2 |
+
|
| 3 |
+
hydra:
|
| 4 |
+
job:
|
| 5 |
+
config:
|
| 6 |
+
override_dirname:
|
| 7 |
+
kv_sep: ':'
|
| 8 |
+
item_sep: '/'
|
| 9 |
+
exclude_keys:
|
| 10 |
+
- run_config
|
| 11 |
+
- distributed_training.distributed_port
|
| 12 |
+
- distributed_training.distributed_world_size
|
| 13 |
+
- model.pretrained_model_path
|
| 14 |
+
- model.target_network_path
|
| 15 |
+
- next_script
|
| 16 |
+
- task.cache_in_scratch
|
| 17 |
+
- task.data
|
| 18 |
+
- checkpoint.save_interval_updates
|
| 19 |
+
- checkpoint.keep_interval_updates
|
| 20 |
+
- checkpoint.save_on_overflow
|
| 21 |
+
- common.log_interval
|
| 22 |
+
- common.user_dir
|
| 23 |
+
sweep:
|
| 24 |
+
dir: /checkpoint/${env:USER}/${env:PREFIX}/${hydra.job.config_name}_${hydra.launcher.gpus_per_node}/${hydra.job.override_dirname}
|
| 25 |
+
subdir: ''
|
| 26 |
+
launcher:
|
| 27 |
+
submitit_folder: ${hydra.sweep.dir}
|
| 28 |
+
timeout_min: 4320
|
| 29 |
+
cpus_per_task: 10
|
| 30 |
+
gpus_per_node: 8
|
| 31 |
+
tasks_per_node: 8
|
| 32 |
+
mem_gb: 450
|
| 33 |
+
nodes: 2
|
| 34 |
+
name: ${env:PREFIX}_${hydra.job.config_name}
|
| 35 |
+
partition: devlab,learnlab,learnfair,scavenge
|
| 36 |
+
constraint: volta32gb,ib4
|
| 37 |
+
max_num_timeout: 30
|
data/fairseq/examples/data2vec/config/audio/pretraining/run_config/slurm_2_aws.yaml
ADDED
|
@@ -0,0 +1,37 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# @package _global_
|
| 2 |
+
|
| 3 |
+
hydra:
|
| 4 |
+
job:
|
| 5 |
+
config:
|
| 6 |
+
override_dirname:
|
| 7 |
+
kv_sep: ':'
|
| 8 |
+
item_sep: '/'
|
| 9 |
+
exclude_keys:
|
| 10 |
+
- run_config
|
| 11 |
+
- distributed_training.distributed_port
|
| 12 |
+
- distributed_training.distributed_world_size
|
| 13 |
+
- model.pretrained_model_path
|
| 14 |
+
- model.target_network_path
|
| 15 |
+
- next_script
|
| 16 |
+
- task.cache_in_scratch
|
| 17 |
+
- task.data
|
| 18 |
+
- task.post_save_script
|
| 19 |
+
- checkpoint.save_interval_updates
|
| 20 |
+
- checkpoint.keep_interval_updates
|
| 21 |
+
- checkpoint.save_on_overflow
|
| 22 |
+
- common.log_interval
|
| 23 |
+
- common.user_dir
|
| 24 |
+
sweep:
|
| 25 |
+
dir: /fsx-wav2vec/${env:USER}/${env:PREFIX}/${hydra.job.config_name}_${hydra.launcher.gpus_per_node}/${hydra.job.override_dirname}
|
| 26 |
+
subdir: ''
|
| 27 |
+
launcher:
|
| 28 |
+
submitit_folder: ${hydra.sweep.dir}
|
| 29 |
+
timeout_min: 4320
|
| 30 |
+
cpus_per_task: 10
|
| 31 |
+
gpus_per_node: 8
|
| 32 |
+
tasks_per_node: 8
|
| 33 |
+
mem_gb: 0
|
| 34 |
+
nodes: 2
|
| 35 |
+
name: ${env:PREFIX}_${hydra.job.config_name}
|
| 36 |
+
partition: wav2vec,learnlab,learnfair
|
| 37 |
+
max_num_timeout: 30
|
data/fairseq/examples/data2vec/config/audio/pretraining/run_config/slurm_3.yaml
ADDED
|
@@ -0,0 +1,36 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# @package _global_
|
| 2 |
+
|
| 3 |
+
hydra:
|
| 4 |
+
job:
|
| 5 |
+
config:
|
| 6 |
+
override_dirname:
|
| 7 |
+
kv_sep: ':'
|
| 8 |
+
item_sep: '/'
|
| 9 |
+
exclude_keys:
|
| 10 |
+
- run_config
|
| 11 |
+
- distributed_training.distributed_port
|
| 12 |
+
- distributed_training.distributed_world_size
|
| 13 |
+
- model.pretrained_model_path
|
| 14 |
+
- model.target_network_path
|
| 15 |
+
- next_script
|
| 16 |
+
- task.cache_in_scratch
|
| 17 |
+
- task.data
|
| 18 |
+
- checkpoint.save_interval_updates
|
| 19 |
+
- checkpoint.keep_interval_updates
|
| 20 |
+
- checkpoint.save_on_overflow
|
| 21 |
+
- common.log_interval
|
| 22 |
+
sweep:
|
| 23 |
+
dir: /checkpoint/${env:USER}/${env:PREFIX}/${hydra.job.config_name}_${hydra.launcher.gpus_per_node}/${hydra.job.override_dirname}
|
| 24 |
+
subdir: ''
|
| 25 |
+
launcher:
|
| 26 |
+
submitit_folder: ${hydra.sweep.dir}
|
| 27 |
+
timeout_min: 4320
|
| 28 |
+
cpus_per_task: 80
|
| 29 |
+
gpus_per_node: 8
|
| 30 |
+
tasks_per_node: 1
|
| 31 |
+
mem_gb: 450
|
| 32 |
+
nodes: 3
|
| 33 |
+
name: ${env:PREFIX}_${hydra.job.config_name}
|
| 34 |
+
partition: devlab,learnlab,learnfair,scavenge
|
| 35 |
+
constraint: volta32gb,ib4
|
| 36 |
+
max_num_timeout: 30
|
data/fairseq/examples/data2vec/config/audio/pretraining/run_config/slurm_4.yaml
ADDED
|
@@ -0,0 +1,36 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# @package _global_
|
| 2 |
+
|
| 3 |
+
hydra:
|
| 4 |
+
job:
|
| 5 |
+
config:
|
| 6 |
+
override_dirname:
|
| 7 |
+
kv_sep: ':'
|
| 8 |
+
item_sep: '/'
|
| 9 |
+
exclude_keys:
|
| 10 |
+
- run_config
|
| 11 |
+
- distributed_training.distributed_port
|
| 12 |
+
- distributed_training.distributed_world_size
|
| 13 |
+
- model.pretrained_model_path
|
| 14 |
+
- model.target_network_path
|
| 15 |
+
- next_script
|
| 16 |
+
- task.cache_in_scratch
|
| 17 |
+
- task.data
|
| 18 |
+
- checkpoint.save_interval_updates
|
| 19 |
+
- checkpoint.keep_interval_updates
|
| 20 |
+
- checkpoint.save_on_overflow
|
| 21 |
+
- common.log_interval
|
| 22 |
+
sweep:
|
| 23 |
+
dir: /checkpoint/${env:USER}/${env:PREFIX}/${hydra.job.config_name}_${hydra.launcher.gpus_per_node}/${hydra.job.override_dirname}
|
| 24 |
+
subdir: ''
|
| 25 |
+
launcher:
|
| 26 |
+
submitit_folder: ${hydra.sweep.dir}
|
| 27 |
+
timeout_min: 4320
|
| 28 |
+
cpus_per_task: 10
|
| 29 |
+
gpus_per_node: 8
|
| 30 |
+
tasks_per_node: 8
|
| 31 |
+
mem_gb: 450
|
| 32 |
+
nodes: 4
|
| 33 |
+
name: ${env:PREFIX}_${hydra.job.config_name}
|
| 34 |
+
partition: devlab,learnlab,learnfair,scavenge
|
| 35 |
+
constraint: volta32gb,ib4
|
| 36 |
+
max_num_timeout: 30
|
data/fairseq/examples/data2vec/config/audio/pretraining/run_config/slurm_4_aws.yaml
ADDED
|
@@ -0,0 +1,37 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# @package _global_
|
| 2 |
+
|
| 3 |
+
hydra:
|
| 4 |
+
job:
|
| 5 |
+
config:
|
| 6 |
+
override_dirname:
|
| 7 |
+
kv_sep: ':'
|
| 8 |
+
item_sep: '/'
|
| 9 |
+
exclude_keys:
|
| 10 |
+
- run_config
|
| 11 |
+
- distributed_training.distributed_port
|
| 12 |
+
- distributed_training.distributed_world_size
|
| 13 |
+
- model.pretrained_model_path
|
| 14 |
+
- model.target_network_path
|
| 15 |
+
- next_script
|
| 16 |
+
- task.cache_in_scratch
|
| 17 |
+
- task.data
|
| 18 |
+
- task.post_save_script
|
| 19 |
+
- checkpoint.save_interval_updates
|
| 20 |
+
- checkpoint.keep_interval_updates
|
| 21 |
+
- checkpoint.save_on_overflow
|
| 22 |
+
- common.log_interval
|
| 23 |
+
- common.user_dir
|
| 24 |
+
sweep:
|
| 25 |
+
dir: /fsx-wav2vec/${env:USER}/${env:PREFIX}/${hydra.job.config_name}_${hydra.launcher.gpus_per_node}/${hydra.job.override_dirname}
|
| 26 |
+
subdir: ''
|
| 27 |
+
launcher:
|
| 28 |
+
submitit_folder: ${hydra.sweep.dir}
|
| 29 |
+
timeout_min: 4320
|
| 30 |
+
cpus_per_task: 10
|
| 31 |
+
gpus_per_node: 8
|
| 32 |
+
tasks_per_node: 8
|
| 33 |
+
mem_gb: 0
|
| 34 |
+
nodes: 4
|
| 35 |
+
name: ${env:PREFIX}_${hydra.job.config_name}
|
| 36 |
+
partition: wav2vec,learnlab,learnfair
|
| 37 |
+
max_num_timeout: 30
|
data/fairseq/examples/data2vec/config/audio/pretraining/run_config/slurm_6_aws.yaml
ADDED
|
@@ -0,0 +1,36 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# @package _global_
|
| 2 |
+
|
| 3 |
+
hydra:
|
| 4 |
+
job:
|
| 5 |
+
config:
|
| 6 |
+
override_dirname:
|
| 7 |
+
kv_sep: ':'
|
| 8 |
+
item_sep: '/'
|
| 9 |
+
exclude_keys:
|
| 10 |
+
- run_config
|
| 11 |
+
- distributed_training.distributed_port
|
| 12 |
+
- distributed_training.distributed_world_size
|
| 13 |
+
- model.pretrained_model_path
|
| 14 |
+
- model.target_network_path
|
| 15 |
+
- next_script
|
| 16 |
+
- task.cache_in_scratch
|
| 17 |
+
- task.data
|
| 18 |
+
- checkpoint.save_interval_updates
|
| 19 |
+
- checkpoint.keep_interval_updates
|
| 20 |
+
- checkpoint.save_on_overflow
|
| 21 |
+
- common.log_interval
|
| 22 |
+
- common.user_dir
|
| 23 |
+
sweep:
|
| 24 |
+
dir: /fsx-wav2vec/${env:USER}/${env:PREFIX}/${hydra.job.config_name}_${hydra.launcher.gpus_per_node}/${hydra.job.override_dirname}
|
| 25 |
+
subdir: ''
|
| 26 |
+
launcher:
|
| 27 |
+
submitit_folder: ${hydra.sweep.dir}
|
| 28 |
+
timeout_min: 4320
|
| 29 |
+
cpus_per_task: 10
|
| 30 |
+
gpus_per_node: 8
|
| 31 |
+
tasks_per_node: 8
|
| 32 |
+
mem_gb: 0
|
| 33 |
+
nodes: 6
|
| 34 |
+
name: ${env:PREFIX}_${hydra.job.config_name}
|
| 35 |
+
partition: wav2vec,learnlab,learnfair
|
| 36 |
+
max_num_timeout: 30
|
data/fairseq/examples/data2vec/config/audio/pretraining/run_config/slurm_8_aws.yaml
ADDED
|
@@ -0,0 +1,36 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# @package _global_
|
| 2 |
+
|
| 3 |
+
hydra:
|
| 4 |
+
job:
|
| 5 |
+
config:
|
| 6 |
+
override_dirname:
|
| 7 |
+
kv_sep: ':'
|
| 8 |
+
item_sep: '/'
|
| 9 |
+
exclude_keys:
|
| 10 |
+
- run_config
|
| 11 |
+
- distributed_training.distributed_port
|
| 12 |
+
- distributed_training.distributed_world_size
|
| 13 |
+
- model.pretrained_model_path
|
| 14 |
+
- model.target_network_path
|
| 15 |
+
- next_script
|
| 16 |
+
- task.cache_in_scratch
|
| 17 |
+
- task.data
|
| 18 |
+
- checkpoint.save_interval_updates
|
| 19 |
+
- checkpoint.keep_interval_updates
|
| 20 |
+
- checkpoint.save_on_overflow
|
| 21 |
+
- common.log_interval
|
| 22 |
+
- common.user_dir
|
| 23 |
+
sweep:
|
| 24 |
+
dir: /fsx-wav2vec/${env:USER}/${env:PREFIX}/${hydra.job.config_name}_${hydra.launcher.gpus_per_node}/${hydra.job.override_dirname}
|
| 25 |
+
subdir: ''
|
| 26 |
+
launcher:
|
| 27 |
+
submitit_folder: ${hydra.sweep.dir}
|
| 28 |
+
timeout_min: 4320
|
| 29 |
+
cpus_per_task: 10
|
| 30 |
+
gpus_per_node: 8
|
| 31 |
+
tasks_per_node: 8
|
| 32 |
+
mem_gb: 0
|
| 33 |
+
nodes: 8
|
| 34 |
+
name: ${env:PREFIX}_${hydra.job.config_name}
|
| 35 |
+
partition: wav2vec,learnlab,learnfair
|
| 36 |
+
max_num_timeout: 30
|
data/fairseq/examples/data2vec/config/text/pretraining/base.yaml
ADDED
|
@@ -0,0 +1,77 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# @package _group_
|
| 2 |
+
common:
|
| 3 |
+
fp16: true
|
| 4 |
+
log_format: json
|
| 5 |
+
log_interval: 200
|
| 6 |
+
tensorboard_logdir: tb
|
| 7 |
+
|
| 8 |
+
checkpoint:
|
| 9 |
+
no_epoch_checkpoints: true
|
| 10 |
+
save_interval_updates: 50000
|
| 11 |
+
keep_interval_updates: 1
|
| 12 |
+
|
| 13 |
+
distributed_training:
|
| 14 |
+
distributed_world_size: 16
|
| 15 |
+
ddp_backend: legacy_ddp
|
| 16 |
+
|
| 17 |
+
task:
|
| 18 |
+
_name: masked_lm
|
| 19 |
+
data: ???
|
| 20 |
+
sample_break_mode: complete_doc
|
| 21 |
+
tokens_per_sample: 512
|
| 22 |
+
include_target_tokens: true
|
| 23 |
+
random_token_prob: 0
|
| 24 |
+
leave_unmasked_prob: 0
|
| 25 |
+
mask_prob: 0.35
|
| 26 |
+
mask_multiple_length: 4
|
| 27 |
+
|
| 28 |
+
criterion: model
|
| 29 |
+
|
| 30 |
+
dataset:
|
| 31 |
+
max_tokens: 8192
|
| 32 |
+
ignore_unused_valid_subsets: true
|
| 33 |
+
skip_invalid_size_inputs_valid_test: true
|
| 34 |
+
|
| 35 |
+
optimizer:
|
| 36 |
+
_name: adam
|
| 37 |
+
weight_decay: 0.01
|
| 38 |
+
adam_betas: (0.9,0.98)
|
| 39 |
+
adam_eps: 1e-06
|
| 40 |
+
|
| 41 |
+
lr_scheduler:
|
| 42 |
+
_name: cosine
|
| 43 |
+
warmup_updates: 10000
|
| 44 |
+
|
| 45 |
+
optimization:
|
| 46 |
+
clip_norm: 5
|
| 47 |
+
lr: [0.0002]
|
| 48 |
+
max_update: 1000000
|
| 49 |
+
update_freq: [1]
|
| 50 |
+
|
| 51 |
+
model:
|
| 52 |
+
_name: data2vec_text
|
| 53 |
+
head_layers: 2
|
| 54 |
+
average_top_k_layers: 10
|
| 55 |
+
layer_norm_target_layer: true
|
| 56 |
+
loss_scale: 1
|
| 57 |
+
ema_decay: 0.999
|
| 58 |
+
ema_end_decay: 0.9999
|
| 59 |
+
ema_anneal_end_step: 300000
|
| 60 |
+
loss_beta: 4
|
| 61 |
+
ema_transformer_layers_only: true
|
| 62 |
+
|
| 63 |
+
transformer:
|
| 64 |
+
dropout: 0.1
|
| 65 |
+
attention_dropout: 0.1
|
| 66 |
+
layernorm_embedding: true
|
| 67 |
+
activation_fn: gelu
|
| 68 |
+
no_scale_embedding: true
|
| 69 |
+
max_source_positions: 512
|
| 70 |
+
encoder:
|
| 71 |
+
embed_dim: 768
|
| 72 |
+
ffn_embed_dim: 3072
|
| 73 |
+
layers: 12
|
| 74 |
+
attention_heads: 12
|
| 75 |
+
normalize_before: false
|
| 76 |
+
learned_pos: true
|
| 77 |
+
layerdrop: 0
|
data/fairseq/examples/data2vec/config/text/pretraining/run_config/local.yaml
ADDED
|
@@ -0,0 +1,15 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# @package _global_
|
| 2 |
+
hydra:
|
| 3 |
+
sweep:
|
| 4 |
+
dir: ${env:PWD}/tmp_dbg/${now:%H-%M-%S}
|
| 5 |
+
|
| 6 |
+
distributed_training:
|
| 7 |
+
distributed_world_size: 1
|
| 8 |
+
nprocs_per_node: 1
|
| 9 |
+
distributed_port: -1
|
| 10 |
+
|
| 11 |
+
common:
|
| 12 |
+
log_interval: 1
|
| 13 |
+
|
| 14 |
+
dataset:
|
| 15 |
+
num_workers: 0
|
data/fairseq/examples/data2vec/config/text/pretraining/run_config/slurm_1_aws.yaml
ADDED
|
@@ -0,0 +1,37 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# @package _global_
|
| 2 |
+
|
| 3 |
+
hydra:
|
| 4 |
+
job:
|
| 5 |
+
config:
|
| 6 |
+
override_dirname:
|
| 7 |
+
kv_sep: '_'
|
| 8 |
+
item_sep: '/'
|
| 9 |
+
exclude_keys:
|
| 10 |
+
- run_config
|
| 11 |
+
- distributed_training.distributed_port
|
| 12 |
+
- distributed_training.distributed_world_size
|
| 13 |
+
- model.pretrained_model_path
|
| 14 |
+
- model.target_network_path
|
| 15 |
+
- next_script
|
| 16 |
+
- task.cache_in_scratch
|
| 17 |
+
- task.data
|
| 18 |
+
- checkpoint.save_interval_updates
|
| 19 |
+
- checkpoint.keep_interval_updates
|
| 20 |
+
- checkpoint.save_on_overflow
|
| 21 |
+
- common.log_interval
|
| 22 |
+
- common.user_dir
|
| 23 |
+
sweep:
|
| 24 |
+
dir: /fsx-wav2vec/${env:USER}/${env:PREFIX}/${hydra.job.config_name}_${hydra.launcher.gpus_per_node}/${hydra.job.override_dirname}
|
| 25 |
+
subdir: ''
|
| 26 |
+
launcher:
|
| 27 |
+
submitit_folder: ${hydra.sweep.dir}/submitit
|
| 28 |
+
timeout_min: 4320
|
| 29 |
+
cpus_per_task: 80
|
| 30 |
+
gpus_per_node: 8
|
| 31 |
+
tasks_per_node: 1
|
| 32 |
+
mem_gb: 0
|
| 33 |
+
nodes: 1
|
| 34 |
+
name: ${env:PREFIX}_${hydra.job.config_name}
|
| 35 |
+
partition: wav2vec
|
| 36 |
+
max_num_timeout: 30
|
| 37 |
+
exclude: a100-st-p4d24xlarge-471
|
data/fairseq/examples/data2vec/config/text/pretraining/run_config/slurm_2.yaml
ADDED
|
@@ -0,0 +1,37 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# @package _global_
|
| 2 |
+
|
| 3 |
+
hydra:
|
| 4 |
+
job:
|
| 5 |
+
config:
|
| 6 |
+
override_dirname:
|
| 7 |
+
kv_sep: ':'
|
| 8 |
+
item_sep: '/'
|
| 9 |
+
exclude_keys:
|
| 10 |
+
- run_config
|
| 11 |
+
- distributed_training.distributed_port
|
| 12 |
+
- distributed_training.distributed_world_size
|
| 13 |
+
- model.pretrained_model_path
|
| 14 |
+
- model.target_network_path
|
| 15 |
+
- next_script
|
| 16 |
+
- task.cache_in_scratch
|
| 17 |
+
- task.data
|
| 18 |
+
- checkpoint.save_interval_updates
|
| 19 |
+
- checkpoint.keep_interval_updates
|
| 20 |
+
- checkpoint.save_on_overflow
|
| 21 |
+
- common.log_interval
|
| 22 |
+
- common.user_dir
|
| 23 |
+
sweep:
|
| 24 |
+
dir: /checkpoint/${env:USER}/${env:PREFIX}/${hydra.job.config_name}_${hydra.launcher.gpus_per_node}/${hydra.job.override_dirname}
|
| 25 |
+
subdir: ''
|
| 26 |
+
launcher:
|
| 27 |
+
submitit_folder: ${hydra.sweep.dir}
|
| 28 |
+
timeout_min: 4320
|
| 29 |
+
cpus_per_task: 80
|
| 30 |
+
gpus_per_node: 8
|
| 31 |
+
tasks_per_node: 1
|
| 32 |
+
mem_gb: 450
|
| 33 |
+
nodes: 2
|
| 34 |
+
name: ${env:PREFIX}_${hydra.job.config_name}
|
| 35 |
+
partition: devlab,learnlab,learnfair,scavenge
|
| 36 |
+
constraint: volta32gb,ib4
|
| 37 |
+
max_num_timeout: 30
|
data/fairseq/examples/data2vec/config/text/pretraining/run_config/slurm_2_aws.yaml
ADDED
|
@@ -0,0 +1,37 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# @package _global_
|
| 2 |
+
|
| 3 |
+
hydra:
|
| 4 |
+
job:
|
| 5 |
+
config:
|
| 6 |
+
override_dirname:
|
| 7 |
+
kv_sep: '_'
|
| 8 |
+
item_sep: '/'
|
| 9 |
+
exclude_keys:
|
| 10 |
+
- run_config
|
| 11 |
+
- distributed_training.distributed_port
|
| 12 |
+
- distributed_training.distributed_world_size
|
| 13 |
+
- model.pretrained_model_path
|
| 14 |
+
- model.target_network_path
|
| 15 |
+
- next_script
|
| 16 |
+
- task.cache_in_scratch
|
| 17 |
+
- task.data
|
| 18 |
+
- checkpoint.save_interval_updates
|
| 19 |
+
- checkpoint.keep_interval_updates
|
| 20 |
+
- checkpoint.save_on_overflow
|
| 21 |
+
- common.log_interval
|
| 22 |
+
- common.user_dir
|
| 23 |
+
sweep:
|
| 24 |
+
dir: /fsx-wav2vec/${env:USER}/${env:PREFIX}/${hydra.job.config_name}_${hydra.launcher.gpus_per_node}/${hydra.job.override_dirname}
|
| 25 |
+
subdir: ''
|
| 26 |
+
launcher:
|
| 27 |
+
submitit_folder: ${hydra.sweep.dir}/submitit
|
| 28 |
+
timeout_min: 4320
|
| 29 |
+
cpus_per_task: 10
|
| 30 |
+
gpus_per_node: 8
|
| 31 |
+
tasks_per_node: 8
|
| 32 |
+
mem_gb: 0
|
| 33 |
+
nodes: 2
|
| 34 |
+
name: ${env:PREFIX}_${hydra.job.config_name}
|
| 35 |
+
partition: wav2vec
|
| 36 |
+
max_num_timeout: 30
|
| 37 |
+
exclude: a100-st-p4d24xlarge-471
|
data/fairseq/examples/data2vec/config/text/pretraining/run_config/slurm_3.yaml
ADDED
|
@@ -0,0 +1,36 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# @package _global_
|
| 2 |
+
|
| 3 |
+
hydra:
|
| 4 |
+
job:
|
| 5 |
+
config:
|
| 6 |
+
override_dirname:
|
| 7 |
+
kv_sep: ':'
|
| 8 |
+
item_sep: '/'
|
| 9 |
+
exclude_keys:
|
| 10 |
+
- run_config
|
| 11 |
+
- distributed_training.distributed_port
|
| 12 |
+
- distributed_training.distributed_world_size
|
| 13 |
+
- model.pretrained_model_path
|
| 14 |
+
- model.target_network_path
|
| 15 |
+
- next_script
|
| 16 |
+
- task.cache_in_scratch
|
| 17 |
+
- task.data
|
| 18 |
+
- checkpoint.save_interval_updates
|
| 19 |
+
- checkpoint.keep_interval_updates
|
| 20 |
+
- checkpoint.save_on_overflow
|
| 21 |
+
- common.log_interval
|
| 22 |
+
sweep:
|
| 23 |
+
dir: /checkpoint/${env:USER}/${env:PREFIX}/${hydra.job.config_name}_${hydra.launcher.gpus_per_node}/${hydra.job.override_dirname}
|
| 24 |
+
subdir: ''
|
| 25 |
+
launcher:
|
| 26 |
+
submitit_folder: ${hydra.sweep.dir}
|
| 27 |
+
timeout_min: 4320
|
| 28 |
+
cpus_per_task: 10
|
| 29 |
+
gpus_per_node: 8
|
| 30 |
+
tasks_per_node: 8
|
| 31 |
+
mem_gb: 450
|
| 32 |
+
nodes: 3
|
| 33 |
+
name: ${env:PREFIX}_${hydra.job.config_name}
|
| 34 |
+
partition: devlab,learnlab,learnfair,scavenge
|
| 35 |
+
constraint: volta32gb,ib4
|
| 36 |
+
max_num_timeout: 30
|