sonithoiningombam commited on
Commit
3bac3f1
·
verified ·
1 Parent(s): 99b970a

Add files using upload-large-folder tool

Browse files
This view is limited to 50 files because it contains too many changes.   See raw diff
Files changed (50) hide show
  1. data/fairseq/examples/.gitignore +2 -0
  2. data/fairseq/examples/__init__.py +9 -0
  3. data/fairseq/examples/bart/README.glue.md +99 -0
  4. data/fairseq/examples/bart/README.md +228 -0
  5. data/fairseq/examples/bart/README.summarization.md +102 -0
  6. data/fairseq/examples/bart/summarize.py +100 -0
  7. data/fairseq/fairseq_cli/__init__.py +0 -0
  8. data/fairseq/fairseq_cli/__pycache__/__init__.cpython-38.pyc +0 -0
  9. data/fairseq/fairseq_cli/__pycache__/generate.cpython-38.pyc +0 -0
  10. data/fairseq/fairseq_cli/__pycache__/preprocess.cpython-38.pyc +0 -0
  11. data/fairseq/fairseq_cli/__pycache__/train.cpython-38.pyc +0 -0
  12. data/fairseq/fairseq_cli/eval_lm.py +347 -0
  13. data/fairseq/fairseq_cli/generate.py +417 -0
  14. data/fairseq/fairseq_cli/hydra_train.py +91 -0
  15. data/fairseq/fairseq_cli/hydra_validate.py +188 -0
  16. data/fairseq/fairseq_cli/interactive.py +317 -0
  17. data/fairseq/fairseq_cli/preprocess.py +393 -0
  18. data/fairseq/fairseq_cli/score.py +102 -0
  19. data/fairseq/fairseq_cli/train.py +581 -0
  20. data/fairseq/fairseq_cli/validate.py +153 -0
  21. data/fairseq/tests/distributed/__init__.py +0 -0
  22. data/fairseq/tests/distributed/test_bmuf.py +204 -0
  23. data/fairseq/tests/distributed/test_distributed_timeout_wrapper.py +52 -0
  24. data/fairseq/tests/distributed/test_module_proxy_wrapper.py +74 -0
  25. data/fairseq/tests/distributed/test_utils.py +124 -0
  26. data/fairseq/tests/distributed/utils.py +65 -0
  27. data/fairseq/tests/gpu/__init__.py +0 -0
  28. data/fairseq/tests/gpu/test_binaries_gpu.py +590 -0
  29. data/fairseq/tests/gpu/test_ema_gpu.py +215 -0
  30. data/fairseq/tests/gpu/transformer_quantization_config.yaml +28 -0
  31. data/fairseq/tests/speech/__init__.py +210 -0
  32. data/fairseq/tests/speech/test_convtransformer_simul_trans.py +33 -0
  33. data/fairseq/tests/speech/test_dual_input_wav_transformer.py +76 -0
  34. data/fairseq/tests/speech/test_dualinput_s2t_transformer.py +110 -0
  35. data/fairseq/tests/speech/test_fastspeech2.py +53 -0
  36. data/fairseq/tests/speech/test_s2s_transformer.py +51 -0
  37. data/fairseq/tests/speech/test_s2t_conformer.py +23 -0
  38. data/fairseq/tests/speech/test_s2t_transformer.py +23 -0
  39. data/fairseq/tests/speech/test_tts_transformer.py +53 -0
  40. data/fairseq/tests/speech/test_wav2vec2.py +90 -0
  41. data/fairseq/tests/speech/test_xm_transformer.py +29 -0
  42. data/fairseq/tests/speech_recognition/__init__.py +0 -0
  43. data/fairseq/tests/speech_recognition/asr_test_base.py +557 -0
  44. data/fairseq/tests/speech_recognition/test_collaters.py +58 -0
  45. data/fairseq/tests/speech_recognition/test_cross_entropy.py +37 -0
  46. data/fairseq/tests/speech_recognition/test_data_utils.py +62 -0
  47. data/fairseq/tests/speech_recognition/test_vggtransformer.py +135 -0
  48. data/fairseq/tests/tasks/test_denoising.py +96 -0
  49. data/fairseq/tests/tasks/test_masked_lm.py +78 -0
  50. data/fairseq/tests/tasks/test_multilingual_denoising.py +98 -0
data/fairseq/examples/.gitignore ADDED
@@ -0,0 +1,2 @@
 
 
 
1
+ !*/*.sh
2
+ !*/*.md
data/fairseq/examples/__init__.py ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright (c) Facebook, Inc. and its affiliates.
2
+ #
3
+ # This source code is licensed under the MIT license found in the
4
+ # LICENSE file in the root directory of this source tree.
5
+
6
+ try:
7
+ from fairseq.version import __version__ # noqa
8
+ except ImportError:
9
+ pass
data/fairseq/examples/bart/README.glue.md ADDED
@@ -0,0 +1,99 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Fine-tuning BART on GLUE tasks
2
+
3
+ ### 1) Download the data from GLUE website (https://gluebenchmark.com/tasks) using following commands:
4
+ ```bash
5
+ wget https://gist.githubusercontent.com/W4ngatang/60c2bdb54d156a41194446737ce03e2e/raw/17b8dd0d724281ed7c3b2aeeda662b92809aadd5/download_glue_data.py
6
+ python download_glue_data.py --data_dir glue_data --tasks all
7
+ ```
8
+
9
+ ### 2) Preprocess GLUE task data (same as RoBERTa):
10
+ ```bash
11
+ ./examples/roberta/preprocess_GLUE_tasks.sh glue_data <glue_task_name>
12
+ ```
13
+ `glue_task_name` is one of the following:
14
+ `{ALL, QQP, MNLI, QNLI, MRPC, RTE, STS-B, SST-2, CoLA}`
15
+ Use `ALL` for preprocessing all the glue tasks.
16
+
17
+ ### 3) Fine-tuning on GLUE task:
18
+ Example fine-tuning cmd for `RTE` task
19
+ ```bash
20
+ TOTAL_NUM_UPDATES=2036 # 10 epochs through RTE for bsz 16
21
+ WARMUP_UPDATES=61 # 6 percent of the number of updates
22
+ LR=1e-05 # Peak LR for polynomial LR scheduler.
23
+ NUM_CLASSES=2
24
+ MAX_SENTENCES=16 # Batch size.
25
+ BART_PATH=/path/to/bart/model.pt
26
+
27
+ CUDA_VISIBLE_DEVICES=0,1 fairseq-train RTE-bin/ \
28
+ --restore-file $BART_PATH \
29
+ --batch-size $MAX_SENTENCES \
30
+ --max-tokens 4400 \
31
+ --task sentence_prediction \
32
+ --add-prev-output-tokens \
33
+ --layernorm-embedding \
34
+ --share-all-embeddings \
35
+ --share-decoder-input-output-embed \
36
+ --reset-optimizer --reset-dataloader --reset-meters \
37
+ --required-batch-size-multiple 1 \
38
+ --init-token 0 \
39
+ --arch bart_large \
40
+ --criterion sentence_prediction \
41
+ --num-classes $NUM_CLASSES \
42
+ --dropout 0.1 --attention-dropout 0.1 \
43
+ --weight-decay 0.01 --optimizer adam --adam-betas "(0.9, 0.98)" --adam-eps 1e-08 \
44
+ --clip-norm 0.0 \
45
+ --lr-scheduler polynomial_decay --lr $LR --total-num-update $TOTAL_NUM_UPDATES --warmup-updates $WARMUP_UPDATES \
46
+ --fp16 --fp16-init-scale 4 --threshold-loss-scale 1 --fp16-scale-window 128 \
47
+ --max-epoch 10 \
48
+ --find-unused-parameters \
49
+ --best-checkpoint-metric accuracy --maximize-best-checkpoint-metric;
50
+ ```
51
+
52
+ For each of the GLUE task, you will need to use following cmd-line arguments:
53
+
54
+ Model | MNLI | QNLI | QQP | RTE | SST-2 | MRPC | CoLA | STS-B
55
+ ---|---|---|---|---|---|---|---|---
56
+ `--num-classes` | 3 | 2 | 2 | 2 | 2 | 2 | 2 | 1
57
+ `--lr` | 5e-6 | 1e-5 | 1e-5 | 1e-5 | 5e-6 | 2e-5 | 2e-5 | 2e-5
58
+ `bsz` | 128 | 32 | 32 | 32 | 128 | 64 | 64 | 32
59
+ `--total-num-update` | 30968 | 33112 | 113272 | 1018 | 5233 | 1148 | 1334 | 1799
60
+ `--warmup-updates` | 1858 | 1986 | 6796 | 61 | 314 | 68 | 80 | 107
61
+
62
+ For `STS-B` additionally add `--regression-target --best-checkpoint-metric loss` and remove `--maximize-best-checkpoint-metric`.
63
+
64
+ **Note:**
65
+
66
+ a) `--total-num-updates` is used by `--polynomial_decay` scheduler and is calculated for `--max-epoch=10` and `--batch-size=32/64/128` depending on the task.
67
+
68
+ b) Above cmd-args and hyperparams are tested on Nvidia `V100` GPU with `32gb` of memory for each task. Depending on the GPU memory resources available to you, you can use increase `--update-freq` and reduce `--batch-size`.
69
+
70
+ ### Inference on GLUE task
71
+ After training the model as mentioned in previous step, you can perform inference with checkpoints in `checkpoints/` directory using following python code snippet:
72
+
73
+ ```python
74
+ from fairseq.models.bart import BARTModel
75
+
76
+ bart = BARTModel.from_pretrained(
77
+ 'checkpoints/',
78
+ checkpoint_file='checkpoint_best.pt',
79
+ data_name_or_path='RTE-bin'
80
+ )
81
+
82
+ label_fn = lambda label: bart.task.label_dictionary.string(
83
+ [label + bart.task.label_dictionary.nspecial]
84
+ )
85
+ ncorrect, nsamples = 0, 0
86
+ bart.cuda()
87
+ bart.eval()
88
+ with open('glue_data/RTE/dev.tsv') as fin:
89
+ fin.readline()
90
+ for index, line in enumerate(fin):
91
+ tokens = line.strip().split('\t')
92
+ sent1, sent2, target = tokens[1], tokens[2], tokens[3]
93
+ tokens = bart.encode(sent1, sent2)
94
+ prediction = bart.predict('sentence_classification_head', tokens).argmax().item()
95
+ prediction_label = label_fn(prediction)
96
+ ncorrect += int(prediction_label == target)
97
+ nsamples += 1
98
+ print('| Accuracy: ', float(ncorrect)/float(nsamples))
99
+ ```
data/fairseq/examples/bart/README.md ADDED
@@ -0,0 +1,228 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # BART: Denoising Sequence-to-Sequence Pre-training for Natural Language Generation, Translation, and Comprehension
2
+
3
+ [https://arxiv.org/abs/1910.13461](https://arxiv.org/abs/1910.13461)
4
+
5
+ ## Introduction
6
+
7
+ BART is sequence-to-sequence model trained with denoising as pretraining objective. We show that this pretraining objective is more generic and show that we can match [RoBERTa](../roberta) results on SQuAD and GLUE and gain state-of-the-art results on summarization (XSum, CNN dataset), long form generative question answering (ELI5) and dialog response genration (ConvAI2). See the associated paper for more details.
8
+
9
+ ## Pre-trained models
10
+
11
+ Model | Description | # params | Download
12
+ ---|---|---|---
13
+ `bart.base` | BART model with 6 encoder and decoder layers | 140M | [bart.base.tar.gz](https://dl.fbaipublicfiles.com/fairseq/models/bart.base.tar.gz)
14
+ `bart.large` | BART model with 12 encoder and decoder layers | 400M | [bart.large.tar.gz](https://dl.fbaipublicfiles.com/fairseq/models/bart.large.tar.gz)
15
+ `bart.large.mnli` | `bart.large` finetuned on `MNLI` | 400M | [bart.large.mnli.tar.gz](https://dl.fbaipublicfiles.com/fairseq/models/bart.large.mnli.tar.gz)
16
+ `bart.large.cnn` | `bart.large` finetuned on `CNN-DM` | 400M | [bart.large.cnn.tar.gz](https://dl.fbaipublicfiles.com/fairseq/models/bart.large.cnn.tar.gz)
17
+ `bart.large.xsum` | `bart.large` finetuned on `Xsum` | 400M | [bart.large.xsum.tar.gz](https://dl.fbaipublicfiles.com/fairseq/models/bart.large.xsum.tar.gz)
18
+
19
+ ## Results
20
+
21
+ **[GLUE (Wang et al., 2019)](https://gluebenchmark.com/)**
22
+ _(dev set, single model, single-task finetuning)_
23
+
24
+ Model | MNLI | QNLI | QQP | RTE | SST-2 | MRPC | CoLA | STS-B
25
+ ---|---|---|---|---|---|---|---|---
26
+ `roberta.large` | 90.2 | 94.7 | 92.2 | 86.6 | 96.4 | 90.9 | 68.0 | 92.4
27
+ `bart.large` | 89.9 | 94.9 | 92.5 | 87.0 | 96.6 | 90.4 | 62.8 | 91.2
28
+
29
+ **[SQuAD (Rajpurkar et al., 2018)](https://rajpurkar.github.io/SQuAD-explorer/)**
30
+ _(dev set, no additional data used)_
31
+
32
+ Model | SQuAD 1.1 EM/F1 | SQuAD 2.0 EM/F1
33
+ ---|---|---
34
+ `roberta.large` | 88.9/94.6 | 86.5/89.4
35
+ `bart.large` | 88.8/94.6 | 86.1/89.2
36
+
37
+ **[CNN/Daily Mail](http://nlpprogress.com/english/summarization.html)**
38
+ _(test set, no additional data used)_
39
+
40
+ Model | R1 | R2 | RL
41
+ ---|---|---|---
42
+ `BERTSUMEXTABS` | 42.13 | 19.60 | 39.18
43
+ `bart.large` | 44.16 | 21.28 | 40.90
44
+
45
+ ## Example usage
46
+
47
+ ##### Load BART from torch.hub (PyTorch >= 1.1):
48
+ ```python
49
+ import torch
50
+ bart = torch.hub.load('pytorch/fairseq', 'bart.large')
51
+ bart.eval() # disable dropout (or leave in train mode to finetune)
52
+ ```
53
+
54
+ ##### Load BART (for PyTorch 1.0 or custom models):
55
+ ```python
56
+ # Download bart.large model
57
+ wget https://dl.fbaipublicfiles.com/fairseq/models/bart.large.tar.gz
58
+ tar -xzvf bart.large.tar.gz
59
+
60
+ # Load the model in fairseq
61
+ from fairseq.models.bart import BARTModel
62
+ bart = BARTModel.from_pretrained('/path/to/bart.large', checkpoint_file='model.pt')
63
+ bart.eval() # disable dropout (or leave in train mode to finetune)
64
+ ```
65
+
66
+ ##### Apply Byte-Pair Encoding (BPE) to input text:
67
+ ```python
68
+ tokens = bart.encode('Hello world!')
69
+ assert tokens.tolist() == [0, 31414, 232, 328, 2]
70
+ bart.decode(tokens) # 'Hello world!'
71
+ ```
72
+
73
+ ##### Extract features from BART:
74
+ ```python
75
+ # Extract the last layer's features
76
+ last_layer_features = bart.extract_features(tokens)
77
+ assert last_layer_features.size() == torch.Size([1, 5, 1024])
78
+
79
+ # Extract all layer's features from decoder (layer 0 is the embedding layer)
80
+ all_layers = bart.extract_features(tokens, return_all_hiddens=True)
81
+ assert len(all_layers) == 13
82
+ assert torch.all(all_layers[-1] == last_layer_features)
83
+ ```
84
+
85
+ ##### Use BART for sentence-pair classification tasks:
86
+ ```python
87
+ # Download BART already finetuned for MNLI
88
+ bart = torch.hub.load('pytorch/fairseq', 'bart.large.mnli')
89
+ bart.eval() # disable dropout for evaluation
90
+
91
+ # Encode a pair of sentences and make a prediction
92
+ tokens = bart.encode('BART is a seq2seq model.', 'BART is not sequence to sequence.')
93
+ bart.predict('mnli', tokens).argmax() # 0: contradiction
94
+
95
+ # Encode another pair of sentences
96
+ tokens = bart.encode('BART is denoising autoencoder.', 'BART is version of autoencoder.')
97
+ bart.predict('mnli', tokens).argmax() # 2: entailment
98
+ ```
99
+
100
+ ##### Register a new (randomly initialized) classification head:
101
+ ```python
102
+ bart.register_classification_head('new_task', num_classes=3)
103
+ logprobs = bart.predict('new_task', tokens)
104
+ ```
105
+
106
+ ##### Batched prediction:
107
+ ```python
108
+ import torch
109
+ from fairseq.data.data_utils import collate_tokens
110
+
111
+ bart = torch.hub.load('pytorch/fairseq', 'bart.large.mnli')
112
+ bart.eval()
113
+
114
+ batch_of_pairs = [
115
+ ['BART is a seq2seq model.', 'BART is not sequence to sequence.'],
116
+ ['BART is denoising autoencoder.', 'BART is version of autoencoder.'],
117
+ ]
118
+
119
+ batch = collate_tokens(
120
+ [bart.encode(pair[0], pair[1]) for pair in batch_of_pairs], pad_idx=1
121
+ )
122
+
123
+ logprobs = bart.predict('mnli', batch)
124
+ print(logprobs.argmax(dim=1))
125
+ # tensor([0, 2])
126
+ ```
127
+
128
+ ##### Using the GPU:
129
+ ```python
130
+ bart.cuda()
131
+ bart.predict('new_task', tokens)
132
+ ```
133
+
134
+ #### Filling masks:
135
+
136
+ BART can be used to fill multiple `<mask>` tokens in the input.
137
+ ```python
138
+ bart = torch.hub.load('pytorch/fairseq', 'bart.base')
139
+ bart.eval()
140
+ bart.fill_mask(['The cat <mask> on the <mask>.'], topk=3, beam=10)
141
+ # [[('The cat was on the ground.', tensor(-0.6183)), ('The cat was on the floor.', tensor(-0.6798)), ('The cat sleeps on the couch.', tensor(-0.6830))]]
142
+ ```
143
+
144
+ Note that by default we enforce the output length to match the input length.
145
+ This can be disabled by setting ``match_source_len=False``:
146
+ ```
147
+ bart.fill_mask(['The cat <mask> on the <mask>.'], topk=3, beam=10, match_source_len=False)
148
+ # [[('The cat was on the ground.', tensor(-0.6185)), ('The cat was asleep on the couch.', tensor(-0.6276)), ('The cat was on the floor.', tensor(-0.6800))]]
149
+ ```
150
+
151
+ Example code to fill masks for a batch of sentences using GPU
152
+ ```
153
+ bart.cuda()
154
+ bart.fill_mask(['The cat <mask> on the <mask>.', 'The dog <mask> on the <mask>.'], topk=3, beam=10)
155
+ # [[('The cat was on the ground.', tensor(-0.6183)), ('The cat was on the floor.', tensor(-0.6798)), ('The cat sleeps on the couch.', tensor(-0.6830))], [('The dog was on the ground.', tensor(-0.6190)), ('The dog lay on the ground.', tensor(-0.6711)),
156
+ ('The dog was asleep on the couch', tensor(-0.6796))]]
157
+ ```
158
+
159
+ #### Evaluating the `bart.large.mnli` model:
160
+
161
+ Example python code snippet to evaluate accuracy on the MNLI `dev_matched` set.
162
+ ```python
163
+ label_map = {0: 'contradiction', 1: 'neutral', 2: 'entailment'}
164
+ ncorrect, nsamples = 0, 0
165
+ bart.cuda()
166
+ bart.eval()
167
+ with open('glue_data/MNLI/dev_matched.tsv') as fin:
168
+ fin.readline()
169
+ for index, line in enumerate(fin):
170
+ tokens = line.strip().split('\t')
171
+ sent1, sent2, target = tokens[8], tokens[9], tokens[-1]
172
+ tokens = bart.encode(sent1, sent2)
173
+ prediction = bart.predict('mnli', tokens).argmax().item()
174
+ prediction_label = label_map[prediction]
175
+ ncorrect += int(prediction_label == target)
176
+ nsamples += 1
177
+ print('| Accuracy: ', float(ncorrect)/float(nsamples))
178
+ # Expected output: 0.9010
179
+ ```
180
+
181
+ #### Evaluating the `bart.large.cnn` model:
182
+ - Follow instructions [here](https://github.com/abisee/cnn-dailymail) to download and process into data-files such that `test.source` and `test.target` has one line for each non-tokenized sample.
183
+ - For simpler preprocessing, you can also `wget https://cdn-datasets.huggingface.co/summarization/cnn_dm_v2.tgz`, although there is no guarantee of identical scores
184
+ - `huggingface/transformers` has a simpler interface that supports [single-gpu](https://github.com/huggingface/transformers/blob/master/examples/legacy/seq2seq/run_eval.py) and [multi-gpu](https://github.com/huggingface/transformers/blob/master/examples/legacy/seq2seq/run_distributed_eval.py) beam search.
185
+ In `huggingface/transformers`, the BART models' paths are `facebook/bart-large-cnn` and `facebook/bart-large-xsum`.
186
+
187
+ In `fairseq`, summaries can be generated using:
188
+
189
+ ```bash
190
+ cp data-bin/cnn_dm/dict.source.txt checkpoints/
191
+ python examples/bart/summarize.py \
192
+ --model-dir pytorch/fairseq \
193
+ --model-file bart.large.cnn \
194
+ --src cnn_dm/test.source \
195
+ --out cnn_dm/test.hypo
196
+ ```
197
+
198
+ For calculating rouge, install `files2rouge` from [here](https://github.com/pltrdy/files2rouge).
199
+
200
+ ```bash
201
+ export CLASSPATH=/path/to/stanford-corenlp-full-2016-10-31/stanford-corenlp-3.7.0.jar
202
+
203
+ # Tokenize hypothesis and target files.
204
+ cat test.hypo | java edu.stanford.nlp.process.PTBTokenizer -ioFileList -preserveLines > test.hypo.tokenized
205
+ cat test.target | java edu.stanford.nlp.process.PTBTokenizer -ioFileList -preserveLines > test.hypo.target
206
+ files2rouge test.hypo.tokenized test.hypo.target
207
+ # Expected output: (ROUGE-2 Average_F: 0.21238)
208
+ ```
209
+
210
+
211
+ ## Finetuning
212
+
213
+ - [Finetuning on GLUE](README.glue.md)
214
+ - [Finetuning on CNN-DM](README.summarization.md)
215
+
216
+ ## Citation
217
+
218
+ ```bibtex
219
+ @article{lewis2019bart,
220
+ title = {BART: Denoising Sequence-to-Sequence Pre-training for Natural
221
+ Language Generation, Translation, and Comprehension},
222
+ author = {Mike Lewis and Yinhan Liu and Naman Goyal and Marjan Ghazvininejad and
223
+ Abdelrahman Mohamed and Omer Levy and Veselin Stoyanov
224
+ and Luke Zettlemoyer },
225
+ journal={arXiv preprint arXiv:1910.13461},
226
+ year = {2019},
227
+ }
228
+ ```
data/fairseq/examples/bart/README.summarization.md ADDED
@@ -0,0 +1,102 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Fine-tuning BART on CNN-Dailymail summarization task
2
+
3
+ ### 1) Download the CNN and Daily Mail data and preprocess it into data files with non-tokenized cased samples.
4
+
5
+ Follow the instructions [here](https://github.com/abisee/cnn-dailymail) to download the original CNN and Daily Mail datasets. To preprocess the data, refer to the pointers in [this issue](https://github.com/pytorch/fairseq/issues/1391) or check out the code [here](https://github.com/artmatsak/cnn-dailymail).
6
+
7
+ Follow the instructions [here](https://github.com/EdinburghNLP/XSum) to download the original Extreme Summarization datasets, or check out the code [here](https://github.com/EdinburghNLP/XSum/tree/master/XSum-Dataset), Please keep the raw dataset and make sure no tokenization nor BPE on the dataset.
8
+
9
+ ### 2) BPE preprocess:
10
+
11
+ ```bash
12
+ wget -N 'https://dl.fbaipublicfiles.com/fairseq/gpt2_bpe/encoder.json'
13
+ wget -N 'https://dl.fbaipublicfiles.com/fairseq/gpt2_bpe/vocab.bpe'
14
+ wget -N 'https://dl.fbaipublicfiles.com/fairseq/gpt2_bpe/dict.txt'
15
+
16
+ TASK=cnn_dm
17
+ for SPLIT in train val
18
+ do
19
+ for LANG in source target
20
+ do
21
+ python -m examples.roberta.multiprocessing_bpe_encoder \
22
+ --encoder-json encoder.json \
23
+ --vocab-bpe vocab.bpe \
24
+ --inputs "$TASK/$SPLIT.$LANG" \
25
+ --outputs "$TASK/$SPLIT.bpe.$LANG" \
26
+ --workers 60 \
27
+ --keep-empty;
28
+ done
29
+ done
30
+ ```
31
+
32
+ ### 3) Binarize dataset:
33
+ ```bash
34
+ fairseq-preprocess \
35
+ --source-lang "source" \
36
+ --target-lang "target" \
37
+ --trainpref "${TASK}/train.bpe" \
38
+ --validpref "${TASK}/val.bpe" \
39
+ --destdir "${TASK}-bin/" \
40
+ --workers 60 \
41
+ --srcdict dict.txt \
42
+ --tgtdict dict.txt;
43
+ ```
44
+
45
+ ### 4) Fine-tuning on CNN-DM summarization task:
46
+ Example fine-tuning CNN-DM
47
+ ```bash
48
+ TOTAL_NUM_UPDATES=20000
49
+ WARMUP_UPDATES=500
50
+ LR=3e-05
51
+ MAX_TOKENS=2048
52
+ UPDATE_FREQ=4
53
+ BART_PATH=/path/to/bart/model.pt
54
+
55
+ CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 fairseq-train cnn_dm-bin \
56
+ --restore-file $BART_PATH \
57
+ --max-tokens $MAX_TOKENS \
58
+ --task translation \
59
+ --source-lang source --target-lang target \
60
+ --truncate-source \
61
+ --layernorm-embedding \
62
+ --share-all-embeddings \
63
+ --share-decoder-input-output-embed \
64
+ --reset-optimizer --reset-dataloader --reset-meters \
65
+ --required-batch-size-multiple 1 \
66
+ --arch bart_large \
67
+ --criterion label_smoothed_cross_entropy \
68
+ --label-smoothing 0.1 \
69
+ --dropout 0.1 --attention-dropout 0.1 \
70
+ --weight-decay 0.01 --optimizer adam --adam-betas "(0.9, 0.999)" --adam-eps 1e-08 \
71
+ --clip-norm 0.1 \
72
+ --lr-scheduler polynomial_decay --lr $LR --total-num-update $TOTAL_NUM_UPDATES --warmup-updates $WARMUP_UPDATES \
73
+ --fp16 --update-freq $UPDATE_FREQ \
74
+ --skip-invalid-size-inputs-valid-test \
75
+ --find-unused-parameters;
76
+ ```
77
+ Above is expected to run on `1` node with `8 32gb-V100`.
78
+ Expected training time is about `5 hours`. Training time can be reduced with distributed training on `4` nodes and `--update-freq 1`.
79
+
80
+ Use TOTAL_NUM_UPDATES=15000 UPDATE_FREQ=2 for Xsum task
81
+
82
+ ### Inference for CNN-DM test data using above trained checkpoint.
83
+ After training the model as mentioned in previous step, you can perform inference with checkpoints in `checkpoints/` directory using `eval_cnn.py`, for example
84
+
85
+ ```bash
86
+ cp data-bin/cnn_dm/dict.source.txt checkpoints/
87
+ python examples/bart/summarize.py \
88
+ --model-dir checkpoints \
89
+ --model-file checkpoint_best.pt \
90
+ --src cnn_dm/test.source \
91
+ --out cnn_dm/test.hypo
92
+ ```
93
+ For XSUM, which uses beam=6, lenpen=1.0, max_len_b=60, min_len=10:
94
+ ```bash
95
+ cp data-bin/cnn_dm/dict.source.txt checkpoints/
96
+ python examples/bart/summarize.py \
97
+ --model-dir checkpoints \
98
+ --model-file checkpoint_best.pt \
99
+ --src cnn_dm/test.source \
100
+ --out cnn_dm/test.hypo \
101
+ --xsum-kwargs
102
+ ```
data/fairseq/examples/bart/summarize.py ADDED
@@ -0,0 +1,100 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright (c) Facebook, Inc. and its affiliates.
2
+ #
3
+ # This source code is licensed under the MIT license found in the
4
+ # LICENSE file in the root directory of this source tree.
5
+
6
+ import torch
7
+ from fairseq.models.bart import BARTModel
8
+ import argparse
9
+
10
+ XSUM_KWARGS = dict(beam=6, lenpen=1.0, max_len_b=60, min_len=10, no_repeat_ngram_size=3)
11
+ CNN_KWARGS = dict(beam=4, lenpen=2.0, max_len_b=140, min_len=55, no_repeat_ngram_size=3)
12
+
13
+
14
+ @torch.no_grad()
15
+ def generate(bart, infile, outfile="bart_hypo.txt", bsz=32, n_obs=None, **eval_kwargs):
16
+ count = 1
17
+
18
+ # if n_obs is not None: bsz = min(bsz, n_obs)
19
+
20
+ with open(infile) as source, open(outfile, "w") as fout:
21
+ sline = source.readline().strip()
22
+ slines = [sline]
23
+ for sline in source:
24
+ if n_obs is not None and count > n_obs:
25
+ break
26
+ if count % bsz == 0:
27
+ hypotheses_batch = bart.sample(slines, **eval_kwargs)
28
+ for hypothesis in hypotheses_batch:
29
+ fout.write(hypothesis + "\n")
30
+ fout.flush()
31
+ slines = []
32
+
33
+ slines.append(sline.strip())
34
+ count += 1
35
+
36
+ if slines != []:
37
+ hypotheses_batch = bart.sample(slines, **eval_kwargs)
38
+ for hypothesis in hypotheses_batch:
39
+ fout.write(hypothesis + "\n")
40
+ fout.flush()
41
+
42
+
43
+ def main():
44
+ """
45
+ Usage::
46
+
47
+ python examples/bart/summarize.py \
48
+ --model-dir $HOME/bart.large.cnn \
49
+ --model-file model.pt \
50
+ --src $HOME/data-bin/cnn_dm/test.source
51
+ """
52
+ parser = argparse.ArgumentParser()
53
+ parser.add_argument(
54
+ "--model-dir",
55
+ required=True,
56
+ type=str,
57
+ default="bart.large.cnn/",
58
+ help="path containing model file and src_dict.txt",
59
+ )
60
+ parser.add_argument(
61
+ "--model-file",
62
+ default="checkpoint_best.pt",
63
+ help="where in model_dir are weights saved",
64
+ )
65
+ parser.add_argument(
66
+ "--src", default="test.source", help="text to summarize", type=str
67
+ )
68
+ parser.add_argument(
69
+ "--out", default="test.hypo", help="where to save summaries", type=str
70
+ )
71
+ parser.add_argument("--bsz", default=32, help="where to save summaries", type=int)
72
+ parser.add_argument(
73
+ "--n", default=None, help="how many examples to summarize", type=int
74
+ )
75
+ parser.add_argument(
76
+ "--xsum-kwargs",
77
+ action="store_true",
78
+ default=False,
79
+ help="if true use XSUM_KWARGS else CNN_KWARGS",
80
+ )
81
+ args = parser.parse_args()
82
+ eval_kwargs = XSUM_KWARGS if args.xsum_kwargs else CNN_KWARGS
83
+ if args.model_dir == "pytorch/fairseq":
84
+ bart = torch.hub.load("pytorch/fairseq", args.model_file)
85
+ else:
86
+ bart = BARTModel.from_pretrained(
87
+ args.model_dir,
88
+ checkpoint_file=args.model_file,
89
+ data_name_or_path=args.model_dir,
90
+ )
91
+ bart = bart.eval()
92
+ if torch.cuda.is_available():
93
+ bart = bart.cuda().half()
94
+ generate(
95
+ bart, args.src, bsz=args.bsz, n_obs=args.n, outfile=args.out, **eval_kwargs
96
+ )
97
+
98
+
99
+ if __name__ == "__main__":
100
+ main()
data/fairseq/fairseq_cli/__init__.py ADDED
File without changes
data/fairseq/fairseq_cli/__pycache__/__init__.cpython-38.pyc ADDED
Binary file (158 Bytes). View file
 
data/fairseq/fairseq_cli/__pycache__/generate.cpython-38.pyc ADDED
Binary file (9.07 kB). View file
 
data/fairseq/fairseq_cli/__pycache__/preprocess.cpython-38.pyc ADDED
Binary file (8.11 kB). View file
 
data/fairseq/fairseq_cli/__pycache__/train.cpython-38.pyc ADDED
Binary file (12.8 kB). View file
 
data/fairseq/fairseq_cli/eval_lm.py ADDED
@@ -0,0 +1,347 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env python3 -u
2
+ # Copyright (c) Facebook, Inc. and its affiliates.
3
+ #
4
+ # This source code is licensed under the MIT license found in the
5
+ # LICENSE file in the root directory of this source tree.
6
+
7
+ """
8
+ Evaluate the perplexity of a trained language model.
9
+ """
10
+
11
+ import logging
12
+ import math
13
+ import os
14
+ import sys
15
+ from argparse import Namespace
16
+ from typing import Iterable, List, Optional
17
+
18
+ import torch
19
+ from omegaconf import DictConfig
20
+
21
+ import fairseq
22
+ from fairseq import checkpoint_utils, distributed_utils, options, tasks, utils
23
+ from fairseq.dataclass.utils import convert_namespace_to_omegaconf
24
+ from fairseq.logging import progress_bar
25
+ from fairseq.logging.meters import StopwatchMeter
26
+ from fairseq.sequence_scorer import SequenceScorer
27
+
28
+ logging.basicConfig(
29
+ format="%(asctime)s | %(levelname)s | %(name)s | %(message)s",
30
+ datefmt="%Y-%m-%d %H:%M:%S",
31
+ level=os.environ.get("LOGLEVEL", "INFO").upper(),
32
+ stream=sys.stdout,
33
+ )
34
+ logger = logging.getLogger("fairseq_cli.eval_lm")
35
+
36
+
37
+ def eval_lm(
38
+ models: List[fairseq.models.FairseqModel],
39
+ source_dictionary: fairseq.data.Dictionary,
40
+ batch_iterator: Iterable,
41
+ post_process: Optional[str] = None,
42
+ output_word_probs: bool = False,
43
+ output_word_stats: bool = False,
44
+ target_dictionary: Optional[fairseq.data.Dictionary] = None,
45
+ softmax_batch: int = 0,
46
+ remove_bos_token: bool = False,
47
+ device: Optional[torch.device] = None,
48
+ ):
49
+ """
50
+ Args:
51
+ models (List[~fairseq.models.FairseqModel]): list of models to
52
+ evaluate. Models are essentially `nn.Module` instances, but
53
+ must be compatible with fairseq's `SequenceScorer`.
54
+ source_dictionary (~fairseq.data.Dictionary): dictionary for
55
+ applying any relevant post processing or outputing word
56
+ probs/stats.
57
+ batch_iterator (Iterable): yield batches of data
58
+ post_process (Optional[str]): post-process text by removing BPE,
59
+ letter segmentation, etc. Valid options can be found in
60
+ fairseq.data.utils.post_process, although not all options
61
+ are implemented here.
62
+ output_word_probs (Optional[bool]): output words and their
63
+ predicted log probabilities
64
+ output_word_stats (Optional[bool]): output word statistics such
65
+ as word count and average probability
66
+ target_dictionary (Optional[~fairseq.data.Dictionary]): output
67
+ dictionary (defaults to *source_dictionary*)
68
+ softmax_batch (Optional[bool]): if BxT is more than this, will
69
+ batch the softmax over vocab to this amount of tokens, in
70
+ order to fit into GPU memory
71
+ remove_bos_token (Optional[bool]): if True, confirm that the
72
+ first token is the beginning-of-sentence symbol (according
73
+ to the relevant dictionary) and remove it from the output
74
+ device (Optional[torch.device]): device to use for evaluation
75
+ (defaults to device of first model parameter)
76
+ """
77
+ if target_dictionary is None:
78
+ target_dictionary = source_dictionary
79
+ if device is None:
80
+ device = next(models[0].parameters()).device
81
+
82
+ gen_timer = StopwatchMeter()
83
+ scorer = SequenceScorer(target_dictionary, softmax_batch)
84
+
85
+ score_sum = 0.0
86
+ count = 0
87
+
88
+ if post_process is not None:
89
+ if post_process in {"subword_nmt", "@@ "}:
90
+ bpe_cont = post_process.rstrip()
91
+ bpe_toks = {
92
+ i
93
+ for i in range(len(source_dictionary))
94
+ if source_dictionary[i].endswith(bpe_cont)
95
+ }
96
+ else:
97
+ raise NotImplementedError(
98
+ f"--post-process={post_process} is not implemented"
99
+ )
100
+ bpe_len = len(bpe_cont)
101
+ else:
102
+ bpe_toks = None
103
+ bpe_len = 0
104
+
105
+ word_stats = dict()
106
+
107
+ for sample in batch_iterator:
108
+ if "net_input" not in sample:
109
+ continue
110
+
111
+ sample = utils.move_to_cuda(sample, device=device)
112
+
113
+ gen_timer.start()
114
+ hypos = scorer.generate(models, sample)
115
+ gen_timer.stop(sample["ntokens"])
116
+
117
+ for i, hypos_i in enumerate(hypos):
118
+ hypo = hypos_i[0]
119
+ sample_id = sample["id"][i]
120
+
121
+ tokens = hypo["tokens"]
122
+ tgt_len = tokens.numel()
123
+ pos_scores = hypo["positional_scores"].float()
124
+
125
+ if remove_bos_token:
126
+ assert hypo["tokens"][0].item() == target_dictionary.bos()
127
+ tokens = tokens[1:]
128
+ pos_scores = pos_scores[1:]
129
+
130
+ skipped_toks = 0
131
+ if bpe_toks is not None:
132
+ for i in range(tgt_len - 1):
133
+ if tokens[i].item() in bpe_toks:
134
+ skipped_toks += 1
135
+ pos_scores[i + 1] += pos_scores[i]
136
+ pos_scores[i] = 0
137
+
138
+ inf_scores = pos_scores.eq(float("inf")) | pos_scores.eq(float("-inf"))
139
+ if inf_scores.any():
140
+ logger.info(
141
+ "skipping tokens with inf scores:",
142
+ target_dictionary.string(tokens[inf_scores.nonzero()]),
143
+ )
144
+ pos_scores = pos_scores[(~inf_scores).nonzero()]
145
+ score_sum += pos_scores.sum().cpu()
146
+ count += pos_scores.numel() - skipped_toks
147
+
148
+ if output_word_probs or output_word_stats:
149
+ w = ""
150
+ word_prob = []
151
+ is_bpe = False
152
+ for i in range(len(tokens)):
153
+ w_ind = tokens[i].item()
154
+ w += source_dictionary[w_ind]
155
+ if bpe_toks is not None and w_ind in bpe_toks:
156
+ w = w[:-bpe_len]
157
+ is_bpe = True
158
+ else:
159
+ word_prob.append((w, pos_scores[i].item()))
160
+
161
+ next_prob = None
162
+ ind = i + 1
163
+ while ind < len(tokens):
164
+ if pos_scores[ind].item() != 0:
165
+ next_prob = pos_scores[ind]
166
+ break
167
+ ind += 1
168
+
169
+ word_stats.setdefault(w, WordStat(w, is_bpe)).add(
170
+ pos_scores[i].item(), next_prob
171
+ )
172
+ is_bpe = False
173
+ w = ""
174
+ if output_word_probs:
175
+ logger.info(
176
+ str(int(sample_id))
177
+ + " "
178
+ + (
179
+ "\t".join(
180
+ "{} [{:2f}]".format(x[0], x[1]) for x in word_prob
181
+ )
182
+ )
183
+ )
184
+
185
+ avg_nll_loss = (
186
+ -score_sum / count / math.log(2) if count > 0 else 0
187
+ ) # convert to base 2
188
+ logger.info(
189
+ "Evaluated {:,} tokens in {:.1f}s ({:.2f} tokens/s)".format(
190
+ gen_timer.n, gen_timer.sum, 1.0 / gen_timer.avg if gen_timer.avg > 0 else 0
191
+ )
192
+ )
193
+
194
+ if output_word_stats:
195
+ for ws in sorted(word_stats.values(), key=lambda x: x.count, reverse=True):
196
+ logger.info(ws)
197
+
198
+ return {
199
+ "loss": avg_nll_loss,
200
+ "perplexity": 2**avg_nll_loss,
201
+ }
202
+
203
+
204
+ class WordStat(object):
205
+ def __init__(self, word, is_bpe):
206
+ self.word = word
207
+ self.is_bpe = is_bpe
208
+ self.log_prob = 0
209
+ self.next_word_prob = 0
210
+ self.count = 0
211
+ self.missing_next_words = 0
212
+
213
+ def add(self, log_prob, next_word_prob):
214
+ """increments counters for the sum of log probs of current word and next
215
+ word (given context ending at current word). Since the next word might be at the end of the example,
216
+ or it might be not counted because it is not an ending subword unit,
217
+ also keeps track of how many of those we have seen"""
218
+ if next_word_prob is not None:
219
+ self.next_word_prob += next_word_prob
220
+ else:
221
+ self.missing_next_words += 1
222
+ self.log_prob += log_prob
223
+ self.count += 1
224
+
225
+ def __str__(self):
226
+ return "{}\t{}\t{}\t{}\t{}\t{}".format(
227
+ self.word,
228
+ self.count,
229
+ self.log_prob,
230
+ self.is_bpe,
231
+ self.next_word_prob,
232
+ self.count - self.missing_next_words,
233
+ )
234
+
235
+
236
+ def main(cfg: DictConfig, **unused_kwargs):
237
+ if isinstance(cfg, Namespace):
238
+ cfg = convert_namespace_to_omegaconf(cfg)
239
+
240
+ utils.import_user_module(cfg.common)
241
+
242
+ logger.info(cfg)
243
+
244
+ if cfg.eval_lm.context_window > 0:
245
+ # reduce tokens per sample by the required context window size
246
+ cfg.task.tokens_per_sample -= cfg.eval_lm.context_window
247
+
248
+ # Initialize the task using the current *cfg*
249
+ task = tasks.setup_task(cfg.task)
250
+
251
+ # Load ensemble
252
+ logger.info("loading model(s) from {}".format(cfg.common_eval.path))
253
+ models, model_args, task = checkpoint_utils.load_model_ensemble_and_task(
254
+ [cfg.common_eval.path],
255
+ arg_overrides=eval(cfg.common_eval.model_overrides),
256
+ suffix=cfg.checkpoint.checkpoint_suffix,
257
+ strict=(cfg.checkpoint.checkpoint_shard_count == 1),
258
+ num_shards=cfg.checkpoint.checkpoint_shard_count,
259
+ task=task,
260
+ )
261
+
262
+ use_fp16 = cfg.common.fp16
263
+ use_cuda = torch.cuda.is_available() and not cfg.common.cpu
264
+ if use_cuda:
265
+ torch.cuda.set_device(cfg.distributed_training.device_id)
266
+
267
+ # Optimize ensemble for generation and set the source and dest dicts on the model
268
+ # (required by scorer)
269
+ for model in models:
270
+ if use_fp16:
271
+ model.half()
272
+ if use_cuda and not cfg.distributed_training.pipeline_model_parallel:
273
+ model.cuda()
274
+ model.prepare_for_inference_(cfg)
275
+
276
+ assert len(models) > 0
277
+
278
+ logger.info(
279
+ "num. model params: {:,}".format(sum(p.numel() for p in models[0].parameters()))
280
+ )
281
+
282
+ # Load dataset splits
283
+ task.load_dataset(cfg.dataset.gen_subset)
284
+ dataset = task.dataset(cfg.dataset.gen_subset)
285
+ logger.info(
286
+ "{} {} {:,} examples".format(
287
+ cfg.task.data, cfg.dataset.gen_subset, len(dataset)
288
+ )
289
+ )
290
+
291
+ itr = task.eval_lm_dataloader(
292
+ dataset=dataset,
293
+ max_tokens=cfg.dataset.max_tokens or 36000,
294
+ batch_size=cfg.dataset.batch_size,
295
+ max_positions=utils.resolve_max_positions(
296
+ *[model.max_positions() for model in models]
297
+ ),
298
+ num_shards=max(
299
+ cfg.dataset.num_shards,
300
+ cfg.distributed_training.distributed_world_size,
301
+ ),
302
+ shard_id=max(
303
+ cfg.dataset.shard_id,
304
+ cfg.distributed_training.distributed_rank,
305
+ ),
306
+ num_workers=cfg.dataset.num_workers,
307
+ data_buffer_size=cfg.dataset.data_buffer_size,
308
+ context_window=cfg.eval_lm.context_window,
309
+ )
310
+
311
+ itr = progress_bar.progress_bar(
312
+ itr,
313
+ log_format=cfg.common.log_format,
314
+ log_interval=cfg.common.log_interval,
315
+ default_log_format=("tqdm" if not cfg.common.no_progress_bar else "simple"),
316
+ )
317
+
318
+ results = eval_lm(
319
+ models=models,
320
+ source_dictionary=task.source_dictionary,
321
+ batch_iterator=itr,
322
+ post_process=cfg.common_eval.post_process,
323
+ output_word_probs=cfg.eval_lm.output_word_probs,
324
+ output_word_stats=cfg.eval_lm.output_word_stats,
325
+ target_dictionary=task.target_dictionary,
326
+ softmax_batch=cfg.eval_lm.softmax_batch,
327
+ remove_bos_token=getattr(cfg.task, "add_bos_token", False),
328
+ )
329
+
330
+ logger.info(
331
+ "Loss (base 2): {:.4f}, Perplexity: {:.2f}".format(
332
+ results["loss"], results["perplexity"]
333
+ )
334
+ )
335
+
336
+ return results
337
+
338
+
339
+ def cli_main():
340
+ parser = options.get_eval_lm_parser()
341
+ args = options.parse_args_and_arch(parser)
342
+
343
+ distributed_utils.call_main(convert_namespace_to_omegaconf(args), main)
344
+
345
+
346
+ if __name__ == "__main__":
347
+ cli_main()
data/fairseq/fairseq_cli/generate.py ADDED
@@ -0,0 +1,417 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env python3 -u
2
+ # Copyright (c) Facebook, Inc. and its affiliates.
3
+ #
4
+ # This source code is licensed under the MIT license found in the
5
+ # LICENSE file in the root directory of this source tree.
6
+ """
7
+ Translate pre-processed data with a trained model.
8
+ """
9
+
10
+ import ast
11
+ import logging
12
+ import math
13
+ import os
14
+ import sys
15
+ from argparse import Namespace
16
+ from itertools import chain
17
+
18
+ import numpy as np
19
+ import torch
20
+ from omegaconf import DictConfig
21
+
22
+ from fairseq import checkpoint_utils, options, scoring, tasks, utils
23
+ from fairseq.dataclass.utils import convert_namespace_to_omegaconf
24
+ from fairseq.logging import progress_bar
25
+ from fairseq.logging.meters import StopwatchMeter, TimeMeter
26
+
27
+
28
+ def main(cfg: DictConfig):
29
+
30
+ if isinstance(cfg, Namespace):
31
+ cfg = convert_namespace_to_omegaconf(cfg)
32
+
33
+ assert cfg.common_eval.path is not None, "--path required for generation!"
34
+ assert (
35
+ not cfg.generation.sampling or cfg.generation.nbest == cfg.generation.beam
36
+ ), "--sampling requires --nbest to be equal to --beam"
37
+ assert (
38
+ cfg.generation.replace_unk is None or cfg.dataset.dataset_impl == "raw"
39
+ ), "--replace-unk requires a raw text dataset (--dataset-impl=raw)"
40
+
41
+ if cfg.common_eval.results_path is not None:
42
+ os.makedirs(cfg.common_eval.results_path, exist_ok=True)
43
+ output_path = os.path.join(
44
+ cfg.common_eval.results_path,
45
+ "generate-{}.txt".format(cfg.dataset.gen_subset),
46
+ )
47
+ with open(output_path, "w", buffering=1, encoding="utf-8") as h:
48
+ return _main(cfg, h)
49
+ else:
50
+ return _main(cfg, sys.stdout)
51
+
52
+
53
+ def get_symbols_to_strip_from_output(generator):
54
+ if hasattr(generator, "symbols_to_strip_from_output"):
55
+ return generator.symbols_to_strip_from_output
56
+ else:
57
+ return {generator.eos}
58
+
59
+
60
+ def _main(cfg: DictConfig, output_file):
61
+ logging.basicConfig(
62
+ format="%(asctime)s | %(levelname)s | %(name)s | %(message)s",
63
+ datefmt="%Y-%m-%d %H:%M:%S",
64
+ level=os.environ.get("LOGLEVEL", "INFO").upper(),
65
+ stream=output_file,
66
+ )
67
+ logger = logging.getLogger("fairseq_cli.generate")
68
+
69
+ utils.import_user_module(cfg.common)
70
+
71
+ if cfg.dataset.max_tokens is None and cfg.dataset.batch_size is None:
72
+ cfg.dataset.max_tokens = 12000
73
+ logger.info(cfg)
74
+
75
+ # Fix seed for stochastic decoding
76
+ if cfg.common.seed is not None and not cfg.generation.no_seed_provided:
77
+ np.random.seed(cfg.common.seed)
78
+ utils.set_torch_seed(cfg.common.seed)
79
+
80
+ use_cuda = torch.cuda.is_available() and not cfg.common.cpu
81
+
82
+ # Load dataset splits
83
+ task = tasks.setup_task(cfg.task)
84
+
85
+ # Set dictionaries
86
+ try:
87
+ src_dict = getattr(task, "source_dictionary", None)
88
+ except NotImplementedError:
89
+ src_dict = None
90
+ tgt_dict = task.target_dictionary
91
+
92
+ overrides = ast.literal_eval(cfg.common_eval.model_overrides)
93
+
94
+ # Load ensemble
95
+ logger.info("loading model(s) from {}".format(cfg.common_eval.path))
96
+ models, saved_cfg = checkpoint_utils.load_model_ensemble(
97
+ utils.split_paths(cfg.common_eval.path),
98
+ arg_overrides=overrides,
99
+ task=task,
100
+ suffix=cfg.checkpoint.checkpoint_suffix,
101
+ strict=(cfg.checkpoint.checkpoint_shard_count == 1),
102
+ num_shards=cfg.checkpoint.checkpoint_shard_count,
103
+ )
104
+
105
+ # loading the dataset should happen after the checkpoint has been loaded so we can give it the saved task config
106
+ task.load_dataset(cfg.dataset.gen_subset, task_cfg=saved_cfg.task)
107
+
108
+ if cfg.generation.lm_path is not None:
109
+ overrides["data"] = cfg.task.data
110
+
111
+ try:
112
+ lms, _ = checkpoint_utils.load_model_ensemble(
113
+ [cfg.generation.lm_path], arg_overrides=overrides, task=None
114
+ )
115
+ except:
116
+ logger.warning(
117
+ f"Failed to load language model! Please make sure that the language model dict is the same "
118
+ f"as target dict and is located in the data dir ({cfg.task.data})"
119
+ )
120
+ raise
121
+
122
+ assert len(lms) == 1
123
+ else:
124
+ lms = [None]
125
+
126
+ # Optimize ensemble for generation
127
+ for model in chain(models, lms):
128
+ if model is None:
129
+ continue
130
+ if cfg.common.fp16:
131
+ model.half()
132
+ if use_cuda and not cfg.distributed_training.pipeline_model_parallel:
133
+ model.cuda()
134
+ model.prepare_for_inference_(cfg)
135
+
136
+ # Load alignment dictionary for unknown word replacement
137
+ # (None if no unknown word replacement, empty if no path to align dictionary)
138
+ align_dict = utils.load_align_dict(cfg.generation.replace_unk)
139
+
140
+ # Load dataset (possibly sharded)
141
+ itr = task.get_batch_iterator(
142
+ dataset=task.dataset(cfg.dataset.gen_subset),
143
+ max_tokens=cfg.dataset.max_tokens,
144
+ max_sentences=cfg.dataset.batch_size,
145
+ max_positions=utils.resolve_max_positions(
146
+ task.max_positions(), *[m.max_positions() for m in models]
147
+ ),
148
+ ignore_invalid_inputs=cfg.dataset.skip_invalid_size_inputs_valid_test,
149
+ required_batch_size_multiple=cfg.dataset.required_batch_size_multiple,
150
+ seed=cfg.common.seed,
151
+ num_shards=cfg.distributed_training.distributed_world_size,
152
+ shard_id=cfg.distributed_training.distributed_rank,
153
+ num_workers=cfg.dataset.num_workers,
154
+ data_buffer_size=cfg.dataset.data_buffer_size,
155
+ ).next_epoch_itr(shuffle=False)
156
+ progress = progress_bar.progress_bar(
157
+ itr,
158
+ log_format=cfg.common.log_format,
159
+ log_interval=cfg.common.log_interval,
160
+ default_log_format=("tqdm" if not cfg.common.no_progress_bar else "simple"),
161
+ )
162
+
163
+ # Initialize generator
164
+ gen_timer = StopwatchMeter()
165
+
166
+ extra_gen_cls_kwargs = {"lm_model": lms[0], "lm_weight": cfg.generation.lm_weight}
167
+ generator = task.build_generator(
168
+ models, cfg.generation, extra_gen_cls_kwargs=extra_gen_cls_kwargs
169
+ )
170
+
171
+ # Handle tokenization and BPE
172
+ tokenizer = task.build_tokenizer(cfg.tokenizer)
173
+ bpe = task.build_bpe(cfg.bpe)
174
+
175
+ def decode_fn(x):
176
+ if bpe is not None:
177
+ x = bpe.decode(x)
178
+ if tokenizer is not None:
179
+ x = tokenizer.decode(x)
180
+ return x
181
+
182
+ scorer = scoring.build_scorer(cfg.scoring, tgt_dict)
183
+
184
+ num_sentences = 0
185
+ has_target = True
186
+ wps_meter = TimeMeter()
187
+ for sample in progress:
188
+ sample = utils.move_to_cuda(sample) if use_cuda else sample
189
+ if "net_input" not in sample:
190
+ continue
191
+
192
+ prefix_tokens = None
193
+ if cfg.generation.prefix_size > 0:
194
+ prefix_tokens = sample["target"][:, : cfg.generation.prefix_size]
195
+
196
+ constraints = None
197
+ if "constraints" in sample:
198
+ constraints = sample["constraints"]
199
+
200
+ gen_timer.start()
201
+ hypos = task.inference_step(
202
+ generator,
203
+ models,
204
+ sample,
205
+ prefix_tokens=prefix_tokens,
206
+ constraints=constraints,
207
+ )
208
+ num_generated_tokens = sum(len(h[0]["tokens"]) for h in hypos)
209
+ gen_timer.stop(num_generated_tokens)
210
+
211
+ for i, sample_id in enumerate(sample["id"].tolist()):
212
+ has_target = sample["target"] is not None
213
+
214
+ # Remove padding
215
+ if "src_tokens" in sample["net_input"]:
216
+ src_tokens = utils.strip_pad(
217
+ sample["net_input"]["src_tokens"][i, :], tgt_dict.pad()
218
+ )
219
+ else:
220
+ src_tokens = None
221
+
222
+ target_tokens = None
223
+ if has_target:
224
+ target_tokens = (
225
+ utils.strip_pad(sample["target"][i, :], tgt_dict.pad()).int().cpu()
226
+ )
227
+
228
+ # Either retrieve the original sentences or regenerate them from tokens.
229
+ if align_dict is not None:
230
+ src_str = task.dataset(cfg.dataset.gen_subset).src.get_original_text(
231
+ sample_id
232
+ )
233
+ target_str = task.dataset(cfg.dataset.gen_subset).tgt.get_original_text(
234
+ sample_id
235
+ )
236
+ else:
237
+ if src_dict is not None:
238
+ src_str = src_dict.string(src_tokens, cfg.common_eval.post_process)
239
+ else:
240
+ src_str = ""
241
+ if has_target:
242
+ target_str = tgt_dict.string(
243
+ target_tokens,
244
+ cfg.common_eval.post_process,
245
+ escape_unk=True,
246
+ extra_symbols_to_ignore=get_symbols_to_strip_from_output(
247
+ generator
248
+ ),
249
+ )
250
+
251
+ src_str = decode_fn(src_str)
252
+ if has_target:
253
+ target_str = decode_fn(target_str)
254
+
255
+ if not cfg.common_eval.quiet:
256
+ if src_dict is not None:
257
+ print("S-{}\t{}".format(sample_id, src_str), file=output_file)
258
+ if has_target:
259
+ print("T-{}\t{}".format(sample_id, target_str), file=output_file)
260
+
261
+ # Process top predictions
262
+ for j, hypo in enumerate(hypos[i][: cfg.generation.nbest]):
263
+ hypo_tokens, hypo_str, alignment = utils.post_process_prediction(
264
+ hypo_tokens=hypo["tokens"].int().cpu(),
265
+ src_str=src_str,
266
+ alignment=hypo["alignment"],
267
+ align_dict=align_dict,
268
+ tgt_dict=tgt_dict,
269
+ remove_bpe=cfg.common_eval.post_process,
270
+ extra_symbols_to_ignore=get_symbols_to_strip_from_output(generator),
271
+ )
272
+ detok_hypo_str = decode_fn(hypo_str)
273
+ if not cfg.common_eval.quiet:
274
+ score = hypo["score"] / math.log(2) # convert to base 2
275
+ # original hypothesis (after tokenization and BPE)
276
+ print(
277
+ "H-{}\t{}\t{}".format(sample_id, score, hypo_str),
278
+ file=output_file,
279
+ )
280
+ # detokenized hypothesis
281
+ print(
282
+ "D-{}\t{}\t{}".format(sample_id, score, detok_hypo_str),
283
+ file=output_file,
284
+ )
285
+ print(
286
+ "P-{}\t{}".format(
287
+ sample_id,
288
+ " ".join(
289
+ map(
290
+ lambda x: "{:.4f}".format(x),
291
+ # convert from base e to base 2
292
+ hypo["positional_scores"]
293
+ .div_(math.log(2))
294
+ .tolist(),
295
+ )
296
+ ),
297
+ ),
298
+ file=output_file,
299
+ )
300
+
301
+ if cfg.generation.print_alignment == "hard":
302
+ print(
303
+ "A-{}\t{}".format(
304
+ sample_id,
305
+ " ".join(
306
+ [
307
+ "{}-{}".format(src_idx, tgt_idx)
308
+ for src_idx, tgt_idx in alignment
309
+ ]
310
+ ),
311
+ ),
312
+ file=output_file,
313
+ )
314
+ if cfg.generation.print_alignment == "soft":
315
+ print(
316
+ "A-{}\t{}".format(
317
+ sample_id,
318
+ " ".join(
319
+ [",".join(src_probs) for src_probs in alignment]
320
+ ),
321
+ ),
322
+ file=output_file,
323
+ )
324
+
325
+ if cfg.generation.print_step:
326
+ print(
327
+ "I-{}\t{}".format(sample_id, hypo["steps"]),
328
+ file=output_file,
329
+ )
330
+
331
+ if cfg.generation.retain_iter_history:
332
+ for step, h in enumerate(hypo["history"]):
333
+ _, h_str, _ = utils.post_process_prediction(
334
+ hypo_tokens=h["tokens"].int().cpu(),
335
+ src_str=src_str,
336
+ alignment=None,
337
+ align_dict=None,
338
+ tgt_dict=tgt_dict,
339
+ remove_bpe=None,
340
+ )
341
+ print(
342
+ "E-{}_{}\t{}".format(sample_id, step, h_str),
343
+ file=output_file,
344
+ )
345
+
346
+ # Score only the top hypothesis
347
+ if has_target and j == 0:
348
+ if (
349
+ align_dict is not None
350
+ or cfg.common_eval.post_process is not None
351
+ ):
352
+ # Convert back to tokens for evaluation with unk replacement and/or without BPE
353
+ target_tokens = tgt_dict.encode_line(
354
+ target_str, add_if_not_exist=True
355
+ )
356
+ hypo_tokens = tgt_dict.encode_line(
357
+ detok_hypo_str, add_if_not_exist=True
358
+ )
359
+ if hasattr(scorer, "add_string"):
360
+ scorer.add_string(target_str, detok_hypo_str)
361
+ else:
362
+ scorer.add(target_tokens, hypo_tokens)
363
+
364
+ wps_meter.update(num_generated_tokens)
365
+ progress.log({"wps": round(wps_meter.avg)})
366
+ num_sentences += (
367
+ sample["nsentences"] if "nsentences" in sample else sample["id"].numel()
368
+ )
369
+
370
+ logger.info("NOTE: hypothesis and token scores are output in base 2")
371
+ logger.info(
372
+ "Translated {:,} sentences ({:,} tokens) in {:.1f}s ({:.2f} sentences/s, {:.2f} tokens/s)".format(
373
+ num_sentences,
374
+ gen_timer.n,
375
+ gen_timer.sum,
376
+ num_sentences / gen_timer.sum,
377
+ 1.0 / gen_timer.avg,
378
+ )
379
+ )
380
+ if has_target:
381
+ if cfg.bpe and not cfg.generation.sacrebleu:
382
+ if cfg.common_eval.post_process:
383
+ logger.warning(
384
+ "BLEU score is being computed by splitting detokenized string on spaces, this is probably not what you want. Use --sacrebleu for standard 13a BLEU tokenization"
385
+ )
386
+ else:
387
+ logger.warning(
388
+ "If you are using BPE on the target side, the BLEU score is computed on BPE tokens, not on proper words. Use --sacrebleu for standard 13a BLEU tokenization"
389
+ )
390
+ # use print to be consistent with other main outputs: S-, H-, T-, D- and so on
391
+ print(
392
+ "Generate {} with beam={}: {}".format(
393
+ cfg.dataset.gen_subset, cfg.generation.beam, scorer.result_string()
394
+ ),
395
+ file=output_file,
396
+ )
397
+
398
+ return scorer
399
+
400
+
401
+ def cli_main():
402
+ parser = options.get_generation_parser()
403
+ # TODO: replace this workaround with refactoring of `AudioPretraining`
404
+ parser.add_argument(
405
+ "--arch",
406
+ "-a",
407
+ metavar="ARCH",
408
+ default="wav2vec2",
409
+ help="Model architecture. For constructing tasks that rely on "
410
+ "model args (e.g. `AudioPretraining`)",
411
+ )
412
+ args = options.parse_args_and_arch(parser)
413
+ main(args)
414
+
415
+
416
+ if __name__ == "__main__":
417
+ cli_main()
data/fairseq/fairseq_cli/hydra_train.py ADDED
@@ -0,0 +1,91 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env python3 -u
2
+ # Copyright (c) Facebook, Inc. and its affiliates.
3
+ #
4
+ # This source code is licensed under the MIT license found in the
5
+ # LICENSE file in the root directory of this source tree.
6
+
7
+ import logging
8
+ import os
9
+
10
+ import hydra
11
+ import torch
12
+ from hydra.core.hydra_config import HydraConfig
13
+ from omegaconf import OmegaConf, open_dict
14
+
15
+ from fairseq import distributed_utils, metrics
16
+ from fairseq.dataclass.configs import FairseqConfig
17
+ from fairseq.dataclass.initialize import add_defaults, hydra_init
18
+ from fairseq.dataclass.utils import omegaconf_no_object_check
19
+ from fairseq.utils import reset_logging
20
+ from fairseq_cli.train import main as pre_main
21
+
22
+ logger = logging.getLogger("fairseq_cli.hydra_train")
23
+
24
+
25
+ @hydra.main(config_path=os.path.join("..", "fairseq", "config"), config_name="config")
26
+ def hydra_main(cfg: FairseqConfig) -> float:
27
+ _hydra_main(cfg)
28
+
29
+
30
+ def _hydra_main(cfg: FairseqConfig, **kwargs) -> float:
31
+ add_defaults(cfg)
32
+
33
+ if cfg.common.reset_logging:
34
+ reset_logging() # Hydra hijacks logging, fix that
35
+ else:
36
+ # check if directly called or called through hydra_main
37
+ if HydraConfig.initialized():
38
+ with open_dict(cfg):
39
+ # make hydra logging work with ddp (see # see https://github.com/facebookresearch/hydra/issues/1126)
40
+ cfg.job_logging_cfg = OmegaConf.to_container(
41
+ HydraConfig.get().job_logging, resolve=True
42
+ )
43
+
44
+ with omegaconf_no_object_check():
45
+ cfg = OmegaConf.create(
46
+ OmegaConf.to_container(cfg, resolve=True, enum_to_str=True)
47
+ )
48
+ OmegaConf.set_struct(cfg, True)
49
+
50
+ try:
51
+ if cfg.common.profile:
52
+ with torch.cuda.profiler.profile():
53
+ with torch.autograd.profiler.emit_nvtx():
54
+ distributed_utils.call_main(cfg, pre_main, **kwargs)
55
+ else:
56
+ distributed_utils.call_main(cfg, pre_main, **kwargs)
57
+ except BaseException as e:
58
+ if not cfg.common.suppress_crashes:
59
+ raise
60
+ else:
61
+ logger.error("Crashed! " + str(e))
62
+
63
+ # get best val and return - useful for sweepers
64
+ try:
65
+ best_val = metrics.get_smoothed_value(
66
+ "valid", cfg.checkpoint.best_checkpoint_metric
67
+ )
68
+ except:
69
+ best_val = None
70
+
71
+ if best_val is None:
72
+ best_val = float("inf")
73
+
74
+ return best_val
75
+
76
+
77
+ def cli_main():
78
+ try:
79
+ from hydra._internal.utils import get_args
80
+
81
+ cfg_name = get_args().config_name or "config"
82
+ except:
83
+ logger.warning("Failed to get config name from hydra args")
84
+ cfg_name = "config"
85
+
86
+ hydra_init(cfg_name)
87
+ hydra_main()
88
+
89
+
90
+ if __name__ == "__main__":
91
+ cli_main()
data/fairseq/fairseq_cli/hydra_validate.py ADDED
@@ -0,0 +1,188 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env python3 -u
2
+ # Copyright (c) Facebook, Inc. and its affiliates.
3
+ #
4
+ # This source code is licensed under the MIT license found in the
5
+ # LICENSE file in the root directory of this source tree.
6
+
7
+ import logging
8
+ import os
9
+ import sys
10
+ from itertools import chain
11
+
12
+ import torch
13
+ from hydra.core.hydra_config import HydraConfig
14
+ from omegaconf import OmegaConf, open_dict
15
+ import hydra
16
+
17
+ from fairseq import checkpoint_utils, distributed_utils, utils
18
+ from fairseq.dataclass.configs import FairseqConfig
19
+ from fairseq.dataclass.initialize import add_defaults, hydra_init
20
+ from fairseq.dataclass.utils import omegaconf_no_object_check
21
+ from fairseq.distributed import utils as distributed_utils
22
+ from fairseq.logging import metrics, progress_bar
23
+ from fairseq.utils import reset_logging
24
+
25
+ logging.basicConfig(
26
+ format="%(asctime)s | %(levelname)s | %(name)s | %(message)s",
27
+ datefmt="%Y-%m-%d %H:%M:%S",
28
+ level=os.environ.get("LOGLEVEL", "INFO").upper(),
29
+ stream=sys.stdout,
30
+ )
31
+ logger = logging.getLogger("fairseq_cli.validate")
32
+
33
+
34
+ @hydra.main(config_path=os.path.join("..", "fairseq", "config"), config_name="config")
35
+ def hydra_main(cfg: FairseqConfig) -> float:
36
+ return _hydra_main(cfg)
37
+
38
+
39
+ def _hydra_main(cfg: FairseqConfig, **kwargs) -> float:
40
+ add_defaults(cfg)
41
+
42
+ if cfg.common.reset_logging:
43
+ reset_logging() # Hydra hijacks logging, fix that
44
+ else:
45
+ # check if directly called or called through hydra_main
46
+ if HydraConfig.initialized():
47
+ with open_dict(cfg):
48
+ # make hydra logging work with ddp (see # see https://github.com/facebookresearch/hydra/issues/1126)
49
+ cfg.job_logging_cfg = OmegaConf.to_container(
50
+ HydraConfig.get().job_logging, resolve=True
51
+ )
52
+
53
+ with omegaconf_no_object_check():
54
+ cfg = OmegaConf.create(
55
+ OmegaConf.to_container(cfg, resolve=True, enum_to_str=True)
56
+ )
57
+ OmegaConf.set_struct(cfg, True)
58
+
59
+ assert (
60
+ cfg.dataset.max_tokens is not None or cfg.dataset.batch_size is not None
61
+ ), "Must specify batch size either with --max-tokens or --batch-size"
62
+
63
+ distributed_utils.call_main(cfg, validate, **kwargs)
64
+
65
+
66
+ def validate(cfg):
67
+ utils.import_user_module(cfg.common)
68
+
69
+ use_fp16 = cfg.common.fp16
70
+ use_cuda = torch.cuda.is_available() and not cfg.common.cpu
71
+
72
+ if use_cuda:
73
+ torch.cuda.set_device(cfg.distributed_training.device_id)
74
+
75
+ if cfg.distributed_training.distributed_world_size > 1:
76
+ data_parallel_world_size = distributed_utils.get_data_parallel_world_size()
77
+ data_parallel_rank = distributed_utils.get_data_parallel_rank()
78
+ else:
79
+ data_parallel_world_size = 1
80
+ data_parallel_rank = 0
81
+
82
+ overrides = {"task": {"data": cfg.task.data}}
83
+
84
+ # Load ensemble
85
+ logger.info("loading model(s) from {}".format(cfg.common_eval.path))
86
+ models, saved_cfg, task = checkpoint_utils.load_model_ensemble_and_task(
87
+ [cfg.common_eval.path],
88
+ arg_overrides=overrides,
89
+ suffix=cfg.checkpoint.checkpoint_suffix,
90
+ )
91
+ model = models[0]
92
+
93
+ # Move models to GPU
94
+ for model in models:
95
+ model.eval()
96
+ if use_fp16:
97
+ model.half()
98
+ if use_cuda:
99
+ model.cuda()
100
+
101
+ # Print args
102
+ logger.info(saved_cfg)
103
+
104
+ # Build criterion
105
+ criterion = task.build_criterion(saved_cfg.criterion, from_checkpoint=True)
106
+ criterion.eval()
107
+
108
+ for subset in cfg.dataset.valid_subset.split(","):
109
+ try:
110
+ task.load_dataset(subset, combine=False, epoch=1, task_cfg=saved_cfg.task)
111
+ dataset = task.dataset(subset)
112
+ except KeyError:
113
+ raise Exception("Cannot find dataset: " + subset)
114
+
115
+ # Initialize data iterator
116
+ itr = task.get_batch_iterator(
117
+ dataset=dataset,
118
+ max_tokens=cfg.dataset.max_tokens,
119
+ max_sentences=cfg.dataset.batch_size,
120
+ max_positions=utils.resolve_max_positions(
121
+ task.max_positions(),
122
+ *[m.max_positions() for m in models],
123
+ ),
124
+ ignore_invalid_inputs=cfg.dataset.skip_invalid_size_inputs_valid_test,
125
+ required_batch_size_multiple=cfg.dataset.required_batch_size_multiple,
126
+ seed=cfg.common.seed,
127
+ num_shards=data_parallel_world_size,
128
+ shard_id=data_parallel_rank,
129
+ num_workers=cfg.dataset.num_workers,
130
+ data_buffer_size=cfg.dataset.data_buffer_size,
131
+ ).next_epoch_itr(shuffle=False)
132
+ progress = progress_bar.progress_bar(
133
+ itr,
134
+ log_format=cfg.common.log_format,
135
+ log_interval=cfg.common.log_interval,
136
+ prefix=f"valid on '{subset}' subset",
137
+ default_log_format=("tqdm" if not cfg.common.no_progress_bar else "simple"),
138
+ )
139
+
140
+ def apply_half(t):
141
+ if t.dtype is torch.float32:
142
+ return t.to(dtype=torch.half)
143
+ return t
144
+
145
+ log_outputs = []
146
+ for i, sample in enumerate(progress):
147
+ sample = utils.move_to_cuda(sample) if use_cuda else sample
148
+
149
+ if use_fp16:
150
+ sample = utils.apply_to_sample(apply_half, sample)
151
+
152
+ _loss, _sample_size, log_output = task.valid_step(sample, model, criterion)
153
+ with metrics.aggregate() as agg:
154
+ task.reduce_metrics([log_output], criterion)
155
+ progress.log(agg.get_smoothed_values(), step=i)
156
+ # progress.log(log_output, step=i) from vision
157
+ log_outputs.append(log_output)
158
+
159
+ if data_parallel_world_size > 1:
160
+ log_outputs = distributed_utils.all_gather_list(
161
+ log_outputs,
162
+ max_size=cfg.common.all_gather_list_size,
163
+ group=distributed_utils.get_data_parallel_group(),
164
+ )
165
+ log_outputs = list(chain.from_iterable(log_outputs))
166
+
167
+ with metrics.aggregate() as agg:
168
+ task.reduce_metrics(log_outputs, criterion)
169
+ log_output = agg.get_smoothed_values()
170
+
171
+ progress.print(log_output, tag=subset, step=i)
172
+
173
+
174
+ def cli_main():
175
+ try:
176
+ from hydra._internal.utils import get_args
177
+
178
+ cfg_name = get_args().config_name or "config"
179
+ except:
180
+ logger.warning("Failed to get config name from hydra args")
181
+ cfg_name = "config"
182
+
183
+ hydra_init(cfg_name)
184
+ hydra_main()
185
+
186
+
187
+ if __name__ == "__main__":
188
+ cli_main()
data/fairseq/fairseq_cli/interactive.py ADDED
@@ -0,0 +1,317 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env python3 -u
2
+ # Copyright (c) Facebook, Inc. and its affiliates.
3
+ #
4
+ # This source code is licensed under the MIT license found in the
5
+ # LICENSE file in the root directory of this source tree.
6
+ """
7
+ Translate raw text with a trained model. Batches data on-the-fly.
8
+ """
9
+
10
+ import ast
11
+ import fileinput
12
+ import logging
13
+ import math
14
+ import os
15
+ import sys
16
+ import time
17
+ from argparse import Namespace
18
+ from collections import namedtuple
19
+
20
+ import numpy as np
21
+ import torch
22
+
23
+ from fairseq import checkpoint_utils, distributed_utils, options, tasks, utils
24
+ from fairseq.dataclass.configs import FairseqConfig
25
+ from fairseq.dataclass.utils import convert_namespace_to_omegaconf
26
+ from fairseq.token_generation_constraints import pack_constraints, unpack_constraints
27
+ from fairseq_cli.generate import get_symbols_to_strip_from_output
28
+
29
+ logging.basicConfig(
30
+ format="%(asctime)s | %(levelname)s | %(name)s | %(message)s",
31
+ datefmt="%Y-%m-%d %H:%M:%S",
32
+ level=os.environ.get("LOGLEVEL", "INFO").upper(),
33
+ stream=sys.stdout,
34
+ )
35
+ logger = logging.getLogger("fairseq_cli.interactive")
36
+
37
+
38
+ Batch = namedtuple("Batch", "ids src_tokens src_lengths constraints")
39
+ Translation = namedtuple("Translation", "src_str hypos pos_scores alignments")
40
+
41
+
42
+ def buffered_read(input, buffer_size):
43
+ buffer = []
44
+ with fileinput.input(files=[input], openhook=fileinput.hook_encoded("utf-8")) as h:
45
+ for src_str in h:
46
+ buffer.append(src_str.strip())
47
+ if len(buffer) >= buffer_size:
48
+ yield buffer
49
+ buffer = []
50
+
51
+ if len(buffer) > 0:
52
+ yield buffer
53
+
54
+
55
+ def make_batches(lines, cfg, task, max_positions, encode_fn):
56
+ def encode_fn_target(x):
57
+ return encode_fn(x)
58
+
59
+ if cfg.generation.constraints:
60
+ # Strip (tab-delimited) contraints, if present, from input lines,
61
+ # store them in batch_constraints
62
+ batch_constraints = [list() for _ in lines]
63
+ for i, line in enumerate(lines):
64
+ if "\t" in line:
65
+ lines[i], *batch_constraints[i] = line.split("\t")
66
+
67
+ # Convert each List[str] to List[Tensor]
68
+ for i, constraint_list in enumerate(batch_constraints):
69
+ batch_constraints[i] = [
70
+ task.target_dictionary.encode_line(
71
+ encode_fn_target(constraint),
72
+ append_eos=False,
73
+ add_if_not_exist=False,
74
+ )
75
+ for constraint in constraint_list
76
+ ]
77
+
78
+ if cfg.generation.constraints:
79
+ constraints_tensor = pack_constraints(batch_constraints)
80
+ else:
81
+ constraints_tensor = None
82
+
83
+ tokens, lengths = task.get_interactive_tokens_and_lengths(lines, encode_fn)
84
+
85
+ itr = task.get_batch_iterator(
86
+ dataset=task.build_dataset_for_inference(
87
+ tokens, lengths, constraints=constraints_tensor
88
+ ),
89
+ max_tokens=cfg.dataset.max_tokens,
90
+ max_sentences=cfg.dataset.batch_size,
91
+ max_positions=max_positions,
92
+ ignore_invalid_inputs=cfg.dataset.skip_invalid_size_inputs_valid_test,
93
+ ).next_epoch_itr(shuffle=False)
94
+ for batch in itr:
95
+ ids = batch["id"]
96
+ src_tokens = batch["net_input"]["src_tokens"]
97
+ src_lengths = batch["net_input"]["src_lengths"]
98
+ constraints = batch.get("constraints", None)
99
+
100
+ yield Batch(
101
+ ids=ids,
102
+ src_tokens=src_tokens,
103
+ src_lengths=src_lengths,
104
+ constraints=constraints,
105
+ )
106
+
107
+
108
+ def main(cfg: FairseqConfig):
109
+ if isinstance(cfg, Namespace):
110
+ cfg = convert_namespace_to_omegaconf(cfg)
111
+
112
+ start_time = time.time()
113
+ total_translate_time = 0
114
+
115
+ utils.import_user_module(cfg.common)
116
+
117
+ if cfg.interactive.buffer_size < 1:
118
+ cfg.interactive.buffer_size = 1
119
+ if cfg.dataset.max_tokens is None and cfg.dataset.batch_size is None:
120
+ cfg.dataset.batch_size = 1
121
+
122
+ assert (
123
+ not cfg.generation.sampling or cfg.generation.nbest == cfg.generation.beam
124
+ ), "--sampling requires --nbest to be equal to --beam"
125
+ assert (
126
+ not cfg.dataset.batch_size
127
+ or cfg.dataset.batch_size <= cfg.interactive.buffer_size
128
+ ), "--batch-size cannot be larger than --buffer-size"
129
+
130
+ logger.info(cfg)
131
+
132
+ # Fix seed for stochastic decoding
133
+ if cfg.common.seed is not None and not cfg.generation.no_seed_provided:
134
+ np.random.seed(cfg.common.seed)
135
+ utils.set_torch_seed(cfg.common.seed)
136
+
137
+ use_cuda = torch.cuda.is_available() and not cfg.common.cpu
138
+
139
+ # Setup task, e.g., translation
140
+ task = tasks.setup_task(cfg.task)
141
+
142
+ # Load ensemble
143
+ overrides = ast.literal_eval(cfg.common_eval.model_overrides)
144
+ logger.info("loading model(s) from {}".format(cfg.common_eval.path))
145
+ models, _model_args = checkpoint_utils.load_model_ensemble(
146
+ utils.split_paths(cfg.common_eval.path),
147
+ arg_overrides=overrides,
148
+ task=task,
149
+ suffix=cfg.checkpoint.checkpoint_suffix,
150
+ strict=(cfg.checkpoint.checkpoint_shard_count == 1),
151
+ num_shards=cfg.checkpoint.checkpoint_shard_count,
152
+ )
153
+
154
+ # Set dictionaries
155
+ src_dict = task.source_dictionary
156
+ tgt_dict = task.target_dictionary
157
+
158
+ # Optimize ensemble for generation
159
+ for model in models:
160
+ if model is None:
161
+ continue
162
+ if cfg.common.fp16:
163
+ model.half()
164
+ if use_cuda and not cfg.distributed_training.pipeline_model_parallel:
165
+ model.cuda()
166
+ model.prepare_for_inference_(cfg)
167
+
168
+ # Initialize generator
169
+ generator = task.build_generator(models, cfg.generation)
170
+
171
+ # Handle tokenization and BPE
172
+ tokenizer = task.build_tokenizer(cfg.tokenizer)
173
+ bpe = task.build_bpe(cfg.bpe)
174
+
175
+ def encode_fn(x):
176
+ if tokenizer is not None:
177
+ x = tokenizer.encode(x)
178
+ if bpe is not None:
179
+ x = bpe.encode(x)
180
+ return x
181
+
182
+ def decode_fn(x):
183
+ if bpe is not None:
184
+ x = bpe.decode(x)
185
+ if tokenizer is not None:
186
+ x = tokenizer.decode(x)
187
+ return x
188
+
189
+ # Load alignment dictionary for unknown word replacement
190
+ # (None if no unknown word replacement, empty if no path to align dictionary)
191
+ align_dict = utils.load_align_dict(cfg.generation.replace_unk)
192
+
193
+ max_positions = utils.resolve_max_positions(
194
+ task.max_positions(), *[model.max_positions() for model in models]
195
+ )
196
+
197
+ if cfg.generation.constraints:
198
+ logger.warning(
199
+ "NOTE: Constrained decoding currently assumes a shared subword vocabulary."
200
+ )
201
+
202
+ if cfg.interactive.buffer_size > 1:
203
+ logger.info("Sentence buffer size: %s", cfg.interactive.buffer_size)
204
+ logger.info("NOTE: hypothesis and token scores are output in base 2")
205
+ logger.info("Type the input sentence and press return:")
206
+ start_id = 0
207
+ for inputs in buffered_read(cfg.interactive.input, cfg.interactive.buffer_size):
208
+ results = []
209
+ for batch in make_batches(inputs, cfg, task, max_positions, encode_fn):
210
+ bsz = batch.src_tokens.size(0)
211
+ src_tokens = batch.src_tokens
212
+ src_lengths = batch.src_lengths
213
+ constraints = batch.constraints
214
+ if use_cuda:
215
+ src_tokens = src_tokens.cuda()
216
+ src_lengths = src_lengths.cuda()
217
+ if constraints is not None:
218
+ constraints = constraints.cuda()
219
+
220
+ sample = {
221
+ "net_input": {
222
+ "src_tokens": src_tokens,
223
+ "src_lengths": src_lengths,
224
+ },
225
+ }
226
+ translate_start_time = time.time()
227
+ translations = task.inference_step(
228
+ generator, models, sample, constraints=constraints
229
+ )
230
+ translate_time = time.time() - translate_start_time
231
+ total_translate_time += translate_time
232
+ list_constraints = [[] for _ in range(bsz)]
233
+ if cfg.generation.constraints:
234
+ list_constraints = [unpack_constraints(c) for c in constraints]
235
+ for i, (id, hypos) in enumerate(zip(batch.ids.tolist(), translations)):
236
+ src_tokens_i = utils.strip_pad(src_tokens[i], tgt_dict.pad())
237
+ constraints = list_constraints[i]
238
+ results.append(
239
+ (
240
+ start_id + id,
241
+ src_tokens_i,
242
+ hypos,
243
+ {
244
+ "constraints": constraints,
245
+ "time": translate_time / len(translations),
246
+ },
247
+ )
248
+ )
249
+
250
+ # sort output to match input order
251
+ for id_, src_tokens, hypos, info in sorted(results, key=lambda x: x[0]):
252
+ src_str = ""
253
+ if src_dict is not None:
254
+ src_str = src_dict.string(src_tokens, cfg.common_eval.post_process)
255
+ print("S-{}\t{}".format(id_, src_str))
256
+ print("W-{}\t{:.3f}\tseconds".format(id_, info["time"]))
257
+ for constraint in info["constraints"]:
258
+ print(
259
+ "C-{}\t{}".format(
260
+ id_,
261
+ tgt_dict.string(constraint, cfg.common_eval.post_process),
262
+ )
263
+ )
264
+
265
+ # Process top predictions
266
+ for hypo in hypos[: min(len(hypos), cfg.generation.nbest)]:
267
+ hypo_tokens, hypo_str, alignment = utils.post_process_prediction(
268
+ hypo_tokens=hypo["tokens"].int().cpu(),
269
+ src_str=src_str,
270
+ alignment=hypo["alignment"],
271
+ align_dict=align_dict,
272
+ tgt_dict=tgt_dict,
273
+ remove_bpe=cfg.common_eval.post_process,
274
+ extra_symbols_to_ignore=get_symbols_to_strip_from_output(generator),
275
+ )
276
+ detok_hypo_str = decode_fn(hypo_str)
277
+ score = hypo["score"] / math.log(2) # convert to base 2
278
+ # original hypothesis (after tokenization and BPE)
279
+ print("H-{}\t{}\t{}".format(id_, score, hypo_str))
280
+ # detokenized hypothesis
281
+ print("D-{}\t{}\t{}".format(id_, score, detok_hypo_str))
282
+ print(
283
+ "P-{}\t{}".format(
284
+ id_,
285
+ " ".join(
286
+ map(
287
+ lambda x: "{:.4f}".format(x),
288
+ # convert from base e to base 2
289
+ hypo["positional_scores"].div_(math.log(2)).tolist(),
290
+ )
291
+ ),
292
+ )
293
+ )
294
+ if cfg.generation.print_alignment:
295
+ alignment_str = " ".join(
296
+ ["{}-{}".format(src, tgt) for src, tgt in alignment]
297
+ )
298
+ print("A-{}\t{}".format(id_, alignment_str))
299
+
300
+ # update running id_ counter
301
+ start_id += len(inputs)
302
+
303
+ logger.info(
304
+ "Total time: {:.3f} seconds; translation time: {:.3f}".format(
305
+ time.time() - start_time, total_translate_time
306
+ )
307
+ )
308
+
309
+
310
+ def cli_main():
311
+ parser = options.get_interactive_generation_parser()
312
+ args = options.parse_args_and_arch(parser)
313
+ distributed_utils.call_main(convert_namespace_to_omegaconf(args), main)
314
+
315
+
316
+ if __name__ == "__main__":
317
+ cli_main()
data/fairseq/fairseq_cli/preprocess.py ADDED
@@ -0,0 +1,393 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env python3
2
+ # Copyright (c) Facebook, Inc. and its affiliates.
3
+ #
4
+ # This source code is licensed under the MIT license found in the
5
+ # LICENSE file in the root directory of this source tree.
6
+ """
7
+ Data pre-processing: build vocabularies and binarize training data.
8
+ """
9
+
10
+ import logging
11
+ import os
12
+ import shutil
13
+ import sys
14
+ import typing as tp
15
+ from argparse import Namespace
16
+ from itertools import zip_longest
17
+
18
+ from fairseq import options, tasks, utils
19
+ from fairseq.binarizer import (
20
+ AlignmentDatasetBinarizer,
21
+ FileBinarizer,
22
+ VocabularyDatasetBinarizer,
23
+ )
24
+ from fairseq.data import Dictionary
25
+
26
+ logging.basicConfig(
27
+ format="%(asctime)s | %(levelname)s | %(name)s | %(message)s",
28
+ datefmt="%Y-%m-%d %H:%M:%S",
29
+ level=os.environ.get("LOGLEVEL", "INFO").upper(),
30
+ stream=sys.stdout,
31
+ )
32
+ logger = logging.getLogger("fairseq_cli.preprocess")
33
+
34
+ #####################################################################
35
+ # file name tools
36
+ #####################################################################
37
+
38
+
39
+ def _train_path(lang, trainpref):
40
+ return "{}{}".format(trainpref, ("." + lang) if lang else "")
41
+
42
+
43
+ def _file_name(prefix, lang):
44
+ fname = prefix
45
+ if lang is not None:
46
+ fname += ".{lang}".format(lang=lang)
47
+ return fname
48
+
49
+
50
+ def _dest_path(prefix, lang, destdir):
51
+ return os.path.join(destdir, _file_name(prefix, lang))
52
+
53
+
54
+ def _dict_path(lang, destdir):
55
+ return _dest_path("dict", lang, destdir) + ".txt"
56
+
57
+
58
+ def dataset_dest_prefix(args, output_prefix, lang):
59
+ base = os.path.join(args.destdir, output_prefix)
60
+ if lang is not None:
61
+ lang_part = f".{args.source_lang}-{args.target_lang}.{lang}"
62
+ elif args.only_source:
63
+ lang_part = ""
64
+ else:
65
+ lang_part = f".{args.source_lang}-{args.target_lang}"
66
+
67
+ return "{}{}".format(base, lang_part)
68
+
69
+
70
+ def dataset_dest_file(args, output_prefix, lang, extension):
71
+ return "{}.{}".format(dataset_dest_prefix(args, output_prefix, lang), extension)
72
+
73
+
74
+ #####################################################################
75
+ # dictionary tools
76
+ #####################################################################
77
+
78
+
79
+ def _build_dictionary(
80
+ filenames,
81
+ task,
82
+ args,
83
+ src=False,
84
+ tgt=False,
85
+ ):
86
+ assert src ^ tgt
87
+ return task.build_dictionary(
88
+ filenames,
89
+ workers=args.workers,
90
+ threshold=args.thresholdsrc if src else args.thresholdtgt,
91
+ nwords=args.nwordssrc if src else args.nwordstgt,
92
+ padding_factor=args.padding_factor,
93
+ )
94
+
95
+
96
+ #####################################################################
97
+ # bin file creation logic
98
+ #####################################################################
99
+
100
+
101
+ def _make_binary_dataset(
102
+ vocab: Dictionary,
103
+ input_prefix: str,
104
+ output_prefix: str,
105
+ lang: tp.Optional[str],
106
+ num_workers: int,
107
+ args: Namespace,
108
+ ):
109
+ logger.info("[{}] Dictionary: {} types".format(lang, len(vocab)))
110
+
111
+ binarizer = VocabularyDatasetBinarizer(
112
+ vocab,
113
+ append_eos=True,
114
+ )
115
+
116
+ input_file = "{}{}".format(input_prefix, ("." + lang) if lang is not None else "")
117
+ full_output_prefix = dataset_dest_prefix(args, output_prefix, lang)
118
+
119
+ final_summary = FileBinarizer.multiprocess_dataset(
120
+ input_file,
121
+ args.dataset_impl,
122
+ binarizer,
123
+ full_output_prefix,
124
+ vocab_size=len(vocab),
125
+ num_workers=num_workers,
126
+ )
127
+
128
+ logger.info(f"[{lang}] {input_file}: {final_summary} (by {vocab.unk_word})")
129
+
130
+
131
+ def _make_binary_alignment_dataset(
132
+ input_prefix: str, output_prefix: str, num_workers: int, args: Namespace
133
+ ):
134
+
135
+ binarizer = AlignmentDatasetBinarizer(utils.parse_alignment)
136
+
137
+ input_file = input_prefix
138
+ full_output_prefix = dataset_dest_prefix(args, output_prefix, lang=None)
139
+
140
+ final_summary = FileBinarizer.multiprocess_dataset(
141
+ input_file,
142
+ args.dataset_impl,
143
+ binarizer,
144
+ full_output_prefix,
145
+ vocab_size=None,
146
+ num_workers=num_workers,
147
+ )
148
+
149
+ logger.info(
150
+ "[alignments] {}: parsed {} alignments".format(
151
+ input_file, final_summary.num_seq
152
+ )
153
+ )
154
+
155
+
156
+ #####################################################################
157
+ # routing logic
158
+ #####################################################################
159
+
160
+
161
+ def _make_dataset(
162
+ vocab: Dictionary,
163
+ input_prefix: str,
164
+ output_prefix: str,
165
+ lang: tp.Optional[str],
166
+ args: Namespace,
167
+ num_workers: int,
168
+ ):
169
+ if args.dataset_impl == "raw":
170
+ # Copy original text file to destination folder
171
+ output_text_file = _dest_path(
172
+ output_prefix + ".{}-{}".format(args.source_lang, args.target_lang),
173
+ lang,
174
+ args.destdir,
175
+ )
176
+ shutil.copyfile(_file_name(input_prefix, lang), output_text_file)
177
+ else:
178
+ _make_binary_dataset(
179
+ vocab, input_prefix, output_prefix, lang, num_workers, args
180
+ )
181
+
182
+
183
+ def _make_all(lang, vocab, args):
184
+ if args.trainpref:
185
+ _make_dataset(
186
+ vocab, args.trainpref, "train", lang, args=args, num_workers=args.workers
187
+ )
188
+ if args.validpref:
189
+ for k, validpref in enumerate(args.validpref.split(",")):
190
+ outprefix = "valid{}".format(k) if k > 0 else "valid"
191
+ _make_dataset(
192
+ vocab, validpref, outprefix, lang, args=args, num_workers=args.workers
193
+ )
194
+ if args.testpref:
195
+ for k, testpref in enumerate(args.testpref.split(",")):
196
+ outprefix = "test{}".format(k) if k > 0 else "test"
197
+ _make_dataset(
198
+ vocab, testpref, outprefix, lang, args=args, num_workers=args.workers
199
+ )
200
+
201
+
202
+ def _make_all_alignments(args):
203
+ if args.trainpref and os.path.exists(args.trainpref + "." + args.align_suffix):
204
+ _make_binary_alignment_dataset(
205
+ args.trainpref + "." + args.align_suffix,
206
+ "train.align",
207
+ num_workers=args.workers,
208
+ args=args,
209
+ )
210
+ if args.validpref and os.path.exists(args.validpref + "." + args.align_suffix):
211
+ _make_binary_alignment_dataset(
212
+ args.validpref + "." + args.align_suffix,
213
+ "valid.align",
214
+ num_workers=args.workers,
215
+ args=args,
216
+ )
217
+ if args.testpref and os.path.exists(args.testpref + "." + args.align_suffix):
218
+ _make_binary_alignment_dataset(
219
+ args.testpref + "." + args.align_suffix,
220
+ "test.align",
221
+ num_workers=args.workers,
222
+ args=args,
223
+ )
224
+
225
+
226
+ #####################################################################
227
+ # align
228
+ #####################################################################
229
+
230
+
231
+ def _align_files(args, src_dict, tgt_dict):
232
+ assert args.trainpref, "--trainpref must be set if --alignfile is specified"
233
+ src_file_name = _train_path(args.source_lang, args.trainpref)
234
+ tgt_file_name = _train_path(args.target_lang, args.trainpref)
235
+ freq_map = {}
236
+ with open(args.alignfile, "r", encoding="utf-8") as align_file:
237
+ with open(src_file_name, "r", encoding="utf-8") as src_file:
238
+ with open(tgt_file_name, "r", encoding="utf-8") as tgt_file:
239
+ for a, s, t in zip_longest(align_file, src_file, tgt_file):
240
+ si = src_dict.encode_line(s, add_if_not_exist=False)
241
+ ti = tgt_dict.encode_line(t, add_if_not_exist=False)
242
+ ai = list(map(lambda x: tuple(x.split("-")), a.split()))
243
+ for sai, tai in ai:
244
+ srcidx = si[int(sai)]
245
+ tgtidx = ti[int(tai)]
246
+ if srcidx != src_dict.unk() and tgtidx != tgt_dict.unk():
247
+ assert srcidx != src_dict.pad()
248
+ assert srcidx != src_dict.eos()
249
+ assert tgtidx != tgt_dict.pad()
250
+ assert tgtidx != tgt_dict.eos()
251
+ if srcidx not in freq_map:
252
+ freq_map[srcidx] = {}
253
+ if tgtidx not in freq_map[srcidx]:
254
+ freq_map[srcidx][tgtidx] = 1
255
+ else:
256
+ freq_map[srcidx][tgtidx] += 1
257
+ align_dict = {}
258
+ for srcidx in freq_map.keys():
259
+ align_dict[srcidx] = max(freq_map[srcidx], key=freq_map[srcidx].get)
260
+ with open(
261
+ os.path.join(
262
+ args.destdir,
263
+ "alignment.{}-{}.txt".format(args.source_lang, args.target_lang),
264
+ ),
265
+ "w",
266
+ encoding="utf-8",
267
+ ) as f:
268
+ for k, v in align_dict.items():
269
+ print("{} {}".format(src_dict[k], tgt_dict[v]), file=f)
270
+
271
+
272
+ #####################################################################
273
+ # MAIN
274
+ #####################################################################
275
+
276
+
277
+ def main(args):
278
+ # setup some basic things
279
+ utils.import_user_module(args)
280
+
281
+ os.makedirs(args.destdir, exist_ok=True)
282
+
283
+ logger.addHandler(
284
+ logging.FileHandler(
285
+ filename=os.path.join(args.destdir, "preprocess.log"),
286
+ )
287
+ )
288
+ logger.info(args)
289
+
290
+ assert (
291
+ args.dataset_impl != "huffman"
292
+ ), "preprocessing.py doesn't support Huffman yet, use HuffmanCodeBuilder directly."
293
+
294
+ # build dictionaries
295
+
296
+ target = not args.only_source
297
+
298
+ if not args.srcdict and os.path.exists(_dict_path(args.source_lang, args.destdir)):
299
+ raise FileExistsError(_dict_path(args.source_lang, args.destdir))
300
+
301
+ if (
302
+ target
303
+ and not args.tgtdict
304
+ and os.path.exists(_dict_path(args.target_lang, args.destdir))
305
+ ):
306
+ raise FileExistsError(_dict_path(args.target_lang, args.destdir))
307
+
308
+ task = tasks.get_task(args.task)
309
+
310
+ if args.joined_dictionary:
311
+ assert (
312
+ not args.srcdict or not args.tgtdict
313
+ ), "cannot use both --srcdict and --tgtdict with --joined-dictionary"
314
+
315
+ if args.srcdict:
316
+ src_dict = task.load_dictionary(args.srcdict)
317
+ elif args.tgtdict:
318
+ src_dict = task.load_dictionary(args.tgtdict)
319
+ else:
320
+ assert (
321
+ args.trainpref
322
+ ), "--trainpref must be set if --srcdict is not specified"
323
+ src_dict = _build_dictionary(
324
+ {
325
+ _train_path(lang, args.trainpref)
326
+ for lang in [args.source_lang, args.target_lang]
327
+ },
328
+ task=task,
329
+ args=args,
330
+ src=True,
331
+ )
332
+ tgt_dict = src_dict
333
+ else:
334
+ if args.srcdict:
335
+ src_dict = task.load_dictionary(args.srcdict)
336
+ else:
337
+ assert (
338
+ args.trainpref
339
+ ), "--trainpref must be set if --srcdict is not specified"
340
+ src_dict = _build_dictionary(
341
+ [_train_path(args.source_lang, args.trainpref)],
342
+ task=task,
343
+ args=args,
344
+ src=True,
345
+ )
346
+
347
+ if target:
348
+ if args.tgtdict:
349
+ tgt_dict = task.load_dictionary(args.tgtdict)
350
+ else:
351
+ assert (
352
+ args.trainpref
353
+ ), "--trainpref must be set if --tgtdict is not specified"
354
+ tgt_dict = _build_dictionary(
355
+ [_train_path(args.target_lang, args.trainpref)],
356
+ task=task,
357
+ args=args,
358
+ tgt=True,
359
+ )
360
+ else:
361
+ tgt_dict = None
362
+
363
+ # save dictionaries
364
+
365
+ src_dict.save(_dict_path(args.source_lang, args.destdir))
366
+ if target and tgt_dict is not None:
367
+ tgt_dict.save(_dict_path(args.target_lang, args.destdir))
368
+
369
+ if args.dict_only:
370
+ return
371
+
372
+ _make_all(args.source_lang, src_dict, args)
373
+ if target:
374
+ _make_all(args.target_lang, tgt_dict, args)
375
+
376
+ # align the datasets if needed
377
+ if args.align_suffix:
378
+ _make_all_alignments(args)
379
+
380
+ logger.info("Wrote preprocessed data to {}".format(args.destdir))
381
+
382
+ if args.alignfile:
383
+ _align_files(args, src_dict=src_dict, tgt_dict=tgt_dict)
384
+
385
+
386
+ def cli_main():
387
+ parser = options.get_preprocessing_parser()
388
+ args = parser.parse_args()
389
+ main(args)
390
+
391
+
392
+ if __name__ == "__main__":
393
+ cli_main()
data/fairseq/fairseq_cli/score.py ADDED
@@ -0,0 +1,102 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env python3
2
+ # Copyright (c) Facebook, Inc. and its affiliates.
3
+ #
4
+ # This source code is licensed under the MIT license found in the
5
+ # LICENSE file in the root directory of this source tree.
6
+ """
7
+ BLEU scoring of generated translations against reference translations.
8
+ """
9
+
10
+ import argparse
11
+ import os
12
+ import sys
13
+
14
+ from fairseq.data import dictionary
15
+ from fairseq.scoring import bleu
16
+
17
+
18
+ def get_parser():
19
+ parser = argparse.ArgumentParser(
20
+ description="Command-line script for BLEU scoring."
21
+ )
22
+ # fmt: off
23
+ parser.add_argument('-s', '--sys', default='-', help='system output')
24
+ parser.add_argument('-r', '--ref', required=True, help='references')
25
+ parser.add_argument('-o', '--order', default=4, metavar='N',
26
+ type=int, help='consider ngrams up to this order')
27
+ parser.add_argument('--ignore-case', action='store_true',
28
+ help='case-insensitive scoring')
29
+ parser.add_argument('--sacrebleu', action='store_true',
30
+ help='score with sacrebleu')
31
+ parser.add_argument('--sentence-bleu', action='store_true',
32
+ help='report sentence-level BLEUs (i.e., with +1 smoothing)')
33
+ # fmt: on
34
+ return parser
35
+
36
+
37
+ def cli_main():
38
+ parser = get_parser()
39
+ args = parser.parse_args()
40
+ print(args)
41
+
42
+ assert args.sys == "-" or os.path.exists(
43
+ args.sys
44
+ ), "System output file {} does not exist".format(args.sys)
45
+ assert os.path.exists(args.ref), "Reference file {} does not exist".format(args.ref)
46
+
47
+ dict = dictionary.Dictionary()
48
+
49
+ def readlines(fd):
50
+ for line in fd.readlines():
51
+ if args.ignore_case:
52
+ yield line.lower()
53
+ else:
54
+ yield line
55
+
56
+ if args.sacrebleu:
57
+ import sacrebleu
58
+
59
+ def score(fdsys):
60
+ with open(args.ref) as fdref:
61
+ print(sacrebleu.corpus_bleu(fdsys, [fdref]).format())
62
+
63
+ elif args.sentence_bleu:
64
+
65
+ def score(fdsys):
66
+ with open(args.ref) as fdref:
67
+ scorer = bleu.Scorer(dict.pad(), dict.eos(), dict.unk())
68
+ for i, (sys_tok, ref_tok) in enumerate(
69
+ zip(readlines(fdsys), readlines(fdref))
70
+ ):
71
+ scorer.reset(one_init=True)
72
+ sys_tok = dict.encode_line(sys_tok)
73
+ ref_tok = dict.encode_line(ref_tok)
74
+ scorer.add(ref_tok, sys_tok)
75
+ print(i, scorer.result_string(args.order))
76
+
77
+ else:
78
+
79
+ def score(fdsys):
80
+ with open(args.ref) as fdref:
81
+ scorer = bleu.Scorer(
82
+ bleu.BleuConfig(
83
+ pad=dict.pad(),
84
+ eos=dict.eos(),
85
+ unk=dict.unk(),
86
+ )
87
+ )
88
+ for sys_tok, ref_tok in zip(readlines(fdsys), readlines(fdref)):
89
+ sys_tok = dict.encode_line(sys_tok)
90
+ ref_tok = dict.encode_line(ref_tok)
91
+ scorer.add(ref_tok, sys_tok)
92
+ print(scorer.result_string(args.order))
93
+
94
+ if args.sys == "-":
95
+ score(sys.stdin)
96
+ else:
97
+ with open(args.sys, "r") as f:
98
+ score(f)
99
+
100
+
101
+ if __name__ == "__main__":
102
+ cli_main()
data/fairseq/fairseq_cli/train.py ADDED
@@ -0,0 +1,581 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env python3 -u
2
+ # Copyright (c) Facebook, Inc. and its affiliates.
3
+ #
4
+ # This source code is licensed under the MIT license found in the
5
+ # LICENSE file in the root directory of this source tree.
6
+ """
7
+ Train a new model on one or across multiple GPUs.
8
+ """
9
+
10
+ import argparse
11
+ import logging
12
+ import math
13
+ import os
14
+ import sys
15
+ from typing import Any, Callable, Dict, List, Optional, Tuple
16
+
17
+ # We need to setup root logger before importing any fairseq libraries.
18
+ logging.basicConfig(
19
+ format="%(asctime)s | %(levelname)s | %(name)s | %(message)s",
20
+ datefmt="%Y-%m-%d %H:%M:%S",
21
+ level=os.environ.get("LOGLEVEL", "INFO").upper(),
22
+ stream=sys.stdout,
23
+ )
24
+ logger = logging.getLogger("fairseq_cli.train")
25
+
26
+ import numpy as np
27
+ import torch
28
+ from omegaconf import DictConfig, OmegaConf
29
+
30
+ from fairseq import checkpoint_utils, options, quantization_utils, tasks, utils
31
+ from fairseq.data import data_utils, iterators
32
+ from fairseq.data.plasma_utils import PlasmaStore
33
+ from fairseq.dataclass.configs import FairseqConfig
34
+ from fairseq.dataclass.initialize import add_defaults
35
+ from fairseq.dataclass.utils import convert_namespace_to_omegaconf
36
+ from fairseq.distributed import fsdp_enable_wrap, fsdp_wrap
37
+ from fairseq.distributed import utils as distributed_utils
38
+ from fairseq.file_io import PathManager
39
+ from fairseq.logging import meters, metrics, progress_bar
40
+ from fairseq.model_parallel.megatron_trainer import MegatronTrainer
41
+ from fairseq.trainer import Trainer
42
+
43
+
44
+ def main(cfg: FairseqConfig) -> None:
45
+ if isinstance(cfg, argparse.Namespace):
46
+ cfg = convert_namespace_to_omegaconf(cfg)
47
+
48
+ utils.import_user_module(cfg.common)
49
+ add_defaults(cfg)
50
+
51
+ if (
52
+ distributed_utils.is_master(cfg.distributed_training)
53
+ and "job_logging_cfg" in cfg
54
+ ):
55
+ # make hydra logging work with ddp (see # see https://github.com/facebookresearch/hydra/issues/1126)
56
+ logging.config.dictConfig(OmegaConf.to_container(cfg.job_logging_cfg))
57
+
58
+ assert (
59
+ cfg.dataset.max_tokens is not None or cfg.dataset.batch_size is not None
60
+ ), "Must specify batch size either with --max-tokens or --batch-size"
61
+ metrics.reset()
62
+
63
+ if cfg.common.log_file is not None:
64
+ handler = logging.FileHandler(filename=cfg.common.log_file)
65
+ logger.addHandler(handler)
66
+
67
+ np.random.seed(cfg.common.seed)
68
+ utils.set_torch_seed(cfg.common.seed)
69
+
70
+ if distributed_utils.is_master(cfg.distributed_training):
71
+ checkpoint_utils.verify_checkpoint_directory(cfg.checkpoint.save_dir)
72
+
73
+ # Print args
74
+ logger.info(cfg)
75
+
76
+ if cfg.checkpoint.write_checkpoints_asynchronously:
77
+ try:
78
+ import iopath # noqa: F401
79
+ except ImportError:
80
+ logging.exception(
81
+ "Asynchronous checkpoint writing is specified but iopath is "
82
+ "not installed: `pip install iopath`"
83
+ )
84
+ return
85
+
86
+ # Setup task, e.g., translation, language modeling, etc.
87
+ task = tasks.setup_task(cfg.task)
88
+
89
+ assert cfg.criterion, "Please specify criterion to train a model"
90
+
91
+ # Build model and criterion
92
+ if cfg.distributed_training.ddp_backend == "fully_sharded":
93
+ with fsdp_enable_wrap(cfg.distributed_training):
94
+ model = fsdp_wrap(task.build_model(cfg.model))
95
+ else:
96
+ model = task.build_model(cfg.model)
97
+ criterion = task.build_criterion(cfg.criterion)
98
+ logger.info(model)
99
+ logger.info("task: {}".format(task.__class__.__name__))
100
+ logger.info("model: {}".format(model.__class__.__name__))
101
+ logger.info("criterion: {}".format(criterion.__class__.__name__))
102
+ logger.info(
103
+ "num. shared model params: {:,} (num. trained: {:,})".format(
104
+ sum(
105
+ p.numel() for p in model.parameters() if not getattr(p, "expert", False)
106
+ ),
107
+ sum(
108
+ p.numel()
109
+ for p in model.parameters()
110
+ if not getattr(p, "expert", False) and p.requires_grad
111
+ ),
112
+ )
113
+ )
114
+
115
+ logger.info(
116
+ "num. expert model params: {} (num. trained: {})".format(
117
+ sum(p.numel() for p in model.parameters() if getattr(p, "expert", False)),
118
+ sum(
119
+ p.numel()
120
+ for p in model.parameters()
121
+ if getattr(p, "expert", False) and p.requires_grad
122
+ ),
123
+ )
124
+ )
125
+
126
+ # Load valid dataset (we load training data below, based on the latest checkpoint)
127
+ # We load the valid dataset AFTER building the model
128
+ if not cfg.dataset.disable_validation:
129
+ data_utils.raise_if_valid_subsets_unintentionally_ignored(cfg)
130
+ if cfg.dataset.combine_valid_subsets:
131
+ task.load_dataset("valid", combine=True, epoch=1)
132
+ else:
133
+ for valid_sub_split in cfg.dataset.valid_subset.split(","):
134
+ task.load_dataset(valid_sub_split, combine=False, epoch=1)
135
+
136
+ # (optionally) Configure quantization
137
+ if cfg.common.quantization_config_path is not None:
138
+ quantizer = quantization_utils.Quantizer(
139
+ config_path=cfg.common.quantization_config_path,
140
+ max_epoch=cfg.optimization.max_epoch,
141
+ max_update=cfg.optimization.max_update,
142
+ )
143
+ else:
144
+ quantizer = None
145
+
146
+ # Build trainer
147
+ if cfg.common.model_parallel_size == 1:
148
+ trainer = Trainer(cfg, task, model, criterion, quantizer)
149
+ else:
150
+ trainer = MegatronTrainer(cfg, task, model, criterion)
151
+ logger.info(
152
+ "training on {} devices (GPUs/TPUs)".format(
153
+ cfg.distributed_training.distributed_world_size
154
+ )
155
+ )
156
+ logger.info(
157
+ "max tokens per device = {} and max sentences per device = {}".format(
158
+ cfg.dataset.max_tokens,
159
+ cfg.dataset.batch_size,
160
+ )
161
+ )
162
+
163
+ # Load the latest checkpoint if one is available and restore the
164
+ # corresponding train iterator
165
+ extra_state, epoch_itr = checkpoint_utils.load_checkpoint(
166
+ cfg.checkpoint,
167
+ trainer,
168
+ # don't cache epoch iterators for sharded datasets
169
+ disable_iterator_cache=task.has_sharded_data("train"),
170
+ )
171
+ if cfg.common.tpu:
172
+ import torch_xla.core.xla_model as xm
173
+
174
+ xm.rendezvous("load_checkpoint") # wait for all workers
175
+
176
+ max_epoch = cfg.optimization.max_epoch or math.inf
177
+ lr = trainer.get_lr()
178
+
179
+ # TODO: a dry run on validation set to pin the memory
180
+ valid_subsets = cfg.dataset.valid_subset.split(",")
181
+ if not cfg.dataset.disable_validation:
182
+ for subset in valid_subsets:
183
+ logger.info('begin dry-run validation on "{}" subset'.format(subset))
184
+ itr = trainer.get_valid_iterator(subset).next_epoch_itr(
185
+ shuffle=False, set_dataset_epoch=False # use a fixed valid set
186
+ )
187
+ if cfg.common.tpu:
188
+ itr = utils.tpu_data_loader(itr)
189
+ for _ in itr:
190
+ pass
191
+ # TODO: end of dry run section
192
+
193
+ train_meter = meters.StopwatchMeter()
194
+ train_meter.start()
195
+ while epoch_itr.next_epoch_idx <= max_epoch:
196
+ if lr <= cfg.optimization.stop_min_lr:
197
+ logger.info(
198
+ f"stopping training because current learning rate ({lr}) is smaller "
199
+ "than or equal to minimum learning rate "
200
+ f"(--stop-min-lr={cfg.optimization.stop_min_lr})"
201
+ )
202
+ break
203
+
204
+ # train for one epoch
205
+ valid_losses, should_stop = train(cfg, trainer, task, epoch_itr)
206
+ if should_stop:
207
+ break
208
+
209
+ # only use first validation loss to update the learning rate
210
+ lr = trainer.lr_step(epoch_itr.epoch, valid_losses[0])
211
+
212
+ epoch_itr = trainer.get_train_iterator(
213
+ epoch_itr.next_epoch_idx,
214
+ # sharded data: get train iterator for next epoch
215
+ load_dataset=task.has_sharded_data("train"),
216
+ # don't cache epoch iterators for sharded datasets
217
+ disable_iterator_cache=task.has_sharded_data("train"),
218
+ )
219
+ train_meter.stop()
220
+ logger.info("done training in {:.1f} seconds".format(train_meter.sum))
221
+
222
+ # ioPath implementation to wait for all asynchronous file writes to complete.
223
+ if cfg.checkpoint.write_checkpoints_asynchronously:
224
+ logger.info(
225
+ "ioPath PathManager waiting for all asynchronous checkpoint "
226
+ "writes to finish."
227
+ )
228
+ PathManager.async_close()
229
+ logger.info("ioPath PathManager finished waiting.")
230
+
231
+
232
+ def should_stop_early(cfg: DictConfig, valid_loss: float) -> bool:
233
+ # skip check if no validation was done in the current epoch
234
+ if valid_loss is None:
235
+ return False
236
+ if cfg.checkpoint.patience <= 0:
237
+ return False
238
+
239
+ def is_better(a, b):
240
+ return a > b if cfg.checkpoint.maximize_best_checkpoint_metric else a < b
241
+
242
+ prev_best = getattr(should_stop_early, "best", None)
243
+ if prev_best is None or is_better(valid_loss, prev_best):
244
+ should_stop_early.best = valid_loss
245
+ should_stop_early.num_runs = 0
246
+ return False
247
+ else:
248
+ should_stop_early.num_runs += 1
249
+ if should_stop_early.num_runs >= cfg.checkpoint.patience:
250
+ logger.info(
251
+ "early stop since valid performance hasn't improved for last {} runs".format(
252
+ cfg.checkpoint.patience
253
+ )
254
+ )
255
+ return True
256
+ else:
257
+ return False
258
+
259
+
260
+ @metrics.aggregate("train")
261
+ def train(
262
+ cfg: DictConfig, trainer: Trainer, task: tasks.FairseqTask, epoch_itr
263
+ ) -> Tuple[List[Optional[float]], bool]:
264
+ """Train the model for one epoch and return validation losses."""
265
+ # Initialize data iterator
266
+ itr = epoch_itr.next_epoch_itr(
267
+ fix_batches_to_gpus=cfg.distributed_training.fix_batches_to_gpus,
268
+ shuffle=(epoch_itr.next_epoch_idx > cfg.dataset.curriculum),
269
+ )
270
+ update_freq = (
271
+ cfg.optimization.update_freq[epoch_itr.epoch - 1]
272
+ if epoch_itr.epoch <= len(cfg.optimization.update_freq)
273
+ else cfg.optimization.update_freq[-1]
274
+ )
275
+ itr = iterators.GroupedIterator(
276
+ itr,
277
+ update_freq,
278
+ skip_remainder_batch=cfg.optimization.skip_remainder_batch,
279
+ )
280
+ if cfg.common.tpu:
281
+ itr = utils.tpu_data_loader(itr)
282
+ progress = progress_bar.progress_bar(
283
+ itr,
284
+ log_format=cfg.common.log_format,
285
+ log_file=cfg.common.log_file,
286
+ log_interval=cfg.common.log_interval,
287
+ epoch=epoch_itr.epoch,
288
+ aim_repo=(
289
+ cfg.common.aim_repo
290
+ if distributed_utils.is_master(cfg.distributed_training)
291
+ else None
292
+ ),
293
+ aim_run_hash=(
294
+ cfg.common.aim_run_hash
295
+ if distributed_utils.is_master(cfg.distributed_training)
296
+ else None
297
+ ),
298
+ aim_param_checkpoint_dir=cfg.checkpoint.save_dir,
299
+ tensorboard_logdir=(
300
+ cfg.common.tensorboard_logdir
301
+ if distributed_utils.is_master(cfg.distributed_training)
302
+ else None
303
+ ),
304
+ default_log_format=("tqdm" if not cfg.common.no_progress_bar else "simple"),
305
+ wandb_project=(
306
+ cfg.common.wandb_project
307
+ if distributed_utils.is_master(cfg.distributed_training)
308
+ else None
309
+ ),
310
+ wandb_run_name=os.environ.get(
311
+ "WANDB_NAME", os.path.basename(cfg.checkpoint.save_dir)
312
+ ),
313
+ azureml_logging=(
314
+ cfg.common.azureml_logging
315
+ if distributed_utils.is_master(cfg.distributed_training)
316
+ else False
317
+ ),
318
+ )
319
+ progress.update_config(_flatten_config(cfg))
320
+
321
+ trainer.begin_epoch(epoch_itr.epoch)
322
+
323
+ valid_subsets = cfg.dataset.valid_subset.split(",")
324
+ should_stop = False
325
+ num_updates = trainer.get_num_updates()
326
+ logger.info("Start iterating over samples")
327
+ for i, samples in enumerate(progress):
328
+ with metrics.aggregate("train_inner"), torch.autograd.profiler.record_function(
329
+ "train_step-%d" % i
330
+ ):
331
+ log_output = trainer.train_step(samples)
332
+
333
+ if log_output is not None: # not OOM, overflow, ...
334
+ # log mid-epoch stats
335
+ num_updates = trainer.get_num_updates()
336
+ if num_updates % cfg.common.log_interval == 0:
337
+ stats = get_training_stats(metrics.get_smoothed_values("train_inner"))
338
+ progress.log(stats, tag="train_inner", step=num_updates)
339
+
340
+ # reset mid-epoch stats after each log interval
341
+ # the end-of-epoch stats will still be preserved
342
+ metrics.reset_meters("train_inner")
343
+
344
+ end_of_epoch = not itr.has_next()
345
+ valid_losses, should_stop = validate_and_save(
346
+ cfg, trainer, task, epoch_itr, valid_subsets, end_of_epoch
347
+ )
348
+
349
+ if should_stop:
350
+ break
351
+
352
+ # log end-of-epoch stats
353
+ logger.info("end of epoch {} (average epoch stats below)".format(epoch_itr.epoch))
354
+ stats = get_training_stats(metrics.get_smoothed_values("train"))
355
+ progress.print(stats, tag="train", step=num_updates)
356
+
357
+ # reset epoch-level meters
358
+ metrics.reset_meters("train")
359
+ return valid_losses, should_stop
360
+
361
+
362
+ def _flatten_config(cfg: DictConfig):
363
+ config = OmegaConf.to_container(cfg)
364
+ # remove any legacy Namespaces and replace with a single "args"
365
+ namespace = None
366
+ for k, v in list(config.items()):
367
+ if isinstance(v, argparse.Namespace):
368
+ namespace = v
369
+ del config[k]
370
+ if namespace is not None:
371
+ config["args"] = vars(namespace)
372
+ return config
373
+
374
+
375
+ def validate_and_save(
376
+ cfg: DictConfig,
377
+ trainer: Trainer,
378
+ task: tasks.FairseqTask,
379
+ epoch_itr,
380
+ valid_subsets: List[str],
381
+ end_of_epoch: bool,
382
+ ) -> Tuple[List[Optional[float]], bool]:
383
+ num_updates = trainer.get_num_updates()
384
+ max_update = cfg.optimization.max_update or math.inf
385
+
386
+ # Stopping conditions (and an additional one based on validation loss later
387
+ # on)
388
+ should_stop = False
389
+ if num_updates >= max_update:
390
+ should_stop = True
391
+ logger.info(
392
+ f"Stopping training due to "
393
+ f"num_updates: {num_updates} >= max_update: {max_update}"
394
+ )
395
+
396
+ training_time_hours = trainer.cumulative_training_time() / (60 * 60)
397
+ if (
398
+ cfg.optimization.stop_time_hours > 0
399
+ and training_time_hours > cfg.optimization.stop_time_hours
400
+ ):
401
+ should_stop = True
402
+ logger.info(
403
+ f"Stopping training due to "
404
+ f"cumulative_training_time: {training_time_hours} > "
405
+ f"stop_time_hours: {cfg.optimization.stop_time_hours} hour(s)"
406
+ )
407
+
408
+ do_save = (
409
+ (end_of_epoch and epoch_itr.epoch % cfg.checkpoint.save_interval == 0)
410
+ or should_stop
411
+ or (
412
+ cfg.checkpoint.save_interval_updates > 0
413
+ and num_updates > 0
414
+ and num_updates % cfg.checkpoint.save_interval_updates == 0
415
+ and num_updates >= cfg.dataset.validate_after_updates
416
+ )
417
+ )
418
+ do_validate = (
419
+ (
420
+ (not end_of_epoch and do_save) # validate during mid-epoch saves
421
+ or (end_of_epoch and epoch_itr.epoch % cfg.dataset.validate_interval == 0)
422
+ or should_stop
423
+ or (
424
+ cfg.dataset.validate_interval_updates > 0
425
+ and num_updates > 0
426
+ and num_updates % cfg.dataset.validate_interval_updates == 0
427
+ )
428
+ )
429
+ and not cfg.dataset.disable_validation
430
+ and num_updates >= cfg.dataset.validate_after_updates
431
+ )
432
+
433
+ # Validate
434
+ valid_losses = [None]
435
+ if do_validate:
436
+ valid_losses = validate(cfg, trainer, task, epoch_itr, valid_subsets)
437
+
438
+ should_stop |= should_stop_early(cfg, valid_losses[0])
439
+
440
+ # Save checkpoint
441
+ if do_save or should_stop:
442
+ cp_path = checkpoint_utils.save_checkpoint(
443
+ cfg.checkpoint, trainer, epoch_itr, valid_losses[0]
444
+ )
445
+ if cp_path is not None and hasattr(task, "post_save"):
446
+ task.post_save(cp_path, num_updates)
447
+
448
+ return valid_losses, should_stop
449
+
450
+
451
+ def get_training_stats(stats: Dict[str, Any]) -> Dict[str, Any]:
452
+ stats["wall"] = round(metrics.get_meter("default", "wall").elapsed_time, 0)
453
+ return stats
454
+
455
+
456
+ def validate(
457
+ cfg: DictConfig,
458
+ trainer: Trainer,
459
+ task: tasks.FairseqTask,
460
+ epoch_itr,
461
+ subsets: List[str],
462
+ ) -> List[Optional[float]]:
463
+ """Evaluate the model on the validation set(s) and return the losses."""
464
+
465
+ if cfg.dataset.fixed_validation_seed is not None:
466
+ # set fixed seed for every validation
467
+ utils.set_torch_seed(cfg.dataset.fixed_validation_seed)
468
+
469
+ trainer.begin_valid_epoch(epoch_itr.epoch)
470
+ valid_losses = []
471
+ for subset_idx, subset in enumerate(subsets):
472
+ logger.info('begin validation on "{}" subset'.format(subset))
473
+
474
+ # Initialize data iterator
475
+ itr = trainer.get_valid_iterator(subset).next_epoch_itr(
476
+ shuffle=False, set_dataset_epoch=False # use a fixed valid set
477
+ )
478
+ if cfg.common.tpu:
479
+ itr = utils.tpu_data_loader(itr)
480
+ progress = progress_bar.progress_bar(
481
+ itr,
482
+ log_format=cfg.common.log_format,
483
+ log_interval=cfg.common.log_interval,
484
+ epoch=epoch_itr.epoch,
485
+ prefix=f"valid on '{subset}' subset",
486
+ aim_repo=(
487
+ cfg.common.aim_repo
488
+ if distributed_utils.is_master(cfg.distributed_training)
489
+ else None
490
+ ),
491
+ aim_run_hash=(
492
+ cfg.common.aim_run_hash
493
+ if distributed_utils.is_master(cfg.distributed_training)
494
+ else None
495
+ ),
496
+ aim_param_checkpoint_dir=cfg.checkpoint.save_dir,
497
+ tensorboard_logdir=(
498
+ cfg.common.tensorboard_logdir
499
+ if distributed_utils.is_master(cfg.distributed_training)
500
+ else None
501
+ ),
502
+ default_log_format=("tqdm" if not cfg.common.no_progress_bar else "simple"),
503
+ wandb_project=(
504
+ cfg.common.wandb_project
505
+ if distributed_utils.is_master(cfg.distributed_training)
506
+ else None
507
+ ),
508
+ wandb_run_name=os.environ.get(
509
+ "WANDB_NAME", os.path.basename(cfg.checkpoint.save_dir)
510
+ ),
511
+ )
512
+
513
+ # create a new root metrics aggregator so validation metrics
514
+ # don't pollute other aggregators (e.g., train meters)
515
+ with metrics.aggregate(new_root=True) as agg:
516
+ for i, sample in enumerate(progress):
517
+ if (
518
+ cfg.dataset.max_valid_steps is not None
519
+ and i > cfg.dataset.max_valid_steps
520
+ ):
521
+ break
522
+ trainer.valid_step(sample)
523
+
524
+ # log validation stats
525
+ # only tracking the best metric on the 1st validation subset
526
+ tracking_best = subset_idx == 0
527
+ stats = get_valid_stats(cfg, trainer, agg.get_smoothed_values(), tracking_best)
528
+
529
+ if hasattr(task, "post_validate"):
530
+ task.post_validate(trainer.get_model(), stats, agg)
531
+
532
+ progress.print(stats, tag=subset, step=trainer.get_num_updates())
533
+
534
+ valid_losses.append(stats[cfg.checkpoint.best_checkpoint_metric])
535
+ return valid_losses
536
+
537
+
538
+ def get_valid_stats(
539
+ cfg: DictConfig,
540
+ trainer: Trainer,
541
+ stats: Dict[str, Any],
542
+ tracking_best: bool,
543
+ ) -> Dict[str, Any]:
544
+ stats["num_updates"] = trainer.get_num_updates()
545
+ if tracking_best and hasattr(checkpoint_utils.save_checkpoint, "best"):
546
+ key = "best_{0}".format(cfg.checkpoint.best_checkpoint_metric)
547
+ best_function = max if cfg.checkpoint.maximize_best_checkpoint_metric else min
548
+ stats[key] = best_function(
549
+ checkpoint_utils.save_checkpoint.best,
550
+ stats[cfg.checkpoint.best_checkpoint_metric],
551
+ )
552
+ return stats
553
+
554
+
555
+ def cli_main(
556
+ modify_parser: Optional[Callable[[argparse.ArgumentParser], None]] = None
557
+ ) -> None:
558
+ parser = options.get_training_parser()
559
+ args = options.parse_args_and_arch(parser, modify_parser=modify_parser)
560
+
561
+ cfg = convert_namespace_to_omegaconf(args)
562
+
563
+ if cfg.common.use_plasma_view:
564
+ server = PlasmaStore(path=cfg.common.plasma_path)
565
+ logger.info(
566
+ f"Started plasma server pid {server.server.pid} {cfg.common.plasma_path}"
567
+ )
568
+
569
+ if args.profile:
570
+ with torch.cuda.profiler.profile():
571
+ with torch.autograd.profiler.emit_nvtx():
572
+ distributed_utils.call_main(cfg, main)
573
+ else:
574
+ distributed_utils.call_main(cfg, main)
575
+
576
+ # if cfg.common.use_plasma_view:
577
+ # server.server.kill()
578
+
579
+
580
+ if __name__ == "__main__":
581
+ cli_main()
data/fairseq/fairseq_cli/validate.py ADDED
@@ -0,0 +1,153 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env python3 -u
2
+ # Copyright (c) Facebook, Inc. and its affiliates.
3
+ #
4
+ # This source code is licensed under the MIT license found in the
5
+ # LICENSE file in the root directory of this source tree.
6
+
7
+ import logging
8
+ import os
9
+ import sys
10
+ from argparse import Namespace
11
+ from itertools import chain
12
+
13
+ import torch
14
+ from omegaconf import DictConfig
15
+
16
+ from fairseq import checkpoint_utils, distributed_utils, options, utils
17
+ from fairseq.dataclass.utils import convert_namespace_to_omegaconf
18
+ from fairseq.logging import metrics, progress_bar
19
+ from fairseq.utils import reset_logging
20
+
21
+ logging.basicConfig(
22
+ format="%(asctime)s | %(levelname)s | %(name)s | %(message)s",
23
+ datefmt="%Y-%m-%d %H:%M:%S",
24
+ level=os.environ.get("LOGLEVEL", "INFO").upper(),
25
+ stream=sys.stdout,
26
+ )
27
+ logger = logging.getLogger("fairseq_cli.validate")
28
+
29
+
30
+ def main(cfg: DictConfig, override_args=None):
31
+ if isinstance(cfg, Namespace):
32
+ cfg = convert_namespace_to_omegaconf(cfg)
33
+
34
+ utils.import_user_module(cfg.common)
35
+
36
+ reset_logging()
37
+
38
+ assert (
39
+ cfg.dataset.max_tokens is not None or cfg.dataset.batch_size is not None
40
+ ), "Must specify batch size either with --max-tokens or --batch-size"
41
+
42
+ use_fp16 = cfg.common.fp16
43
+ use_cuda = torch.cuda.is_available() and not cfg.common.cpu
44
+
45
+ if use_cuda:
46
+ torch.cuda.set_device(cfg.distributed_training.device_id)
47
+
48
+ if cfg.distributed_training.distributed_world_size > 1:
49
+ data_parallel_world_size = distributed_utils.get_data_parallel_world_size()
50
+ data_parallel_rank = distributed_utils.get_data_parallel_rank()
51
+ else:
52
+ data_parallel_world_size = 1
53
+ data_parallel_rank = 0
54
+
55
+ if override_args is not None:
56
+ overrides = vars(override_args)
57
+ overrides.update(eval(getattr(override_args, "model_overrides", "{}")))
58
+ else:
59
+ overrides = None
60
+
61
+ # Load ensemble
62
+ logger.info("loading model(s) from {}".format(cfg.common_eval.path))
63
+ models, saved_cfg, task = checkpoint_utils.load_model_ensemble_and_task(
64
+ [cfg.common_eval.path],
65
+ arg_overrides=overrides,
66
+ suffix=cfg.checkpoint.checkpoint_suffix,
67
+ )
68
+ model = models[0]
69
+
70
+ # Move models to GPU
71
+ for model in models:
72
+ model.eval()
73
+ if use_fp16:
74
+ model.half()
75
+ if use_cuda:
76
+ model.cuda()
77
+
78
+ # Print args
79
+ logger.info(saved_cfg)
80
+
81
+ # Build criterion
82
+ criterion = task.build_criterion(saved_cfg.criterion)
83
+ criterion.eval()
84
+
85
+ for subset in cfg.dataset.valid_subset.split(","):
86
+ try:
87
+ task.load_dataset(subset, combine=False, epoch=1, task_cfg=saved_cfg.task)
88
+ dataset = task.dataset(subset)
89
+ except KeyError:
90
+ raise Exception("Cannot find dataset: " + subset)
91
+
92
+ # Initialize data iterator
93
+ itr = task.get_batch_iterator(
94
+ dataset=dataset,
95
+ max_tokens=cfg.dataset.max_tokens,
96
+ max_sentences=cfg.dataset.batch_size,
97
+ max_positions=utils.resolve_max_positions(
98
+ task.max_positions(),
99
+ *[m.max_positions() for m in models],
100
+ ),
101
+ ignore_invalid_inputs=cfg.dataset.skip_invalid_size_inputs_valid_test,
102
+ required_batch_size_multiple=cfg.dataset.required_batch_size_multiple,
103
+ seed=cfg.common.seed,
104
+ num_shards=data_parallel_world_size,
105
+ shard_id=data_parallel_rank,
106
+ num_workers=cfg.dataset.num_workers,
107
+ data_buffer_size=cfg.dataset.data_buffer_size,
108
+ ).next_epoch_itr(shuffle=False)
109
+ progress = progress_bar.progress_bar(
110
+ itr,
111
+ log_format=cfg.common.log_format,
112
+ log_interval=cfg.common.log_interval,
113
+ prefix=f"valid on '{subset}' subset",
114
+ default_log_format=("tqdm" if not cfg.common.no_progress_bar else "simple"),
115
+ )
116
+
117
+ log_outputs = []
118
+ for i, sample in enumerate(progress):
119
+ sample = utils.move_to_cuda(sample) if use_cuda else sample
120
+ _loss, _sample_size, log_output = task.valid_step(sample, model, criterion)
121
+ progress.log(log_output, step=i)
122
+ log_outputs.append(log_output)
123
+
124
+ if data_parallel_world_size > 1:
125
+ log_outputs = distributed_utils.all_gather_list(
126
+ log_outputs,
127
+ max_size=cfg.common.all_gather_list_size,
128
+ group=distributed_utils.get_data_parallel_group(),
129
+ )
130
+ log_outputs = list(chain.from_iterable(log_outputs))
131
+
132
+ with metrics.aggregate() as agg:
133
+ task.reduce_metrics(log_outputs, criterion)
134
+ log_output = agg.get_smoothed_values()
135
+
136
+ progress.print(log_output, tag=subset, step=i)
137
+
138
+
139
+ def cli_main():
140
+ parser = options.get_validation_parser()
141
+ args = options.parse_args_and_arch(parser)
142
+
143
+ # only override args that are explicitly given on the command line
144
+ override_parser = options.get_validation_parser()
145
+ override_args = options.parse_args_and_arch(override_parser, suppress_defaults=True)
146
+
147
+ distributed_utils.call_main(
148
+ convert_namespace_to_omegaconf(args), main, override_args=override_args
149
+ )
150
+
151
+
152
+ if __name__ == "__main__":
153
+ cli_main()
data/fairseq/tests/distributed/__init__.py ADDED
File without changes
data/fairseq/tests/distributed/test_bmuf.py ADDED
@@ -0,0 +1,204 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright (c) Facebook, Inc. and its affiliates.
2
+ #
3
+ # This source code is licensed under the MIT license found in the
4
+ # LICENSE file in the root directory of this source tree.
5
+
6
+ import argparse
7
+ import functools
8
+ import random
9
+ import unittest
10
+ from multiprocessing import Manager
11
+
12
+ import torch
13
+ import torch.nn as nn
14
+ from omegaconf import OmegaConf
15
+
16
+ from fairseq import optim
17
+ from fairseq.distributed import utils as distributed_utils
18
+
19
+
20
+ class Model(nn.Module):
21
+ def __init__(self, input_size, output_size):
22
+ super(Model, self).__init__()
23
+ self.fc = nn.Linear(input_size, output_size)
24
+
25
+ def forward(self, input):
26
+ output = self.fc(input)
27
+ return output
28
+
29
+
30
+ def setup_model_loss_criterion(cfg, args, rank, is_cuda):
31
+ """
32
+ setup model, criterion and optimizer based on input args
33
+ """
34
+ args.distributed_rank = rank
35
+ cfg.distributed_training.distributed_rank = args.distributed_rank
36
+ if cfg.distributed_training.distributed_world_size > 1:
37
+ distributed_utils.distributed_init(cfg)
38
+ torch.manual_seed(1)
39
+ model = Model(args.input_size, args.nb_classes)
40
+ loss_fn = nn.CrossEntropyLoss()
41
+ if is_cuda:
42
+ model = model.cuda()
43
+ loss_fn = loss_fn.cuda()
44
+
45
+ optimizer = optim.sgd.SGD(args, model.parameters())
46
+ optimizer = optim.FairseqBMUF(cfg=cfg.bmuf, optimizer=optimizer)
47
+
48
+ return model, loss_fn, optimizer
49
+
50
+
51
+ def train_step(input, target, model, loss_fn, optimizer, **unused):
52
+ """Do forward, backward and parameter update."""
53
+ model.train()
54
+ output = model(input)
55
+ loss = loss_fn(output, target)
56
+ optimizer.backward(loss)
57
+ optimizer.step()
58
+
59
+
60
+ def single_gpu_training(cfg, args, rank, iterations, shared_results):
61
+
62
+ is_cuda = torch.cuda.is_available()
63
+ if is_cuda:
64
+ torch.cuda.set_device(rank)
65
+
66
+ model, loss_fn, optimizer = setup_model_loss_criterion(cfg, args, rank, is_cuda)
67
+
68
+ for _ in range(iterations):
69
+ input = torch.randn(1, args.input_size)
70
+ target = torch.empty(args.batch_size, dtype=torch.long).random_(args.nb_classes)
71
+
72
+ if is_cuda:
73
+ input = input.cuda()
74
+ target = target.cuda()
75
+ train_step(input, target, model, loss_fn, optimizer)
76
+
77
+ results = []
78
+ for param in model.parameters():
79
+ if len(results) == 0:
80
+ results = param.flatten().cpu().data
81
+ else:
82
+ results = torch.cat((results, param.flatten().cpu().data), 0)
83
+
84
+ shared_results[rank] = results
85
+
86
+
87
+ def setup_args():
88
+ args = argparse.Namespace()
89
+ args.global_sync_iter = 20
90
+ args.block_momentum = 0.875
91
+ args.block_lr = 0.5
92
+ args.input_size = 5
93
+ args.nb_classes = 2
94
+ args.batch_size = 1
95
+ args.lr = [1e-3]
96
+ args.momentum = 0
97
+ args.weight_decay = 0
98
+ args.warmup_iterations = 0
99
+ args.use_nbm = True
100
+ args.average_sync = True
101
+ args.global_sync_iter = 1
102
+ args.model_parallel_size = 1
103
+ args.distributed_backend = "gloo"
104
+
105
+ args.distributed_world_size = 2
106
+ port = random.randint(10000, 20000)
107
+ args.distributed_init_method = "tcp://localhost:{port}".format(port=port)
108
+ args.distributed_init_host = "localhost"
109
+ args.distributed_port = port + 1
110
+ args.local_world_size = args.distributed_world_size
111
+
112
+ cfg = OmegaConf.create()
113
+ cfg.optimization = OmegaConf.create()
114
+ cfg.common = OmegaConf.create()
115
+ cfg.distributed_training = OmegaConf.create()
116
+ cfg.dataset = OmegaConf.create()
117
+ cfg.bmuf = OmegaConf.create()
118
+ cfg.optimizer = OmegaConf.create()
119
+
120
+ cfg.bmuf.global_sync_iter = args.global_sync_iter
121
+ cfg.bmuf.block_momentum = args.block_momentum
122
+ cfg.bmuf.block_lr = args.block_lr
123
+ cfg.dataset.batch_size = args.batch_size
124
+ cfg.optimization.lr = args.lr
125
+ cfg.optimizer.momentum = args.momentum
126
+ cfg.optimizer.weight_decay = args.weight_decay
127
+ cfg.bmuf.warmup_iterations = args.warmup_iterations
128
+ cfg.bmuf.use_nbm = args.use_nbm
129
+ cfg.bmuf.average_sync = args.average_sync
130
+ cfg.common.model_parallel_size = args.model_parallel_size
131
+ cfg.distributed_training.distributed_backend = args.distributed_backend
132
+ cfg.distributed_training.distributed_world_size = args.distributed_world_size
133
+ cfg.bmuf.distributed_world_size = args.distributed_world_size
134
+ cfg.distributed_training.distributed_init_method = args.distributed_init_method
135
+ cfg.distributed_training.distributed_port = args.distributed_port
136
+
137
+ return cfg, args
138
+
139
+
140
+ @unittest.skipIf(torch.cuda.device_count() < 2, "test requires 2 GPUs")
141
+ class TestBMUF(unittest.TestCase):
142
+ def bmuf_process(self, cfg, args, iterations):
143
+ results = Manager().dict()
144
+ torch.multiprocessing.spawn(
145
+ fn=functools.partial(single_gpu_training, cfg, args),
146
+ args=(iterations, results),
147
+ nprocs=args.distributed_world_size,
148
+ join=True,
149
+ )
150
+ return results
151
+
152
+ def test_bmuf_sync(self):
153
+ # Train model for 1 iteration and do bmuf sync without doing warmup
154
+ cfg, args = setup_args()
155
+ iterations = 1
156
+ results = self.bmuf_process(cfg, args, iterations)
157
+ # Make sure params in both machines are same
158
+ assert len(results) == 2
159
+ self.assertAlmostEqual(results[0], results[1])
160
+
161
+ def test_warmup_sync(self):
162
+ # Train model for 20 iteration and do warmup sync without doing bmuf sync
163
+ cfg, args = setup_args()
164
+ args.warmup_iterations = 20
165
+ cfg.bmuf.warmup_iterations = args.warmup_iterations
166
+ iterations = 20
167
+ results = self.bmuf_process(cfg, args, iterations)
168
+ # Make sure params in both machines are same
169
+ assert len(results) == 2
170
+ self.assertAlmostEqual(results[0], results[1])
171
+
172
+ def test_warmup_sync_bmuf_sync(self):
173
+ # Train model for 25 iteration and do warmup sync after 20 iteration
174
+ # and bmuf sync after 25 iteration
175
+ cfg, args = setup_args()
176
+ args.warmup_iterations = 20
177
+ args.global_sync_iter = 5
178
+ cfg.bmuf.warmup_iterations = args.warmup_iterations
179
+ cfg.bmuf.global_sync_iter = args.global_sync_iter
180
+ iterations = 25
181
+ results = self.bmuf_process(cfg, args, iterations)
182
+ # Make sure params in both machines are same
183
+ assert len(results) == 2
184
+ self.assertAlmostEqual(results[0], results[1])
185
+
186
+ def test_single_gpu_bmuf(self):
187
+ # Train model for 5 iterations and use GPU 1
188
+ cfg, args = setup_args()
189
+ args.distributed_world_size = 1
190
+ args.warmup_iterations = 5
191
+ cfg.distributed_training.distributed_world_size = args.distributed_world_size
192
+ cfg.bmuf.distributed_world_size = args.distributed_world_size
193
+ cfg.bmuf.warmup_iterations = args.warmup_iterations
194
+ iterations = 20
195
+ results = self.bmuf_process(cfg, args, iterations)
196
+ assert len(results) == 1
197
+
198
+ def assertAlmostEqual(self, t1, t2):
199
+ self.assertEqual(t1.size(), t2.size(), "size mismatch")
200
+ self.assertLess((t1 - t2).abs().max(), 1e-4)
201
+
202
+
203
+ if __name__ == "__main__":
204
+ unittest.main()
data/fairseq/tests/distributed/test_distributed_timeout_wrapper.py ADDED
@@ -0,0 +1,52 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright (c) Facebook, Inc. and its affiliates.
2
+ #
3
+ # This source code is licensed under the MIT license found in the
4
+ # LICENSE file in the root directory of this source tree.
5
+
6
+ import logging
7
+ import signal
8
+ import time
9
+ import unittest
10
+
11
+ import torch
12
+ from torch import nn
13
+
14
+ from fairseq.distributed import DistributedTimeoutWrapper
15
+
16
+
17
+ class ModuleWithDelay(nn.Module):
18
+ def __init__(self, delay):
19
+ super().__init__()
20
+ self.delay = delay
21
+
22
+ def forward(self, x):
23
+ time.sleep(self.delay)
24
+ return x
25
+
26
+
27
+ class TestDistributedTimeoutWrapper(unittest.TestCase):
28
+ def setUp(self):
29
+ logging.disable(logging.CRITICAL)
30
+
31
+ def tearDown(self):
32
+ logging.disable(logging.NOTSET)
33
+
34
+ def test_no_timeout(self):
35
+ module = DistributedTimeoutWrapper(ModuleWithDelay(1), 0, signal.SIGINT)
36
+ module(torch.rand(5))
37
+ module.stop_timeout()
38
+
39
+ def test_timeout_safe(self):
40
+ module = DistributedTimeoutWrapper(ModuleWithDelay(1), 10, signal.SIGINT)
41
+ module(torch.rand(5))
42
+ module.stop_timeout()
43
+
44
+ def test_timeout_killed(self):
45
+ with self.assertRaises(KeyboardInterrupt):
46
+ module = DistributedTimeoutWrapper(ModuleWithDelay(5), 1, signal.SIGINT)
47
+ module(torch.rand(5))
48
+ module.stop_timeout()
49
+
50
+
51
+ if __name__ == "__main__":
52
+ unittest.main()
data/fairseq/tests/distributed/test_module_proxy_wrapper.py ADDED
@@ -0,0 +1,74 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright (c) Facebook, Inc. and its affiliates.
2
+ #
3
+ # This source code is licensed under the MIT license found in the
4
+ # LICENSE file in the root directory of this source tree.
5
+
6
+ import unittest
7
+
8
+ import torch
9
+ from torch import nn
10
+
11
+ from fairseq.distributed import ModuleProxyWrapper
12
+
13
+ from .utils import objects_are_equal
14
+
15
+
16
+ class MockDDPWrapper(nn.Module):
17
+ """A simple wrapper with an interface similar to DistributedDataParallel."""
18
+
19
+ def __init__(self, module):
20
+ super().__init__()
21
+ self.module = module
22
+
23
+ def forward(self, x):
24
+ return self.module(x)
25
+
26
+
27
+ class Model(nn.Module):
28
+ def __init__(self):
29
+ super().__init__()
30
+ self.linear = nn.Linear(5, 10)
31
+ self.xyz = "hello"
32
+
33
+ def forward(self, x):
34
+ return self.linear(x)
35
+
36
+ def get_xyz(self):
37
+ return self.xyz
38
+
39
+
40
+ class TestModuleProxyWrapper(unittest.TestCase):
41
+ def _get_module(self):
42
+ module = Model()
43
+ wrapped_module = MockDDPWrapper(module)
44
+ wrapped_module = ModuleProxyWrapper(wrapped_module)
45
+ return wrapped_module, module
46
+
47
+ def test_getattr_forwarding(self):
48
+ wrapped_module, module = self._get_module()
49
+ assert module.xyz == "hello"
50
+ assert module.get_xyz() == "hello"
51
+ assert wrapped_module.xyz == "hello"
52
+
53
+ wrapped_module.xyz = "world"
54
+ assert wrapped_module.xyz == "world"
55
+ assert module.get_xyz() == "hello"
56
+
57
+ def test_state_dict(self):
58
+ wrapped_module, module = self._get_module()
59
+ assert objects_are_equal(wrapped_module.state_dict(), module.state_dict())
60
+
61
+ def test_load_state_dict(self):
62
+ wrapped_module, module = self._get_module()
63
+ wrapped_module.load_state_dict(module.state_dict())
64
+ input = torch.rand(4, 5)
65
+ torch.testing.assert_allclose(wrapped_module(input), module(input))
66
+
67
+ def test_forward(self):
68
+ wrapped_module, module = self._get_module()
69
+ input = torch.rand(4, 5)
70
+ torch.testing.assert_allclose(wrapped_module(input), module(input))
71
+
72
+
73
+ if __name__ == "__main__":
74
+ unittest.main()
data/fairseq/tests/distributed/test_utils.py ADDED
@@ -0,0 +1,124 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright (c) Facebook, Inc. and its affiliates.
2
+ #
3
+ # This source code is licensed under the MIT license found in the
4
+ # LICENSE file in the root directory of this source tree.
5
+
6
+ import functools
7
+ import sys
8
+ import unittest
9
+
10
+ import torch
11
+
12
+ from fairseq.distributed import utils as dist_utils
13
+
14
+ from .utils import objects_are_equal, spawn_and_init
15
+
16
+
17
+ class DistributedTest(unittest.TestCase):
18
+ def setUp(self):
19
+ if not torch.cuda.is_available():
20
+ raise unittest.SkipTest("CUDA not available, skipping test")
21
+ if sys.platform == "win32":
22
+ raise unittest.SkipTest("NCCL doesn't support Windows, skipping test")
23
+ if torch.cuda.device_count() < 2:
24
+ raise unittest.SkipTest("distributed tests require 2+ GPUs, skipping")
25
+
26
+
27
+ class TestBroadcastObject(DistributedTest):
28
+ def test_str(self):
29
+ spawn_and_init(
30
+ functools.partial(
31
+ TestBroadcastObject._test_broadcast_object, "hello world"
32
+ ),
33
+ world_size=2,
34
+ )
35
+
36
+ def test_tensor(self):
37
+ spawn_and_init(
38
+ functools.partial(
39
+ TestBroadcastObject._test_broadcast_object,
40
+ torch.rand(5),
41
+ ),
42
+ world_size=2,
43
+ )
44
+
45
+ def test_complex(self):
46
+ spawn_and_init(
47
+ functools.partial(
48
+ TestBroadcastObject._test_broadcast_object,
49
+ {
50
+ "a": "1",
51
+ "b": [2, torch.rand(2, 3), 3],
52
+ "c": (torch.rand(2, 3), 4),
53
+ "d": {5, torch.rand(5)},
54
+ "e": torch.rand(5),
55
+ "f": torch.rand(5).int().cuda(),
56
+ },
57
+ ),
58
+ world_size=2,
59
+ )
60
+
61
+ @staticmethod
62
+ def _test_broadcast_object(ref_obj, rank, group):
63
+ obj = dist_utils.broadcast_object(
64
+ ref_obj if rank == 0 else None, src_rank=0, group=group
65
+ )
66
+ assert objects_are_equal(ref_obj, obj)
67
+
68
+
69
+ class TestAllGatherList(DistributedTest):
70
+ def test_str_equality(self):
71
+ spawn_and_init(
72
+ functools.partial(
73
+ TestAllGatherList._test_all_gather_list_equality,
74
+ "hello world",
75
+ ),
76
+ world_size=2,
77
+ )
78
+
79
+ def test_tensor_equality(self):
80
+ spawn_and_init(
81
+ functools.partial(
82
+ TestAllGatherList._test_all_gather_list_equality,
83
+ torch.rand(5),
84
+ ),
85
+ world_size=2,
86
+ )
87
+
88
+ def test_complex_equality(self):
89
+ spawn_and_init(
90
+ functools.partial(
91
+ TestAllGatherList._test_all_gather_list_equality,
92
+ {
93
+ "a": "1",
94
+ "b": [2, torch.rand(2, 3), 3],
95
+ "c": (torch.rand(2, 3), 4),
96
+ "d": {5, torch.rand(5)},
97
+ "e": torch.rand(5),
98
+ "f": torch.rand(5).int(),
99
+ },
100
+ ),
101
+ world_size=2,
102
+ )
103
+
104
+ @staticmethod
105
+ def _test_all_gather_list_equality(ref_obj, rank, group):
106
+ objs = dist_utils.all_gather_list(ref_obj, group)
107
+ for obj in objs:
108
+ assert objects_are_equal(ref_obj, obj)
109
+
110
+ def test_rank_tensor(self):
111
+ spawn_and_init(
112
+ TestAllGatherList._test_all_gather_list_rank_tensor, world_size=2
113
+ )
114
+
115
+ @staticmethod
116
+ def _test_all_gather_list_rank_tensor(rank, group):
117
+ obj = torch.tensor([rank])
118
+ objs = dist_utils.all_gather_list(obj, group)
119
+ for i, obj in enumerate(objs):
120
+ assert obj.item() == i
121
+
122
+
123
+ if __name__ == "__main__":
124
+ unittest.main()
data/fairseq/tests/distributed/utils.py ADDED
@@ -0,0 +1,65 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright (c) Facebook, Inc. and its affiliates.
2
+ #
3
+ # This source code is licensed under the MIT license found in the
4
+ # LICENSE file in the root directory of this source tree.
5
+
6
+ import functools
7
+ import tempfile
8
+
9
+ import torch
10
+
11
+
12
+ def spawn_and_init(fn, world_size, args=None):
13
+ if args is None:
14
+ args = ()
15
+ with tempfile.NamedTemporaryFile(delete=False) as tmp_file:
16
+ torch.multiprocessing.spawn(
17
+ fn=functools.partial(init_and_run, fn, args),
18
+ args=(
19
+ world_size,
20
+ tmp_file.name,
21
+ ),
22
+ nprocs=world_size,
23
+ join=True,
24
+ )
25
+
26
+
27
+ def distributed_init(rank, world_size, tmp_file):
28
+ torch.distributed.init_process_group(
29
+ backend="nccl",
30
+ init_method="file://{}".format(tmp_file),
31
+ world_size=world_size,
32
+ rank=rank,
33
+ )
34
+ torch.cuda.set_device(rank)
35
+
36
+
37
+ def init_and_run(fn, args, rank, world_size, tmp_file):
38
+ distributed_init(rank, world_size, tmp_file)
39
+ group = torch.distributed.new_group()
40
+ fn(rank, group, *args)
41
+
42
+
43
+ def objects_are_equal(a, b) -> bool:
44
+ if type(a) is not type(b):
45
+ return False
46
+ if isinstance(a, dict):
47
+ if set(a.keys()) != set(b.keys()):
48
+ return False
49
+ for k in a.keys():
50
+ if not objects_are_equal(a[k], b[k]):
51
+ return False
52
+ return True
53
+ elif isinstance(a, (list, tuple, set)):
54
+ if len(a) != len(b):
55
+ return False
56
+ return all(objects_are_equal(x, y) for x, y in zip(a, b))
57
+ elif torch.is_tensor(a):
58
+ return (
59
+ a.size() == b.size()
60
+ and a.dtype == b.dtype
61
+ and a.device == b.device
62
+ and torch.all(a == b)
63
+ )
64
+ else:
65
+ return a == b
data/fairseq/tests/gpu/__init__.py ADDED
File without changes
data/fairseq/tests/gpu/test_binaries_gpu.py ADDED
@@ -0,0 +1,590 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright (c) Facebook, Inc. and its affiliates.
2
+ #
3
+ # This source code is licensed under the MIT license found in the
4
+ # LICENSE file in the root directory of this source tree.
5
+
6
+ import contextlib
7
+ import json
8
+ import logging
9
+ import os
10
+ import tempfile
11
+ import unittest
12
+ from io import StringIO
13
+
14
+ import torch
15
+
16
+ from fairseq import options
17
+ from fairseq_cli import train
18
+ from tests.utils import (
19
+ create_dummy_data,
20
+ generate_main,
21
+ preprocess_lm_data,
22
+ preprocess_translation_data,
23
+ train_language_model,
24
+ train_translation_model,
25
+ )
26
+
27
+
28
+ @unittest.skipIf(not torch.cuda.is_available(), "test requires a GPU")
29
+ class TestMultiGPU(unittest.TestCase):
30
+ @staticmethod
31
+ def parse_logs(logfile):
32
+ logs = []
33
+ for ln in open(logfile, "r").readlines():
34
+ try:
35
+ logs.append(json.loads(ln))
36
+ except json.JSONDecodeError:
37
+ continue
38
+ return logs
39
+
40
+ @property
41
+ def world_size(self):
42
+ return torch.cuda.device_count()
43
+
44
+ def train_flags(self, mu):
45
+ return [
46
+ "--memory-efficient-fp16",
47
+ "--update-freq",
48
+ "1",
49
+ "--seed",
50
+ "1",
51
+ "--log-format",
52
+ "json",
53
+ "--max-update",
54
+ str(mu),
55
+ "--tokens-per-sample",
56
+ "20",
57
+ "--batch-size",
58
+ "2",
59
+ "--share-decoder-input-output-embed",
60
+ "--optimizer",
61
+ "adam",
62
+ "--max-valid-steps",
63
+ "1",
64
+ "--pad-to-fixed-length",
65
+ "--sample-break-mode",
66
+ "none",
67
+ ]
68
+
69
+ def _test_resume_multilingual_training(
70
+ self, extra_clargs, arch="transformer_lm_gpt2_tiny"
71
+ ):
72
+ languages = ["en_XX", "fr_XX", "zh_CN"]
73
+ save_interval = 5
74
+ mu = 10
75
+ flags = (
76
+ self.train_flags(mu)
77
+ + ["--save-interval-updates", str(save_interval), "--log-interval", "1"]
78
+ + extra_clargs
79
+ )
80
+ with contextlib.redirect_stdout(StringIO()):
81
+ with tempfile.TemporaryDirectory("test_fp16") as data_dir:
82
+ log = os.path.join(data_dir, "train.log")
83
+ create_dummy_data(
84
+ data_dir,
85
+ num_examples=int(
86
+ mu * 20 * self.world_size * 1.5
87
+ ), # make sure enough data for max updates
88
+ languages=languages,
89
+ )
90
+ preprocess_lm_data(data_dir, languages)
91
+ train_language_model(
92
+ data_dir,
93
+ arch,
94
+ flags + ["--log-file", log],
95
+ task="multilingual_language_modeling",
96
+ world_size=self.world_size,
97
+ )
98
+ log2 = os.path.join(data_dir, "resume.log")
99
+ ckpt_name = f"checkpoint_1_{save_interval}.pt"
100
+ restore_file = os.path.join(data_dir, ckpt_name)
101
+ train_language_model(
102
+ data_dir,
103
+ arch,
104
+ flags
105
+ + ["--log-file", log2, "--restore-file", restore_file, "--no-save"],
106
+ task="multilingual_language_modeling",
107
+ world_size=self.world_size,
108
+ )
109
+
110
+ l1 = self.parse_logs(log)
111
+ assert (
112
+ int(l1[-1]["train_num_updates"]) == mu
113
+ ), f"The first run did not complete {mu} updates. Add more data"
114
+ l2 = self.parse_logs(log2)
115
+
116
+ if int(l2[0]["num_updates"]) != save_interval + 1:
117
+ all_ckpt_files = [
118
+ x for x in os.listdir(data_dir) if x.endswith(".pt")
119
+ ]
120
+ import shutil
121
+
122
+ shutil.move(data_dir, "last_failed_resume")
123
+ raise AssertionError(
124
+ f"Likely failed to load {ckpt_name}. {all_ckpt_files} \n LOGS: {l1} \n\n {l2}. "
125
+ )
126
+ for k in [
127
+ "train_loss",
128
+ "train_num_updates",
129
+ "train_ppl",
130
+ "train_gnorm",
131
+ ]:
132
+ from_scratch, resumed = float(l1[-1][k]), float(l2[-1][k])
133
+ # This fails without rounding!
134
+ assert (
135
+ from_scratch == resumed
136
+ ), f"difference at {k} {from_scratch} != {resumed}"
137
+
138
+
139
+ @unittest.skipIf(not torch.cuda.is_available(), "test requires a GPU")
140
+ class TestTranslationGPU(unittest.TestCase):
141
+ def setUp(self):
142
+ logging.disable(logging.CRITICAL)
143
+
144
+ def tearDown(self):
145
+ logging.disable(logging.NOTSET)
146
+
147
+ def test_fp16_multigpu(self):
148
+ self._test_multigpu("test_fp16", ["--fp16"])
149
+
150
+ def test_slowmo_multigpu(self):
151
+ self._test_multigpu(
152
+ "test_slowmo", ["--ddp-backend", "slowmo", "--nprocs-per-node", "1"]
153
+ )
154
+
155
+ def test_slowmo_single_node_multigpu(self):
156
+ self._test_multigpu(
157
+ "test_slowmo_single_node",
158
+ ["--ddp-backend", "slowmo", "--nprocs-per-node", "2"],
159
+ )
160
+
161
+ def _test_multigpu(self, test_name, test_args):
162
+ with contextlib.redirect_stdout(StringIO()):
163
+ with tempfile.TemporaryDirectory(test_name) as data_dir:
164
+ log = os.path.join(data_dir, "train.log")
165
+ create_dummy_data(data_dir)
166
+ preprocess_translation_data(data_dir)
167
+ train_translation_model(
168
+ data_dir,
169
+ "fconv_iwslt_de_en",
170
+ test_args + ["--log-file", log],
171
+ world_size=min(torch.cuda.device_count(), 2),
172
+ )
173
+ generate_main(data_dir)
174
+ assert os.path.exists(log)
175
+
176
+ @staticmethod
177
+ def parse_logs(logfile):
178
+ logs = []
179
+ for ln in open(logfile, "r").readlines():
180
+ try:
181
+ logs.append(json.loads(ln))
182
+ except json.JSONDecodeError:
183
+ continue
184
+ return logs
185
+
186
+ def test_resume_training_fsdp(self):
187
+ self._test_resume_training(["--ddp-backend", "fully_sharded"])
188
+
189
+ def test_resume_training_fsdp_sharded_state(self):
190
+ self._test_resume_training(
191
+ ["--ddp-backend", "fully_sharded", "--use-sharded-state"]
192
+ )
193
+
194
+ def test_resume_training_noc10d(self):
195
+ self._test_resume_training([])
196
+
197
+ def _test_resume_training(self, extra_clargs, arch="fconv_iwslt_de_en"):
198
+ flags = [
199
+ "--fp16",
200
+ "--log-format",
201
+ "json",
202
+ "--max-update",
203
+ "10",
204
+ "--save-interval-updates",
205
+ "2",
206
+ "--log-interval",
207
+ "1",
208
+ ] + extra_clargs
209
+ world_size = min(torch.cuda.device_count(), 2)
210
+ with contextlib.redirect_stdout(StringIO()):
211
+ with tempfile.TemporaryDirectory("test_fp16") as data_dir:
212
+ log = os.path.join(data_dir, "train.log")
213
+ create_dummy_data(data_dir)
214
+ preprocess_translation_data(data_dir)
215
+ train_translation_model(
216
+ data_dir,
217
+ arch,
218
+ flags + ["--log-file", log],
219
+ world_size=world_size,
220
+ )
221
+ log2 = os.path.join(data_dir, "resume.log")
222
+ restore_file = os.path.join(data_dir, "checkpoint_1_2.pt")
223
+ train_translation_model(
224
+ data_dir,
225
+ arch,
226
+ flags + ["--log-file", log2, "--restore-file", restore_file],
227
+ world_size=world_size,
228
+ )
229
+
230
+ l1 = self.parse_logs(log)
231
+ l2 = self.parse_logs(log2)
232
+ assert int(l2[0]["num_updates"]) == 3, f"{l1}\n\n {l2}"
233
+ for k in [
234
+ "train_loss",
235
+ "train_num_updates",
236
+ "train_ppl",
237
+ "train_gnorm",
238
+ ]:
239
+ from_scratch, resumed = l1[-1][k], l2[-1][k]
240
+ assert (
241
+ from_scratch == resumed
242
+ ), f"difference at {k} {from_scratch} != {resumed}"
243
+
244
+ def test_memory_efficient_fp16(self):
245
+ with contextlib.redirect_stdout(StringIO()):
246
+ with tempfile.TemporaryDirectory("test_memory_efficient_fp16") as data_dir:
247
+ create_dummy_data(data_dir)
248
+ preprocess_translation_data(data_dir)
249
+ train_translation_model(
250
+ data_dir, "fconv_iwslt_de_en", ["--memory-efficient-fp16"]
251
+ )
252
+ generate_main(data_dir)
253
+
254
+ def test_transformer_fp16(self):
255
+ with contextlib.redirect_stdout(StringIO()):
256
+ with tempfile.TemporaryDirectory("test_transformer") as data_dir:
257
+ create_dummy_data(data_dir)
258
+ preprocess_translation_data(data_dir)
259
+ train_translation_model(
260
+ data_dir,
261
+ "transformer_iwslt_de_en",
262
+ [
263
+ "--encoder-layers",
264
+ "2",
265
+ "--decoder-layers",
266
+ "2",
267
+ "--encoder-embed-dim",
268
+ "64",
269
+ "--decoder-embed-dim",
270
+ "64",
271
+ "--fp16",
272
+ ],
273
+ run_validation=True,
274
+ )
275
+ generate_main(data_dir)
276
+
277
+ @unittest.skipIf(not torch.cuda.is_available(), "test requires a GPU")
278
+ def test_amp(self):
279
+ with contextlib.redirect_stdout(StringIO()):
280
+ with tempfile.TemporaryDirectory("test_amp") as data_dir:
281
+ create_dummy_data(data_dir)
282
+ preprocess_translation_data(data_dir)
283
+ train_translation_model(data_dir, "fconv_iwslt_de_en", ["--amp"])
284
+ generate_main(data_dir)
285
+
286
+ @unittest.skipIf(not torch.cuda.is_available(), "test requires a GPU")
287
+ def test_transformer_amp(self):
288
+ with contextlib.redirect_stdout(StringIO()):
289
+ with tempfile.TemporaryDirectory("test_transformer") as data_dir:
290
+ create_dummy_data(data_dir)
291
+ preprocess_translation_data(data_dir)
292
+ train_translation_model(
293
+ data_dir,
294
+ "transformer_iwslt_de_en",
295
+ [
296
+ "--encoder-layers",
297
+ "2",
298
+ "--decoder-layers",
299
+ "2",
300
+ "--encoder-embed-dim",
301
+ "64",
302
+ "--decoder-embed-dim",
303
+ "64",
304
+ "--amp",
305
+ ],
306
+ run_validation=True,
307
+ )
308
+ generate_main(data_dir)
309
+
310
+ @unittest.skipIf(not torch.cuda.is_available(), "test requires a GPU")
311
+ def test_levenshtein_transformer(self):
312
+ with contextlib.redirect_stdout(StringIO()):
313
+ with tempfile.TemporaryDirectory(
314
+ "test_levenshtein_transformer"
315
+ ) as data_dir:
316
+ create_dummy_data(data_dir)
317
+ preprocess_translation_data(data_dir, ["--joined-dictionary"])
318
+ train_translation_model(
319
+ data_dir,
320
+ "levenshtein_transformer",
321
+ [
322
+ "--apply-bert-init",
323
+ "--early-exit",
324
+ "6,6,6",
325
+ "--criterion",
326
+ "nat_loss",
327
+ ],
328
+ task="translation_lev",
329
+ )
330
+ gen_config = [
331
+ "--task",
332
+ "translation_lev",
333
+ "--iter-decode-max-iter",
334
+ "9",
335
+ "--iter-decode-eos-penalty",
336
+ "0",
337
+ "--print-step",
338
+ ]
339
+ # non-ensemble generation
340
+ generate_main(data_dir, gen_config)
341
+ # ensemble generation
342
+ generate_main(
343
+ data_dir,
344
+ gen_config,
345
+ path=os.pathsep.join(
346
+ [
347
+ os.path.join(data_dir, "checkpoint_last.pt"),
348
+ os.path.join(data_dir, "checkpoint_last.pt"),
349
+ ]
350
+ ),
351
+ )
352
+
353
+ def test_fsdp_checkpoint_generate(self):
354
+ with contextlib.redirect_stdout(StringIO()):
355
+ with tempfile.TemporaryDirectory("test_fsdp_sharded") as data_dir:
356
+ log = os.path.join(data_dir, "train.log")
357
+ create_dummy_data(data_dir)
358
+ preprocess_translation_data(data_dir)
359
+ world_size = min(torch.cuda.device_count(), 2)
360
+ train_translation_model(
361
+ data_dir,
362
+ "fconv_iwslt_de_en",
363
+ ["--log-file", log, "--ddp-backend", "fully_sharded"],
364
+ world_size=world_size,
365
+ )
366
+ generate_main(data_dir)
367
+ assert os.path.exists(log)
368
+
369
+ def test_fsdp_sharded_checkpoint_generate(self):
370
+ with contextlib.redirect_stdout(StringIO()):
371
+ with tempfile.TemporaryDirectory("test_fsdp_sharded") as data_dir:
372
+ log = os.path.join(data_dir, "train.log")
373
+ create_dummy_data(data_dir)
374
+ preprocess_translation_data(data_dir)
375
+ world_size = min(torch.cuda.device_count(), 2)
376
+ train_translation_model(
377
+ data_dir,
378
+ "fconv_iwslt_de_en",
379
+ [
380
+ "--log-file",
381
+ log,
382
+ "--ddp-backend",
383
+ "fully_sharded",
384
+ "--use-sharded-state",
385
+ ],
386
+ world_size=world_size,
387
+ )
388
+ generate_main(data_dir, ["--checkpoint-shard-count", str(world_size)])
389
+ assert os.path.exists(log)
390
+
391
+
392
+ def _quantize_language_model(data_dir, arch, extra_flags=None, run_validation=False):
393
+ train_parser = options.get_training_parser()
394
+ train_args = options.parse_args_and_arch(
395
+ train_parser,
396
+ [
397
+ "--task",
398
+ "language_modeling",
399
+ data_dir,
400
+ "--arch",
401
+ arch,
402
+ "--optimizer",
403
+ "adam",
404
+ "--lr",
405
+ "0.0001",
406
+ "--criterion",
407
+ "adaptive_loss",
408
+ "--adaptive-softmax-cutoff",
409
+ "5,10,15",
410
+ "--max-tokens",
411
+ "500",
412
+ "--tokens-per-sample",
413
+ "500",
414
+ "--save-dir",
415
+ data_dir,
416
+ "--max-epoch",
417
+ "1",
418
+ "--no-progress-bar",
419
+ "--distributed-world-size",
420
+ "1",
421
+ "--ddp-backend",
422
+ "no_c10d",
423
+ "--num-workers",
424
+ "0",
425
+ ]
426
+ + (extra_flags or []),
427
+ )
428
+ train.main(train_args)
429
+
430
+ # try scalar quantization
431
+ scalar_quant_train_parser = options.get_training_parser()
432
+ scalar_quant_train_args = options.parse_args_and_arch(
433
+ scalar_quant_train_parser,
434
+ [
435
+ "--task",
436
+ "language_modeling",
437
+ data_dir,
438
+ "--arch",
439
+ arch,
440
+ "--optimizer",
441
+ "adam",
442
+ "--lr",
443
+ "0.0001",
444
+ "--criterion",
445
+ "adaptive_loss",
446
+ "--adaptive-softmax-cutoff",
447
+ "5,10,15",
448
+ "--max-tokens",
449
+ "500",
450
+ "--tokens-per-sample",
451
+ "500",
452
+ "--save-dir",
453
+ data_dir,
454
+ "--max-update",
455
+ "3",
456
+ "--no-progress-bar",
457
+ "--distributed-world-size",
458
+ "1",
459
+ "--ddp-backend",
460
+ "no_c10d",
461
+ "--num-workers",
462
+ "0",
463
+ "--quant-noise-scalar",
464
+ "0.5",
465
+ ]
466
+ + (extra_flags or []),
467
+ )
468
+ train.main(scalar_quant_train_args)
469
+
470
+ # try iterative PQ quantization
471
+ quantize_parser = options.get_training_parser()
472
+ quantize_args = options.parse_args_and_arch(
473
+ quantize_parser,
474
+ [
475
+ "--task",
476
+ "language_modeling",
477
+ data_dir,
478
+ "--arch",
479
+ arch,
480
+ "--optimizer",
481
+ "adam",
482
+ "--lr",
483
+ "0.0001",
484
+ "--criterion",
485
+ "adaptive_loss",
486
+ "--adaptive-softmax-cutoff",
487
+ "5,10,15",
488
+ "--max-tokens",
489
+ "50",
490
+ "--tokens-per-sample",
491
+ "50",
492
+ "--max-update",
493
+ "6",
494
+ "--no-progress-bar",
495
+ "--distributed-world-size",
496
+ "1",
497
+ "--ddp-backend",
498
+ "no_c10d",
499
+ "--num-workers",
500
+ "0",
501
+ "--restore-file",
502
+ os.path.join(data_dir, "checkpoint_last.pt"),
503
+ "--reset-optimizer",
504
+ "--quantization-config-path",
505
+ os.path.join(
506
+ os.path.dirname(__file__), "transformer_quantization_config.yaml"
507
+ ),
508
+ ]
509
+ + (extra_flags or []),
510
+ )
511
+ train.main(quantize_args)
512
+
513
+
514
+ @unittest.skipIf(
515
+ int(torch.__version__[2]) < 10, reason="quantized kernels are only supported on CPU"
516
+ )
517
+ @unittest.skipIf(not torch.cuda.is_available(), "test requires a GPU")
518
+ class TestQuantization(unittest.TestCase):
519
+ def setUp(self):
520
+ logging.disable(logging.CRITICAL)
521
+
522
+ def tearDown(self):
523
+ logging.disable(logging.NOTSET)
524
+
525
+ def test_quantization(self):
526
+ with contextlib.redirect_stdout(StringIO()):
527
+ with tempfile.TemporaryDirectory("test_quantization") as data_dir:
528
+ create_dummy_data(data_dir)
529
+ preprocess_lm_data(data_dir)
530
+ # tests both scalar and iterative PQ quantization
531
+ _quantize_language_model(data_dir, "transformer_lm")
532
+
533
+
534
+ @unittest.skipIf(not torch.cuda.is_available(), "test requires a GPU")
535
+ class TestOptimizersGPU(unittest.TestCase):
536
+ def setUp(self):
537
+ logging.disable(logging.CRITICAL)
538
+
539
+ def tearDown(self):
540
+ logging.disable(logging.NOTSET)
541
+
542
+ def test_flat_grads(self):
543
+ with contextlib.redirect_stdout(StringIO()):
544
+ with tempfile.TemporaryDirectory("test_flat_grads") as data_dir:
545
+ # Use just a bit of data and tiny model to keep this test runtime reasonable
546
+ create_dummy_data(data_dir, num_examples=10, maxlen=5)
547
+ preprocess_translation_data(data_dir)
548
+ with self.assertRaises(RuntimeError):
549
+ # adafactor isn't compatible with flat grads, which
550
+ # are used by default with --fp16
551
+ train_translation_model(
552
+ data_dir,
553
+ "lstm",
554
+ [
555
+ "--required-batch-size-multiple",
556
+ "1",
557
+ "--encoder-layers",
558
+ "1",
559
+ "--encoder-hidden-size",
560
+ "32",
561
+ "--decoder-layers",
562
+ "1",
563
+ "--optimizer",
564
+ "adafactor",
565
+ "--fp16",
566
+ ],
567
+ )
568
+ # but it should pass once we set --fp16-no-flatten-grads
569
+ train_translation_model(
570
+ data_dir,
571
+ "lstm",
572
+ [
573
+ "--required-batch-size-multiple",
574
+ "1",
575
+ "--encoder-layers",
576
+ "1",
577
+ "--encoder-hidden-size",
578
+ "32",
579
+ "--decoder-layers",
580
+ "1",
581
+ "--optimizer",
582
+ "adafactor",
583
+ "--fp16",
584
+ "--fp16-no-flatten-grads",
585
+ ],
586
+ )
587
+
588
+
589
+ if __name__ == "__main__":
590
+ unittest.main()
data/fairseq/tests/gpu/test_ema_gpu.py ADDED
@@ -0,0 +1,215 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright (c) Facebook, Inc. and its affiliates.
2
+ #
3
+ # This source code is licensed under the MIT license found in the
4
+ # LICENSE file in the root directory of this source tree.
5
+
6
+ import unittest
7
+ from copy import deepcopy
8
+ from dataclasses import dataclass
9
+ from typing import Optional
10
+
11
+ import torch
12
+
13
+ from fairseq.models.ema import EMA
14
+
15
+
16
+ class DummyModule(torch.nn.Module):
17
+ def __init__(self) -> None:
18
+ """LightningModule for testing purposes
19
+
20
+ Args:
21
+ epoch_min_loss_override (int, optional): Pass in an epoch that will be set to the minimum
22
+ validation loss for testing purposes (zero based). If None this is ignored. Defaults to None.
23
+ """
24
+ super().__init__()
25
+ self.layer = torch.nn.Linear(in_features=32, out_features=2)
26
+ self.another_layer = torch.nn.Linear(in_features=2, out_features=2)
27
+
28
+ def forward(self, x: torch.Tensor) -> torch.Tensor:
29
+ x = self.layer(x)
30
+ return self.another_layer(x)
31
+
32
+
33
+ @dataclass
34
+ class EMAConfig(object):
35
+ ema_decay: float = 0.99
36
+ ema_start_update: int = 0
37
+ ema_fp32: bool = False
38
+ ema_seed_model: Optional[str] = None
39
+ ema_update_freq: int = 1
40
+
41
+
42
+ @unittest.skipIf(not torch.cuda.is_available(), "test requires a GPU")
43
+ class TestEMAGPU(unittest.TestCase):
44
+ def assertTorchAllClose(self, x, y, atol=1e-8, rtol=1e-5, msg=None):
45
+ diff = x.float() - y.float()
46
+ diff_norm = torch.norm(diff)
47
+ other_norm = torch.norm(y.float())
48
+
49
+ if msg is None:
50
+ msg = "|input - other| > {} + {} * |other|".format(atol, rtol)
51
+
52
+ self.assertLessEqual(
53
+ diff_norm,
54
+ atol + rtol * other_norm,
55
+ msg=msg,
56
+ )
57
+
58
+ def test_ema(self):
59
+ model = DummyModule().cuda()
60
+ optimizer = torch.optim.SGD(model.parameters(), lr=0.01)
61
+ state = deepcopy(model.state_dict())
62
+ config = EMAConfig()
63
+ ema = EMA(model, config)
64
+
65
+ # set decay
66
+ ema._set_decay(config.ema_decay)
67
+ self.assertEqual(ema.get_decay(), config.ema_decay)
68
+
69
+ # get model
70
+ self.assertEqual(ema.get_model(), ema.model)
71
+
72
+ # Since fp32 params is not used, it should be of size 0
73
+ self.assertEqual(len(ema.fp32_params), 0)
74
+
75
+ # EMA step
76
+ x = torch.randn(32).cuda()
77
+ y = model(x)
78
+ loss = y.sum()
79
+ loss.backward()
80
+ optimizer.step()
81
+
82
+ ema.step(model)
83
+
84
+ ema_state_dict = ema.get_model().state_dict()
85
+
86
+ for key, param in model.state_dict().items():
87
+ prev_param = state[key]
88
+ ema_param = ema_state_dict[key]
89
+
90
+ if "version" in key:
91
+ # Do not decay a model.version pytorch param
92
+ continue
93
+ self.assertTorchAllClose(
94
+ ema_param,
95
+ config.ema_decay * prev_param + (1 - config.ema_decay) * param,
96
+ )
97
+
98
+ # Since fp32 params is not used, it should be of size 0
99
+ self.assertEqual(len(ema.fp32_params), 0)
100
+
101
+ # Load EMA into model
102
+ model2 = DummyModule().cuda()
103
+ ema.reverse(model2)
104
+
105
+ for key, param in model2.state_dict().items():
106
+ ema_param = ema_state_dict[key]
107
+ self.assertTrue(torch.allclose(ema_param, param))
108
+
109
+ def test_ema_fp32(self):
110
+ model = DummyModule().cuda().half()
111
+ optimizer = torch.optim.SGD(model.parameters(), lr=0.01)
112
+ state = deepcopy(model.state_dict())
113
+ config = EMAConfig(ema_fp32=True)
114
+ ema = EMA(model, config)
115
+
116
+ x = torch.randn(32).cuda()
117
+ y = model(x.half())
118
+ loss = y.sum()
119
+ loss.backward()
120
+ optimizer.step()
121
+
122
+ ema.step(model)
123
+
124
+ for key, param in model.state_dict().items():
125
+ prev_param = state[key]
126
+ ema_param = ema.get_model().state_dict()[key]
127
+
128
+ if "version" in key:
129
+ # Do not decay a model.version pytorch param
130
+ continue
131
+ self.assertIn(key, ema.fp32_params)
132
+
133
+ # EMA update is done in fp32, and hence the EMA param must be
134
+ # closer to the EMA update done in fp32 than in fp16.
135
+ self.assertLessEqual(
136
+ torch.norm(
137
+ ema_param.float()
138
+ - (
139
+ config.ema_decay * prev_param.float()
140
+ + (1 - config.ema_decay) * param.float()
141
+ )
142
+ .half()
143
+ .float()
144
+ ),
145
+ torch.norm(
146
+ ema_param.float()
147
+ - (
148
+ config.ema_decay * prev_param + (1 - config.ema_decay) * param
149
+ ).float()
150
+ ),
151
+ )
152
+ self.assertTorchAllClose(
153
+ ema_param,
154
+ (
155
+ config.ema_decay * prev_param.float()
156
+ + (1 - config.ema_decay) * param.float()
157
+ ).half(),
158
+ )
159
+
160
+ def test_ema_fp16(self):
161
+ model = DummyModule().cuda().half()
162
+ optimizer = torch.optim.SGD(model.parameters(), lr=0.01)
163
+ state = deepcopy(model.state_dict())
164
+ config = EMAConfig(ema_fp32=False)
165
+ ema = EMA(model, config)
166
+
167
+ # Since fp32 params is not used, it should be of size 0
168
+ self.assertEqual(len(ema.fp32_params), 0)
169
+
170
+ x = torch.randn(32).cuda()
171
+ y = model(x.half())
172
+ loss = y.sum()
173
+ loss.backward()
174
+ optimizer.step()
175
+
176
+ ema.step(model)
177
+
178
+ for key, param in model.state_dict().items():
179
+ prev_param = state[key]
180
+ ema_param = ema.get_model().state_dict()[key]
181
+
182
+ if "version" in key:
183
+ # Do not decay a model.version pytorch param
184
+ continue
185
+
186
+ # EMA update is done in fp16, and hence the EMA param must be
187
+ # closer to the EMA update done in fp16 than in fp32.
188
+ self.assertLessEqual(
189
+ torch.norm(
190
+ ema_param.float()
191
+ - (
192
+ config.ema_decay * prev_param + (1 - config.ema_decay) * param
193
+ ).float()
194
+ ),
195
+ torch.norm(
196
+ ema_param.float()
197
+ - (
198
+ config.ema_decay * prev_param.float()
199
+ + (1 - config.ema_decay) * param.float()
200
+ )
201
+ .half()
202
+ .float()
203
+ ),
204
+ )
205
+ self.assertTorchAllClose(
206
+ ema_param,
207
+ config.ema_decay * prev_param + (1 - config.ema_decay) * param,
208
+ )
209
+
210
+ # Since fp32 params is not used, it should be of size 0
211
+ self.assertEqual(len(ema.fp32_params), 0)
212
+
213
+
214
+ if __name__ == "__main__":
215
+ unittest.main()
data/fairseq/tests/gpu/transformer_quantization_config.yaml ADDED
@@ -0,0 +1,28 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright (c) Facebook, Inc. and its affiliates.
2
+ #
3
+ # This source code is licensed under the MIT license found in the
4
+ # LICENSE file in the root directory of this source tree.
5
+
6
+ # This file defines example configuration arguments for quantizing
7
+ # a transformer model with product quantization
8
+
9
+ n_centroids:
10
+ Linear:
11
+ key: in_features
12
+ value: {"*": 8}
13
+ Embedding:
14
+ key: embedding_dim
15
+ value: {"*": 8}
16
+
17
+ block_sizes:
18
+ Linear:
19
+ key: fuzzy_name
20
+ value: {fc: 8, attn: 4, emb: 4}
21
+ Embedding:
22
+ key: fuzzy_name
23
+ value: {emb: 8}
24
+
25
+ layers_to_quantize:
26
+ - decoder\\.layers\\.\d+\\.fc[12]
27
+ - decoder\\.embed_tokens\\.embeddings\\.[012]\\.[01]
28
+ - decoder\\.layers\\.\d+\\.self_attn\\.(k_proj|v_proj|q_proj|out_proj)
data/fairseq/tests/speech/__init__.py ADDED
@@ -0,0 +1,210 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright (c) Facebook, Inc. and its affiliates.
2
+ #
3
+ # This source code is licensed under the MIT license found in the
4
+ # LICENSE file in the root directory of this source tree.
5
+
6
+ from argparse import Namespace
7
+ import os
8
+ import re
9
+ import unittest
10
+ from pathlib import Path
11
+ from tqdm import tqdm
12
+ from typing import List, Dict, Optional
13
+ import torch
14
+ from fairseq.checkpoint_utils import load_model_ensemble_and_task
15
+ from fairseq.scoring.wer import WerScorer
16
+ from fairseq.scoring.bleu import SacrebleuScorer
17
+ from fairseq import utils
18
+ import zipfile
19
+
20
+ S3_BASE_URL = "https://dl.fbaipublicfiles.com/fairseq"
21
+
22
+
23
+ class TestFairseqSpeech(unittest.TestCase):
24
+ @classmethod
25
+ def download(cls, base_url: str, out_root: Path, filename: str):
26
+ url = f"{base_url}/{filename}"
27
+ path = out_root / filename
28
+ if not path.exists():
29
+ torch.hub.download_url_to_file(url, path.as_posix(), progress=True)
30
+ return path
31
+
32
+ def _set_up(self, dataset_id: str, s3_dir: str, data_filenames: List[str]):
33
+ self.use_cuda = torch.cuda.is_available()
34
+ self.root = Path.home() / ".cache" / "fairseq" / dataset_id
35
+ self.root.mkdir(exist_ok=True, parents=True)
36
+ os.chdir(self.root)
37
+ self.base_url = (
38
+ s3_dir if re.search("^https:", s3_dir) else f"{S3_BASE_URL}/{s3_dir}"
39
+ )
40
+ for filename in data_filenames:
41
+ self.download(self.base_url, self.root, filename)
42
+
43
+ def set_up_librispeech(self):
44
+ self._set_up(
45
+ "librispeech",
46
+ "s2t/librispeech",
47
+ [
48
+ "cfg_librispeech.yaml",
49
+ "spm_librispeech_unigram10000.model",
50
+ "spm_librispeech_unigram10000.txt",
51
+ "librispeech_test-other.tsv",
52
+ "librispeech_test-other.zip",
53
+ ],
54
+ )
55
+
56
+ def set_up_ljspeech(self):
57
+ self._set_up(
58
+ "ljspeech",
59
+ "s2/ljspeech",
60
+ [
61
+ "cfg_ljspeech_g2p.yaml",
62
+ "ljspeech_g2p_gcmvn_stats.npz",
63
+ "ljspeech_g2p.txt",
64
+ "ljspeech_test.tsv",
65
+ "ljspeech_test.zip",
66
+ ],
67
+ )
68
+
69
+ def set_up_sotasty_es_en(self):
70
+ self._set_up(
71
+ "sotasty_es_en",
72
+ "s2t/big/es-en",
73
+ [
74
+ "cfg_es_en.yaml",
75
+ "spm_bpe32768_es_en.model",
76
+ "spm_bpe32768_es_en.txt",
77
+ "sotasty_es_en_test_ted.tsv",
78
+ "sotasty_es_en_test_ted.zip",
79
+ ],
80
+ )
81
+
82
+ def set_up_mustc_de_fbank(self):
83
+ self._set_up(
84
+ "mustc_de_fbank",
85
+ "https://dl.fbaipublicfiles.com/joint_speech_text_4_s2t/must_c/en_de",
86
+ [
87
+ "config.yaml",
88
+ "spm.model",
89
+ "dict.txt",
90
+ "src_dict.txt",
91
+ "tgt_dict.txt",
92
+ "tst-COMMON.tsv",
93
+ "tst-COMMON.zip",
94
+ ],
95
+ )
96
+
97
+ def download_and_load_checkpoint(
98
+ self,
99
+ checkpoint_filename: str,
100
+ arg_overrides: Optional[Dict[str, str]] = None,
101
+ strict: bool = True,
102
+ ):
103
+ path = self.download(self.base_url, self.root, checkpoint_filename)
104
+ _arg_overrides = arg_overrides or {}
105
+ _arg_overrides["data"] = self.root.as_posix()
106
+ models, cfg, task = load_model_ensemble_and_task(
107
+ [path.as_posix()], arg_overrides=_arg_overrides, strict=strict
108
+ )
109
+ if self.use_cuda:
110
+ for model in models:
111
+ model.cuda()
112
+
113
+ return models, cfg, task, self.build_generator(task, models, cfg)
114
+
115
+ def build_generator(
116
+ self,
117
+ task,
118
+ models,
119
+ cfg,
120
+ ):
121
+ return task.build_generator(models, cfg)
122
+
123
+ @classmethod
124
+ def get_batch_iterator(cls, task, test_split, max_tokens, max_positions):
125
+ task.load_dataset(test_split)
126
+ return task.get_batch_iterator(
127
+ dataset=task.dataset(test_split),
128
+ max_tokens=max_tokens,
129
+ max_positions=max_positions,
130
+ num_workers=1,
131
+ ).next_epoch_itr(shuffle=False)
132
+
133
+ @classmethod
134
+ def get_wer_scorer(
135
+ cls, tokenizer="none", lowercase=False, remove_punct=False, char_level=False
136
+ ):
137
+ scorer_args = {
138
+ "wer_tokenizer": tokenizer,
139
+ "wer_lowercase": lowercase,
140
+ "wer_remove_punct": remove_punct,
141
+ "wer_char_level": char_level,
142
+ }
143
+ return WerScorer(Namespace(**scorer_args))
144
+
145
+ @classmethod
146
+ def get_bleu_scorer(cls, tokenizer="13a", lowercase=False, char_level=False):
147
+ scorer_args = {
148
+ "sacrebleu_tokenizer": tokenizer,
149
+ "sacrebleu_lowercase": lowercase,
150
+ "sacrebleu_char_level": char_level,
151
+ }
152
+ return SacrebleuScorer(Namespace(**scorer_args))
153
+
154
+ @torch.no_grad()
155
+ def base_test(
156
+ self,
157
+ ckpt_name,
158
+ reference_score,
159
+ score_delta=0.3,
160
+ dataset="librispeech_test-other",
161
+ max_tokens=65_536,
162
+ max_positions=(4_096, 1_024),
163
+ arg_overrides=None,
164
+ strict=True,
165
+ score_type="wer",
166
+ ):
167
+ models, _, task, generator = self.download_and_load_checkpoint(
168
+ ckpt_name, arg_overrides=arg_overrides, strict=strict
169
+ )
170
+ if not self.use_cuda:
171
+ return
172
+
173
+ batch_iterator = self.get_batch_iterator(
174
+ task, dataset, max_tokens, max_positions
175
+ )
176
+ if score_type == "bleu":
177
+ scorer = self.get_bleu_scorer()
178
+ elif score_type == "wer":
179
+ scorer = self.get_wer_scorer()
180
+ else:
181
+ raise Exception(f"Unsupported score type {score_type}")
182
+
183
+ progress = tqdm(enumerate(batch_iterator), total=len(batch_iterator))
184
+ for batch_idx, sample in progress:
185
+ sample = utils.move_to_cuda(sample) if self.use_cuda else sample
186
+ hypo = task.inference_step(generator, models, sample)
187
+ for i, sample_id in enumerate(sample["id"].tolist()):
188
+ tgt_str, hypo_str = self.postprocess_tokens(
189
+ task,
190
+ sample["target"][i, :],
191
+ hypo[i][0]["tokens"].int().cpu(),
192
+ )
193
+ if batch_idx == 0 and i < 3:
194
+ print(f"T-{sample_id} {tgt_str}")
195
+ print(f"H-{sample_id} {hypo_str}")
196
+ scorer.add_string(tgt_str, hypo_str)
197
+
198
+ print(scorer.result_string() + f" (reference: {reference_score})")
199
+ self.assertAlmostEqual(scorer.score(), reference_score, delta=score_delta)
200
+
201
+ def postprocess_tokens(self, task, target, hypo_tokens):
202
+ tgt_tokens = utils.strip_pad(target, task.tgt_dict.pad()).int().cpu()
203
+ tgt_str = task.tgt_dict.string(tgt_tokens, "sentencepiece")
204
+ hypo_str = task.tgt_dict.string(hypo_tokens, "sentencepiece")
205
+ return tgt_str, hypo_str
206
+
207
+ def unzip_files(self, zip_file_name):
208
+ zip_file_path = self.root / zip_file_name
209
+ with zipfile.ZipFile(zip_file_path, "r") as zip_ref:
210
+ zip_ref.extractall(self.root / zip_file_name.strip(".zip"))
data/fairseq/tests/speech/test_convtransformer_simul_trans.py ADDED
@@ -0,0 +1,33 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright (c) Facebook, Inc. and its affiliates.
2
+ #
3
+ # This source code is licensed under the MIT license found in the
4
+ # LICENSE file in the root directory of this source tree.
5
+
6
+ import unittest
7
+ from tests.speech import TestFairseqSpeech
8
+
9
+ S3_BASE_URL = "https://dl.fbaipublicfiles.com/fairseq/"
10
+
11
+
12
+ class TestConvtransformerSimulTrans(TestFairseqSpeech):
13
+ def setUp(self):
14
+ self._set_up(
15
+ "simul",
16
+ "speech_tests/simul",
17
+ ["config_gcmvn_specaug.yaml", "dict.txt", "dev.tsv"],
18
+ )
19
+
20
+ def test_waitk_checkpoint(self):
21
+ """Only test model loading since fairseq currently doesn't support inference of simultaneous models"""
22
+ _, _, _, _ = self.download_and_load_checkpoint(
23
+ "checkpoint_best.pt",
24
+ arg_overrides={
25
+ "config_yaml": "config_gcmvn_specaug.yaml",
26
+ "load_pretrained_encoder_from": None,
27
+ },
28
+ )
29
+ return
30
+
31
+
32
+ if __name__ == "__main__":
33
+ unittest.main()
data/fairseq/tests/speech/test_dual_input_wav_transformer.py ADDED
@@ -0,0 +1,76 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright (c) Facebook, Inc. and its affiliates.
2
+ #
3
+ # This source code is licensed under the MIT license found in the
4
+ # LICENSE file in the root directory of this source tree.
5
+
6
+ import unittest
7
+ from collections import namedtuple
8
+ from pathlib import Path
9
+
10
+ import torch
11
+ from tqdm import tqdm
12
+
13
+ import fairseq
14
+ from fairseq import utils
15
+ from fairseq.checkpoint_utils import load_model_ensemble_and_task
16
+ from fairseq.scoring.bleu import SacrebleuScorer
17
+ from fairseq.tasks import import_tasks
18
+ from tests.speech import S3_BASE_URL, TestFairseqSpeech
19
+
20
+
21
+ @unittest.skipIf(not torch.cuda.is_available(), "test requires a GPU")
22
+ class TestLibrispeechDualInputWavTransformer(TestFairseqSpeech):
23
+ def setUp(self):
24
+ dataset_id = "librispeech_wvtrasnformer"
25
+ base_url = "https://dl.fbaipublicfiles.com/joint_speech_text_4_s2t/acl2022/librispeech/finetuned"
26
+ data_filenames = [
27
+ "checkpoint_ave_10.pt",
28
+ "spm.model",
29
+ "src_dict.txt",
30
+ "tgt_dict.txt",
31
+ "config.yaml",
32
+ ]
33
+ self._set_up(
34
+ dataset_id,
35
+ "s2t",
36
+ [
37
+ "librispeech_flac_test-other.tsv",
38
+ "librispeech_flac_test-other.zip",
39
+ ],
40
+ )
41
+ for filename in data_filenames:
42
+ self.download(base_url, self.root, filename)
43
+
44
+ def import_user_module(self):
45
+ user_dir = (
46
+ Path(fairseq.__file__).parent.parent / "examples/speech_text_joint_to_text"
47
+ )
48
+ Arg = namedtuple("Arg", ["user_dir"])
49
+ arg = Arg(user_dir.__str__())
50
+ utils.import_user_module(arg)
51
+
52
+ @torch.no_grad()
53
+ def test_librispeech_dualinput_wav_transformer_checkpoint(self):
54
+ self.import_user_module()
55
+ checkpoint_filename = "checkpoint_ave_10.pt"
56
+ arg_overrides = {
57
+ "config_yaml": "config.yaml",
58
+ "load_pretrained_speech_text_encoder": "",
59
+ "load_pretrained_speech_text_decoder": "",
60
+ "beam": 10,
61
+ "nbest": 1,
62
+ "lenpen": 1.0,
63
+ "load_speech_only": True,
64
+ }
65
+ self.base_test(
66
+ checkpoint_filename,
67
+ 4.6,
68
+ dataset="librispeech_flac_test-other",
69
+ max_tokens=800000,
70
+ max_positions=(800000, 1024),
71
+ arg_overrides=arg_overrides,
72
+ )
73
+
74
+
75
+ if __name__ == "__main__":
76
+ unittest.main()
data/fairseq/tests/speech/test_dualinput_s2t_transformer.py ADDED
@@ -0,0 +1,110 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright (c) Facebook, Inc. and its affiliates.
2
+ #
3
+ # This source code is licensed under the MIT license found in the
4
+ # LICENSE file in the root directory of this source tree.
5
+
6
+ import unittest
7
+ from argparse import Namespace
8
+ from collections import namedtuple
9
+ from pathlib import Path
10
+
11
+ import torch
12
+ from tqdm import tqdm
13
+
14
+ import fairseq
15
+ from fairseq import utils
16
+ from fairseq.checkpoint_utils import load_model_ensemble_and_task
17
+ from fairseq.scoring.bleu import SacrebleuScorer
18
+ from fairseq.tasks import import_tasks
19
+ from tests.speech import TestFairseqSpeech
20
+
21
+
22
+ @unittest.skipIf(not torch.cuda.is_available(), "test requires a GPU")
23
+ class TestDualInputS2TTransformer(TestFairseqSpeech):
24
+ def setUp(self):
25
+ self.set_up_mustc_de_fbank()
26
+
27
+ def import_user_module(self):
28
+ user_dir = (
29
+ Path(fairseq.__file__).parent.parent / "examples/speech_text_joint_to_text"
30
+ )
31
+ Arg = namedtuple("Arg", ["user_dir"])
32
+ arg = Arg(user_dir.__str__())
33
+ utils.import_user_module(arg)
34
+
35
+ @torch.no_grad()
36
+ def test_mustc_de_fbank_dualinput_s2t_transformer_checkpoint(self):
37
+ self.import_user_module()
38
+ checkpoint_filename = "checkpoint_ave_10.pt"
39
+ path = self.download(self.base_url, self.root, checkpoint_filename)
40
+ models, cfg, task = load_model_ensemble_and_task(
41
+ [path.as_posix()],
42
+ arg_overrides={
43
+ "data": self.root.as_posix(),
44
+ "config_yaml": "config.yaml",
45
+ "load_pretrain_speech_encoder": "",
46
+ "load_pretrain_text_encoder_last": "",
47
+ "load_pretrain_decoder": "",
48
+ "beam": 10,
49
+ "nbest": 1,
50
+ "lenpen": 1.0,
51
+ "load_speech_only": True,
52
+ },
53
+ )
54
+ if self.use_cuda:
55
+ for model in models:
56
+ model.cuda()
57
+ generator = task.build_generator(models, cfg)
58
+ test_split = "tst-COMMON"
59
+ task.load_dataset(test_split)
60
+ batch_iterator = task.get_batch_iterator(
61
+ dataset=task.dataset(test_split),
62
+ max_tokens=250_000,
63
+ max_positions=(10_000, 1_024),
64
+ num_workers=1,
65
+ ).next_epoch_itr(shuffle=False)
66
+
67
+ tokenizer = task.build_tokenizer(cfg.tokenizer)
68
+ bpe = task.build_bpe(cfg.bpe)
69
+
70
+ def decode_fn(x):
71
+ if bpe is not None:
72
+ x = bpe.decode(x)
73
+ if tokenizer is not None:
74
+ x = tokenizer.decode(x)
75
+ return x
76
+
77
+ scorer_args = {
78
+ "sacrebleu_tokenizer": "13a",
79
+ "sacrebleu_lowercase": False,
80
+ "sacrebleu_char_level": False,
81
+ }
82
+ scorer = SacrebleuScorer(Namespace(**scorer_args))
83
+ progress = tqdm(enumerate(batch_iterator), total=len(batch_iterator))
84
+ for batch_idx, sample in progress:
85
+ sample = utils.move_to_cuda(sample) if self.use_cuda else sample
86
+ hypo = task.inference_step(generator, models, sample)
87
+ for i, sample_id in enumerate(sample["id"].tolist()):
88
+ tgt_tokens = (
89
+ utils.strip_pad(sample["target"][i, :], task.tgt_dict.pad())
90
+ .int()
91
+ .cpu()
92
+ )
93
+
94
+ tgt_str = task.tgt_dict.string(tgt_tokens, "sentencepiece")
95
+ hypo_str = task.tgt_dict.string(
96
+ hypo[i][0]["tokens"].int().cpu(), "sentencepiece"
97
+ )
98
+ if batch_idx == 0 and i < 3:
99
+ print(f"T-{sample_id} {tgt_str}")
100
+ print(f"D-{sample_id} {hypo_str}")
101
+ scorer.add_string(tgt_str, hypo_str)
102
+ reference_bleu = 27.3
103
+ result = scorer.result_string()
104
+ print(result + f" (reference: {reference_bleu})")
105
+ res_bleu = float(result.split()[2])
106
+ self.assertAlmostEqual(res_bleu, reference_bleu, delta=0.3)
107
+
108
+
109
+ if __name__ == "__main__":
110
+ unittest.main()
data/fairseq/tests/speech/test_fastspeech2.py ADDED
@@ -0,0 +1,53 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright (c) Facebook, Inc. and its affiliates.
2
+ #
3
+ # This source code is licensed under the MIT license found in the
4
+ # LICENSE file in the root directory of this source tree.
5
+
6
+ import unittest
7
+
8
+ import torch
9
+ from tqdm import tqdm
10
+
11
+ from fairseq import utils
12
+ from fairseq.tasks.text_to_speech import batch_mel_cepstral_distortion
13
+ from tests.speech import TestFairseqSpeech
14
+
15
+
16
+ @unittest.skipIf(not torch.cuda.is_available(), "test requires a GPU")
17
+ class TestFastSpeech2(TestFairseqSpeech):
18
+ def setUp(self):
19
+ self.set_up_ljspeech()
20
+
21
+ @torch.no_grad()
22
+ def test_ljspeech_fastspeech2_checkpoint(self):
23
+ models, cfg, task, generator = self.download_and_load_checkpoint(
24
+ "ljspeech_fastspeech2_g2p.pt",
25
+ arg_overrides={
26
+ "config_yaml": "cfg_ljspeech_g2p.yaml",
27
+ "vocoder": "griffin_lim",
28
+ "fp16": False,
29
+ },
30
+ )
31
+
32
+ batch_iterator = self.get_batch_iterator(task, "ljspeech_test", 65_536, 4_096)
33
+ progress = tqdm(batch_iterator, total=len(batch_iterator))
34
+ mcd, n_samples = 0.0, 0
35
+ for sample in progress:
36
+ sample = utils.move_to_cuda(sample) if self.use_cuda else sample
37
+ hypos = generator.generate(models[0], sample, has_targ=True)
38
+ rets = batch_mel_cepstral_distortion(
39
+ [hypo["targ_waveform"] for hypo in hypos],
40
+ [hypo["waveform"] for hypo in hypos],
41
+ sr=task.sr,
42
+ )
43
+ mcd += sum(d.item() for d, _ in rets)
44
+ n_samples += len(sample["id"].tolist())
45
+
46
+ mcd = round(mcd / n_samples, 1)
47
+ reference_mcd = 3.2
48
+ print(f"MCD: {mcd} (reference: {reference_mcd})")
49
+ self.assertAlmostEqual(mcd, reference_mcd, delta=0.1)
50
+
51
+
52
+ if __name__ == "__main__":
53
+ unittest.main()
data/fairseq/tests/speech/test_s2s_transformer.py ADDED
@@ -0,0 +1,51 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright (c) Facebook, Inc. and its affiliates.
2
+ #
3
+ # This source code is licensed under the MIT license found in the
4
+ # LICENSE file in the root directory of this source tree.
5
+
6
+ import unittest
7
+ from tests.speech import TestFairseqSpeech
8
+ from fairseq import utils
9
+
10
+ S3_BASE_URL = "https://dl.fbaipublicfiles.com/fairseq/"
11
+
12
+
13
+ class TestS2STransformer(TestFairseqSpeech):
14
+ def setUp(self):
15
+ self._set_up(
16
+ "s2s",
17
+ "speech_tests/s2s",
18
+ [
19
+ "dev_shuf200.tsv",
20
+ "src_feat.zip",
21
+ "config_specaug_lb.yaml",
22
+ "vocoder",
23
+ "vocoder_config.json",
24
+ ],
25
+ )
26
+
27
+ def test_s2s_transformer_checkpoint(self):
28
+ self.base_test(
29
+ ckpt_name="s2u_transformer_reduced_fisher.pt",
30
+ reference_score=38.3,
31
+ dataset="dev_shuf200",
32
+ arg_overrides={
33
+ "config_yaml": "config_specaug_lb.yaml",
34
+ "multitask_config_yaml": None,
35
+ "target_is_code": True,
36
+ "target_code_size": 100,
37
+ "eval_inference": False,
38
+ },
39
+ score_type="bleu",
40
+ strict=False,
41
+ )
42
+
43
+ def postprocess_tokens(self, task, target, hypo_tokens):
44
+ tgt_tokens = utils.strip_pad(target, task.tgt_dict.pad()).int().cpu()
45
+ tgt_str = task.tgt_dict.string(tgt_tokens)
46
+ hypo_str = task.tgt_dict.string(hypo_tokens)
47
+ return tgt_str, hypo_str
48
+
49
+
50
+ if __name__ == "__main__":
51
+ unittest.main()
data/fairseq/tests/speech/test_s2t_conformer.py ADDED
@@ -0,0 +1,23 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright (c) Facebook, Inc. and its affiliates.
2
+ #
3
+ # This source code is licensed under the MIT license found in the
4
+ # LICENSE file in the root directory of this source tree.
5
+
6
+ import unittest
7
+ from tests.speech import TestFairseqSpeech
8
+
9
+
10
+ class TestS2TConformer(TestFairseqSpeech):
11
+ def setUp(self):
12
+ self.set_up_librispeech()
13
+
14
+ def test_librispeech_s2t_conformer_s_checkpoint(self):
15
+ self.base_test(
16
+ ckpt_name="librispeech_conformer_rel_pos_s.pt",
17
+ reference_score=12,
18
+ arg_overrides={"config_yaml": "cfg_librispeech.yaml"},
19
+ )
20
+
21
+
22
+ if __name__ == "__main__":
23
+ unittest.main()
data/fairseq/tests/speech/test_s2t_transformer.py ADDED
@@ -0,0 +1,23 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright (c) Facebook, Inc. and its affiliates.
2
+ #
3
+ # This source code is licensed under the MIT license found in the
4
+ # LICENSE file in the root directory of this source tree.
5
+
6
+ import unittest
7
+ from tests.speech import TestFairseqSpeech
8
+
9
+
10
+ class TestS2TTransformer(TestFairseqSpeech):
11
+ def setUp(self):
12
+ self.set_up_librispeech()
13
+
14
+ def test_librispeech_s2t_transformer_s_checkpoint(self):
15
+ self.base_test(
16
+ ckpt_name="librispeech_transformer_s.pt",
17
+ reference_score=9,
18
+ arg_overrides={"config_yaml": "cfg_librispeech.yaml"},
19
+ )
20
+
21
+
22
+ if __name__ == "__main__":
23
+ unittest.main()
data/fairseq/tests/speech/test_tts_transformer.py ADDED
@@ -0,0 +1,53 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright (c) Facebook, Inc. and its affiliates.
2
+ #
3
+ # This source code is licensed under the MIT license found in the
4
+ # LICENSE file in the root directory of this source tree.
5
+
6
+ import unittest
7
+
8
+ import torch
9
+ from tqdm import tqdm
10
+
11
+ from fairseq import utils
12
+ from fairseq.tasks.text_to_speech import batch_mel_cepstral_distortion
13
+ from tests.speech import TestFairseqSpeech
14
+
15
+
16
+ @unittest.skipIf(not torch.cuda.is_available(), "test requires a GPU")
17
+ class TestTTSTransformer(TestFairseqSpeech):
18
+ def setUp(self):
19
+ self.set_up_ljspeech()
20
+
21
+ @torch.no_grad()
22
+ def test_ljspeech_tts_transformer_checkpoint(self):
23
+ models, cfg, task, generator = self.download_and_load_checkpoint(
24
+ "ljspeech_transformer_g2p.pt",
25
+ arg_overrides={
26
+ "config_yaml": "cfg_ljspeech_g2p.yaml",
27
+ "vocoder": "griffin_lim",
28
+ "fp16": False,
29
+ },
30
+ )
31
+
32
+ batch_iterator = self.get_batch_iterator(task, "ljspeech_test", 65_536, 1024)
33
+ progress = tqdm(batch_iterator, total=len(batch_iterator))
34
+ mcd, n_samples = 0.0, 0
35
+ for sample in progress:
36
+ sample = utils.move_to_cuda(sample) if self.use_cuda else sample
37
+ hypos = generator.generate(models[0], sample, has_targ=True)
38
+ rets = batch_mel_cepstral_distortion(
39
+ [hypo["targ_waveform"] for hypo in hypos],
40
+ [hypo["waveform"] for hypo in hypos],
41
+ sr=task.sr,
42
+ )
43
+ mcd += sum(d.item() for d, _ in rets)
44
+ n_samples += len(sample["id"].tolist())
45
+
46
+ mcd = round(mcd / n_samples, 1)
47
+ reference_mcd = 3.3
48
+ print(f"MCD: {mcd} (reference: {reference_mcd})")
49
+ self.assertAlmostEqual(mcd, reference_mcd, delta=0.1)
50
+
51
+
52
+ if __name__ == "__main__":
53
+ unittest.main()
data/fairseq/tests/speech/test_wav2vec2.py ADDED
@@ -0,0 +1,90 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright (c) Facebook, Inc. and its affiliates.
2
+ #
3
+ # This source code is licensed under the MIT license found in the
4
+ # LICENSE file in the root directory of this source tree.
5
+
6
+ import unittest
7
+ import torch
8
+ from tests.speech import TestFairseqSpeech
9
+ from fairseq.data.data_utils import post_process
10
+ from fairseq import utils
11
+ from omegaconf import open_dict
12
+
13
+ S3_BASE_URL = "https://dl.fbaipublicfiles.com/fairseq"
14
+
15
+
16
+ @unittest.skipIf(not torch.cuda.is_available(), "test requires a GPU")
17
+ class TestWav2Vec2(TestFairseqSpeech):
18
+ def setUp(self):
19
+ self._set_up(
20
+ "librispeech_w2v2",
21
+ "conformer/wav2vec2/librispeech",
22
+ [
23
+ "test_librispeech-other.ltr",
24
+ "test_librispeech-other.tsv",
25
+ "test_librispeech-other_small.ltr_100",
26
+ "test_librispeech-other_small.tsv",
27
+ "test-other.zip",
28
+ "dict.ltr.txt",
29
+ "dict.ltr_100.txt",
30
+ ],
31
+ )
32
+ self.unzip_files(
33
+ "test-other.zip",
34
+ )
35
+
36
+ def test_transformer_w2v2(self):
37
+ self.base_test(
38
+ ckpt_name="transformer_oss_small_100h.pt",
39
+ reference_score=38,
40
+ score_delta=1,
41
+ dataset="test_librispeech-other",
42
+ max_tokens=1000000,
43
+ max_positions=(700000, 1000),
44
+ arg_overrides={
45
+ "task": "audio_finetuning",
46
+ "labels": "ltr",
47
+ "nbest": 1,
48
+ "tpu": False,
49
+ },
50
+ strict=False,
51
+ )
52
+
53
+ def test_conformer_w2v2(self):
54
+ self.base_test(
55
+ ckpt_name="conformer_LS_PT_LS_FT_rope.pt",
56
+ reference_score=4.5,
57
+ score_delta=1,
58
+ dataset="test_librispeech-other_small",
59
+ max_tokens=1000000,
60
+ max_positions=(700000, 1000),
61
+ arg_overrides={
62
+ "task": "audio_finetuning",
63
+ "labels": "ltr_100",
64
+ "nbest": 1,
65
+ "tpu": False,
66
+ },
67
+ strict=True,
68
+ )
69
+
70
+ def build_generator(self, task, models, cfg):
71
+ try:
72
+ from examples.speech_recognition.w2l_decoder import W2lViterbiDecoder
73
+ except Exception:
74
+ raise Exception("Cannot run this test without flashlight dependency")
75
+ with open_dict(cfg):
76
+ cfg.nbest = 1
77
+ return W2lViterbiDecoder(cfg, task.target_dictionary)
78
+
79
+ def postprocess_tokens(self, task, target, hypo_tokens):
80
+ tgt_tokens = utils.strip_pad(target, task.target_dictionary.pad()).int().cpu()
81
+ tgt_str = task.target_dictionary.string(tgt_tokens)
82
+ tgt_str = post_process(tgt_str, "letter")
83
+
84
+ hypo_pieces = task.target_dictionary.string(hypo_tokens)
85
+ hypo_str = post_process(hypo_pieces, "letter")
86
+ return tgt_str, hypo_str
87
+
88
+
89
+ if __name__ == "__main__":
90
+ unittest.main()
data/fairseq/tests/speech/test_xm_transformer.py ADDED
@@ -0,0 +1,29 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright (c) Facebook, Inc. and its affiliates.
2
+ #
3
+ # This source code is licensed under the MIT license found in the
4
+ # LICENSE file in the root directory of this source tree.
5
+
6
+ import unittest
7
+ from tests.speech import TestFairseqSpeech
8
+
9
+
10
+ class TestXMTransformer(TestFairseqSpeech):
11
+ def setUp(self):
12
+ self.set_up_sotasty_es_en()
13
+
14
+ # TODO: investigate increases BLEU score (30.42 -> 31.74)
15
+ def test_sotasty_es_en_600m_checkpoint(self):
16
+ self.base_test(
17
+ ckpt_name="xm_transformer_600m_es_en_md.pt",
18
+ reference_score=31.74,
19
+ score_delta=0.2,
20
+ max_tokens=3_000_000,
21
+ max_positions=(1_000_000, 1_024),
22
+ dataset="sotasty_es_en_test_ted",
23
+ arg_overrides={"config_yaml": "cfg_es_en.yaml"},
24
+ score_type="bleu",
25
+ )
26
+
27
+
28
+ if __name__ == "__main__":
29
+ unittest.main()
data/fairseq/tests/speech_recognition/__init__.py ADDED
File without changes
data/fairseq/tests/speech_recognition/asr_test_base.py ADDED
@@ -0,0 +1,557 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env python3
2
+
3
+ import argparse
4
+ import os
5
+ import unittest
6
+ from inspect import currentframe, getframeinfo
7
+
8
+ import numpy as np
9
+ import torch
10
+ from examples.speech_recognition.data.data_utils import lengths_to_encoder_padding_mask
11
+ from fairseq.data import data_utils as fairseq_data_utils
12
+ from fairseq.data.dictionary import Dictionary
13
+ from fairseq.models import (
14
+ BaseFairseqModel,
15
+ FairseqDecoder,
16
+ FairseqEncoder,
17
+ FairseqEncoderDecoderModel,
18
+ FairseqEncoderModel,
19
+ FairseqModel,
20
+ )
21
+ from fairseq.tasks.fairseq_task import LegacyFairseqTask
22
+
23
+
24
+ DEFAULT_TEST_VOCAB_SIZE = 100
25
+
26
+
27
+ # ///////////////////////////////////////////////////////////////////////////
28
+ # utility function to setup dummy dict/task/input
29
+ # ///////////////////////////////////////////////////////////////////////////
30
+
31
+
32
+ def get_dummy_dictionary(vocab_size=DEFAULT_TEST_VOCAB_SIZE):
33
+ dummy_dict = Dictionary()
34
+ # add dummy symbol to satisfy vocab size
35
+ for id, _ in enumerate(range(vocab_size)):
36
+ dummy_dict.add_symbol("{}".format(id), 1000)
37
+ return dummy_dict
38
+
39
+
40
+ class DummyTask(LegacyFairseqTask):
41
+ def __init__(self, args):
42
+ super().__init__(args)
43
+ self.dictionary = get_dummy_dictionary()
44
+ if getattr(self.args, "ctc", False):
45
+ self.dictionary.add_symbol("<ctc_blank>")
46
+ self.tgt_dict = self.dictionary
47
+
48
+ @property
49
+ def target_dictionary(self):
50
+ return self.dictionary
51
+
52
+
53
+ def get_dummy_task_and_parser():
54
+ """
55
+ to build a fariseq model, we need some dummy parse and task. This function
56
+ is used to create dummy task and parser to faciliate model/criterion test
57
+
58
+ Note: we use FbSpeechRecognitionTask as the dummy task. You may want
59
+ to use other task by providing another function
60
+ """
61
+ parser = argparse.ArgumentParser(
62
+ description="test_dummy_s2s_task", argument_default=argparse.SUPPRESS
63
+ )
64
+ DummyTask.add_args(parser)
65
+ args = parser.parse_args([])
66
+ task = DummyTask.setup_task(args)
67
+ return task, parser
68
+
69
+
70
+ def get_dummy_input(T=100, D=80, B=5, K=100):
71
+ forward_input = {}
72
+ # T max sequence length
73
+ # D feature vector dimension
74
+ # B batch size
75
+ # K target dimension size
76
+ feature = torch.randn(B, T, D)
77
+ # this (B, T, D) layout is just a convention, you can override it by
78
+ # write your own _prepare_forward_input function
79
+ src_lengths = torch.from_numpy(
80
+ np.random.randint(low=1, high=T, size=B, dtype=np.int64)
81
+ )
82
+ src_lengths[0] = T # make sure the maximum length matches
83
+ prev_output_tokens = []
84
+ for b in range(B):
85
+ token_length = np.random.randint(low=1, high=src_lengths[b].item() + 1)
86
+ tokens = np.random.randint(low=0, high=K, size=token_length, dtype=np.int64)
87
+ prev_output_tokens.append(torch.from_numpy(tokens))
88
+
89
+ prev_output_tokens = fairseq_data_utils.collate_tokens(
90
+ prev_output_tokens,
91
+ pad_idx=1,
92
+ eos_idx=2,
93
+ left_pad=False,
94
+ move_eos_to_beginning=False,
95
+ )
96
+ src_lengths, sorted_order = src_lengths.sort(descending=True)
97
+ forward_input["src_tokens"] = feature.index_select(0, sorted_order)
98
+ forward_input["src_lengths"] = src_lengths
99
+ forward_input["prev_output_tokens"] = prev_output_tokens
100
+
101
+ return forward_input
102
+
103
+
104
+ def get_dummy_encoder_output(encoder_out_shape=(100, 80, 5)):
105
+ """
106
+ This only provides an example to generate dummy encoder output
107
+ """
108
+ (T, B, D) = encoder_out_shape
109
+ encoder_out = {}
110
+
111
+ encoder_out["encoder_out"] = torch.from_numpy(
112
+ np.random.randn(*encoder_out_shape).astype(np.float32)
113
+ )
114
+ seq_lengths = torch.from_numpy(np.random.randint(low=1, high=T, size=B))
115
+ # some dummy mask
116
+ encoder_out["encoder_padding_mask"] = torch.arange(T).view(1, T).expand(
117
+ B, -1
118
+ ) >= seq_lengths.view(B, 1).expand(-1, T)
119
+ encoder_out["encoder_padding_mask"].t_()
120
+
121
+ # encoer_padding_mask is (T, B) tensor, with (t, b)-th element indicate
122
+ # whether encoder_out[t, b] is valid (=0) or not (=1)
123
+ return encoder_out
124
+
125
+
126
+ def _current_postion_info():
127
+ cf = currentframe()
128
+ frameinfo = " (at {}:{})".format(
129
+ os.path.basename(getframeinfo(cf).filename), cf.f_back.f_lineno
130
+ )
131
+ return frameinfo
132
+
133
+
134
+ def check_encoder_output(encoder_output, batch_size=None):
135
+ """we expect encoder_output to be a dict with the following
136
+ key/value pairs:
137
+ - encoder_out: a Torch.Tensor
138
+ - encoder_padding_mask: a binary Torch.Tensor
139
+ """
140
+ if not isinstance(encoder_output, dict):
141
+ msg = (
142
+ "FairseqEncoderModel.forward(...) must be a dict" + _current_postion_info()
143
+ )
144
+ return False, msg
145
+
146
+ if "encoder_out" not in encoder_output:
147
+ msg = (
148
+ "FairseqEncoderModel.forward(...) must contain encoder_out"
149
+ + _current_postion_info()
150
+ )
151
+ return False, msg
152
+
153
+ if "encoder_padding_mask" not in encoder_output:
154
+ msg = (
155
+ "FairseqEncoderModel.forward(...) must contain encoder_padding_mask"
156
+ + _current_postion_info()
157
+ )
158
+ return False, msg
159
+
160
+ if not isinstance(encoder_output["encoder_out"], torch.Tensor):
161
+ msg = "encoder_out must be a torch.Tensor" + _current_postion_info()
162
+ return False, msg
163
+
164
+ if encoder_output["encoder_out"].dtype != torch.float32:
165
+ msg = "encoder_out must have float32 dtype" + _current_postion_info()
166
+ return False, msg
167
+
168
+ mask = encoder_output["encoder_padding_mask"]
169
+ if mask is not None:
170
+ if not isinstance(mask, torch.Tensor):
171
+ msg = (
172
+ "encoder_padding_mask must be a torch.Tensor" + _current_postion_info()
173
+ )
174
+ return False, msg
175
+ if mask.dtype != torch.uint8 and (
176
+ not hasattr(torch, "bool") or mask.dtype != torch.bool
177
+ ):
178
+ msg = (
179
+ "encoder_padding_mask must have dtype of uint8"
180
+ + _current_postion_info()
181
+ )
182
+ return False, msg
183
+
184
+ if mask.dim() != 2:
185
+ msg = (
186
+ "we expect encoder_padding_mask to be a 2-d tensor, in shape (T, B)"
187
+ + _current_postion_info()
188
+ )
189
+ return False, msg
190
+
191
+ if batch_size is not None and mask.size(1) != batch_size:
192
+ msg = (
193
+ "we expect encoder_padding_mask to be a 2-d tensor, with size(1)"
194
+ + " being the batch size"
195
+ + _current_postion_info()
196
+ )
197
+ return False, msg
198
+ return True, None
199
+
200
+
201
+ def check_decoder_output(decoder_output):
202
+ """we expect output from a decoder is a tuple with the following constraint:
203
+ - the first element is a torch.Tensor
204
+ - the second element can be anything (reserved for future use)
205
+ """
206
+ if not isinstance(decoder_output, tuple):
207
+ msg = "FariseqDecoder output must be a tuple" + _current_postion_info()
208
+ return False, msg
209
+
210
+ if len(decoder_output) != 2:
211
+ msg = "FairseqDecoder output must be 2-elem tuple" + _current_postion_info()
212
+ return False, msg
213
+
214
+ if not isinstance(decoder_output[0], torch.Tensor):
215
+ msg = (
216
+ "FariseqDecoder output[0] must be a torch.Tensor" + _current_postion_info()
217
+ )
218
+ return False, msg
219
+
220
+ return True, None
221
+
222
+
223
+ # ///////////////////////////////////////////////////////////////////////////
224
+ # Base Test class
225
+ # ///////////////////////////////////////////////////////////////////////////
226
+
227
+
228
+ class TestBaseFairseqModelBase(unittest.TestCase):
229
+ """
230
+ This class is used to facilitate writing unittest for any class derived from
231
+ `BaseFairseqModel`.
232
+ """
233
+
234
+ @classmethod
235
+ def setUpClass(cls):
236
+ if cls is TestBaseFairseqModelBase:
237
+ raise unittest.SkipTest("Skipping test case in base")
238
+ super().setUpClass()
239
+
240
+ def setUpModel(self, model):
241
+ self.assertTrue(isinstance(model, BaseFairseqModel))
242
+ self.model = model
243
+
244
+ def setupInput(self):
245
+ pass
246
+
247
+ def setUp(self):
248
+ self.model = None
249
+ self.forward_input = None
250
+ pass
251
+
252
+
253
+ class TestFairseqEncoderDecoderModelBase(TestBaseFairseqModelBase):
254
+ """
255
+ base code to test FairseqEncoderDecoderModel (formally known as
256
+ `FairseqModel`) must be derived from this base class
257
+ """
258
+
259
+ @classmethod
260
+ def setUpClass(cls):
261
+ if cls is TestFairseqEncoderDecoderModelBase:
262
+ raise unittest.SkipTest("Skipping test case in base")
263
+ super().setUpClass()
264
+
265
+ def setUpModel(self, model_cls, extra_args_setters=None):
266
+ self.assertTrue(
267
+ issubclass(model_cls, (FairseqEncoderDecoderModel, FairseqModel)),
268
+ msg="This class only tests for FairseqModel subclasses",
269
+ )
270
+
271
+ task, parser = get_dummy_task_and_parser()
272
+ model_cls.add_args(parser)
273
+
274
+ args = parser.parse_args([])
275
+
276
+ if extra_args_setters is not None:
277
+ for args_setter in extra_args_setters:
278
+ args_setter(args)
279
+ model = model_cls.build_model(args, task)
280
+ self.model = model
281
+
282
+ def setUpInput(self, input=None):
283
+ self.forward_input = get_dummy_input() if input is None else input
284
+
285
+ def setUp(self):
286
+ super().setUp()
287
+
288
+ def test_forward(self):
289
+ if self.model and self.forward_input:
290
+ forward_output = self.model.forward(**self.forward_input)
291
+ # for FairseqEncoderDecoderModel, forward returns a tuple of two
292
+ # elements, the first one is a Torch.Tensor
293
+ succ, msg = check_decoder_output(forward_output)
294
+ if not succ:
295
+ self.assertTrue(succ, msg=msg)
296
+ self.forward_output = forward_output
297
+
298
+ def test_get_normalized_probs(self):
299
+ if self.model and self.forward_input:
300
+ forward_output = self.model.forward(**self.forward_input)
301
+ logprob = self.model.get_normalized_probs(forward_output, log_probs=True)
302
+ prob = self.model.get_normalized_probs(forward_output, log_probs=False)
303
+
304
+ # in order for different models/criterion to play with each other
305
+ # we need to know whether the logprob or prob output is batch_first
306
+ # or not. We assume an additional attribute will be attached to logprob
307
+ # or prob. If you find your code failed here, simply override
308
+ # FairseqModel.get_normalized_probs, see example at
309
+ # https://fburl.com/batch_first_example
310
+ self.assertTrue(hasattr(logprob, "batch_first"))
311
+ self.assertTrue(hasattr(prob, "batch_first"))
312
+
313
+ self.assertTrue(torch.is_tensor(logprob))
314
+ self.assertTrue(torch.is_tensor(prob))
315
+
316
+
317
+ class TestFairseqEncoderModelBase(TestBaseFairseqModelBase):
318
+ """
319
+ base class to test FairseqEncoderModel
320
+ """
321
+
322
+ @classmethod
323
+ def setUpClass(cls):
324
+ if cls is TestFairseqEncoderModelBase:
325
+ raise unittest.SkipTest("Skipping test case in base")
326
+ super().setUpClass()
327
+
328
+ def setUpModel(self, model_cls, extra_args_setters=None):
329
+ self.assertTrue(
330
+ issubclass(model_cls, FairseqEncoderModel),
331
+ msg="This class is only used for testing FairseqEncoderModel",
332
+ )
333
+ task, parser = get_dummy_task_and_parser()
334
+ model_cls.add_args(parser)
335
+ args = parser.parse_args([])
336
+ if extra_args_setters is not None:
337
+ for args_setter in extra_args_setters:
338
+ args_setter(args)
339
+
340
+ model = model_cls.build_model(args, task)
341
+ self.model = model
342
+
343
+ def setUpInput(self, input=None):
344
+ self.forward_input = get_dummy_input() if input is None else input
345
+ # get_dummy_input() is originally for s2s, here we delete extra dict
346
+ # items, so it can be used for EncoderModel / Encoder as well
347
+ self.forward_input.pop("prev_output_tokens", None)
348
+
349
+ def setUp(self):
350
+ super().setUp()
351
+
352
+ def test_forward(self):
353
+ if self.forward_input and self.model:
354
+ bsz = self.forward_input["src_tokens"].size(0)
355
+ forward_output = self.model.forward(**self.forward_input)
356
+
357
+ # we expect forward_output to be a dict with the following
358
+ # key/value pairs:
359
+ # - encoder_out: a Torch.Tensor
360
+ # - encoder_padding_mask: a binary Torch.Tensor
361
+ succ, msg = check_encoder_output(forward_output, batch_size=bsz)
362
+ if not succ:
363
+ self.assertTrue(succ, msg=msg)
364
+ self.forward_output = forward_output
365
+
366
+ def test_get_normalized_probs(self):
367
+ if self.model and self.forward_input:
368
+ forward_output = self.model.forward(**self.forward_input)
369
+ logprob = self.model.get_normalized_probs(forward_output, log_probs=True)
370
+ prob = self.model.get_normalized_probs(forward_output, log_probs=False)
371
+
372
+ # in order for different models/criterion to play with each other
373
+ # we need to know whether the logprob or prob output is batch_first
374
+ # or not. We assume an additional attribute will be attached to logprob
375
+ # or prob. If you find your code failed here, simply override
376
+ # FairseqModel.get_normalized_probs, see example at
377
+ # https://fburl.com/batch_first_example
378
+ self.assertTrue(hasattr(logprob, "batch_first"))
379
+ self.assertTrue(hasattr(prob, "batch_first"))
380
+
381
+ self.assertTrue(torch.is_tensor(logprob))
382
+ self.assertTrue(torch.is_tensor(prob))
383
+
384
+
385
+ class TestFairseqEncoderBase(unittest.TestCase):
386
+ """
387
+ base class to test FairseqEncoder
388
+ """
389
+
390
+ @classmethod
391
+ def setUpClass(cls):
392
+ if cls is TestFairseqEncoderBase:
393
+ raise unittest.SkipTest("Skipping test case in base")
394
+ super().setUpClass()
395
+
396
+ def setUpEncoder(self, encoder):
397
+ self.assertTrue(
398
+ isinstance(encoder, FairseqEncoder),
399
+ msg="This class is only used for test FairseqEncoder",
400
+ )
401
+ self.encoder = encoder
402
+
403
+ def setUpInput(self, input=None):
404
+ self.forward_input = get_dummy_input() if input is None else input
405
+ # get_dummy_input() is originally for s2s, here we delete extra dict
406
+ # items, so it can be used for EncoderModel / Encoder as well
407
+ self.forward_input.pop("prev_output_tokens", None)
408
+
409
+ def setUp(self):
410
+ self.encoder = None
411
+ self.forward_input = None
412
+
413
+ def test_forward(self):
414
+ if self.encoder and self.forward_input:
415
+ bsz = self.forward_input["src_tokens"].size(0)
416
+
417
+ forward_output = self.encoder.forward(**self.forward_input)
418
+ succ, msg = check_encoder_output(forward_output, batch_size=bsz)
419
+ if not succ:
420
+ self.assertTrue(succ, msg=msg)
421
+ self.forward_output = forward_output
422
+
423
+
424
+ class TestFairseqDecoderBase(unittest.TestCase):
425
+ """
426
+ base class to test FairseqDecoder
427
+ """
428
+
429
+ @classmethod
430
+ def setUpClass(cls):
431
+ if cls is TestFairseqDecoderBase:
432
+ raise unittest.SkipTest("Skipping test case in base")
433
+ super().setUpClass()
434
+
435
+ def setUpDecoder(self, decoder):
436
+ self.assertTrue(
437
+ isinstance(decoder, FairseqDecoder),
438
+ msg="This class is only used for test FairseqDecoder",
439
+ )
440
+ self.decoder = decoder
441
+
442
+ def setUpInput(self, input=None):
443
+ self.forward_input = get_dummy_encoder_output() if input is None else input
444
+
445
+ def setUpPrevOutputTokens(self, tokens=None):
446
+ if tokens is None:
447
+ self.encoder_input = get_dummy_input()
448
+ self.prev_output_tokens = self.encoder_input["prev_output_tokens"]
449
+ else:
450
+ self.prev_output_tokens = tokens
451
+
452
+ def setUp(self):
453
+ self.decoder = None
454
+ self.forward_input = None
455
+ self.prev_output_tokens = None
456
+
457
+ def test_forward(self):
458
+ if (
459
+ self.decoder is not None
460
+ and self.forward_input is not None
461
+ and self.prev_output_tokens is not None
462
+ ):
463
+ forward_output = self.decoder.forward(
464
+ prev_output_tokens=self.prev_output_tokens,
465
+ encoder_out=self.forward_input,
466
+ )
467
+ succ, msg = check_decoder_output(forward_output)
468
+ if not succ:
469
+ self.assertTrue(succ, msg=msg)
470
+ self.forward_input = forward_output
471
+
472
+
473
+ class DummyEncoderModel(FairseqEncoderModel):
474
+ def __init__(self, encoder):
475
+ super().__init__(encoder)
476
+
477
+ @classmethod
478
+ def build_model(cls, args, task):
479
+ return cls(DummyEncoder())
480
+
481
+ def get_logits(self, net_output):
482
+ # Inverse of sigmoid to use with BinaryCrossEntropyWithLogitsCriterion as
483
+ # F.binary_cross_entropy_with_logits combines sigmoid and CE
484
+ return torch.log(
485
+ torch.div(net_output["encoder_out"], 1 - net_output["encoder_out"])
486
+ )
487
+
488
+ def get_normalized_probs(self, net_output, log_probs, sample=None):
489
+ lprobs = super().get_normalized_probs(net_output, log_probs, sample=sample)
490
+ lprobs.batch_first = True
491
+ return lprobs
492
+
493
+
494
+ class DummyEncoder(FairseqEncoder):
495
+ def __init__(self):
496
+ super().__init__(None)
497
+
498
+ def forward(self, src_tokens, src_lengths):
499
+ mask, max_len = lengths_to_encoder_padding_mask(src_lengths)
500
+ return {"encoder_out": src_tokens, "encoder_padding_mask": mask}
501
+
502
+
503
+ class CrossEntropyCriterionTestBase(unittest.TestCase):
504
+ @classmethod
505
+ def setUpClass(cls):
506
+ if cls is CrossEntropyCriterionTestBase:
507
+ raise unittest.SkipTest("Skipping base class test case")
508
+ super().setUpClass()
509
+
510
+ def setUpArgs(self):
511
+ args = argparse.Namespace()
512
+ args.sentence_avg = False
513
+ args.threshold = 0.1 # to use with BinaryCrossEntropyWithLogitsCriterion
514
+ return args
515
+
516
+ def setUp(self):
517
+ args = self.setUpArgs()
518
+ self.model = DummyEncoderModel(encoder=DummyEncoder())
519
+ self.criterion = self.criterion_cls.build_criterion(args, task=DummyTask(args))
520
+
521
+ def get_src_tokens(self, correct_prediction, aggregate):
522
+ """
523
+ correct_prediction: True if the net_output (src_tokens) should
524
+ predict the correct target
525
+ aggregate: True if the criterion expects net_output (src_tokens)
526
+ aggregated across time axis
527
+ """
528
+ predicted_idx = 0 if correct_prediction else 1
529
+ if aggregate:
530
+ src_tokens = torch.zeros((2, 2), dtype=torch.float)
531
+ for b in range(2):
532
+ src_tokens[b][predicted_idx] = 1.0
533
+ else:
534
+ src_tokens = torch.zeros((2, 10, 2), dtype=torch.float)
535
+ for b in range(2):
536
+ for t in range(10):
537
+ src_tokens[b][t][predicted_idx] = 1.0
538
+ return src_tokens
539
+
540
+ def get_target(self, soft_target):
541
+ if soft_target:
542
+ target = torch.zeros((2, 2), dtype=torch.float)
543
+ for b in range(2):
544
+ target[b][0] = 1.0
545
+ else:
546
+ target = torch.zeros((2, 10), dtype=torch.long)
547
+ return target
548
+
549
+ def get_test_sample(self, correct, soft_target, aggregate):
550
+ src_tokens = self.get_src_tokens(correct, aggregate)
551
+ target = self.get_target(soft_target)
552
+ L = src_tokens.size(1)
553
+ return {
554
+ "net_input": {"src_tokens": src_tokens, "src_lengths": torch.tensor([L])},
555
+ "target": target,
556
+ "ntokens": src_tokens.size(0) * src_tokens.size(1),
557
+ }
data/fairseq/tests/speech_recognition/test_collaters.py ADDED
@@ -0,0 +1,58 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env python3
2
+ # Copyright (c) Facebook, Inc. and its affiliates.
3
+ #
4
+ # This source code is licensed under the MIT license found in the
5
+ # LICENSE file in the root directory of this source tree.
6
+
7
+ import unittest
8
+
9
+ import numpy as np
10
+ import torch
11
+ from examples.speech_recognition.data.collaters import Seq2SeqCollater
12
+
13
+
14
+ class TestSeq2SeqCollator(unittest.TestCase):
15
+ def test_collate(self):
16
+
17
+ eos_idx = 1
18
+ pad_idx = 0
19
+ collater = Seq2SeqCollater(
20
+ feature_index=0, label_index=1, pad_index=pad_idx, eos_index=eos_idx
21
+ )
22
+
23
+ # 2 frames in the first sample and 3 frames in the second one
24
+ frames1 = np.array([[7, 8], [9, 10]])
25
+ frames2 = np.array([[1, 2], [3, 4], [5, 6]])
26
+ target1 = np.array([4, 2, 3, eos_idx])
27
+ target2 = np.array([3, 2, eos_idx])
28
+ sample1 = {"id": 0, "data": [frames1, target1]}
29
+ sample2 = {"id": 1, "data": [frames2, target2]}
30
+ batch = collater.collate([sample1, sample2])
31
+
32
+ # collate sort inputs by frame's length before creating the batch
33
+ self.assertTensorEqual(batch["id"], torch.tensor([1, 0]))
34
+ self.assertEqual(batch["ntokens"], 7)
35
+ self.assertTensorEqual(
36
+ batch["net_input"]["src_tokens"],
37
+ torch.tensor(
38
+ [[[1, 2], [3, 4], [5, 6]], [[7, 8], [9, 10], [pad_idx, pad_idx]]]
39
+ ),
40
+ )
41
+ self.assertTensorEqual(
42
+ batch["net_input"]["prev_output_tokens"],
43
+ torch.tensor([[eos_idx, 3, 2, pad_idx], [eos_idx, 4, 2, 3]]),
44
+ )
45
+ self.assertTensorEqual(batch["net_input"]["src_lengths"], torch.tensor([3, 2]))
46
+ self.assertTensorEqual(
47
+ batch["target"],
48
+ torch.tensor([[3, 2, eos_idx, pad_idx], [4, 2, 3, eos_idx]]),
49
+ )
50
+ self.assertEqual(batch["nsentences"], 2)
51
+
52
+ def assertTensorEqual(self, t1, t2):
53
+ self.assertEqual(t1.size(), t2.size(), "size mismatch")
54
+ self.assertEqual(t1.ne(t2).long().sum(), 0)
55
+
56
+
57
+ if __name__ == "__main__":
58
+ unittest.main()
data/fairseq/tests/speech_recognition/test_cross_entropy.py ADDED
@@ -0,0 +1,37 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env python3
2
+ # Copyright (c) Facebook, Inc. and its affiliates.
3
+ #
4
+ # This source code is licensed under the MIT license found in the
5
+ # LICENSE file in the root directory of this source tree.
6
+
7
+ from examples.speech_recognition.criterions.cross_entropy_acc import (
8
+ CrossEntropyWithAccCriterion,
9
+ )
10
+
11
+ from .asr_test_base import CrossEntropyCriterionTestBase
12
+
13
+
14
+ class CrossEntropyWithAccCriterionTest(CrossEntropyCriterionTestBase):
15
+ def setUp(self):
16
+ self.criterion_cls = CrossEntropyWithAccCriterion
17
+ super().setUp()
18
+
19
+ def test_cross_entropy_all_correct(self):
20
+ sample = self.get_test_sample(correct=True, soft_target=False, aggregate=False)
21
+ loss, sample_size, logging_output = self.criterion(
22
+ self.model, sample, "sum", log_probs=True
23
+ )
24
+ assert logging_output["correct"] == 20
25
+ assert logging_output["total"] == 20
26
+ assert logging_output["sample_size"] == 20
27
+ assert logging_output["ntokens"] == 20
28
+
29
+ def test_cross_entropy_all_wrong(self):
30
+ sample = self.get_test_sample(correct=False, soft_target=False, aggregate=False)
31
+ loss, sample_size, logging_output = self.criterion(
32
+ self.model, sample, "sum", log_probs=True
33
+ )
34
+ assert logging_output["correct"] == 0
35
+ assert logging_output["total"] == 20
36
+ assert logging_output["sample_size"] == 20
37
+ assert logging_output["ntokens"] == 20
data/fairseq/tests/speech_recognition/test_data_utils.py ADDED
@@ -0,0 +1,62 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env python3
2
+ # Copyright (c) Facebook, Inc. and its affiliates.
3
+ #
4
+ # This source code is licensed under the MIT license found in the
5
+ # LICENSE file in the root directory of this source tree.
6
+ import unittest
7
+
8
+ import torch
9
+ from examples.speech_recognition.data import data_utils
10
+
11
+
12
+ class DataUtilsTest(unittest.TestCase):
13
+ def test_normalization(self):
14
+ sample_len1 = torch.tensor(
15
+ [
16
+ [
17
+ -0.7661,
18
+ -1.3889,
19
+ -2.0972,
20
+ -0.9134,
21
+ -0.7071,
22
+ -0.9765,
23
+ -0.8700,
24
+ -0.8283,
25
+ 0.7512,
26
+ 1.3211,
27
+ 2.1532,
28
+ 2.1174,
29
+ 1.2800,
30
+ 1.2633,
31
+ 1.6147,
32
+ 1.6322,
33
+ 2.0723,
34
+ 3.1522,
35
+ 3.2852,
36
+ 2.2309,
37
+ 2.5569,
38
+ 2.2183,
39
+ 2.2862,
40
+ 1.5886,
41
+ 0.8773,
42
+ 0.8725,
43
+ 1.2662,
44
+ 0.9899,
45
+ 1.1069,
46
+ 1.3926,
47
+ 1.2795,
48
+ 1.1199,
49
+ 1.1477,
50
+ 1.2687,
51
+ 1.3843,
52
+ 1.1903,
53
+ 0.8355,
54
+ 1.1367,
55
+ 1.2639,
56
+ 1.4707,
57
+ ]
58
+ ]
59
+ )
60
+ out = data_utils.apply_mv_norm(sample_len1)
61
+ assert not torch.isnan(out).any()
62
+ assert (out == sample_len1).all()
data/fairseq/tests/speech_recognition/test_vggtransformer.py ADDED
@@ -0,0 +1,135 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env python3
2
+
3
+ # import models/encoder/decoder to be tested
4
+ from examples.speech_recognition.models.vggtransformer import (
5
+ TransformerDecoder,
6
+ VGGTransformerEncoder,
7
+ VGGTransformerModel,
8
+ vggtransformer_1,
9
+ vggtransformer_2,
10
+ vggtransformer_base,
11
+ )
12
+
13
+ # import base test class
14
+ from .asr_test_base import (
15
+ DEFAULT_TEST_VOCAB_SIZE,
16
+ TestFairseqDecoderBase,
17
+ TestFairseqEncoderBase,
18
+ TestFairseqEncoderDecoderModelBase,
19
+ get_dummy_dictionary,
20
+ get_dummy_encoder_output,
21
+ get_dummy_input,
22
+ )
23
+
24
+
25
+ class VGGTransformerModelTest_mid(TestFairseqEncoderDecoderModelBase):
26
+ def setUp(self):
27
+ def override_config(args):
28
+ """
29
+ vggtrasformer_1 use 14 layers of transformer,
30
+ for testing purpose, it is too expensive. For fast turn-around
31
+ test, reduce the number of layers to 3.
32
+ """
33
+ args.transformer_enc_config = (
34
+ "((1024, 16, 4096, True, 0.15, 0.15, 0.15),) * 3"
35
+ )
36
+
37
+ super().setUp()
38
+ extra_args_setter = [vggtransformer_1, override_config]
39
+
40
+ self.setUpModel(VGGTransformerModel, extra_args_setter)
41
+ self.setUpInput(get_dummy_input(T=50, D=80, B=5, K=DEFAULT_TEST_VOCAB_SIZE))
42
+
43
+
44
+ class VGGTransformerModelTest_big(TestFairseqEncoderDecoderModelBase):
45
+ def setUp(self):
46
+ def override_config(args):
47
+ """
48
+ vggtrasformer_2 use 16 layers of transformer,
49
+ for testing purpose, it is too expensive. For fast turn-around
50
+ test, reduce the number of layers to 3.
51
+ """
52
+ args.transformer_enc_config = (
53
+ "((1024, 16, 4096, True, 0.15, 0.15, 0.15),) * 3"
54
+ )
55
+
56
+ super().setUp()
57
+ extra_args_setter = [vggtransformer_2, override_config]
58
+
59
+ self.setUpModel(VGGTransformerModel, extra_args_setter)
60
+ self.setUpInput(get_dummy_input(T=50, D=80, B=5, K=DEFAULT_TEST_VOCAB_SIZE))
61
+
62
+
63
+ class VGGTransformerModelTest_base(TestFairseqEncoderDecoderModelBase):
64
+ def setUp(self):
65
+ def override_config(args):
66
+ """
67
+ vggtrasformer_base use 12 layers of transformer,
68
+ for testing purpose, it is too expensive. For fast turn-around
69
+ test, reduce the number of layers to 3.
70
+ """
71
+ args.transformer_enc_config = (
72
+ "((512, 8, 2048, True, 0.15, 0.15, 0.15),) * 3"
73
+ )
74
+
75
+ super().setUp()
76
+ extra_args_setter = [vggtransformer_base, override_config]
77
+
78
+ self.setUpModel(VGGTransformerModel, extra_args_setter)
79
+ self.setUpInput(get_dummy_input(T=50, D=80, B=5, K=DEFAULT_TEST_VOCAB_SIZE))
80
+
81
+
82
+ class VGGTransformerEncoderTest(TestFairseqEncoderBase):
83
+ def setUp(self):
84
+ super().setUp()
85
+
86
+ self.setUpInput(get_dummy_input(T=50, D=80, B=5))
87
+
88
+ def test_forward(self):
89
+ print("1. test standard vggtransformer")
90
+ self.setUpEncoder(VGGTransformerEncoder(input_feat_per_channel=80))
91
+ super().test_forward()
92
+ print("2. test vggtransformer with limited right context")
93
+ self.setUpEncoder(
94
+ VGGTransformerEncoder(
95
+ input_feat_per_channel=80, transformer_context=(-1, 5)
96
+ )
97
+ )
98
+ super().test_forward()
99
+ print("3. test vggtransformer with limited left context")
100
+ self.setUpEncoder(
101
+ VGGTransformerEncoder(
102
+ input_feat_per_channel=80, transformer_context=(5, -1)
103
+ )
104
+ )
105
+ super().test_forward()
106
+ print("4. test vggtransformer with limited right context and sampling")
107
+ self.setUpEncoder(
108
+ VGGTransformerEncoder(
109
+ input_feat_per_channel=80,
110
+ transformer_context=(-1, 12),
111
+ transformer_sampling=(2, 2),
112
+ )
113
+ )
114
+ super().test_forward()
115
+ print("5. test vggtransformer with windowed context and sampling")
116
+ self.setUpEncoder(
117
+ VGGTransformerEncoder(
118
+ input_feat_per_channel=80,
119
+ transformer_context=(12, 12),
120
+ transformer_sampling=(2, 2),
121
+ )
122
+ )
123
+
124
+
125
+ class TransformerDecoderTest(TestFairseqDecoderBase):
126
+ def setUp(self):
127
+ super().setUp()
128
+
129
+ dict = get_dummy_dictionary(vocab_size=DEFAULT_TEST_VOCAB_SIZE)
130
+ decoder = TransformerDecoder(dict)
131
+ dummy_encoder_output = get_dummy_encoder_output(encoder_out_shape=(50, 5, 256))
132
+
133
+ self.setUpDecoder(decoder)
134
+ self.setUpInput(dummy_encoder_output)
135
+ self.setUpPrevOutputTokens()
data/fairseq/tests/tasks/test_denoising.py ADDED
@@ -0,0 +1,96 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright (c) Facebook, Inc. and its affiliates.
2
+ #
3
+ # This source code is licensed under the MIT license found in the
4
+ # LICENSE file in the root directory of this source tree.
5
+
6
+ import os
7
+ import unittest
8
+ from tempfile import TemporaryDirectory
9
+
10
+ from fairseq import options
11
+ from fairseq.binarizer import FileBinarizer, VocabularyDatasetBinarizer
12
+ from fairseq.dataclass.utils import convert_namespace_to_omegaconf
13
+ from fairseq.tasks.denoising import DenoisingTask
14
+ from tests.utils import build_vocab, make_data
15
+
16
+
17
+ class TestDenoising(unittest.TestCase):
18
+ def test_denoising(self):
19
+ with TemporaryDirectory() as dirname:
20
+
21
+ # prep input file
22
+ raw_file = os.path.join(dirname, "raw")
23
+ data = make_data(out_file=raw_file)
24
+ vocab = build_vocab(data)
25
+
26
+ # binarize
27
+ binarizer = VocabularyDatasetBinarizer(vocab, append_eos=False)
28
+ split = "train"
29
+ bin_file = os.path.join(dirname, split)
30
+ dataset_impl = "mmap"
31
+ FileBinarizer.multiprocess_dataset(
32
+ input_file=raw_file,
33
+ binarizer=binarizer,
34
+ dataset_impl=dataset_impl,
35
+ vocab_size=len(vocab),
36
+ output_prefix=bin_file,
37
+ )
38
+
39
+ # setup task
40
+ train_args = options.parse_args_and_arch(
41
+ options.get_training_parser(),
42
+ [
43
+ "--task",
44
+ "denoising",
45
+ "--arch",
46
+ "bart_base",
47
+ "--seed",
48
+ "42",
49
+ "--mask-length",
50
+ "word",
51
+ "--permute-sentences",
52
+ "1",
53
+ "--rotate",
54
+ "0",
55
+ "--replace-length",
56
+ "-1",
57
+ "--mask",
58
+ "0.2",
59
+ dirname,
60
+ ],
61
+ )
62
+ cfg = convert_namespace_to_omegaconf(train_args)
63
+ task = DenoisingTask(cfg.task, binarizer.dict)
64
+
65
+ # load datasets
66
+ original_dataset = task._load_dataset_split(bin_file, 1, False)
67
+ task.load_dataset(split)
68
+ masked_dataset = task.dataset(split)
69
+
70
+ iterator = task.get_batch_iterator(
71
+ dataset=masked_dataset,
72
+ max_tokens=65_536,
73
+ max_positions=4_096,
74
+ ).next_epoch_itr(shuffle=False)
75
+ mask_index = task.source_dictionary.index("<mask>")
76
+ for batch in iterator:
77
+ for sample in range(len(batch)):
78
+ net_input = batch["net_input"]
79
+ masked_src_tokens = net_input["src_tokens"][sample]
80
+ masked_src_length = net_input["src_lengths"][sample]
81
+ masked_tgt_tokens = batch["target"][sample]
82
+
83
+ sample_id = batch["id"][sample]
84
+ original_tokens = original_dataset[sample_id]
85
+ original_tokens = original_tokens.masked_select(
86
+ masked_src_tokens[:masked_src_length] == mask_index
87
+ )
88
+ masked_tokens = masked_tgt_tokens.masked_select(
89
+ masked_src_tokens == mask_index
90
+ )
91
+
92
+ assert masked_tokens.equal(original_tokens)
93
+
94
+
95
+ if __name__ == "__main__":
96
+ unittest.main()
data/fairseq/tests/tasks/test_masked_lm.py ADDED
@@ -0,0 +1,78 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright (c) Facebook, Inc. and its affiliates.
2
+ #
3
+ # This source code is licensed under the MIT license found in the
4
+ # LICENSE file in the root directory of this source tree.
5
+
6
+ import os
7
+ import unittest
8
+ from tempfile import TemporaryDirectory
9
+
10
+ from fairseq.binarizer import FileBinarizer, VocabularyDatasetBinarizer
11
+ from fairseq.tasks.masked_lm import MaskedLMConfig, MaskedLMTask
12
+ from tests.utils import build_vocab, make_data
13
+
14
+
15
+ class TestMaskedLM(unittest.TestCase):
16
+ def test_masks_tokens(self):
17
+ with TemporaryDirectory() as dirname:
18
+
19
+ # prep input file
20
+ raw_file = os.path.join(dirname, "raw")
21
+ data = make_data(out_file=raw_file)
22
+ vocab = build_vocab(data)
23
+
24
+ # binarize
25
+ binarizer = VocabularyDatasetBinarizer(vocab, append_eos=False)
26
+ split = "train"
27
+ bin_file = os.path.join(dirname, split)
28
+ FileBinarizer.multiprocess_dataset(
29
+ input_file=raw_file,
30
+ binarizer=binarizer,
31
+ dataset_impl="mmap",
32
+ vocab_size=len(vocab),
33
+ output_prefix=bin_file,
34
+ )
35
+
36
+ # setup task
37
+ cfg = MaskedLMConfig(
38
+ data=dirname,
39
+ seed=42,
40
+ mask_prob=0.5, # increasing the odds of masking
41
+ random_token_prob=0, # avoiding random tokens for exact match
42
+ leave_unmasked_prob=0, # always masking for exact match
43
+ )
44
+ task = MaskedLMTask(cfg, binarizer.dict)
45
+
46
+ original_dataset = task._load_dataset_split(bin_file, 1, False)
47
+
48
+ # load datasets
49
+ task.load_dataset(split)
50
+ masked_dataset = task.dataset(split)
51
+
52
+ mask_index = task.source_dictionary.index("<mask>")
53
+ iterator = task.get_batch_iterator(
54
+ dataset=masked_dataset,
55
+ max_tokens=65_536,
56
+ max_positions=4_096,
57
+ ).next_epoch_itr(shuffle=False)
58
+ for batch in iterator:
59
+ for sample in range(len(batch)):
60
+ net_input = batch["net_input"]
61
+ masked_src_tokens = net_input["src_tokens"][sample]
62
+ masked_src_length = net_input["src_lengths"][sample]
63
+ masked_tgt_tokens = batch["target"][sample]
64
+
65
+ sample_id = batch["id"][sample]
66
+ original_tokens = original_dataset[sample_id]
67
+ original_tokens = original_tokens.masked_select(
68
+ masked_src_tokens[:masked_src_length] == mask_index
69
+ )
70
+ masked_tokens = masked_tgt_tokens.masked_select(
71
+ masked_tgt_tokens != task.source_dictionary.pad()
72
+ )
73
+
74
+ assert masked_tokens.equal(original_tokens)
75
+
76
+
77
+ if __name__ == "__main__":
78
+ unittest.main()
data/fairseq/tests/tasks/test_multilingual_denoising.py ADDED
@@ -0,0 +1,98 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright (c) Facebook, Inc. and its affiliates.
2
+ #
3
+ # This source code is licensed under the MIT license found in the
4
+ # LICENSE file in the root directory of this source tree.
5
+
6
+ import os
7
+ import unittest
8
+ from tempfile import TemporaryDirectory
9
+
10
+ from fairseq import options
11
+ from fairseq.binarizer import FileBinarizer, VocabularyDatasetBinarizer
12
+ from fairseq.dataclass.utils import convert_namespace_to_omegaconf
13
+ from fairseq.tasks.multilingual_denoising import MultilingualDenoisingTask
14
+ from tests.utils import build_vocab, make_data
15
+
16
+
17
+ class TestMultilingualDenoising(unittest.TestCase):
18
+ def test_multilingual_denoising(self):
19
+ with TemporaryDirectory() as dirname:
20
+
21
+ # prep input file
22
+ lang_dir = os.path.join(dirname, "en")
23
+ os.mkdir(lang_dir)
24
+ raw_file = os.path.join(lang_dir, "raw")
25
+ data = make_data(out_file=raw_file)
26
+ vocab = build_vocab(data)
27
+
28
+ # binarize
29
+ binarizer = VocabularyDatasetBinarizer(vocab, append_eos=False)
30
+ split = "train"
31
+ bin_file = os.path.join(lang_dir, split)
32
+ dataset_impl = "mmap"
33
+ FileBinarizer.multiprocess_dataset(
34
+ input_file=raw_file,
35
+ binarizer=binarizer,
36
+ dataset_impl=dataset_impl,
37
+ vocab_size=len(vocab),
38
+ output_prefix=bin_file,
39
+ )
40
+
41
+ # setup task
42
+ train_args = options.parse_args_and_arch(
43
+ options.get_training_parser(),
44
+ [
45
+ "--task",
46
+ "multilingual_denoising",
47
+ "--arch",
48
+ "bart_base",
49
+ "--seed",
50
+ "42",
51
+ "--mask-length",
52
+ "word",
53
+ "--permute-sentences",
54
+ "1",
55
+ "--rotate",
56
+ "0",
57
+ "--replace-length",
58
+ "-1",
59
+ "--mask",
60
+ "0.2",
61
+ dirname,
62
+ ],
63
+ )
64
+ cfg = convert_namespace_to_omegaconf(train_args)
65
+ task = MultilingualDenoisingTask(cfg.task, binarizer.dict)
66
+
67
+ # load datasets
68
+ original_dataset = task._load_dataset_split(bin_file, 1, False)
69
+ task.load_dataset(split)
70
+ masked_dataset = task.dataset(split)
71
+
72
+ iterator = task.get_batch_iterator(
73
+ dataset=masked_dataset,
74
+ max_tokens=65_536,
75
+ max_positions=4_096,
76
+ ).next_epoch_itr(shuffle=False)
77
+ mask_index = task.source_dictionary.index("<mask>")
78
+ for batch in iterator:
79
+ for sample in range(len(batch)):
80
+ net_input = batch["net_input"]
81
+ masked_src_tokens = net_input["src_tokens"][sample]
82
+ masked_src_length = net_input["src_lengths"][sample]
83
+ masked_tgt_tokens = batch["target"][sample]
84
+
85
+ sample_id = batch["id"][sample]
86
+ original_tokens = original_dataset[sample_id]
87
+ original_tokens = original_tokens.masked_select(
88
+ masked_src_tokens[:masked_src_length] == mask_index
89
+ )
90
+ masked_tokens = masked_tgt_tokens.masked_select(
91
+ masked_src_tokens == mask_index
92
+ )
93
+
94
+ assert masked_tokens.equal(original_tokens)
95
+
96
+
97
+ if __name__ == "__main__":
98
+ unittest.main()