Add files using upload-large-folder tool
Browse filesThis view is limited to 50 files because it contains too many changes. See raw diff
- data/fairseq/examples/.gitignore +2 -0
- data/fairseq/examples/__init__.py +9 -0
- data/fairseq/examples/bart/README.glue.md +99 -0
- data/fairseq/examples/bart/README.md +228 -0
- data/fairseq/examples/bart/README.summarization.md +102 -0
- data/fairseq/examples/bart/summarize.py +100 -0
- data/fairseq/fairseq_cli/__init__.py +0 -0
- data/fairseq/fairseq_cli/__pycache__/__init__.cpython-38.pyc +0 -0
- data/fairseq/fairseq_cli/__pycache__/generate.cpython-38.pyc +0 -0
- data/fairseq/fairseq_cli/__pycache__/preprocess.cpython-38.pyc +0 -0
- data/fairseq/fairseq_cli/__pycache__/train.cpython-38.pyc +0 -0
- data/fairseq/fairseq_cli/eval_lm.py +347 -0
- data/fairseq/fairseq_cli/generate.py +417 -0
- data/fairseq/fairseq_cli/hydra_train.py +91 -0
- data/fairseq/fairseq_cli/hydra_validate.py +188 -0
- data/fairseq/fairseq_cli/interactive.py +317 -0
- data/fairseq/fairseq_cli/preprocess.py +393 -0
- data/fairseq/fairseq_cli/score.py +102 -0
- data/fairseq/fairseq_cli/train.py +581 -0
- data/fairseq/fairseq_cli/validate.py +153 -0
- data/fairseq/tests/distributed/__init__.py +0 -0
- data/fairseq/tests/distributed/test_bmuf.py +204 -0
- data/fairseq/tests/distributed/test_distributed_timeout_wrapper.py +52 -0
- data/fairseq/tests/distributed/test_module_proxy_wrapper.py +74 -0
- data/fairseq/tests/distributed/test_utils.py +124 -0
- data/fairseq/tests/distributed/utils.py +65 -0
- data/fairseq/tests/gpu/__init__.py +0 -0
- data/fairseq/tests/gpu/test_binaries_gpu.py +590 -0
- data/fairseq/tests/gpu/test_ema_gpu.py +215 -0
- data/fairseq/tests/gpu/transformer_quantization_config.yaml +28 -0
- data/fairseq/tests/speech/__init__.py +210 -0
- data/fairseq/tests/speech/test_convtransformer_simul_trans.py +33 -0
- data/fairseq/tests/speech/test_dual_input_wav_transformer.py +76 -0
- data/fairseq/tests/speech/test_dualinput_s2t_transformer.py +110 -0
- data/fairseq/tests/speech/test_fastspeech2.py +53 -0
- data/fairseq/tests/speech/test_s2s_transformer.py +51 -0
- data/fairseq/tests/speech/test_s2t_conformer.py +23 -0
- data/fairseq/tests/speech/test_s2t_transformer.py +23 -0
- data/fairseq/tests/speech/test_tts_transformer.py +53 -0
- data/fairseq/tests/speech/test_wav2vec2.py +90 -0
- data/fairseq/tests/speech/test_xm_transformer.py +29 -0
- data/fairseq/tests/speech_recognition/__init__.py +0 -0
- data/fairseq/tests/speech_recognition/asr_test_base.py +557 -0
- data/fairseq/tests/speech_recognition/test_collaters.py +58 -0
- data/fairseq/tests/speech_recognition/test_cross_entropy.py +37 -0
- data/fairseq/tests/speech_recognition/test_data_utils.py +62 -0
- data/fairseq/tests/speech_recognition/test_vggtransformer.py +135 -0
- data/fairseq/tests/tasks/test_denoising.py +96 -0
- data/fairseq/tests/tasks/test_masked_lm.py +78 -0
- data/fairseq/tests/tasks/test_multilingual_denoising.py +98 -0
data/fairseq/examples/.gitignore
ADDED
|
@@ -0,0 +1,2 @@
|
|
|
|
|
|
|
|
|
|
| 1 |
+
!*/*.sh
|
| 2 |
+
!*/*.md
|
data/fairseq/examples/__init__.py
ADDED
|
@@ -0,0 +1,9 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) Facebook, Inc. and its affiliates.
|
| 2 |
+
#
|
| 3 |
+
# This source code is licensed under the MIT license found in the
|
| 4 |
+
# LICENSE file in the root directory of this source tree.
|
| 5 |
+
|
| 6 |
+
try:
|
| 7 |
+
from fairseq.version import __version__ # noqa
|
| 8 |
+
except ImportError:
|
| 9 |
+
pass
|
data/fairseq/examples/bart/README.glue.md
ADDED
|
@@ -0,0 +1,99 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Fine-tuning BART on GLUE tasks
|
| 2 |
+
|
| 3 |
+
### 1) Download the data from GLUE website (https://gluebenchmark.com/tasks) using following commands:
|
| 4 |
+
```bash
|
| 5 |
+
wget https://gist.githubusercontent.com/W4ngatang/60c2bdb54d156a41194446737ce03e2e/raw/17b8dd0d724281ed7c3b2aeeda662b92809aadd5/download_glue_data.py
|
| 6 |
+
python download_glue_data.py --data_dir glue_data --tasks all
|
| 7 |
+
```
|
| 8 |
+
|
| 9 |
+
### 2) Preprocess GLUE task data (same as RoBERTa):
|
| 10 |
+
```bash
|
| 11 |
+
./examples/roberta/preprocess_GLUE_tasks.sh glue_data <glue_task_name>
|
| 12 |
+
```
|
| 13 |
+
`glue_task_name` is one of the following:
|
| 14 |
+
`{ALL, QQP, MNLI, QNLI, MRPC, RTE, STS-B, SST-2, CoLA}`
|
| 15 |
+
Use `ALL` for preprocessing all the glue tasks.
|
| 16 |
+
|
| 17 |
+
### 3) Fine-tuning on GLUE task:
|
| 18 |
+
Example fine-tuning cmd for `RTE` task
|
| 19 |
+
```bash
|
| 20 |
+
TOTAL_NUM_UPDATES=2036 # 10 epochs through RTE for bsz 16
|
| 21 |
+
WARMUP_UPDATES=61 # 6 percent of the number of updates
|
| 22 |
+
LR=1e-05 # Peak LR for polynomial LR scheduler.
|
| 23 |
+
NUM_CLASSES=2
|
| 24 |
+
MAX_SENTENCES=16 # Batch size.
|
| 25 |
+
BART_PATH=/path/to/bart/model.pt
|
| 26 |
+
|
| 27 |
+
CUDA_VISIBLE_DEVICES=0,1 fairseq-train RTE-bin/ \
|
| 28 |
+
--restore-file $BART_PATH \
|
| 29 |
+
--batch-size $MAX_SENTENCES \
|
| 30 |
+
--max-tokens 4400 \
|
| 31 |
+
--task sentence_prediction \
|
| 32 |
+
--add-prev-output-tokens \
|
| 33 |
+
--layernorm-embedding \
|
| 34 |
+
--share-all-embeddings \
|
| 35 |
+
--share-decoder-input-output-embed \
|
| 36 |
+
--reset-optimizer --reset-dataloader --reset-meters \
|
| 37 |
+
--required-batch-size-multiple 1 \
|
| 38 |
+
--init-token 0 \
|
| 39 |
+
--arch bart_large \
|
| 40 |
+
--criterion sentence_prediction \
|
| 41 |
+
--num-classes $NUM_CLASSES \
|
| 42 |
+
--dropout 0.1 --attention-dropout 0.1 \
|
| 43 |
+
--weight-decay 0.01 --optimizer adam --adam-betas "(0.9, 0.98)" --adam-eps 1e-08 \
|
| 44 |
+
--clip-norm 0.0 \
|
| 45 |
+
--lr-scheduler polynomial_decay --lr $LR --total-num-update $TOTAL_NUM_UPDATES --warmup-updates $WARMUP_UPDATES \
|
| 46 |
+
--fp16 --fp16-init-scale 4 --threshold-loss-scale 1 --fp16-scale-window 128 \
|
| 47 |
+
--max-epoch 10 \
|
| 48 |
+
--find-unused-parameters \
|
| 49 |
+
--best-checkpoint-metric accuracy --maximize-best-checkpoint-metric;
|
| 50 |
+
```
|
| 51 |
+
|
| 52 |
+
For each of the GLUE task, you will need to use following cmd-line arguments:
|
| 53 |
+
|
| 54 |
+
Model | MNLI | QNLI | QQP | RTE | SST-2 | MRPC | CoLA | STS-B
|
| 55 |
+
---|---|---|---|---|---|---|---|---
|
| 56 |
+
`--num-classes` | 3 | 2 | 2 | 2 | 2 | 2 | 2 | 1
|
| 57 |
+
`--lr` | 5e-6 | 1e-5 | 1e-5 | 1e-5 | 5e-6 | 2e-5 | 2e-5 | 2e-5
|
| 58 |
+
`bsz` | 128 | 32 | 32 | 32 | 128 | 64 | 64 | 32
|
| 59 |
+
`--total-num-update` | 30968 | 33112 | 113272 | 1018 | 5233 | 1148 | 1334 | 1799
|
| 60 |
+
`--warmup-updates` | 1858 | 1986 | 6796 | 61 | 314 | 68 | 80 | 107
|
| 61 |
+
|
| 62 |
+
For `STS-B` additionally add `--regression-target --best-checkpoint-metric loss` and remove `--maximize-best-checkpoint-metric`.
|
| 63 |
+
|
| 64 |
+
**Note:**
|
| 65 |
+
|
| 66 |
+
a) `--total-num-updates` is used by `--polynomial_decay` scheduler and is calculated for `--max-epoch=10` and `--batch-size=32/64/128` depending on the task.
|
| 67 |
+
|
| 68 |
+
b) Above cmd-args and hyperparams are tested on Nvidia `V100` GPU with `32gb` of memory for each task. Depending on the GPU memory resources available to you, you can use increase `--update-freq` and reduce `--batch-size`.
|
| 69 |
+
|
| 70 |
+
### Inference on GLUE task
|
| 71 |
+
After training the model as mentioned in previous step, you can perform inference with checkpoints in `checkpoints/` directory using following python code snippet:
|
| 72 |
+
|
| 73 |
+
```python
|
| 74 |
+
from fairseq.models.bart import BARTModel
|
| 75 |
+
|
| 76 |
+
bart = BARTModel.from_pretrained(
|
| 77 |
+
'checkpoints/',
|
| 78 |
+
checkpoint_file='checkpoint_best.pt',
|
| 79 |
+
data_name_or_path='RTE-bin'
|
| 80 |
+
)
|
| 81 |
+
|
| 82 |
+
label_fn = lambda label: bart.task.label_dictionary.string(
|
| 83 |
+
[label + bart.task.label_dictionary.nspecial]
|
| 84 |
+
)
|
| 85 |
+
ncorrect, nsamples = 0, 0
|
| 86 |
+
bart.cuda()
|
| 87 |
+
bart.eval()
|
| 88 |
+
with open('glue_data/RTE/dev.tsv') as fin:
|
| 89 |
+
fin.readline()
|
| 90 |
+
for index, line in enumerate(fin):
|
| 91 |
+
tokens = line.strip().split('\t')
|
| 92 |
+
sent1, sent2, target = tokens[1], tokens[2], tokens[3]
|
| 93 |
+
tokens = bart.encode(sent1, sent2)
|
| 94 |
+
prediction = bart.predict('sentence_classification_head', tokens).argmax().item()
|
| 95 |
+
prediction_label = label_fn(prediction)
|
| 96 |
+
ncorrect += int(prediction_label == target)
|
| 97 |
+
nsamples += 1
|
| 98 |
+
print('| Accuracy: ', float(ncorrect)/float(nsamples))
|
| 99 |
+
```
|
data/fairseq/examples/bart/README.md
ADDED
|
@@ -0,0 +1,228 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# BART: Denoising Sequence-to-Sequence Pre-training for Natural Language Generation, Translation, and Comprehension
|
| 2 |
+
|
| 3 |
+
[https://arxiv.org/abs/1910.13461](https://arxiv.org/abs/1910.13461)
|
| 4 |
+
|
| 5 |
+
## Introduction
|
| 6 |
+
|
| 7 |
+
BART is sequence-to-sequence model trained with denoising as pretraining objective. We show that this pretraining objective is more generic and show that we can match [RoBERTa](../roberta) results on SQuAD and GLUE and gain state-of-the-art results on summarization (XSum, CNN dataset), long form generative question answering (ELI5) and dialog response genration (ConvAI2). See the associated paper for more details.
|
| 8 |
+
|
| 9 |
+
## Pre-trained models
|
| 10 |
+
|
| 11 |
+
Model | Description | # params | Download
|
| 12 |
+
---|---|---|---
|
| 13 |
+
`bart.base` | BART model with 6 encoder and decoder layers | 140M | [bart.base.tar.gz](https://dl.fbaipublicfiles.com/fairseq/models/bart.base.tar.gz)
|
| 14 |
+
`bart.large` | BART model with 12 encoder and decoder layers | 400M | [bart.large.tar.gz](https://dl.fbaipublicfiles.com/fairseq/models/bart.large.tar.gz)
|
| 15 |
+
`bart.large.mnli` | `bart.large` finetuned on `MNLI` | 400M | [bart.large.mnli.tar.gz](https://dl.fbaipublicfiles.com/fairseq/models/bart.large.mnli.tar.gz)
|
| 16 |
+
`bart.large.cnn` | `bart.large` finetuned on `CNN-DM` | 400M | [bart.large.cnn.tar.gz](https://dl.fbaipublicfiles.com/fairseq/models/bart.large.cnn.tar.gz)
|
| 17 |
+
`bart.large.xsum` | `bart.large` finetuned on `Xsum` | 400M | [bart.large.xsum.tar.gz](https://dl.fbaipublicfiles.com/fairseq/models/bart.large.xsum.tar.gz)
|
| 18 |
+
|
| 19 |
+
## Results
|
| 20 |
+
|
| 21 |
+
**[GLUE (Wang et al., 2019)](https://gluebenchmark.com/)**
|
| 22 |
+
_(dev set, single model, single-task finetuning)_
|
| 23 |
+
|
| 24 |
+
Model | MNLI | QNLI | QQP | RTE | SST-2 | MRPC | CoLA | STS-B
|
| 25 |
+
---|---|---|---|---|---|---|---|---
|
| 26 |
+
`roberta.large` | 90.2 | 94.7 | 92.2 | 86.6 | 96.4 | 90.9 | 68.0 | 92.4
|
| 27 |
+
`bart.large` | 89.9 | 94.9 | 92.5 | 87.0 | 96.6 | 90.4 | 62.8 | 91.2
|
| 28 |
+
|
| 29 |
+
**[SQuAD (Rajpurkar et al., 2018)](https://rajpurkar.github.io/SQuAD-explorer/)**
|
| 30 |
+
_(dev set, no additional data used)_
|
| 31 |
+
|
| 32 |
+
Model | SQuAD 1.1 EM/F1 | SQuAD 2.0 EM/F1
|
| 33 |
+
---|---|---
|
| 34 |
+
`roberta.large` | 88.9/94.6 | 86.5/89.4
|
| 35 |
+
`bart.large` | 88.8/94.6 | 86.1/89.2
|
| 36 |
+
|
| 37 |
+
**[CNN/Daily Mail](http://nlpprogress.com/english/summarization.html)**
|
| 38 |
+
_(test set, no additional data used)_
|
| 39 |
+
|
| 40 |
+
Model | R1 | R2 | RL
|
| 41 |
+
---|---|---|---
|
| 42 |
+
`BERTSUMEXTABS` | 42.13 | 19.60 | 39.18
|
| 43 |
+
`bart.large` | 44.16 | 21.28 | 40.90
|
| 44 |
+
|
| 45 |
+
## Example usage
|
| 46 |
+
|
| 47 |
+
##### Load BART from torch.hub (PyTorch >= 1.1):
|
| 48 |
+
```python
|
| 49 |
+
import torch
|
| 50 |
+
bart = torch.hub.load('pytorch/fairseq', 'bart.large')
|
| 51 |
+
bart.eval() # disable dropout (or leave in train mode to finetune)
|
| 52 |
+
```
|
| 53 |
+
|
| 54 |
+
##### Load BART (for PyTorch 1.0 or custom models):
|
| 55 |
+
```python
|
| 56 |
+
# Download bart.large model
|
| 57 |
+
wget https://dl.fbaipublicfiles.com/fairseq/models/bart.large.tar.gz
|
| 58 |
+
tar -xzvf bart.large.tar.gz
|
| 59 |
+
|
| 60 |
+
# Load the model in fairseq
|
| 61 |
+
from fairseq.models.bart import BARTModel
|
| 62 |
+
bart = BARTModel.from_pretrained('/path/to/bart.large', checkpoint_file='model.pt')
|
| 63 |
+
bart.eval() # disable dropout (or leave in train mode to finetune)
|
| 64 |
+
```
|
| 65 |
+
|
| 66 |
+
##### Apply Byte-Pair Encoding (BPE) to input text:
|
| 67 |
+
```python
|
| 68 |
+
tokens = bart.encode('Hello world!')
|
| 69 |
+
assert tokens.tolist() == [0, 31414, 232, 328, 2]
|
| 70 |
+
bart.decode(tokens) # 'Hello world!'
|
| 71 |
+
```
|
| 72 |
+
|
| 73 |
+
##### Extract features from BART:
|
| 74 |
+
```python
|
| 75 |
+
# Extract the last layer's features
|
| 76 |
+
last_layer_features = bart.extract_features(tokens)
|
| 77 |
+
assert last_layer_features.size() == torch.Size([1, 5, 1024])
|
| 78 |
+
|
| 79 |
+
# Extract all layer's features from decoder (layer 0 is the embedding layer)
|
| 80 |
+
all_layers = bart.extract_features(tokens, return_all_hiddens=True)
|
| 81 |
+
assert len(all_layers) == 13
|
| 82 |
+
assert torch.all(all_layers[-1] == last_layer_features)
|
| 83 |
+
```
|
| 84 |
+
|
| 85 |
+
##### Use BART for sentence-pair classification tasks:
|
| 86 |
+
```python
|
| 87 |
+
# Download BART already finetuned for MNLI
|
| 88 |
+
bart = torch.hub.load('pytorch/fairseq', 'bart.large.mnli')
|
| 89 |
+
bart.eval() # disable dropout for evaluation
|
| 90 |
+
|
| 91 |
+
# Encode a pair of sentences and make a prediction
|
| 92 |
+
tokens = bart.encode('BART is a seq2seq model.', 'BART is not sequence to sequence.')
|
| 93 |
+
bart.predict('mnli', tokens).argmax() # 0: contradiction
|
| 94 |
+
|
| 95 |
+
# Encode another pair of sentences
|
| 96 |
+
tokens = bart.encode('BART is denoising autoencoder.', 'BART is version of autoencoder.')
|
| 97 |
+
bart.predict('mnli', tokens).argmax() # 2: entailment
|
| 98 |
+
```
|
| 99 |
+
|
| 100 |
+
##### Register a new (randomly initialized) classification head:
|
| 101 |
+
```python
|
| 102 |
+
bart.register_classification_head('new_task', num_classes=3)
|
| 103 |
+
logprobs = bart.predict('new_task', tokens)
|
| 104 |
+
```
|
| 105 |
+
|
| 106 |
+
##### Batched prediction:
|
| 107 |
+
```python
|
| 108 |
+
import torch
|
| 109 |
+
from fairseq.data.data_utils import collate_tokens
|
| 110 |
+
|
| 111 |
+
bart = torch.hub.load('pytorch/fairseq', 'bart.large.mnli')
|
| 112 |
+
bart.eval()
|
| 113 |
+
|
| 114 |
+
batch_of_pairs = [
|
| 115 |
+
['BART is a seq2seq model.', 'BART is not sequence to sequence.'],
|
| 116 |
+
['BART is denoising autoencoder.', 'BART is version of autoencoder.'],
|
| 117 |
+
]
|
| 118 |
+
|
| 119 |
+
batch = collate_tokens(
|
| 120 |
+
[bart.encode(pair[0], pair[1]) for pair in batch_of_pairs], pad_idx=1
|
| 121 |
+
)
|
| 122 |
+
|
| 123 |
+
logprobs = bart.predict('mnli', batch)
|
| 124 |
+
print(logprobs.argmax(dim=1))
|
| 125 |
+
# tensor([0, 2])
|
| 126 |
+
```
|
| 127 |
+
|
| 128 |
+
##### Using the GPU:
|
| 129 |
+
```python
|
| 130 |
+
bart.cuda()
|
| 131 |
+
bart.predict('new_task', tokens)
|
| 132 |
+
```
|
| 133 |
+
|
| 134 |
+
#### Filling masks:
|
| 135 |
+
|
| 136 |
+
BART can be used to fill multiple `<mask>` tokens in the input.
|
| 137 |
+
```python
|
| 138 |
+
bart = torch.hub.load('pytorch/fairseq', 'bart.base')
|
| 139 |
+
bart.eval()
|
| 140 |
+
bart.fill_mask(['The cat <mask> on the <mask>.'], topk=3, beam=10)
|
| 141 |
+
# [[('The cat was on the ground.', tensor(-0.6183)), ('The cat was on the floor.', tensor(-0.6798)), ('The cat sleeps on the couch.', tensor(-0.6830))]]
|
| 142 |
+
```
|
| 143 |
+
|
| 144 |
+
Note that by default we enforce the output length to match the input length.
|
| 145 |
+
This can be disabled by setting ``match_source_len=False``:
|
| 146 |
+
```
|
| 147 |
+
bart.fill_mask(['The cat <mask> on the <mask>.'], topk=3, beam=10, match_source_len=False)
|
| 148 |
+
# [[('The cat was on the ground.', tensor(-0.6185)), ('The cat was asleep on the couch.', tensor(-0.6276)), ('The cat was on the floor.', tensor(-0.6800))]]
|
| 149 |
+
```
|
| 150 |
+
|
| 151 |
+
Example code to fill masks for a batch of sentences using GPU
|
| 152 |
+
```
|
| 153 |
+
bart.cuda()
|
| 154 |
+
bart.fill_mask(['The cat <mask> on the <mask>.', 'The dog <mask> on the <mask>.'], topk=3, beam=10)
|
| 155 |
+
# [[('The cat was on the ground.', tensor(-0.6183)), ('The cat was on the floor.', tensor(-0.6798)), ('The cat sleeps on the couch.', tensor(-0.6830))], [('The dog was on the ground.', tensor(-0.6190)), ('The dog lay on the ground.', tensor(-0.6711)),
|
| 156 |
+
('The dog was asleep on the couch', tensor(-0.6796))]]
|
| 157 |
+
```
|
| 158 |
+
|
| 159 |
+
#### Evaluating the `bart.large.mnli` model:
|
| 160 |
+
|
| 161 |
+
Example python code snippet to evaluate accuracy on the MNLI `dev_matched` set.
|
| 162 |
+
```python
|
| 163 |
+
label_map = {0: 'contradiction', 1: 'neutral', 2: 'entailment'}
|
| 164 |
+
ncorrect, nsamples = 0, 0
|
| 165 |
+
bart.cuda()
|
| 166 |
+
bart.eval()
|
| 167 |
+
with open('glue_data/MNLI/dev_matched.tsv') as fin:
|
| 168 |
+
fin.readline()
|
| 169 |
+
for index, line in enumerate(fin):
|
| 170 |
+
tokens = line.strip().split('\t')
|
| 171 |
+
sent1, sent2, target = tokens[8], tokens[9], tokens[-1]
|
| 172 |
+
tokens = bart.encode(sent1, sent2)
|
| 173 |
+
prediction = bart.predict('mnli', tokens).argmax().item()
|
| 174 |
+
prediction_label = label_map[prediction]
|
| 175 |
+
ncorrect += int(prediction_label == target)
|
| 176 |
+
nsamples += 1
|
| 177 |
+
print('| Accuracy: ', float(ncorrect)/float(nsamples))
|
| 178 |
+
# Expected output: 0.9010
|
| 179 |
+
```
|
| 180 |
+
|
| 181 |
+
#### Evaluating the `bart.large.cnn` model:
|
| 182 |
+
- Follow instructions [here](https://github.com/abisee/cnn-dailymail) to download and process into data-files such that `test.source` and `test.target` has one line for each non-tokenized sample.
|
| 183 |
+
- For simpler preprocessing, you can also `wget https://cdn-datasets.huggingface.co/summarization/cnn_dm_v2.tgz`, although there is no guarantee of identical scores
|
| 184 |
+
- `huggingface/transformers` has a simpler interface that supports [single-gpu](https://github.com/huggingface/transformers/blob/master/examples/legacy/seq2seq/run_eval.py) and [multi-gpu](https://github.com/huggingface/transformers/blob/master/examples/legacy/seq2seq/run_distributed_eval.py) beam search.
|
| 185 |
+
In `huggingface/transformers`, the BART models' paths are `facebook/bart-large-cnn` and `facebook/bart-large-xsum`.
|
| 186 |
+
|
| 187 |
+
In `fairseq`, summaries can be generated using:
|
| 188 |
+
|
| 189 |
+
```bash
|
| 190 |
+
cp data-bin/cnn_dm/dict.source.txt checkpoints/
|
| 191 |
+
python examples/bart/summarize.py \
|
| 192 |
+
--model-dir pytorch/fairseq \
|
| 193 |
+
--model-file bart.large.cnn \
|
| 194 |
+
--src cnn_dm/test.source \
|
| 195 |
+
--out cnn_dm/test.hypo
|
| 196 |
+
```
|
| 197 |
+
|
| 198 |
+
For calculating rouge, install `files2rouge` from [here](https://github.com/pltrdy/files2rouge).
|
| 199 |
+
|
| 200 |
+
```bash
|
| 201 |
+
export CLASSPATH=/path/to/stanford-corenlp-full-2016-10-31/stanford-corenlp-3.7.0.jar
|
| 202 |
+
|
| 203 |
+
# Tokenize hypothesis and target files.
|
| 204 |
+
cat test.hypo | java edu.stanford.nlp.process.PTBTokenizer -ioFileList -preserveLines > test.hypo.tokenized
|
| 205 |
+
cat test.target | java edu.stanford.nlp.process.PTBTokenizer -ioFileList -preserveLines > test.hypo.target
|
| 206 |
+
files2rouge test.hypo.tokenized test.hypo.target
|
| 207 |
+
# Expected output: (ROUGE-2 Average_F: 0.21238)
|
| 208 |
+
```
|
| 209 |
+
|
| 210 |
+
|
| 211 |
+
## Finetuning
|
| 212 |
+
|
| 213 |
+
- [Finetuning on GLUE](README.glue.md)
|
| 214 |
+
- [Finetuning on CNN-DM](README.summarization.md)
|
| 215 |
+
|
| 216 |
+
## Citation
|
| 217 |
+
|
| 218 |
+
```bibtex
|
| 219 |
+
@article{lewis2019bart,
|
| 220 |
+
title = {BART: Denoising Sequence-to-Sequence Pre-training for Natural
|
| 221 |
+
Language Generation, Translation, and Comprehension},
|
| 222 |
+
author = {Mike Lewis and Yinhan Liu and Naman Goyal and Marjan Ghazvininejad and
|
| 223 |
+
Abdelrahman Mohamed and Omer Levy and Veselin Stoyanov
|
| 224 |
+
and Luke Zettlemoyer },
|
| 225 |
+
journal={arXiv preprint arXiv:1910.13461},
|
| 226 |
+
year = {2019},
|
| 227 |
+
}
|
| 228 |
+
```
|
data/fairseq/examples/bart/README.summarization.md
ADDED
|
@@ -0,0 +1,102 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Fine-tuning BART on CNN-Dailymail summarization task
|
| 2 |
+
|
| 3 |
+
### 1) Download the CNN and Daily Mail data and preprocess it into data files with non-tokenized cased samples.
|
| 4 |
+
|
| 5 |
+
Follow the instructions [here](https://github.com/abisee/cnn-dailymail) to download the original CNN and Daily Mail datasets. To preprocess the data, refer to the pointers in [this issue](https://github.com/pytorch/fairseq/issues/1391) or check out the code [here](https://github.com/artmatsak/cnn-dailymail).
|
| 6 |
+
|
| 7 |
+
Follow the instructions [here](https://github.com/EdinburghNLP/XSum) to download the original Extreme Summarization datasets, or check out the code [here](https://github.com/EdinburghNLP/XSum/tree/master/XSum-Dataset), Please keep the raw dataset and make sure no tokenization nor BPE on the dataset.
|
| 8 |
+
|
| 9 |
+
### 2) BPE preprocess:
|
| 10 |
+
|
| 11 |
+
```bash
|
| 12 |
+
wget -N 'https://dl.fbaipublicfiles.com/fairseq/gpt2_bpe/encoder.json'
|
| 13 |
+
wget -N 'https://dl.fbaipublicfiles.com/fairseq/gpt2_bpe/vocab.bpe'
|
| 14 |
+
wget -N 'https://dl.fbaipublicfiles.com/fairseq/gpt2_bpe/dict.txt'
|
| 15 |
+
|
| 16 |
+
TASK=cnn_dm
|
| 17 |
+
for SPLIT in train val
|
| 18 |
+
do
|
| 19 |
+
for LANG in source target
|
| 20 |
+
do
|
| 21 |
+
python -m examples.roberta.multiprocessing_bpe_encoder \
|
| 22 |
+
--encoder-json encoder.json \
|
| 23 |
+
--vocab-bpe vocab.bpe \
|
| 24 |
+
--inputs "$TASK/$SPLIT.$LANG" \
|
| 25 |
+
--outputs "$TASK/$SPLIT.bpe.$LANG" \
|
| 26 |
+
--workers 60 \
|
| 27 |
+
--keep-empty;
|
| 28 |
+
done
|
| 29 |
+
done
|
| 30 |
+
```
|
| 31 |
+
|
| 32 |
+
### 3) Binarize dataset:
|
| 33 |
+
```bash
|
| 34 |
+
fairseq-preprocess \
|
| 35 |
+
--source-lang "source" \
|
| 36 |
+
--target-lang "target" \
|
| 37 |
+
--trainpref "${TASK}/train.bpe" \
|
| 38 |
+
--validpref "${TASK}/val.bpe" \
|
| 39 |
+
--destdir "${TASK}-bin/" \
|
| 40 |
+
--workers 60 \
|
| 41 |
+
--srcdict dict.txt \
|
| 42 |
+
--tgtdict dict.txt;
|
| 43 |
+
```
|
| 44 |
+
|
| 45 |
+
### 4) Fine-tuning on CNN-DM summarization task:
|
| 46 |
+
Example fine-tuning CNN-DM
|
| 47 |
+
```bash
|
| 48 |
+
TOTAL_NUM_UPDATES=20000
|
| 49 |
+
WARMUP_UPDATES=500
|
| 50 |
+
LR=3e-05
|
| 51 |
+
MAX_TOKENS=2048
|
| 52 |
+
UPDATE_FREQ=4
|
| 53 |
+
BART_PATH=/path/to/bart/model.pt
|
| 54 |
+
|
| 55 |
+
CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 fairseq-train cnn_dm-bin \
|
| 56 |
+
--restore-file $BART_PATH \
|
| 57 |
+
--max-tokens $MAX_TOKENS \
|
| 58 |
+
--task translation \
|
| 59 |
+
--source-lang source --target-lang target \
|
| 60 |
+
--truncate-source \
|
| 61 |
+
--layernorm-embedding \
|
| 62 |
+
--share-all-embeddings \
|
| 63 |
+
--share-decoder-input-output-embed \
|
| 64 |
+
--reset-optimizer --reset-dataloader --reset-meters \
|
| 65 |
+
--required-batch-size-multiple 1 \
|
| 66 |
+
--arch bart_large \
|
| 67 |
+
--criterion label_smoothed_cross_entropy \
|
| 68 |
+
--label-smoothing 0.1 \
|
| 69 |
+
--dropout 0.1 --attention-dropout 0.1 \
|
| 70 |
+
--weight-decay 0.01 --optimizer adam --adam-betas "(0.9, 0.999)" --adam-eps 1e-08 \
|
| 71 |
+
--clip-norm 0.1 \
|
| 72 |
+
--lr-scheduler polynomial_decay --lr $LR --total-num-update $TOTAL_NUM_UPDATES --warmup-updates $WARMUP_UPDATES \
|
| 73 |
+
--fp16 --update-freq $UPDATE_FREQ \
|
| 74 |
+
--skip-invalid-size-inputs-valid-test \
|
| 75 |
+
--find-unused-parameters;
|
| 76 |
+
```
|
| 77 |
+
Above is expected to run on `1` node with `8 32gb-V100`.
|
| 78 |
+
Expected training time is about `5 hours`. Training time can be reduced with distributed training on `4` nodes and `--update-freq 1`.
|
| 79 |
+
|
| 80 |
+
Use TOTAL_NUM_UPDATES=15000 UPDATE_FREQ=2 for Xsum task
|
| 81 |
+
|
| 82 |
+
### Inference for CNN-DM test data using above trained checkpoint.
|
| 83 |
+
After training the model as mentioned in previous step, you can perform inference with checkpoints in `checkpoints/` directory using `eval_cnn.py`, for example
|
| 84 |
+
|
| 85 |
+
```bash
|
| 86 |
+
cp data-bin/cnn_dm/dict.source.txt checkpoints/
|
| 87 |
+
python examples/bart/summarize.py \
|
| 88 |
+
--model-dir checkpoints \
|
| 89 |
+
--model-file checkpoint_best.pt \
|
| 90 |
+
--src cnn_dm/test.source \
|
| 91 |
+
--out cnn_dm/test.hypo
|
| 92 |
+
```
|
| 93 |
+
For XSUM, which uses beam=6, lenpen=1.0, max_len_b=60, min_len=10:
|
| 94 |
+
```bash
|
| 95 |
+
cp data-bin/cnn_dm/dict.source.txt checkpoints/
|
| 96 |
+
python examples/bart/summarize.py \
|
| 97 |
+
--model-dir checkpoints \
|
| 98 |
+
--model-file checkpoint_best.pt \
|
| 99 |
+
--src cnn_dm/test.source \
|
| 100 |
+
--out cnn_dm/test.hypo \
|
| 101 |
+
--xsum-kwargs
|
| 102 |
+
```
|
data/fairseq/examples/bart/summarize.py
ADDED
|
@@ -0,0 +1,100 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) Facebook, Inc. and its affiliates.
|
| 2 |
+
#
|
| 3 |
+
# This source code is licensed under the MIT license found in the
|
| 4 |
+
# LICENSE file in the root directory of this source tree.
|
| 5 |
+
|
| 6 |
+
import torch
|
| 7 |
+
from fairseq.models.bart import BARTModel
|
| 8 |
+
import argparse
|
| 9 |
+
|
| 10 |
+
XSUM_KWARGS = dict(beam=6, lenpen=1.0, max_len_b=60, min_len=10, no_repeat_ngram_size=3)
|
| 11 |
+
CNN_KWARGS = dict(beam=4, lenpen=2.0, max_len_b=140, min_len=55, no_repeat_ngram_size=3)
|
| 12 |
+
|
| 13 |
+
|
| 14 |
+
@torch.no_grad()
|
| 15 |
+
def generate(bart, infile, outfile="bart_hypo.txt", bsz=32, n_obs=None, **eval_kwargs):
|
| 16 |
+
count = 1
|
| 17 |
+
|
| 18 |
+
# if n_obs is not None: bsz = min(bsz, n_obs)
|
| 19 |
+
|
| 20 |
+
with open(infile) as source, open(outfile, "w") as fout:
|
| 21 |
+
sline = source.readline().strip()
|
| 22 |
+
slines = [sline]
|
| 23 |
+
for sline in source:
|
| 24 |
+
if n_obs is not None and count > n_obs:
|
| 25 |
+
break
|
| 26 |
+
if count % bsz == 0:
|
| 27 |
+
hypotheses_batch = bart.sample(slines, **eval_kwargs)
|
| 28 |
+
for hypothesis in hypotheses_batch:
|
| 29 |
+
fout.write(hypothesis + "\n")
|
| 30 |
+
fout.flush()
|
| 31 |
+
slines = []
|
| 32 |
+
|
| 33 |
+
slines.append(sline.strip())
|
| 34 |
+
count += 1
|
| 35 |
+
|
| 36 |
+
if slines != []:
|
| 37 |
+
hypotheses_batch = bart.sample(slines, **eval_kwargs)
|
| 38 |
+
for hypothesis in hypotheses_batch:
|
| 39 |
+
fout.write(hypothesis + "\n")
|
| 40 |
+
fout.flush()
|
| 41 |
+
|
| 42 |
+
|
| 43 |
+
def main():
|
| 44 |
+
"""
|
| 45 |
+
Usage::
|
| 46 |
+
|
| 47 |
+
python examples/bart/summarize.py \
|
| 48 |
+
--model-dir $HOME/bart.large.cnn \
|
| 49 |
+
--model-file model.pt \
|
| 50 |
+
--src $HOME/data-bin/cnn_dm/test.source
|
| 51 |
+
"""
|
| 52 |
+
parser = argparse.ArgumentParser()
|
| 53 |
+
parser.add_argument(
|
| 54 |
+
"--model-dir",
|
| 55 |
+
required=True,
|
| 56 |
+
type=str,
|
| 57 |
+
default="bart.large.cnn/",
|
| 58 |
+
help="path containing model file and src_dict.txt",
|
| 59 |
+
)
|
| 60 |
+
parser.add_argument(
|
| 61 |
+
"--model-file",
|
| 62 |
+
default="checkpoint_best.pt",
|
| 63 |
+
help="where in model_dir are weights saved",
|
| 64 |
+
)
|
| 65 |
+
parser.add_argument(
|
| 66 |
+
"--src", default="test.source", help="text to summarize", type=str
|
| 67 |
+
)
|
| 68 |
+
parser.add_argument(
|
| 69 |
+
"--out", default="test.hypo", help="where to save summaries", type=str
|
| 70 |
+
)
|
| 71 |
+
parser.add_argument("--bsz", default=32, help="where to save summaries", type=int)
|
| 72 |
+
parser.add_argument(
|
| 73 |
+
"--n", default=None, help="how many examples to summarize", type=int
|
| 74 |
+
)
|
| 75 |
+
parser.add_argument(
|
| 76 |
+
"--xsum-kwargs",
|
| 77 |
+
action="store_true",
|
| 78 |
+
default=False,
|
| 79 |
+
help="if true use XSUM_KWARGS else CNN_KWARGS",
|
| 80 |
+
)
|
| 81 |
+
args = parser.parse_args()
|
| 82 |
+
eval_kwargs = XSUM_KWARGS if args.xsum_kwargs else CNN_KWARGS
|
| 83 |
+
if args.model_dir == "pytorch/fairseq":
|
| 84 |
+
bart = torch.hub.load("pytorch/fairseq", args.model_file)
|
| 85 |
+
else:
|
| 86 |
+
bart = BARTModel.from_pretrained(
|
| 87 |
+
args.model_dir,
|
| 88 |
+
checkpoint_file=args.model_file,
|
| 89 |
+
data_name_or_path=args.model_dir,
|
| 90 |
+
)
|
| 91 |
+
bart = bart.eval()
|
| 92 |
+
if torch.cuda.is_available():
|
| 93 |
+
bart = bart.cuda().half()
|
| 94 |
+
generate(
|
| 95 |
+
bart, args.src, bsz=args.bsz, n_obs=args.n, outfile=args.out, **eval_kwargs
|
| 96 |
+
)
|
| 97 |
+
|
| 98 |
+
|
| 99 |
+
if __name__ == "__main__":
|
| 100 |
+
main()
|
data/fairseq/fairseq_cli/__init__.py
ADDED
|
File without changes
|
data/fairseq/fairseq_cli/__pycache__/__init__.cpython-38.pyc
ADDED
|
Binary file (158 Bytes). View file
|
|
|
data/fairseq/fairseq_cli/__pycache__/generate.cpython-38.pyc
ADDED
|
Binary file (9.07 kB). View file
|
|
|
data/fairseq/fairseq_cli/__pycache__/preprocess.cpython-38.pyc
ADDED
|
Binary file (8.11 kB). View file
|
|
|
data/fairseq/fairseq_cli/__pycache__/train.cpython-38.pyc
ADDED
|
Binary file (12.8 kB). View file
|
|
|
data/fairseq/fairseq_cli/eval_lm.py
ADDED
|
@@ -0,0 +1,347 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python3 -u
|
| 2 |
+
# Copyright (c) Facebook, Inc. and its affiliates.
|
| 3 |
+
#
|
| 4 |
+
# This source code is licensed under the MIT license found in the
|
| 5 |
+
# LICENSE file in the root directory of this source tree.
|
| 6 |
+
|
| 7 |
+
"""
|
| 8 |
+
Evaluate the perplexity of a trained language model.
|
| 9 |
+
"""
|
| 10 |
+
|
| 11 |
+
import logging
|
| 12 |
+
import math
|
| 13 |
+
import os
|
| 14 |
+
import sys
|
| 15 |
+
from argparse import Namespace
|
| 16 |
+
from typing import Iterable, List, Optional
|
| 17 |
+
|
| 18 |
+
import torch
|
| 19 |
+
from omegaconf import DictConfig
|
| 20 |
+
|
| 21 |
+
import fairseq
|
| 22 |
+
from fairseq import checkpoint_utils, distributed_utils, options, tasks, utils
|
| 23 |
+
from fairseq.dataclass.utils import convert_namespace_to_omegaconf
|
| 24 |
+
from fairseq.logging import progress_bar
|
| 25 |
+
from fairseq.logging.meters import StopwatchMeter
|
| 26 |
+
from fairseq.sequence_scorer import SequenceScorer
|
| 27 |
+
|
| 28 |
+
logging.basicConfig(
|
| 29 |
+
format="%(asctime)s | %(levelname)s | %(name)s | %(message)s",
|
| 30 |
+
datefmt="%Y-%m-%d %H:%M:%S",
|
| 31 |
+
level=os.environ.get("LOGLEVEL", "INFO").upper(),
|
| 32 |
+
stream=sys.stdout,
|
| 33 |
+
)
|
| 34 |
+
logger = logging.getLogger("fairseq_cli.eval_lm")
|
| 35 |
+
|
| 36 |
+
|
| 37 |
+
def eval_lm(
|
| 38 |
+
models: List[fairseq.models.FairseqModel],
|
| 39 |
+
source_dictionary: fairseq.data.Dictionary,
|
| 40 |
+
batch_iterator: Iterable,
|
| 41 |
+
post_process: Optional[str] = None,
|
| 42 |
+
output_word_probs: bool = False,
|
| 43 |
+
output_word_stats: bool = False,
|
| 44 |
+
target_dictionary: Optional[fairseq.data.Dictionary] = None,
|
| 45 |
+
softmax_batch: int = 0,
|
| 46 |
+
remove_bos_token: bool = False,
|
| 47 |
+
device: Optional[torch.device] = None,
|
| 48 |
+
):
|
| 49 |
+
"""
|
| 50 |
+
Args:
|
| 51 |
+
models (List[~fairseq.models.FairseqModel]): list of models to
|
| 52 |
+
evaluate. Models are essentially `nn.Module` instances, but
|
| 53 |
+
must be compatible with fairseq's `SequenceScorer`.
|
| 54 |
+
source_dictionary (~fairseq.data.Dictionary): dictionary for
|
| 55 |
+
applying any relevant post processing or outputing word
|
| 56 |
+
probs/stats.
|
| 57 |
+
batch_iterator (Iterable): yield batches of data
|
| 58 |
+
post_process (Optional[str]): post-process text by removing BPE,
|
| 59 |
+
letter segmentation, etc. Valid options can be found in
|
| 60 |
+
fairseq.data.utils.post_process, although not all options
|
| 61 |
+
are implemented here.
|
| 62 |
+
output_word_probs (Optional[bool]): output words and their
|
| 63 |
+
predicted log probabilities
|
| 64 |
+
output_word_stats (Optional[bool]): output word statistics such
|
| 65 |
+
as word count and average probability
|
| 66 |
+
target_dictionary (Optional[~fairseq.data.Dictionary]): output
|
| 67 |
+
dictionary (defaults to *source_dictionary*)
|
| 68 |
+
softmax_batch (Optional[bool]): if BxT is more than this, will
|
| 69 |
+
batch the softmax over vocab to this amount of tokens, in
|
| 70 |
+
order to fit into GPU memory
|
| 71 |
+
remove_bos_token (Optional[bool]): if True, confirm that the
|
| 72 |
+
first token is the beginning-of-sentence symbol (according
|
| 73 |
+
to the relevant dictionary) and remove it from the output
|
| 74 |
+
device (Optional[torch.device]): device to use for evaluation
|
| 75 |
+
(defaults to device of first model parameter)
|
| 76 |
+
"""
|
| 77 |
+
if target_dictionary is None:
|
| 78 |
+
target_dictionary = source_dictionary
|
| 79 |
+
if device is None:
|
| 80 |
+
device = next(models[0].parameters()).device
|
| 81 |
+
|
| 82 |
+
gen_timer = StopwatchMeter()
|
| 83 |
+
scorer = SequenceScorer(target_dictionary, softmax_batch)
|
| 84 |
+
|
| 85 |
+
score_sum = 0.0
|
| 86 |
+
count = 0
|
| 87 |
+
|
| 88 |
+
if post_process is not None:
|
| 89 |
+
if post_process in {"subword_nmt", "@@ "}:
|
| 90 |
+
bpe_cont = post_process.rstrip()
|
| 91 |
+
bpe_toks = {
|
| 92 |
+
i
|
| 93 |
+
for i in range(len(source_dictionary))
|
| 94 |
+
if source_dictionary[i].endswith(bpe_cont)
|
| 95 |
+
}
|
| 96 |
+
else:
|
| 97 |
+
raise NotImplementedError(
|
| 98 |
+
f"--post-process={post_process} is not implemented"
|
| 99 |
+
)
|
| 100 |
+
bpe_len = len(bpe_cont)
|
| 101 |
+
else:
|
| 102 |
+
bpe_toks = None
|
| 103 |
+
bpe_len = 0
|
| 104 |
+
|
| 105 |
+
word_stats = dict()
|
| 106 |
+
|
| 107 |
+
for sample in batch_iterator:
|
| 108 |
+
if "net_input" not in sample:
|
| 109 |
+
continue
|
| 110 |
+
|
| 111 |
+
sample = utils.move_to_cuda(sample, device=device)
|
| 112 |
+
|
| 113 |
+
gen_timer.start()
|
| 114 |
+
hypos = scorer.generate(models, sample)
|
| 115 |
+
gen_timer.stop(sample["ntokens"])
|
| 116 |
+
|
| 117 |
+
for i, hypos_i in enumerate(hypos):
|
| 118 |
+
hypo = hypos_i[0]
|
| 119 |
+
sample_id = sample["id"][i]
|
| 120 |
+
|
| 121 |
+
tokens = hypo["tokens"]
|
| 122 |
+
tgt_len = tokens.numel()
|
| 123 |
+
pos_scores = hypo["positional_scores"].float()
|
| 124 |
+
|
| 125 |
+
if remove_bos_token:
|
| 126 |
+
assert hypo["tokens"][0].item() == target_dictionary.bos()
|
| 127 |
+
tokens = tokens[1:]
|
| 128 |
+
pos_scores = pos_scores[1:]
|
| 129 |
+
|
| 130 |
+
skipped_toks = 0
|
| 131 |
+
if bpe_toks is not None:
|
| 132 |
+
for i in range(tgt_len - 1):
|
| 133 |
+
if tokens[i].item() in bpe_toks:
|
| 134 |
+
skipped_toks += 1
|
| 135 |
+
pos_scores[i + 1] += pos_scores[i]
|
| 136 |
+
pos_scores[i] = 0
|
| 137 |
+
|
| 138 |
+
inf_scores = pos_scores.eq(float("inf")) | pos_scores.eq(float("-inf"))
|
| 139 |
+
if inf_scores.any():
|
| 140 |
+
logger.info(
|
| 141 |
+
"skipping tokens with inf scores:",
|
| 142 |
+
target_dictionary.string(tokens[inf_scores.nonzero()]),
|
| 143 |
+
)
|
| 144 |
+
pos_scores = pos_scores[(~inf_scores).nonzero()]
|
| 145 |
+
score_sum += pos_scores.sum().cpu()
|
| 146 |
+
count += pos_scores.numel() - skipped_toks
|
| 147 |
+
|
| 148 |
+
if output_word_probs or output_word_stats:
|
| 149 |
+
w = ""
|
| 150 |
+
word_prob = []
|
| 151 |
+
is_bpe = False
|
| 152 |
+
for i in range(len(tokens)):
|
| 153 |
+
w_ind = tokens[i].item()
|
| 154 |
+
w += source_dictionary[w_ind]
|
| 155 |
+
if bpe_toks is not None and w_ind in bpe_toks:
|
| 156 |
+
w = w[:-bpe_len]
|
| 157 |
+
is_bpe = True
|
| 158 |
+
else:
|
| 159 |
+
word_prob.append((w, pos_scores[i].item()))
|
| 160 |
+
|
| 161 |
+
next_prob = None
|
| 162 |
+
ind = i + 1
|
| 163 |
+
while ind < len(tokens):
|
| 164 |
+
if pos_scores[ind].item() != 0:
|
| 165 |
+
next_prob = pos_scores[ind]
|
| 166 |
+
break
|
| 167 |
+
ind += 1
|
| 168 |
+
|
| 169 |
+
word_stats.setdefault(w, WordStat(w, is_bpe)).add(
|
| 170 |
+
pos_scores[i].item(), next_prob
|
| 171 |
+
)
|
| 172 |
+
is_bpe = False
|
| 173 |
+
w = ""
|
| 174 |
+
if output_word_probs:
|
| 175 |
+
logger.info(
|
| 176 |
+
str(int(sample_id))
|
| 177 |
+
+ " "
|
| 178 |
+
+ (
|
| 179 |
+
"\t".join(
|
| 180 |
+
"{} [{:2f}]".format(x[0], x[1]) for x in word_prob
|
| 181 |
+
)
|
| 182 |
+
)
|
| 183 |
+
)
|
| 184 |
+
|
| 185 |
+
avg_nll_loss = (
|
| 186 |
+
-score_sum / count / math.log(2) if count > 0 else 0
|
| 187 |
+
) # convert to base 2
|
| 188 |
+
logger.info(
|
| 189 |
+
"Evaluated {:,} tokens in {:.1f}s ({:.2f} tokens/s)".format(
|
| 190 |
+
gen_timer.n, gen_timer.sum, 1.0 / gen_timer.avg if gen_timer.avg > 0 else 0
|
| 191 |
+
)
|
| 192 |
+
)
|
| 193 |
+
|
| 194 |
+
if output_word_stats:
|
| 195 |
+
for ws in sorted(word_stats.values(), key=lambda x: x.count, reverse=True):
|
| 196 |
+
logger.info(ws)
|
| 197 |
+
|
| 198 |
+
return {
|
| 199 |
+
"loss": avg_nll_loss,
|
| 200 |
+
"perplexity": 2**avg_nll_loss,
|
| 201 |
+
}
|
| 202 |
+
|
| 203 |
+
|
| 204 |
+
class WordStat(object):
|
| 205 |
+
def __init__(self, word, is_bpe):
|
| 206 |
+
self.word = word
|
| 207 |
+
self.is_bpe = is_bpe
|
| 208 |
+
self.log_prob = 0
|
| 209 |
+
self.next_word_prob = 0
|
| 210 |
+
self.count = 0
|
| 211 |
+
self.missing_next_words = 0
|
| 212 |
+
|
| 213 |
+
def add(self, log_prob, next_word_prob):
|
| 214 |
+
"""increments counters for the sum of log probs of current word and next
|
| 215 |
+
word (given context ending at current word). Since the next word might be at the end of the example,
|
| 216 |
+
or it might be not counted because it is not an ending subword unit,
|
| 217 |
+
also keeps track of how many of those we have seen"""
|
| 218 |
+
if next_word_prob is not None:
|
| 219 |
+
self.next_word_prob += next_word_prob
|
| 220 |
+
else:
|
| 221 |
+
self.missing_next_words += 1
|
| 222 |
+
self.log_prob += log_prob
|
| 223 |
+
self.count += 1
|
| 224 |
+
|
| 225 |
+
def __str__(self):
|
| 226 |
+
return "{}\t{}\t{}\t{}\t{}\t{}".format(
|
| 227 |
+
self.word,
|
| 228 |
+
self.count,
|
| 229 |
+
self.log_prob,
|
| 230 |
+
self.is_bpe,
|
| 231 |
+
self.next_word_prob,
|
| 232 |
+
self.count - self.missing_next_words,
|
| 233 |
+
)
|
| 234 |
+
|
| 235 |
+
|
| 236 |
+
def main(cfg: DictConfig, **unused_kwargs):
|
| 237 |
+
if isinstance(cfg, Namespace):
|
| 238 |
+
cfg = convert_namespace_to_omegaconf(cfg)
|
| 239 |
+
|
| 240 |
+
utils.import_user_module(cfg.common)
|
| 241 |
+
|
| 242 |
+
logger.info(cfg)
|
| 243 |
+
|
| 244 |
+
if cfg.eval_lm.context_window > 0:
|
| 245 |
+
# reduce tokens per sample by the required context window size
|
| 246 |
+
cfg.task.tokens_per_sample -= cfg.eval_lm.context_window
|
| 247 |
+
|
| 248 |
+
# Initialize the task using the current *cfg*
|
| 249 |
+
task = tasks.setup_task(cfg.task)
|
| 250 |
+
|
| 251 |
+
# Load ensemble
|
| 252 |
+
logger.info("loading model(s) from {}".format(cfg.common_eval.path))
|
| 253 |
+
models, model_args, task = checkpoint_utils.load_model_ensemble_and_task(
|
| 254 |
+
[cfg.common_eval.path],
|
| 255 |
+
arg_overrides=eval(cfg.common_eval.model_overrides),
|
| 256 |
+
suffix=cfg.checkpoint.checkpoint_suffix,
|
| 257 |
+
strict=(cfg.checkpoint.checkpoint_shard_count == 1),
|
| 258 |
+
num_shards=cfg.checkpoint.checkpoint_shard_count,
|
| 259 |
+
task=task,
|
| 260 |
+
)
|
| 261 |
+
|
| 262 |
+
use_fp16 = cfg.common.fp16
|
| 263 |
+
use_cuda = torch.cuda.is_available() and not cfg.common.cpu
|
| 264 |
+
if use_cuda:
|
| 265 |
+
torch.cuda.set_device(cfg.distributed_training.device_id)
|
| 266 |
+
|
| 267 |
+
# Optimize ensemble for generation and set the source and dest dicts on the model
|
| 268 |
+
# (required by scorer)
|
| 269 |
+
for model in models:
|
| 270 |
+
if use_fp16:
|
| 271 |
+
model.half()
|
| 272 |
+
if use_cuda and not cfg.distributed_training.pipeline_model_parallel:
|
| 273 |
+
model.cuda()
|
| 274 |
+
model.prepare_for_inference_(cfg)
|
| 275 |
+
|
| 276 |
+
assert len(models) > 0
|
| 277 |
+
|
| 278 |
+
logger.info(
|
| 279 |
+
"num. model params: {:,}".format(sum(p.numel() for p in models[0].parameters()))
|
| 280 |
+
)
|
| 281 |
+
|
| 282 |
+
# Load dataset splits
|
| 283 |
+
task.load_dataset(cfg.dataset.gen_subset)
|
| 284 |
+
dataset = task.dataset(cfg.dataset.gen_subset)
|
| 285 |
+
logger.info(
|
| 286 |
+
"{} {} {:,} examples".format(
|
| 287 |
+
cfg.task.data, cfg.dataset.gen_subset, len(dataset)
|
| 288 |
+
)
|
| 289 |
+
)
|
| 290 |
+
|
| 291 |
+
itr = task.eval_lm_dataloader(
|
| 292 |
+
dataset=dataset,
|
| 293 |
+
max_tokens=cfg.dataset.max_tokens or 36000,
|
| 294 |
+
batch_size=cfg.dataset.batch_size,
|
| 295 |
+
max_positions=utils.resolve_max_positions(
|
| 296 |
+
*[model.max_positions() for model in models]
|
| 297 |
+
),
|
| 298 |
+
num_shards=max(
|
| 299 |
+
cfg.dataset.num_shards,
|
| 300 |
+
cfg.distributed_training.distributed_world_size,
|
| 301 |
+
),
|
| 302 |
+
shard_id=max(
|
| 303 |
+
cfg.dataset.shard_id,
|
| 304 |
+
cfg.distributed_training.distributed_rank,
|
| 305 |
+
),
|
| 306 |
+
num_workers=cfg.dataset.num_workers,
|
| 307 |
+
data_buffer_size=cfg.dataset.data_buffer_size,
|
| 308 |
+
context_window=cfg.eval_lm.context_window,
|
| 309 |
+
)
|
| 310 |
+
|
| 311 |
+
itr = progress_bar.progress_bar(
|
| 312 |
+
itr,
|
| 313 |
+
log_format=cfg.common.log_format,
|
| 314 |
+
log_interval=cfg.common.log_interval,
|
| 315 |
+
default_log_format=("tqdm" if not cfg.common.no_progress_bar else "simple"),
|
| 316 |
+
)
|
| 317 |
+
|
| 318 |
+
results = eval_lm(
|
| 319 |
+
models=models,
|
| 320 |
+
source_dictionary=task.source_dictionary,
|
| 321 |
+
batch_iterator=itr,
|
| 322 |
+
post_process=cfg.common_eval.post_process,
|
| 323 |
+
output_word_probs=cfg.eval_lm.output_word_probs,
|
| 324 |
+
output_word_stats=cfg.eval_lm.output_word_stats,
|
| 325 |
+
target_dictionary=task.target_dictionary,
|
| 326 |
+
softmax_batch=cfg.eval_lm.softmax_batch,
|
| 327 |
+
remove_bos_token=getattr(cfg.task, "add_bos_token", False),
|
| 328 |
+
)
|
| 329 |
+
|
| 330 |
+
logger.info(
|
| 331 |
+
"Loss (base 2): {:.4f}, Perplexity: {:.2f}".format(
|
| 332 |
+
results["loss"], results["perplexity"]
|
| 333 |
+
)
|
| 334 |
+
)
|
| 335 |
+
|
| 336 |
+
return results
|
| 337 |
+
|
| 338 |
+
|
| 339 |
+
def cli_main():
|
| 340 |
+
parser = options.get_eval_lm_parser()
|
| 341 |
+
args = options.parse_args_and_arch(parser)
|
| 342 |
+
|
| 343 |
+
distributed_utils.call_main(convert_namespace_to_omegaconf(args), main)
|
| 344 |
+
|
| 345 |
+
|
| 346 |
+
if __name__ == "__main__":
|
| 347 |
+
cli_main()
|
data/fairseq/fairseq_cli/generate.py
ADDED
|
@@ -0,0 +1,417 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python3 -u
|
| 2 |
+
# Copyright (c) Facebook, Inc. and its affiliates.
|
| 3 |
+
#
|
| 4 |
+
# This source code is licensed under the MIT license found in the
|
| 5 |
+
# LICENSE file in the root directory of this source tree.
|
| 6 |
+
"""
|
| 7 |
+
Translate pre-processed data with a trained model.
|
| 8 |
+
"""
|
| 9 |
+
|
| 10 |
+
import ast
|
| 11 |
+
import logging
|
| 12 |
+
import math
|
| 13 |
+
import os
|
| 14 |
+
import sys
|
| 15 |
+
from argparse import Namespace
|
| 16 |
+
from itertools import chain
|
| 17 |
+
|
| 18 |
+
import numpy as np
|
| 19 |
+
import torch
|
| 20 |
+
from omegaconf import DictConfig
|
| 21 |
+
|
| 22 |
+
from fairseq import checkpoint_utils, options, scoring, tasks, utils
|
| 23 |
+
from fairseq.dataclass.utils import convert_namespace_to_omegaconf
|
| 24 |
+
from fairseq.logging import progress_bar
|
| 25 |
+
from fairseq.logging.meters import StopwatchMeter, TimeMeter
|
| 26 |
+
|
| 27 |
+
|
| 28 |
+
def main(cfg: DictConfig):
|
| 29 |
+
|
| 30 |
+
if isinstance(cfg, Namespace):
|
| 31 |
+
cfg = convert_namespace_to_omegaconf(cfg)
|
| 32 |
+
|
| 33 |
+
assert cfg.common_eval.path is not None, "--path required for generation!"
|
| 34 |
+
assert (
|
| 35 |
+
not cfg.generation.sampling or cfg.generation.nbest == cfg.generation.beam
|
| 36 |
+
), "--sampling requires --nbest to be equal to --beam"
|
| 37 |
+
assert (
|
| 38 |
+
cfg.generation.replace_unk is None or cfg.dataset.dataset_impl == "raw"
|
| 39 |
+
), "--replace-unk requires a raw text dataset (--dataset-impl=raw)"
|
| 40 |
+
|
| 41 |
+
if cfg.common_eval.results_path is not None:
|
| 42 |
+
os.makedirs(cfg.common_eval.results_path, exist_ok=True)
|
| 43 |
+
output_path = os.path.join(
|
| 44 |
+
cfg.common_eval.results_path,
|
| 45 |
+
"generate-{}.txt".format(cfg.dataset.gen_subset),
|
| 46 |
+
)
|
| 47 |
+
with open(output_path, "w", buffering=1, encoding="utf-8") as h:
|
| 48 |
+
return _main(cfg, h)
|
| 49 |
+
else:
|
| 50 |
+
return _main(cfg, sys.stdout)
|
| 51 |
+
|
| 52 |
+
|
| 53 |
+
def get_symbols_to_strip_from_output(generator):
|
| 54 |
+
if hasattr(generator, "symbols_to_strip_from_output"):
|
| 55 |
+
return generator.symbols_to_strip_from_output
|
| 56 |
+
else:
|
| 57 |
+
return {generator.eos}
|
| 58 |
+
|
| 59 |
+
|
| 60 |
+
def _main(cfg: DictConfig, output_file):
|
| 61 |
+
logging.basicConfig(
|
| 62 |
+
format="%(asctime)s | %(levelname)s | %(name)s | %(message)s",
|
| 63 |
+
datefmt="%Y-%m-%d %H:%M:%S",
|
| 64 |
+
level=os.environ.get("LOGLEVEL", "INFO").upper(),
|
| 65 |
+
stream=output_file,
|
| 66 |
+
)
|
| 67 |
+
logger = logging.getLogger("fairseq_cli.generate")
|
| 68 |
+
|
| 69 |
+
utils.import_user_module(cfg.common)
|
| 70 |
+
|
| 71 |
+
if cfg.dataset.max_tokens is None and cfg.dataset.batch_size is None:
|
| 72 |
+
cfg.dataset.max_tokens = 12000
|
| 73 |
+
logger.info(cfg)
|
| 74 |
+
|
| 75 |
+
# Fix seed for stochastic decoding
|
| 76 |
+
if cfg.common.seed is not None and not cfg.generation.no_seed_provided:
|
| 77 |
+
np.random.seed(cfg.common.seed)
|
| 78 |
+
utils.set_torch_seed(cfg.common.seed)
|
| 79 |
+
|
| 80 |
+
use_cuda = torch.cuda.is_available() and not cfg.common.cpu
|
| 81 |
+
|
| 82 |
+
# Load dataset splits
|
| 83 |
+
task = tasks.setup_task(cfg.task)
|
| 84 |
+
|
| 85 |
+
# Set dictionaries
|
| 86 |
+
try:
|
| 87 |
+
src_dict = getattr(task, "source_dictionary", None)
|
| 88 |
+
except NotImplementedError:
|
| 89 |
+
src_dict = None
|
| 90 |
+
tgt_dict = task.target_dictionary
|
| 91 |
+
|
| 92 |
+
overrides = ast.literal_eval(cfg.common_eval.model_overrides)
|
| 93 |
+
|
| 94 |
+
# Load ensemble
|
| 95 |
+
logger.info("loading model(s) from {}".format(cfg.common_eval.path))
|
| 96 |
+
models, saved_cfg = checkpoint_utils.load_model_ensemble(
|
| 97 |
+
utils.split_paths(cfg.common_eval.path),
|
| 98 |
+
arg_overrides=overrides,
|
| 99 |
+
task=task,
|
| 100 |
+
suffix=cfg.checkpoint.checkpoint_suffix,
|
| 101 |
+
strict=(cfg.checkpoint.checkpoint_shard_count == 1),
|
| 102 |
+
num_shards=cfg.checkpoint.checkpoint_shard_count,
|
| 103 |
+
)
|
| 104 |
+
|
| 105 |
+
# loading the dataset should happen after the checkpoint has been loaded so we can give it the saved task config
|
| 106 |
+
task.load_dataset(cfg.dataset.gen_subset, task_cfg=saved_cfg.task)
|
| 107 |
+
|
| 108 |
+
if cfg.generation.lm_path is not None:
|
| 109 |
+
overrides["data"] = cfg.task.data
|
| 110 |
+
|
| 111 |
+
try:
|
| 112 |
+
lms, _ = checkpoint_utils.load_model_ensemble(
|
| 113 |
+
[cfg.generation.lm_path], arg_overrides=overrides, task=None
|
| 114 |
+
)
|
| 115 |
+
except:
|
| 116 |
+
logger.warning(
|
| 117 |
+
f"Failed to load language model! Please make sure that the language model dict is the same "
|
| 118 |
+
f"as target dict and is located in the data dir ({cfg.task.data})"
|
| 119 |
+
)
|
| 120 |
+
raise
|
| 121 |
+
|
| 122 |
+
assert len(lms) == 1
|
| 123 |
+
else:
|
| 124 |
+
lms = [None]
|
| 125 |
+
|
| 126 |
+
# Optimize ensemble for generation
|
| 127 |
+
for model in chain(models, lms):
|
| 128 |
+
if model is None:
|
| 129 |
+
continue
|
| 130 |
+
if cfg.common.fp16:
|
| 131 |
+
model.half()
|
| 132 |
+
if use_cuda and not cfg.distributed_training.pipeline_model_parallel:
|
| 133 |
+
model.cuda()
|
| 134 |
+
model.prepare_for_inference_(cfg)
|
| 135 |
+
|
| 136 |
+
# Load alignment dictionary for unknown word replacement
|
| 137 |
+
# (None if no unknown word replacement, empty if no path to align dictionary)
|
| 138 |
+
align_dict = utils.load_align_dict(cfg.generation.replace_unk)
|
| 139 |
+
|
| 140 |
+
# Load dataset (possibly sharded)
|
| 141 |
+
itr = task.get_batch_iterator(
|
| 142 |
+
dataset=task.dataset(cfg.dataset.gen_subset),
|
| 143 |
+
max_tokens=cfg.dataset.max_tokens,
|
| 144 |
+
max_sentences=cfg.dataset.batch_size,
|
| 145 |
+
max_positions=utils.resolve_max_positions(
|
| 146 |
+
task.max_positions(), *[m.max_positions() for m in models]
|
| 147 |
+
),
|
| 148 |
+
ignore_invalid_inputs=cfg.dataset.skip_invalid_size_inputs_valid_test,
|
| 149 |
+
required_batch_size_multiple=cfg.dataset.required_batch_size_multiple,
|
| 150 |
+
seed=cfg.common.seed,
|
| 151 |
+
num_shards=cfg.distributed_training.distributed_world_size,
|
| 152 |
+
shard_id=cfg.distributed_training.distributed_rank,
|
| 153 |
+
num_workers=cfg.dataset.num_workers,
|
| 154 |
+
data_buffer_size=cfg.dataset.data_buffer_size,
|
| 155 |
+
).next_epoch_itr(shuffle=False)
|
| 156 |
+
progress = progress_bar.progress_bar(
|
| 157 |
+
itr,
|
| 158 |
+
log_format=cfg.common.log_format,
|
| 159 |
+
log_interval=cfg.common.log_interval,
|
| 160 |
+
default_log_format=("tqdm" if not cfg.common.no_progress_bar else "simple"),
|
| 161 |
+
)
|
| 162 |
+
|
| 163 |
+
# Initialize generator
|
| 164 |
+
gen_timer = StopwatchMeter()
|
| 165 |
+
|
| 166 |
+
extra_gen_cls_kwargs = {"lm_model": lms[0], "lm_weight": cfg.generation.lm_weight}
|
| 167 |
+
generator = task.build_generator(
|
| 168 |
+
models, cfg.generation, extra_gen_cls_kwargs=extra_gen_cls_kwargs
|
| 169 |
+
)
|
| 170 |
+
|
| 171 |
+
# Handle tokenization and BPE
|
| 172 |
+
tokenizer = task.build_tokenizer(cfg.tokenizer)
|
| 173 |
+
bpe = task.build_bpe(cfg.bpe)
|
| 174 |
+
|
| 175 |
+
def decode_fn(x):
|
| 176 |
+
if bpe is not None:
|
| 177 |
+
x = bpe.decode(x)
|
| 178 |
+
if tokenizer is not None:
|
| 179 |
+
x = tokenizer.decode(x)
|
| 180 |
+
return x
|
| 181 |
+
|
| 182 |
+
scorer = scoring.build_scorer(cfg.scoring, tgt_dict)
|
| 183 |
+
|
| 184 |
+
num_sentences = 0
|
| 185 |
+
has_target = True
|
| 186 |
+
wps_meter = TimeMeter()
|
| 187 |
+
for sample in progress:
|
| 188 |
+
sample = utils.move_to_cuda(sample) if use_cuda else sample
|
| 189 |
+
if "net_input" not in sample:
|
| 190 |
+
continue
|
| 191 |
+
|
| 192 |
+
prefix_tokens = None
|
| 193 |
+
if cfg.generation.prefix_size > 0:
|
| 194 |
+
prefix_tokens = sample["target"][:, : cfg.generation.prefix_size]
|
| 195 |
+
|
| 196 |
+
constraints = None
|
| 197 |
+
if "constraints" in sample:
|
| 198 |
+
constraints = sample["constraints"]
|
| 199 |
+
|
| 200 |
+
gen_timer.start()
|
| 201 |
+
hypos = task.inference_step(
|
| 202 |
+
generator,
|
| 203 |
+
models,
|
| 204 |
+
sample,
|
| 205 |
+
prefix_tokens=prefix_tokens,
|
| 206 |
+
constraints=constraints,
|
| 207 |
+
)
|
| 208 |
+
num_generated_tokens = sum(len(h[0]["tokens"]) for h in hypos)
|
| 209 |
+
gen_timer.stop(num_generated_tokens)
|
| 210 |
+
|
| 211 |
+
for i, sample_id in enumerate(sample["id"].tolist()):
|
| 212 |
+
has_target = sample["target"] is not None
|
| 213 |
+
|
| 214 |
+
# Remove padding
|
| 215 |
+
if "src_tokens" in sample["net_input"]:
|
| 216 |
+
src_tokens = utils.strip_pad(
|
| 217 |
+
sample["net_input"]["src_tokens"][i, :], tgt_dict.pad()
|
| 218 |
+
)
|
| 219 |
+
else:
|
| 220 |
+
src_tokens = None
|
| 221 |
+
|
| 222 |
+
target_tokens = None
|
| 223 |
+
if has_target:
|
| 224 |
+
target_tokens = (
|
| 225 |
+
utils.strip_pad(sample["target"][i, :], tgt_dict.pad()).int().cpu()
|
| 226 |
+
)
|
| 227 |
+
|
| 228 |
+
# Either retrieve the original sentences or regenerate them from tokens.
|
| 229 |
+
if align_dict is not None:
|
| 230 |
+
src_str = task.dataset(cfg.dataset.gen_subset).src.get_original_text(
|
| 231 |
+
sample_id
|
| 232 |
+
)
|
| 233 |
+
target_str = task.dataset(cfg.dataset.gen_subset).tgt.get_original_text(
|
| 234 |
+
sample_id
|
| 235 |
+
)
|
| 236 |
+
else:
|
| 237 |
+
if src_dict is not None:
|
| 238 |
+
src_str = src_dict.string(src_tokens, cfg.common_eval.post_process)
|
| 239 |
+
else:
|
| 240 |
+
src_str = ""
|
| 241 |
+
if has_target:
|
| 242 |
+
target_str = tgt_dict.string(
|
| 243 |
+
target_tokens,
|
| 244 |
+
cfg.common_eval.post_process,
|
| 245 |
+
escape_unk=True,
|
| 246 |
+
extra_symbols_to_ignore=get_symbols_to_strip_from_output(
|
| 247 |
+
generator
|
| 248 |
+
),
|
| 249 |
+
)
|
| 250 |
+
|
| 251 |
+
src_str = decode_fn(src_str)
|
| 252 |
+
if has_target:
|
| 253 |
+
target_str = decode_fn(target_str)
|
| 254 |
+
|
| 255 |
+
if not cfg.common_eval.quiet:
|
| 256 |
+
if src_dict is not None:
|
| 257 |
+
print("S-{}\t{}".format(sample_id, src_str), file=output_file)
|
| 258 |
+
if has_target:
|
| 259 |
+
print("T-{}\t{}".format(sample_id, target_str), file=output_file)
|
| 260 |
+
|
| 261 |
+
# Process top predictions
|
| 262 |
+
for j, hypo in enumerate(hypos[i][: cfg.generation.nbest]):
|
| 263 |
+
hypo_tokens, hypo_str, alignment = utils.post_process_prediction(
|
| 264 |
+
hypo_tokens=hypo["tokens"].int().cpu(),
|
| 265 |
+
src_str=src_str,
|
| 266 |
+
alignment=hypo["alignment"],
|
| 267 |
+
align_dict=align_dict,
|
| 268 |
+
tgt_dict=tgt_dict,
|
| 269 |
+
remove_bpe=cfg.common_eval.post_process,
|
| 270 |
+
extra_symbols_to_ignore=get_symbols_to_strip_from_output(generator),
|
| 271 |
+
)
|
| 272 |
+
detok_hypo_str = decode_fn(hypo_str)
|
| 273 |
+
if not cfg.common_eval.quiet:
|
| 274 |
+
score = hypo["score"] / math.log(2) # convert to base 2
|
| 275 |
+
# original hypothesis (after tokenization and BPE)
|
| 276 |
+
print(
|
| 277 |
+
"H-{}\t{}\t{}".format(sample_id, score, hypo_str),
|
| 278 |
+
file=output_file,
|
| 279 |
+
)
|
| 280 |
+
# detokenized hypothesis
|
| 281 |
+
print(
|
| 282 |
+
"D-{}\t{}\t{}".format(sample_id, score, detok_hypo_str),
|
| 283 |
+
file=output_file,
|
| 284 |
+
)
|
| 285 |
+
print(
|
| 286 |
+
"P-{}\t{}".format(
|
| 287 |
+
sample_id,
|
| 288 |
+
" ".join(
|
| 289 |
+
map(
|
| 290 |
+
lambda x: "{:.4f}".format(x),
|
| 291 |
+
# convert from base e to base 2
|
| 292 |
+
hypo["positional_scores"]
|
| 293 |
+
.div_(math.log(2))
|
| 294 |
+
.tolist(),
|
| 295 |
+
)
|
| 296 |
+
),
|
| 297 |
+
),
|
| 298 |
+
file=output_file,
|
| 299 |
+
)
|
| 300 |
+
|
| 301 |
+
if cfg.generation.print_alignment == "hard":
|
| 302 |
+
print(
|
| 303 |
+
"A-{}\t{}".format(
|
| 304 |
+
sample_id,
|
| 305 |
+
" ".join(
|
| 306 |
+
[
|
| 307 |
+
"{}-{}".format(src_idx, tgt_idx)
|
| 308 |
+
for src_idx, tgt_idx in alignment
|
| 309 |
+
]
|
| 310 |
+
),
|
| 311 |
+
),
|
| 312 |
+
file=output_file,
|
| 313 |
+
)
|
| 314 |
+
if cfg.generation.print_alignment == "soft":
|
| 315 |
+
print(
|
| 316 |
+
"A-{}\t{}".format(
|
| 317 |
+
sample_id,
|
| 318 |
+
" ".join(
|
| 319 |
+
[",".join(src_probs) for src_probs in alignment]
|
| 320 |
+
),
|
| 321 |
+
),
|
| 322 |
+
file=output_file,
|
| 323 |
+
)
|
| 324 |
+
|
| 325 |
+
if cfg.generation.print_step:
|
| 326 |
+
print(
|
| 327 |
+
"I-{}\t{}".format(sample_id, hypo["steps"]),
|
| 328 |
+
file=output_file,
|
| 329 |
+
)
|
| 330 |
+
|
| 331 |
+
if cfg.generation.retain_iter_history:
|
| 332 |
+
for step, h in enumerate(hypo["history"]):
|
| 333 |
+
_, h_str, _ = utils.post_process_prediction(
|
| 334 |
+
hypo_tokens=h["tokens"].int().cpu(),
|
| 335 |
+
src_str=src_str,
|
| 336 |
+
alignment=None,
|
| 337 |
+
align_dict=None,
|
| 338 |
+
tgt_dict=tgt_dict,
|
| 339 |
+
remove_bpe=None,
|
| 340 |
+
)
|
| 341 |
+
print(
|
| 342 |
+
"E-{}_{}\t{}".format(sample_id, step, h_str),
|
| 343 |
+
file=output_file,
|
| 344 |
+
)
|
| 345 |
+
|
| 346 |
+
# Score only the top hypothesis
|
| 347 |
+
if has_target and j == 0:
|
| 348 |
+
if (
|
| 349 |
+
align_dict is not None
|
| 350 |
+
or cfg.common_eval.post_process is not None
|
| 351 |
+
):
|
| 352 |
+
# Convert back to tokens for evaluation with unk replacement and/or without BPE
|
| 353 |
+
target_tokens = tgt_dict.encode_line(
|
| 354 |
+
target_str, add_if_not_exist=True
|
| 355 |
+
)
|
| 356 |
+
hypo_tokens = tgt_dict.encode_line(
|
| 357 |
+
detok_hypo_str, add_if_not_exist=True
|
| 358 |
+
)
|
| 359 |
+
if hasattr(scorer, "add_string"):
|
| 360 |
+
scorer.add_string(target_str, detok_hypo_str)
|
| 361 |
+
else:
|
| 362 |
+
scorer.add(target_tokens, hypo_tokens)
|
| 363 |
+
|
| 364 |
+
wps_meter.update(num_generated_tokens)
|
| 365 |
+
progress.log({"wps": round(wps_meter.avg)})
|
| 366 |
+
num_sentences += (
|
| 367 |
+
sample["nsentences"] if "nsentences" in sample else sample["id"].numel()
|
| 368 |
+
)
|
| 369 |
+
|
| 370 |
+
logger.info("NOTE: hypothesis and token scores are output in base 2")
|
| 371 |
+
logger.info(
|
| 372 |
+
"Translated {:,} sentences ({:,} tokens) in {:.1f}s ({:.2f} sentences/s, {:.2f} tokens/s)".format(
|
| 373 |
+
num_sentences,
|
| 374 |
+
gen_timer.n,
|
| 375 |
+
gen_timer.sum,
|
| 376 |
+
num_sentences / gen_timer.sum,
|
| 377 |
+
1.0 / gen_timer.avg,
|
| 378 |
+
)
|
| 379 |
+
)
|
| 380 |
+
if has_target:
|
| 381 |
+
if cfg.bpe and not cfg.generation.sacrebleu:
|
| 382 |
+
if cfg.common_eval.post_process:
|
| 383 |
+
logger.warning(
|
| 384 |
+
"BLEU score is being computed by splitting detokenized string on spaces, this is probably not what you want. Use --sacrebleu for standard 13a BLEU tokenization"
|
| 385 |
+
)
|
| 386 |
+
else:
|
| 387 |
+
logger.warning(
|
| 388 |
+
"If you are using BPE on the target side, the BLEU score is computed on BPE tokens, not on proper words. Use --sacrebleu for standard 13a BLEU tokenization"
|
| 389 |
+
)
|
| 390 |
+
# use print to be consistent with other main outputs: S-, H-, T-, D- and so on
|
| 391 |
+
print(
|
| 392 |
+
"Generate {} with beam={}: {}".format(
|
| 393 |
+
cfg.dataset.gen_subset, cfg.generation.beam, scorer.result_string()
|
| 394 |
+
),
|
| 395 |
+
file=output_file,
|
| 396 |
+
)
|
| 397 |
+
|
| 398 |
+
return scorer
|
| 399 |
+
|
| 400 |
+
|
| 401 |
+
def cli_main():
|
| 402 |
+
parser = options.get_generation_parser()
|
| 403 |
+
# TODO: replace this workaround with refactoring of `AudioPretraining`
|
| 404 |
+
parser.add_argument(
|
| 405 |
+
"--arch",
|
| 406 |
+
"-a",
|
| 407 |
+
metavar="ARCH",
|
| 408 |
+
default="wav2vec2",
|
| 409 |
+
help="Model architecture. For constructing tasks that rely on "
|
| 410 |
+
"model args (e.g. `AudioPretraining`)",
|
| 411 |
+
)
|
| 412 |
+
args = options.parse_args_and_arch(parser)
|
| 413 |
+
main(args)
|
| 414 |
+
|
| 415 |
+
|
| 416 |
+
if __name__ == "__main__":
|
| 417 |
+
cli_main()
|
data/fairseq/fairseq_cli/hydra_train.py
ADDED
|
@@ -0,0 +1,91 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python3 -u
|
| 2 |
+
# Copyright (c) Facebook, Inc. and its affiliates.
|
| 3 |
+
#
|
| 4 |
+
# This source code is licensed under the MIT license found in the
|
| 5 |
+
# LICENSE file in the root directory of this source tree.
|
| 6 |
+
|
| 7 |
+
import logging
|
| 8 |
+
import os
|
| 9 |
+
|
| 10 |
+
import hydra
|
| 11 |
+
import torch
|
| 12 |
+
from hydra.core.hydra_config import HydraConfig
|
| 13 |
+
from omegaconf import OmegaConf, open_dict
|
| 14 |
+
|
| 15 |
+
from fairseq import distributed_utils, metrics
|
| 16 |
+
from fairseq.dataclass.configs import FairseqConfig
|
| 17 |
+
from fairseq.dataclass.initialize import add_defaults, hydra_init
|
| 18 |
+
from fairseq.dataclass.utils import omegaconf_no_object_check
|
| 19 |
+
from fairseq.utils import reset_logging
|
| 20 |
+
from fairseq_cli.train import main as pre_main
|
| 21 |
+
|
| 22 |
+
logger = logging.getLogger("fairseq_cli.hydra_train")
|
| 23 |
+
|
| 24 |
+
|
| 25 |
+
@hydra.main(config_path=os.path.join("..", "fairseq", "config"), config_name="config")
|
| 26 |
+
def hydra_main(cfg: FairseqConfig) -> float:
|
| 27 |
+
_hydra_main(cfg)
|
| 28 |
+
|
| 29 |
+
|
| 30 |
+
def _hydra_main(cfg: FairseqConfig, **kwargs) -> float:
|
| 31 |
+
add_defaults(cfg)
|
| 32 |
+
|
| 33 |
+
if cfg.common.reset_logging:
|
| 34 |
+
reset_logging() # Hydra hijacks logging, fix that
|
| 35 |
+
else:
|
| 36 |
+
# check if directly called or called through hydra_main
|
| 37 |
+
if HydraConfig.initialized():
|
| 38 |
+
with open_dict(cfg):
|
| 39 |
+
# make hydra logging work with ddp (see # see https://github.com/facebookresearch/hydra/issues/1126)
|
| 40 |
+
cfg.job_logging_cfg = OmegaConf.to_container(
|
| 41 |
+
HydraConfig.get().job_logging, resolve=True
|
| 42 |
+
)
|
| 43 |
+
|
| 44 |
+
with omegaconf_no_object_check():
|
| 45 |
+
cfg = OmegaConf.create(
|
| 46 |
+
OmegaConf.to_container(cfg, resolve=True, enum_to_str=True)
|
| 47 |
+
)
|
| 48 |
+
OmegaConf.set_struct(cfg, True)
|
| 49 |
+
|
| 50 |
+
try:
|
| 51 |
+
if cfg.common.profile:
|
| 52 |
+
with torch.cuda.profiler.profile():
|
| 53 |
+
with torch.autograd.profiler.emit_nvtx():
|
| 54 |
+
distributed_utils.call_main(cfg, pre_main, **kwargs)
|
| 55 |
+
else:
|
| 56 |
+
distributed_utils.call_main(cfg, pre_main, **kwargs)
|
| 57 |
+
except BaseException as e:
|
| 58 |
+
if not cfg.common.suppress_crashes:
|
| 59 |
+
raise
|
| 60 |
+
else:
|
| 61 |
+
logger.error("Crashed! " + str(e))
|
| 62 |
+
|
| 63 |
+
# get best val and return - useful for sweepers
|
| 64 |
+
try:
|
| 65 |
+
best_val = metrics.get_smoothed_value(
|
| 66 |
+
"valid", cfg.checkpoint.best_checkpoint_metric
|
| 67 |
+
)
|
| 68 |
+
except:
|
| 69 |
+
best_val = None
|
| 70 |
+
|
| 71 |
+
if best_val is None:
|
| 72 |
+
best_val = float("inf")
|
| 73 |
+
|
| 74 |
+
return best_val
|
| 75 |
+
|
| 76 |
+
|
| 77 |
+
def cli_main():
|
| 78 |
+
try:
|
| 79 |
+
from hydra._internal.utils import get_args
|
| 80 |
+
|
| 81 |
+
cfg_name = get_args().config_name or "config"
|
| 82 |
+
except:
|
| 83 |
+
logger.warning("Failed to get config name from hydra args")
|
| 84 |
+
cfg_name = "config"
|
| 85 |
+
|
| 86 |
+
hydra_init(cfg_name)
|
| 87 |
+
hydra_main()
|
| 88 |
+
|
| 89 |
+
|
| 90 |
+
if __name__ == "__main__":
|
| 91 |
+
cli_main()
|
data/fairseq/fairseq_cli/hydra_validate.py
ADDED
|
@@ -0,0 +1,188 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python3 -u
|
| 2 |
+
# Copyright (c) Facebook, Inc. and its affiliates.
|
| 3 |
+
#
|
| 4 |
+
# This source code is licensed under the MIT license found in the
|
| 5 |
+
# LICENSE file in the root directory of this source tree.
|
| 6 |
+
|
| 7 |
+
import logging
|
| 8 |
+
import os
|
| 9 |
+
import sys
|
| 10 |
+
from itertools import chain
|
| 11 |
+
|
| 12 |
+
import torch
|
| 13 |
+
from hydra.core.hydra_config import HydraConfig
|
| 14 |
+
from omegaconf import OmegaConf, open_dict
|
| 15 |
+
import hydra
|
| 16 |
+
|
| 17 |
+
from fairseq import checkpoint_utils, distributed_utils, utils
|
| 18 |
+
from fairseq.dataclass.configs import FairseqConfig
|
| 19 |
+
from fairseq.dataclass.initialize import add_defaults, hydra_init
|
| 20 |
+
from fairseq.dataclass.utils import omegaconf_no_object_check
|
| 21 |
+
from fairseq.distributed import utils as distributed_utils
|
| 22 |
+
from fairseq.logging import metrics, progress_bar
|
| 23 |
+
from fairseq.utils import reset_logging
|
| 24 |
+
|
| 25 |
+
logging.basicConfig(
|
| 26 |
+
format="%(asctime)s | %(levelname)s | %(name)s | %(message)s",
|
| 27 |
+
datefmt="%Y-%m-%d %H:%M:%S",
|
| 28 |
+
level=os.environ.get("LOGLEVEL", "INFO").upper(),
|
| 29 |
+
stream=sys.stdout,
|
| 30 |
+
)
|
| 31 |
+
logger = logging.getLogger("fairseq_cli.validate")
|
| 32 |
+
|
| 33 |
+
|
| 34 |
+
@hydra.main(config_path=os.path.join("..", "fairseq", "config"), config_name="config")
|
| 35 |
+
def hydra_main(cfg: FairseqConfig) -> float:
|
| 36 |
+
return _hydra_main(cfg)
|
| 37 |
+
|
| 38 |
+
|
| 39 |
+
def _hydra_main(cfg: FairseqConfig, **kwargs) -> float:
|
| 40 |
+
add_defaults(cfg)
|
| 41 |
+
|
| 42 |
+
if cfg.common.reset_logging:
|
| 43 |
+
reset_logging() # Hydra hijacks logging, fix that
|
| 44 |
+
else:
|
| 45 |
+
# check if directly called or called through hydra_main
|
| 46 |
+
if HydraConfig.initialized():
|
| 47 |
+
with open_dict(cfg):
|
| 48 |
+
# make hydra logging work with ddp (see # see https://github.com/facebookresearch/hydra/issues/1126)
|
| 49 |
+
cfg.job_logging_cfg = OmegaConf.to_container(
|
| 50 |
+
HydraConfig.get().job_logging, resolve=True
|
| 51 |
+
)
|
| 52 |
+
|
| 53 |
+
with omegaconf_no_object_check():
|
| 54 |
+
cfg = OmegaConf.create(
|
| 55 |
+
OmegaConf.to_container(cfg, resolve=True, enum_to_str=True)
|
| 56 |
+
)
|
| 57 |
+
OmegaConf.set_struct(cfg, True)
|
| 58 |
+
|
| 59 |
+
assert (
|
| 60 |
+
cfg.dataset.max_tokens is not None or cfg.dataset.batch_size is not None
|
| 61 |
+
), "Must specify batch size either with --max-tokens or --batch-size"
|
| 62 |
+
|
| 63 |
+
distributed_utils.call_main(cfg, validate, **kwargs)
|
| 64 |
+
|
| 65 |
+
|
| 66 |
+
def validate(cfg):
|
| 67 |
+
utils.import_user_module(cfg.common)
|
| 68 |
+
|
| 69 |
+
use_fp16 = cfg.common.fp16
|
| 70 |
+
use_cuda = torch.cuda.is_available() and not cfg.common.cpu
|
| 71 |
+
|
| 72 |
+
if use_cuda:
|
| 73 |
+
torch.cuda.set_device(cfg.distributed_training.device_id)
|
| 74 |
+
|
| 75 |
+
if cfg.distributed_training.distributed_world_size > 1:
|
| 76 |
+
data_parallel_world_size = distributed_utils.get_data_parallel_world_size()
|
| 77 |
+
data_parallel_rank = distributed_utils.get_data_parallel_rank()
|
| 78 |
+
else:
|
| 79 |
+
data_parallel_world_size = 1
|
| 80 |
+
data_parallel_rank = 0
|
| 81 |
+
|
| 82 |
+
overrides = {"task": {"data": cfg.task.data}}
|
| 83 |
+
|
| 84 |
+
# Load ensemble
|
| 85 |
+
logger.info("loading model(s) from {}".format(cfg.common_eval.path))
|
| 86 |
+
models, saved_cfg, task = checkpoint_utils.load_model_ensemble_and_task(
|
| 87 |
+
[cfg.common_eval.path],
|
| 88 |
+
arg_overrides=overrides,
|
| 89 |
+
suffix=cfg.checkpoint.checkpoint_suffix,
|
| 90 |
+
)
|
| 91 |
+
model = models[0]
|
| 92 |
+
|
| 93 |
+
# Move models to GPU
|
| 94 |
+
for model in models:
|
| 95 |
+
model.eval()
|
| 96 |
+
if use_fp16:
|
| 97 |
+
model.half()
|
| 98 |
+
if use_cuda:
|
| 99 |
+
model.cuda()
|
| 100 |
+
|
| 101 |
+
# Print args
|
| 102 |
+
logger.info(saved_cfg)
|
| 103 |
+
|
| 104 |
+
# Build criterion
|
| 105 |
+
criterion = task.build_criterion(saved_cfg.criterion, from_checkpoint=True)
|
| 106 |
+
criterion.eval()
|
| 107 |
+
|
| 108 |
+
for subset in cfg.dataset.valid_subset.split(","):
|
| 109 |
+
try:
|
| 110 |
+
task.load_dataset(subset, combine=False, epoch=1, task_cfg=saved_cfg.task)
|
| 111 |
+
dataset = task.dataset(subset)
|
| 112 |
+
except KeyError:
|
| 113 |
+
raise Exception("Cannot find dataset: " + subset)
|
| 114 |
+
|
| 115 |
+
# Initialize data iterator
|
| 116 |
+
itr = task.get_batch_iterator(
|
| 117 |
+
dataset=dataset,
|
| 118 |
+
max_tokens=cfg.dataset.max_tokens,
|
| 119 |
+
max_sentences=cfg.dataset.batch_size,
|
| 120 |
+
max_positions=utils.resolve_max_positions(
|
| 121 |
+
task.max_positions(),
|
| 122 |
+
*[m.max_positions() for m in models],
|
| 123 |
+
),
|
| 124 |
+
ignore_invalid_inputs=cfg.dataset.skip_invalid_size_inputs_valid_test,
|
| 125 |
+
required_batch_size_multiple=cfg.dataset.required_batch_size_multiple,
|
| 126 |
+
seed=cfg.common.seed,
|
| 127 |
+
num_shards=data_parallel_world_size,
|
| 128 |
+
shard_id=data_parallel_rank,
|
| 129 |
+
num_workers=cfg.dataset.num_workers,
|
| 130 |
+
data_buffer_size=cfg.dataset.data_buffer_size,
|
| 131 |
+
).next_epoch_itr(shuffle=False)
|
| 132 |
+
progress = progress_bar.progress_bar(
|
| 133 |
+
itr,
|
| 134 |
+
log_format=cfg.common.log_format,
|
| 135 |
+
log_interval=cfg.common.log_interval,
|
| 136 |
+
prefix=f"valid on '{subset}' subset",
|
| 137 |
+
default_log_format=("tqdm" if not cfg.common.no_progress_bar else "simple"),
|
| 138 |
+
)
|
| 139 |
+
|
| 140 |
+
def apply_half(t):
|
| 141 |
+
if t.dtype is torch.float32:
|
| 142 |
+
return t.to(dtype=torch.half)
|
| 143 |
+
return t
|
| 144 |
+
|
| 145 |
+
log_outputs = []
|
| 146 |
+
for i, sample in enumerate(progress):
|
| 147 |
+
sample = utils.move_to_cuda(sample) if use_cuda else sample
|
| 148 |
+
|
| 149 |
+
if use_fp16:
|
| 150 |
+
sample = utils.apply_to_sample(apply_half, sample)
|
| 151 |
+
|
| 152 |
+
_loss, _sample_size, log_output = task.valid_step(sample, model, criterion)
|
| 153 |
+
with metrics.aggregate() as agg:
|
| 154 |
+
task.reduce_metrics([log_output], criterion)
|
| 155 |
+
progress.log(agg.get_smoothed_values(), step=i)
|
| 156 |
+
# progress.log(log_output, step=i) from vision
|
| 157 |
+
log_outputs.append(log_output)
|
| 158 |
+
|
| 159 |
+
if data_parallel_world_size > 1:
|
| 160 |
+
log_outputs = distributed_utils.all_gather_list(
|
| 161 |
+
log_outputs,
|
| 162 |
+
max_size=cfg.common.all_gather_list_size,
|
| 163 |
+
group=distributed_utils.get_data_parallel_group(),
|
| 164 |
+
)
|
| 165 |
+
log_outputs = list(chain.from_iterable(log_outputs))
|
| 166 |
+
|
| 167 |
+
with metrics.aggregate() as agg:
|
| 168 |
+
task.reduce_metrics(log_outputs, criterion)
|
| 169 |
+
log_output = agg.get_smoothed_values()
|
| 170 |
+
|
| 171 |
+
progress.print(log_output, tag=subset, step=i)
|
| 172 |
+
|
| 173 |
+
|
| 174 |
+
def cli_main():
|
| 175 |
+
try:
|
| 176 |
+
from hydra._internal.utils import get_args
|
| 177 |
+
|
| 178 |
+
cfg_name = get_args().config_name or "config"
|
| 179 |
+
except:
|
| 180 |
+
logger.warning("Failed to get config name from hydra args")
|
| 181 |
+
cfg_name = "config"
|
| 182 |
+
|
| 183 |
+
hydra_init(cfg_name)
|
| 184 |
+
hydra_main()
|
| 185 |
+
|
| 186 |
+
|
| 187 |
+
if __name__ == "__main__":
|
| 188 |
+
cli_main()
|
data/fairseq/fairseq_cli/interactive.py
ADDED
|
@@ -0,0 +1,317 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python3 -u
|
| 2 |
+
# Copyright (c) Facebook, Inc. and its affiliates.
|
| 3 |
+
#
|
| 4 |
+
# This source code is licensed under the MIT license found in the
|
| 5 |
+
# LICENSE file in the root directory of this source tree.
|
| 6 |
+
"""
|
| 7 |
+
Translate raw text with a trained model. Batches data on-the-fly.
|
| 8 |
+
"""
|
| 9 |
+
|
| 10 |
+
import ast
|
| 11 |
+
import fileinput
|
| 12 |
+
import logging
|
| 13 |
+
import math
|
| 14 |
+
import os
|
| 15 |
+
import sys
|
| 16 |
+
import time
|
| 17 |
+
from argparse import Namespace
|
| 18 |
+
from collections import namedtuple
|
| 19 |
+
|
| 20 |
+
import numpy as np
|
| 21 |
+
import torch
|
| 22 |
+
|
| 23 |
+
from fairseq import checkpoint_utils, distributed_utils, options, tasks, utils
|
| 24 |
+
from fairseq.dataclass.configs import FairseqConfig
|
| 25 |
+
from fairseq.dataclass.utils import convert_namespace_to_omegaconf
|
| 26 |
+
from fairseq.token_generation_constraints import pack_constraints, unpack_constraints
|
| 27 |
+
from fairseq_cli.generate import get_symbols_to_strip_from_output
|
| 28 |
+
|
| 29 |
+
logging.basicConfig(
|
| 30 |
+
format="%(asctime)s | %(levelname)s | %(name)s | %(message)s",
|
| 31 |
+
datefmt="%Y-%m-%d %H:%M:%S",
|
| 32 |
+
level=os.environ.get("LOGLEVEL", "INFO").upper(),
|
| 33 |
+
stream=sys.stdout,
|
| 34 |
+
)
|
| 35 |
+
logger = logging.getLogger("fairseq_cli.interactive")
|
| 36 |
+
|
| 37 |
+
|
| 38 |
+
Batch = namedtuple("Batch", "ids src_tokens src_lengths constraints")
|
| 39 |
+
Translation = namedtuple("Translation", "src_str hypos pos_scores alignments")
|
| 40 |
+
|
| 41 |
+
|
| 42 |
+
def buffered_read(input, buffer_size):
|
| 43 |
+
buffer = []
|
| 44 |
+
with fileinput.input(files=[input], openhook=fileinput.hook_encoded("utf-8")) as h:
|
| 45 |
+
for src_str in h:
|
| 46 |
+
buffer.append(src_str.strip())
|
| 47 |
+
if len(buffer) >= buffer_size:
|
| 48 |
+
yield buffer
|
| 49 |
+
buffer = []
|
| 50 |
+
|
| 51 |
+
if len(buffer) > 0:
|
| 52 |
+
yield buffer
|
| 53 |
+
|
| 54 |
+
|
| 55 |
+
def make_batches(lines, cfg, task, max_positions, encode_fn):
|
| 56 |
+
def encode_fn_target(x):
|
| 57 |
+
return encode_fn(x)
|
| 58 |
+
|
| 59 |
+
if cfg.generation.constraints:
|
| 60 |
+
# Strip (tab-delimited) contraints, if present, from input lines,
|
| 61 |
+
# store them in batch_constraints
|
| 62 |
+
batch_constraints = [list() for _ in lines]
|
| 63 |
+
for i, line in enumerate(lines):
|
| 64 |
+
if "\t" in line:
|
| 65 |
+
lines[i], *batch_constraints[i] = line.split("\t")
|
| 66 |
+
|
| 67 |
+
# Convert each List[str] to List[Tensor]
|
| 68 |
+
for i, constraint_list in enumerate(batch_constraints):
|
| 69 |
+
batch_constraints[i] = [
|
| 70 |
+
task.target_dictionary.encode_line(
|
| 71 |
+
encode_fn_target(constraint),
|
| 72 |
+
append_eos=False,
|
| 73 |
+
add_if_not_exist=False,
|
| 74 |
+
)
|
| 75 |
+
for constraint in constraint_list
|
| 76 |
+
]
|
| 77 |
+
|
| 78 |
+
if cfg.generation.constraints:
|
| 79 |
+
constraints_tensor = pack_constraints(batch_constraints)
|
| 80 |
+
else:
|
| 81 |
+
constraints_tensor = None
|
| 82 |
+
|
| 83 |
+
tokens, lengths = task.get_interactive_tokens_and_lengths(lines, encode_fn)
|
| 84 |
+
|
| 85 |
+
itr = task.get_batch_iterator(
|
| 86 |
+
dataset=task.build_dataset_for_inference(
|
| 87 |
+
tokens, lengths, constraints=constraints_tensor
|
| 88 |
+
),
|
| 89 |
+
max_tokens=cfg.dataset.max_tokens,
|
| 90 |
+
max_sentences=cfg.dataset.batch_size,
|
| 91 |
+
max_positions=max_positions,
|
| 92 |
+
ignore_invalid_inputs=cfg.dataset.skip_invalid_size_inputs_valid_test,
|
| 93 |
+
).next_epoch_itr(shuffle=False)
|
| 94 |
+
for batch in itr:
|
| 95 |
+
ids = batch["id"]
|
| 96 |
+
src_tokens = batch["net_input"]["src_tokens"]
|
| 97 |
+
src_lengths = batch["net_input"]["src_lengths"]
|
| 98 |
+
constraints = batch.get("constraints", None)
|
| 99 |
+
|
| 100 |
+
yield Batch(
|
| 101 |
+
ids=ids,
|
| 102 |
+
src_tokens=src_tokens,
|
| 103 |
+
src_lengths=src_lengths,
|
| 104 |
+
constraints=constraints,
|
| 105 |
+
)
|
| 106 |
+
|
| 107 |
+
|
| 108 |
+
def main(cfg: FairseqConfig):
|
| 109 |
+
if isinstance(cfg, Namespace):
|
| 110 |
+
cfg = convert_namespace_to_omegaconf(cfg)
|
| 111 |
+
|
| 112 |
+
start_time = time.time()
|
| 113 |
+
total_translate_time = 0
|
| 114 |
+
|
| 115 |
+
utils.import_user_module(cfg.common)
|
| 116 |
+
|
| 117 |
+
if cfg.interactive.buffer_size < 1:
|
| 118 |
+
cfg.interactive.buffer_size = 1
|
| 119 |
+
if cfg.dataset.max_tokens is None and cfg.dataset.batch_size is None:
|
| 120 |
+
cfg.dataset.batch_size = 1
|
| 121 |
+
|
| 122 |
+
assert (
|
| 123 |
+
not cfg.generation.sampling or cfg.generation.nbest == cfg.generation.beam
|
| 124 |
+
), "--sampling requires --nbest to be equal to --beam"
|
| 125 |
+
assert (
|
| 126 |
+
not cfg.dataset.batch_size
|
| 127 |
+
or cfg.dataset.batch_size <= cfg.interactive.buffer_size
|
| 128 |
+
), "--batch-size cannot be larger than --buffer-size"
|
| 129 |
+
|
| 130 |
+
logger.info(cfg)
|
| 131 |
+
|
| 132 |
+
# Fix seed for stochastic decoding
|
| 133 |
+
if cfg.common.seed is not None and not cfg.generation.no_seed_provided:
|
| 134 |
+
np.random.seed(cfg.common.seed)
|
| 135 |
+
utils.set_torch_seed(cfg.common.seed)
|
| 136 |
+
|
| 137 |
+
use_cuda = torch.cuda.is_available() and not cfg.common.cpu
|
| 138 |
+
|
| 139 |
+
# Setup task, e.g., translation
|
| 140 |
+
task = tasks.setup_task(cfg.task)
|
| 141 |
+
|
| 142 |
+
# Load ensemble
|
| 143 |
+
overrides = ast.literal_eval(cfg.common_eval.model_overrides)
|
| 144 |
+
logger.info("loading model(s) from {}".format(cfg.common_eval.path))
|
| 145 |
+
models, _model_args = checkpoint_utils.load_model_ensemble(
|
| 146 |
+
utils.split_paths(cfg.common_eval.path),
|
| 147 |
+
arg_overrides=overrides,
|
| 148 |
+
task=task,
|
| 149 |
+
suffix=cfg.checkpoint.checkpoint_suffix,
|
| 150 |
+
strict=(cfg.checkpoint.checkpoint_shard_count == 1),
|
| 151 |
+
num_shards=cfg.checkpoint.checkpoint_shard_count,
|
| 152 |
+
)
|
| 153 |
+
|
| 154 |
+
# Set dictionaries
|
| 155 |
+
src_dict = task.source_dictionary
|
| 156 |
+
tgt_dict = task.target_dictionary
|
| 157 |
+
|
| 158 |
+
# Optimize ensemble for generation
|
| 159 |
+
for model in models:
|
| 160 |
+
if model is None:
|
| 161 |
+
continue
|
| 162 |
+
if cfg.common.fp16:
|
| 163 |
+
model.half()
|
| 164 |
+
if use_cuda and not cfg.distributed_training.pipeline_model_parallel:
|
| 165 |
+
model.cuda()
|
| 166 |
+
model.prepare_for_inference_(cfg)
|
| 167 |
+
|
| 168 |
+
# Initialize generator
|
| 169 |
+
generator = task.build_generator(models, cfg.generation)
|
| 170 |
+
|
| 171 |
+
# Handle tokenization and BPE
|
| 172 |
+
tokenizer = task.build_tokenizer(cfg.tokenizer)
|
| 173 |
+
bpe = task.build_bpe(cfg.bpe)
|
| 174 |
+
|
| 175 |
+
def encode_fn(x):
|
| 176 |
+
if tokenizer is not None:
|
| 177 |
+
x = tokenizer.encode(x)
|
| 178 |
+
if bpe is not None:
|
| 179 |
+
x = bpe.encode(x)
|
| 180 |
+
return x
|
| 181 |
+
|
| 182 |
+
def decode_fn(x):
|
| 183 |
+
if bpe is not None:
|
| 184 |
+
x = bpe.decode(x)
|
| 185 |
+
if tokenizer is not None:
|
| 186 |
+
x = tokenizer.decode(x)
|
| 187 |
+
return x
|
| 188 |
+
|
| 189 |
+
# Load alignment dictionary for unknown word replacement
|
| 190 |
+
# (None if no unknown word replacement, empty if no path to align dictionary)
|
| 191 |
+
align_dict = utils.load_align_dict(cfg.generation.replace_unk)
|
| 192 |
+
|
| 193 |
+
max_positions = utils.resolve_max_positions(
|
| 194 |
+
task.max_positions(), *[model.max_positions() for model in models]
|
| 195 |
+
)
|
| 196 |
+
|
| 197 |
+
if cfg.generation.constraints:
|
| 198 |
+
logger.warning(
|
| 199 |
+
"NOTE: Constrained decoding currently assumes a shared subword vocabulary."
|
| 200 |
+
)
|
| 201 |
+
|
| 202 |
+
if cfg.interactive.buffer_size > 1:
|
| 203 |
+
logger.info("Sentence buffer size: %s", cfg.interactive.buffer_size)
|
| 204 |
+
logger.info("NOTE: hypothesis and token scores are output in base 2")
|
| 205 |
+
logger.info("Type the input sentence and press return:")
|
| 206 |
+
start_id = 0
|
| 207 |
+
for inputs in buffered_read(cfg.interactive.input, cfg.interactive.buffer_size):
|
| 208 |
+
results = []
|
| 209 |
+
for batch in make_batches(inputs, cfg, task, max_positions, encode_fn):
|
| 210 |
+
bsz = batch.src_tokens.size(0)
|
| 211 |
+
src_tokens = batch.src_tokens
|
| 212 |
+
src_lengths = batch.src_lengths
|
| 213 |
+
constraints = batch.constraints
|
| 214 |
+
if use_cuda:
|
| 215 |
+
src_tokens = src_tokens.cuda()
|
| 216 |
+
src_lengths = src_lengths.cuda()
|
| 217 |
+
if constraints is not None:
|
| 218 |
+
constraints = constraints.cuda()
|
| 219 |
+
|
| 220 |
+
sample = {
|
| 221 |
+
"net_input": {
|
| 222 |
+
"src_tokens": src_tokens,
|
| 223 |
+
"src_lengths": src_lengths,
|
| 224 |
+
},
|
| 225 |
+
}
|
| 226 |
+
translate_start_time = time.time()
|
| 227 |
+
translations = task.inference_step(
|
| 228 |
+
generator, models, sample, constraints=constraints
|
| 229 |
+
)
|
| 230 |
+
translate_time = time.time() - translate_start_time
|
| 231 |
+
total_translate_time += translate_time
|
| 232 |
+
list_constraints = [[] for _ in range(bsz)]
|
| 233 |
+
if cfg.generation.constraints:
|
| 234 |
+
list_constraints = [unpack_constraints(c) for c in constraints]
|
| 235 |
+
for i, (id, hypos) in enumerate(zip(batch.ids.tolist(), translations)):
|
| 236 |
+
src_tokens_i = utils.strip_pad(src_tokens[i], tgt_dict.pad())
|
| 237 |
+
constraints = list_constraints[i]
|
| 238 |
+
results.append(
|
| 239 |
+
(
|
| 240 |
+
start_id + id,
|
| 241 |
+
src_tokens_i,
|
| 242 |
+
hypos,
|
| 243 |
+
{
|
| 244 |
+
"constraints": constraints,
|
| 245 |
+
"time": translate_time / len(translations),
|
| 246 |
+
},
|
| 247 |
+
)
|
| 248 |
+
)
|
| 249 |
+
|
| 250 |
+
# sort output to match input order
|
| 251 |
+
for id_, src_tokens, hypos, info in sorted(results, key=lambda x: x[0]):
|
| 252 |
+
src_str = ""
|
| 253 |
+
if src_dict is not None:
|
| 254 |
+
src_str = src_dict.string(src_tokens, cfg.common_eval.post_process)
|
| 255 |
+
print("S-{}\t{}".format(id_, src_str))
|
| 256 |
+
print("W-{}\t{:.3f}\tseconds".format(id_, info["time"]))
|
| 257 |
+
for constraint in info["constraints"]:
|
| 258 |
+
print(
|
| 259 |
+
"C-{}\t{}".format(
|
| 260 |
+
id_,
|
| 261 |
+
tgt_dict.string(constraint, cfg.common_eval.post_process),
|
| 262 |
+
)
|
| 263 |
+
)
|
| 264 |
+
|
| 265 |
+
# Process top predictions
|
| 266 |
+
for hypo in hypos[: min(len(hypos), cfg.generation.nbest)]:
|
| 267 |
+
hypo_tokens, hypo_str, alignment = utils.post_process_prediction(
|
| 268 |
+
hypo_tokens=hypo["tokens"].int().cpu(),
|
| 269 |
+
src_str=src_str,
|
| 270 |
+
alignment=hypo["alignment"],
|
| 271 |
+
align_dict=align_dict,
|
| 272 |
+
tgt_dict=tgt_dict,
|
| 273 |
+
remove_bpe=cfg.common_eval.post_process,
|
| 274 |
+
extra_symbols_to_ignore=get_symbols_to_strip_from_output(generator),
|
| 275 |
+
)
|
| 276 |
+
detok_hypo_str = decode_fn(hypo_str)
|
| 277 |
+
score = hypo["score"] / math.log(2) # convert to base 2
|
| 278 |
+
# original hypothesis (after tokenization and BPE)
|
| 279 |
+
print("H-{}\t{}\t{}".format(id_, score, hypo_str))
|
| 280 |
+
# detokenized hypothesis
|
| 281 |
+
print("D-{}\t{}\t{}".format(id_, score, detok_hypo_str))
|
| 282 |
+
print(
|
| 283 |
+
"P-{}\t{}".format(
|
| 284 |
+
id_,
|
| 285 |
+
" ".join(
|
| 286 |
+
map(
|
| 287 |
+
lambda x: "{:.4f}".format(x),
|
| 288 |
+
# convert from base e to base 2
|
| 289 |
+
hypo["positional_scores"].div_(math.log(2)).tolist(),
|
| 290 |
+
)
|
| 291 |
+
),
|
| 292 |
+
)
|
| 293 |
+
)
|
| 294 |
+
if cfg.generation.print_alignment:
|
| 295 |
+
alignment_str = " ".join(
|
| 296 |
+
["{}-{}".format(src, tgt) for src, tgt in alignment]
|
| 297 |
+
)
|
| 298 |
+
print("A-{}\t{}".format(id_, alignment_str))
|
| 299 |
+
|
| 300 |
+
# update running id_ counter
|
| 301 |
+
start_id += len(inputs)
|
| 302 |
+
|
| 303 |
+
logger.info(
|
| 304 |
+
"Total time: {:.3f} seconds; translation time: {:.3f}".format(
|
| 305 |
+
time.time() - start_time, total_translate_time
|
| 306 |
+
)
|
| 307 |
+
)
|
| 308 |
+
|
| 309 |
+
|
| 310 |
+
def cli_main():
|
| 311 |
+
parser = options.get_interactive_generation_parser()
|
| 312 |
+
args = options.parse_args_and_arch(parser)
|
| 313 |
+
distributed_utils.call_main(convert_namespace_to_omegaconf(args), main)
|
| 314 |
+
|
| 315 |
+
|
| 316 |
+
if __name__ == "__main__":
|
| 317 |
+
cli_main()
|
data/fairseq/fairseq_cli/preprocess.py
ADDED
|
@@ -0,0 +1,393 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python3
|
| 2 |
+
# Copyright (c) Facebook, Inc. and its affiliates.
|
| 3 |
+
#
|
| 4 |
+
# This source code is licensed under the MIT license found in the
|
| 5 |
+
# LICENSE file in the root directory of this source tree.
|
| 6 |
+
"""
|
| 7 |
+
Data pre-processing: build vocabularies and binarize training data.
|
| 8 |
+
"""
|
| 9 |
+
|
| 10 |
+
import logging
|
| 11 |
+
import os
|
| 12 |
+
import shutil
|
| 13 |
+
import sys
|
| 14 |
+
import typing as tp
|
| 15 |
+
from argparse import Namespace
|
| 16 |
+
from itertools import zip_longest
|
| 17 |
+
|
| 18 |
+
from fairseq import options, tasks, utils
|
| 19 |
+
from fairseq.binarizer import (
|
| 20 |
+
AlignmentDatasetBinarizer,
|
| 21 |
+
FileBinarizer,
|
| 22 |
+
VocabularyDatasetBinarizer,
|
| 23 |
+
)
|
| 24 |
+
from fairseq.data import Dictionary
|
| 25 |
+
|
| 26 |
+
logging.basicConfig(
|
| 27 |
+
format="%(asctime)s | %(levelname)s | %(name)s | %(message)s",
|
| 28 |
+
datefmt="%Y-%m-%d %H:%M:%S",
|
| 29 |
+
level=os.environ.get("LOGLEVEL", "INFO").upper(),
|
| 30 |
+
stream=sys.stdout,
|
| 31 |
+
)
|
| 32 |
+
logger = logging.getLogger("fairseq_cli.preprocess")
|
| 33 |
+
|
| 34 |
+
#####################################################################
|
| 35 |
+
# file name tools
|
| 36 |
+
#####################################################################
|
| 37 |
+
|
| 38 |
+
|
| 39 |
+
def _train_path(lang, trainpref):
|
| 40 |
+
return "{}{}".format(trainpref, ("." + lang) if lang else "")
|
| 41 |
+
|
| 42 |
+
|
| 43 |
+
def _file_name(prefix, lang):
|
| 44 |
+
fname = prefix
|
| 45 |
+
if lang is not None:
|
| 46 |
+
fname += ".{lang}".format(lang=lang)
|
| 47 |
+
return fname
|
| 48 |
+
|
| 49 |
+
|
| 50 |
+
def _dest_path(prefix, lang, destdir):
|
| 51 |
+
return os.path.join(destdir, _file_name(prefix, lang))
|
| 52 |
+
|
| 53 |
+
|
| 54 |
+
def _dict_path(lang, destdir):
|
| 55 |
+
return _dest_path("dict", lang, destdir) + ".txt"
|
| 56 |
+
|
| 57 |
+
|
| 58 |
+
def dataset_dest_prefix(args, output_prefix, lang):
|
| 59 |
+
base = os.path.join(args.destdir, output_prefix)
|
| 60 |
+
if lang is not None:
|
| 61 |
+
lang_part = f".{args.source_lang}-{args.target_lang}.{lang}"
|
| 62 |
+
elif args.only_source:
|
| 63 |
+
lang_part = ""
|
| 64 |
+
else:
|
| 65 |
+
lang_part = f".{args.source_lang}-{args.target_lang}"
|
| 66 |
+
|
| 67 |
+
return "{}{}".format(base, lang_part)
|
| 68 |
+
|
| 69 |
+
|
| 70 |
+
def dataset_dest_file(args, output_prefix, lang, extension):
|
| 71 |
+
return "{}.{}".format(dataset_dest_prefix(args, output_prefix, lang), extension)
|
| 72 |
+
|
| 73 |
+
|
| 74 |
+
#####################################################################
|
| 75 |
+
# dictionary tools
|
| 76 |
+
#####################################################################
|
| 77 |
+
|
| 78 |
+
|
| 79 |
+
def _build_dictionary(
|
| 80 |
+
filenames,
|
| 81 |
+
task,
|
| 82 |
+
args,
|
| 83 |
+
src=False,
|
| 84 |
+
tgt=False,
|
| 85 |
+
):
|
| 86 |
+
assert src ^ tgt
|
| 87 |
+
return task.build_dictionary(
|
| 88 |
+
filenames,
|
| 89 |
+
workers=args.workers,
|
| 90 |
+
threshold=args.thresholdsrc if src else args.thresholdtgt,
|
| 91 |
+
nwords=args.nwordssrc if src else args.nwordstgt,
|
| 92 |
+
padding_factor=args.padding_factor,
|
| 93 |
+
)
|
| 94 |
+
|
| 95 |
+
|
| 96 |
+
#####################################################################
|
| 97 |
+
# bin file creation logic
|
| 98 |
+
#####################################################################
|
| 99 |
+
|
| 100 |
+
|
| 101 |
+
def _make_binary_dataset(
|
| 102 |
+
vocab: Dictionary,
|
| 103 |
+
input_prefix: str,
|
| 104 |
+
output_prefix: str,
|
| 105 |
+
lang: tp.Optional[str],
|
| 106 |
+
num_workers: int,
|
| 107 |
+
args: Namespace,
|
| 108 |
+
):
|
| 109 |
+
logger.info("[{}] Dictionary: {} types".format(lang, len(vocab)))
|
| 110 |
+
|
| 111 |
+
binarizer = VocabularyDatasetBinarizer(
|
| 112 |
+
vocab,
|
| 113 |
+
append_eos=True,
|
| 114 |
+
)
|
| 115 |
+
|
| 116 |
+
input_file = "{}{}".format(input_prefix, ("." + lang) if lang is not None else "")
|
| 117 |
+
full_output_prefix = dataset_dest_prefix(args, output_prefix, lang)
|
| 118 |
+
|
| 119 |
+
final_summary = FileBinarizer.multiprocess_dataset(
|
| 120 |
+
input_file,
|
| 121 |
+
args.dataset_impl,
|
| 122 |
+
binarizer,
|
| 123 |
+
full_output_prefix,
|
| 124 |
+
vocab_size=len(vocab),
|
| 125 |
+
num_workers=num_workers,
|
| 126 |
+
)
|
| 127 |
+
|
| 128 |
+
logger.info(f"[{lang}] {input_file}: {final_summary} (by {vocab.unk_word})")
|
| 129 |
+
|
| 130 |
+
|
| 131 |
+
def _make_binary_alignment_dataset(
|
| 132 |
+
input_prefix: str, output_prefix: str, num_workers: int, args: Namespace
|
| 133 |
+
):
|
| 134 |
+
|
| 135 |
+
binarizer = AlignmentDatasetBinarizer(utils.parse_alignment)
|
| 136 |
+
|
| 137 |
+
input_file = input_prefix
|
| 138 |
+
full_output_prefix = dataset_dest_prefix(args, output_prefix, lang=None)
|
| 139 |
+
|
| 140 |
+
final_summary = FileBinarizer.multiprocess_dataset(
|
| 141 |
+
input_file,
|
| 142 |
+
args.dataset_impl,
|
| 143 |
+
binarizer,
|
| 144 |
+
full_output_prefix,
|
| 145 |
+
vocab_size=None,
|
| 146 |
+
num_workers=num_workers,
|
| 147 |
+
)
|
| 148 |
+
|
| 149 |
+
logger.info(
|
| 150 |
+
"[alignments] {}: parsed {} alignments".format(
|
| 151 |
+
input_file, final_summary.num_seq
|
| 152 |
+
)
|
| 153 |
+
)
|
| 154 |
+
|
| 155 |
+
|
| 156 |
+
#####################################################################
|
| 157 |
+
# routing logic
|
| 158 |
+
#####################################################################
|
| 159 |
+
|
| 160 |
+
|
| 161 |
+
def _make_dataset(
|
| 162 |
+
vocab: Dictionary,
|
| 163 |
+
input_prefix: str,
|
| 164 |
+
output_prefix: str,
|
| 165 |
+
lang: tp.Optional[str],
|
| 166 |
+
args: Namespace,
|
| 167 |
+
num_workers: int,
|
| 168 |
+
):
|
| 169 |
+
if args.dataset_impl == "raw":
|
| 170 |
+
# Copy original text file to destination folder
|
| 171 |
+
output_text_file = _dest_path(
|
| 172 |
+
output_prefix + ".{}-{}".format(args.source_lang, args.target_lang),
|
| 173 |
+
lang,
|
| 174 |
+
args.destdir,
|
| 175 |
+
)
|
| 176 |
+
shutil.copyfile(_file_name(input_prefix, lang), output_text_file)
|
| 177 |
+
else:
|
| 178 |
+
_make_binary_dataset(
|
| 179 |
+
vocab, input_prefix, output_prefix, lang, num_workers, args
|
| 180 |
+
)
|
| 181 |
+
|
| 182 |
+
|
| 183 |
+
def _make_all(lang, vocab, args):
|
| 184 |
+
if args.trainpref:
|
| 185 |
+
_make_dataset(
|
| 186 |
+
vocab, args.trainpref, "train", lang, args=args, num_workers=args.workers
|
| 187 |
+
)
|
| 188 |
+
if args.validpref:
|
| 189 |
+
for k, validpref in enumerate(args.validpref.split(",")):
|
| 190 |
+
outprefix = "valid{}".format(k) if k > 0 else "valid"
|
| 191 |
+
_make_dataset(
|
| 192 |
+
vocab, validpref, outprefix, lang, args=args, num_workers=args.workers
|
| 193 |
+
)
|
| 194 |
+
if args.testpref:
|
| 195 |
+
for k, testpref in enumerate(args.testpref.split(",")):
|
| 196 |
+
outprefix = "test{}".format(k) if k > 0 else "test"
|
| 197 |
+
_make_dataset(
|
| 198 |
+
vocab, testpref, outprefix, lang, args=args, num_workers=args.workers
|
| 199 |
+
)
|
| 200 |
+
|
| 201 |
+
|
| 202 |
+
def _make_all_alignments(args):
|
| 203 |
+
if args.trainpref and os.path.exists(args.trainpref + "." + args.align_suffix):
|
| 204 |
+
_make_binary_alignment_dataset(
|
| 205 |
+
args.trainpref + "." + args.align_suffix,
|
| 206 |
+
"train.align",
|
| 207 |
+
num_workers=args.workers,
|
| 208 |
+
args=args,
|
| 209 |
+
)
|
| 210 |
+
if args.validpref and os.path.exists(args.validpref + "." + args.align_suffix):
|
| 211 |
+
_make_binary_alignment_dataset(
|
| 212 |
+
args.validpref + "." + args.align_suffix,
|
| 213 |
+
"valid.align",
|
| 214 |
+
num_workers=args.workers,
|
| 215 |
+
args=args,
|
| 216 |
+
)
|
| 217 |
+
if args.testpref and os.path.exists(args.testpref + "." + args.align_suffix):
|
| 218 |
+
_make_binary_alignment_dataset(
|
| 219 |
+
args.testpref + "." + args.align_suffix,
|
| 220 |
+
"test.align",
|
| 221 |
+
num_workers=args.workers,
|
| 222 |
+
args=args,
|
| 223 |
+
)
|
| 224 |
+
|
| 225 |
+
|
| 226 |
+
#####################################################################
|
| 227 |
+
# align
|
| 228 |
+
#####################################################################
|
| 229 |
+
|
| 230 |
+
|
| 231 |
+
def _align_files(args, src_dict, tgt_dict):
|
| 232 |
+
assert args.trainpref, "--trainpref must be set if --alignfile is specified"
|
| 233 |
+
src_file_name = _train_path(args.source_lang, args.trainpref)
|
| 234 |
+
tgt_file_name = _train_path(args.target_lang, args.trainpref)
|
| 235 |
+
freq_map = {}
|
| 236 |
+
with open(args.alignfile, "r", encoding="utf-8") as align_file:
|
| 237 |
+
with open(src_file_name, "r", encoding="utf-8") as src_file:
|
| 238 |
+
with open(tgt_file_name, "r", encoding="utf-8") as tgt_file:
|
| 239 |
+
for a, s, t in zip_longest(align_file, src_file, tgt_file):
|
| 240 |
+
si = src_dict.encode_line(s, add_if_not_exist=False)
|
| 241 |
+
ti = tgt_dict.encode_line(t, add_if_not_exist=False)
|
| 242 |
+
ai = list(map(lambda x: tuple(x.split("-")), a.split()))
|
| 243 |
+
for sai, tai in ai:
|
| 244 |
+
srcidx = si[int(sai)]
|
| 245 |
+
tgtidx = ti[int(tai)]
|
| 246 |
+
if srcidx != src_dict.unk() and tgtidx != tgt_dict.unk():
|
| 247 |
+
assert srcidx != src_dict.pad()
|
| 248 |
+
assert srcidx != src_dict.eos()
|
| 249 |
+
assert tgtidx != tgt_dict.pad()
|
| 250 |
+
assert tgtidx != tgt_dict.eos()
|
| 251 |
+
if srcidx not in freq_map:
|
| 252 |
+
freq_map[srcidx] = {}
|
| 253 |
+
if tgtidx not in freq_map[srcidx]:
|
| 254 |
+
freq_map[srcidx][tgtidx] = 1
|
| 255 |
+
else:
|
| 256 |
+
freq_map[srcidx][tgtidx] += 1
|
| 257 |
+
align_dict = {}
|
| 258 |
+
for srcidx in freq_map.keys():
|
| 259 |
+
align_dict[srcidx] = max(freq_map[srcidx], key=freq_map[srcidx].get)
|
| 260 |
+
with open(
|
| 261 |
+
os.path.join(
|
| 262 |
+
args.destdir,
|
| 263 |
+
"alignment.{}-{}.txt".format(args.source_lang, args.target_lang),
|
| 264 |
+
),
|
| 265 |
+
"w",
|
| 266 |
+
encoding="utf-8",
|
| 267 |
+
) as f:
|
| 268 |
+
for k, v in align_dict.items():
|
| 269 |
+
print("{} {}".format(src_dict[k], tgt_dict[v]), file=f)
|
| 270 |
+
|
| 271 |
+
|
| 272 |
+
#####################################################################
|
| 273 |
+
# MAIN
|
| 274 |
+
#####################################################################
|
| 275 |
+
|
| 276 |
+
|
| 277 |
+
def main(args):
|
| 278 |
+
# setup some basic things
|
| 279 |
+
utils.import_user_module(args)
|
| 280 |
+
|
| 281 |
+
os.makedirs(args.destdir, exist_ok=True)
|
| 282 |
+
|
| 283 |
+
logger.addHandler(
|
| 284 |
+
logging.FileHandler(
|
| 285 |
+
filename=os.path.join(args.destdir, "preprocess.log"),
|
| 286 |
+
)
|
| 287 |
+
)
|
| 288 |
+
logger.info(args)
|
| 289 |
+
|
| 290 |
+
assert (
|
| 291 |
+
args.dataset_impl != "huffman"
|
| 292 |
+
), "preprocessing.py doesn't support Huffman yet, use HuffmanCodeBuilder directly."
|
| 293 |
+
|
| 294 |
+
# build dictionaries
|
| 295 |
+
|
| 296 |
+
target = not args.only_source
|
| 297 |
+
|
| 298 |
+
if not args.srcdict and os.path.exists(_dict_path(args.source_lang, args.destdir)):
|
| 299 |
+
raise FileExistsError(_dict_path(args.source_lang, args.destdir))
|
| 300 |
+
|
| 301 |
+
if (
|
| 302 |
+
target
|
| 303 |
+
and not args.tgtdict
|
| 304 |
+
and os.path.exists(_dict_path(args.target_lang, args.destdir))
|
| 305 |
+
):
|
| 306 |
+
raise FileExistsError(_dict_path(args.target_lang, args.destdir))
|
| 307 |
+
|
| 308 |
+
task = tasks.get_task(args.task)
|
| 309 |
+
|
| 310 |
+
if args.joined_dictionary:
|
| 311 |
+
assert (
|
| 312 |
+
not args.srcdict or not args.tgtdict
|
| 313 |
+
), "cannot use both --srcdict and --tgtdict with --joined-dictionary"
|
| 314 |
+
|
| 315 |
+
if args.srcdict:
|
| 316 |
+
src_dict = task.load_dictionary(args.srcdict)
|
| 317 |
+
elif args.tgtdict:
|
| 318 |
+
src_dict = task.load_dictionary(args.tgtdict)
|
| 319 |
+
else:
|
| 320 |
+
assert (
|
| 321 |
+
args.trainpref
|
| 322 |
+
), "--trainpref must be set if --srcdict is not specified"
|
| 323 |
+
src_dict = _build_dictionary(
|
| 324 |
+
{
|
| 325 |
+
_train_path(lang, args.trainpref)
|
| 326 |
+
for lang in [args.source_lang, args.target_lang]
|
| 327 |
+
},
|
| 328 |
+
task=task,
|
| 329 |
+
args=args,
|
| 330 |
+
src=True,
|
| 331 |
+
)
|
| 332 |
+
tgt_dict = src_dict
|
| 333 |
+
else:
|
| 334 |
+
if args.srcdict:
|
| 335 |
+
src_dict = task.load_dictionary(args.srcdict)
|
| 336 |
+
else:
|
| 337 |
+
assert (
|
| 338 |
+
args.trainpref
|
| 339 |
+
), "--trainpref must be set if --srcdict is not specified"
|
| 340 |
+
src_dict = _build_dictionary(
|
| 341 |
+
[_train_path(args.source_lang, args.trainpref)],
|
| 342 |
+
task=task,
|
| 343 |
+
args=args,
|
| 344 |
+
src=True,
|
| 345 |
+
)
|
| 346 |
+
|
| 347 |
+
if target:
|
| 348 |
+
if args.tgtdict:
|
| 349 |
+
tgt_dict = task.load_dictionary(args.tgtdict)
|
| 350 |
+
else:
|
| 351 |
+
assert (
|
| 352 |
+
args.trainpref
|
| 353 |
+
), "--trainpref must be set if --tgtdict is not specified"
|
| 354 |
+
tgt_dict = _build_dictionary(
|
| 355 |
+
[_train_path(args.target_lang, args.trainpref)],
|
| 356 |
+
task=task,
|
| 357 |
+
args=args,
|
| 358 |
+
tgt=True,
|
| 359 |
+
)
|
| 360 |
+
else:
|
| 361 |
+
tgt_dict = None
|
| 362 |
+
|
| 363 |
+
# save dictionaries
|
| 364 |
+
|
| 365 |
+
src_dict.save(_dict_path(args.source_lang, args.destdir))
|
| 366 |
+
if target and tgt_dict is not None:
|
| 367 |
+
tgt_dict.save(_dict_path(args.target_lang, args.destdir))
|
| 368 |
+
|
| 369 |
+
if args.dict_only:
|
| 370 |
+
return
|
| 371 |
+
|
| 372 |
+
_make_all(args.source_lang, src_dict, args)
|
| 373 |
+
if target:
|
| 374 |
+
_make_all(args.target_lang, tgt_dict, args)
|
| 375 |
+
|
| 376 |
+
# align the datasets if needed
|
| 377 |
+
if args.align_suffix:
|
| 378 |
+
_make_all_alignments(args)
|
| 379 |
+
|
| 380 |
+
logger.info("Wrote preprocessed data to {}".format(args.destdir))
|
| 381 |
+
|
| 382 |
+
if args.alignfile:
|
| 383 |
+
_align_files(args, src_dict=src_dict, tgt_dict=tgt_dict)
|
| 384 |
+
|
| 385 |
+
|
| 386 |
+
def cli_main():
|
| 387 |
+
parser = options.get_preprocessing_parser()
|
| 388 |
+
args = parser.parse_args()
|
| 389 |
+
main(args)
|
| 390 |
+
|
| 391 |
+
|
| 392 |
+
if __name__ == "__main__":
|
| 393 |
+
cli_main()
|
data/fairseq/fairseq_cli/score.py
ADDED
|
@@ -0,0 +1,102 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python3
|
| 2 |
+
# Copyright (c) Facebook, Inc. and its affiliates.
|
| 3 |
+
#
|
| 4 |
+
# This source code is licensed under the MIT license found in the
|
| 5 |
+
# LICENSE file in the root directory of this source tree.
|
| 6 |
+
"""
|
| 7 |
+
BLEU scoring of generated translations against reference translations.
|
| 8 |
+
"""
|
| 9 |
+
|
| 10 |
+
import argparse
|
| 11 |
+
import os
|
| 12 |
+
import sys
|
| 13 |
+
|
| 14 |
+
from fairseq.data import dictionary
|
| 15 |
+
from fairseq.scoring import bleu
|
| 16 |
+
|
| 17 |
+
|
| 18 |
+
def get_parser():
|
| 19 |
+
parser = argparse.ArgumentParser(
|
| 20 |
+
description="Command-line script for BLEU scoring."
|
| 21 |
+
)
|
| 22 |
+
# fmt: off
|
| 23 |
+
parser.add_argument('-s', '--sys', default='-', help='system output')
|
| 24 |
+
parser.add_argument('-r', '--ref', required=True, help='references')
|
| 25 |
+
parser.add_argument('-o', '--order', default=4, metavar='N',
|
| 26 |
+
type=int, help='consider ngrams up to this order')
|
| 27 |
+
parser.add_argument('--ignore-case', action='store_true',
|
| 28 |
+
help='case-insensitive scoring')
|
| 29 |
+
parser.add_argument('--sacrebleu', action='store_true',
|
| 30 |
+
help='score with sacrebleu')
|
| 31 |
+
parser.add_argument('--sentence-bleu', action='store_true',
|
| 32 |
+
help='report sentence-level BLEUs (i.e., with +1 smoothing)')
|
| 33 |
+
# fmt: on
|
| 34 |
+
return parser
|
| 35 |
+
|
| 36 |
+
|
| 37 |
+
def cli_main():
|
| 38 |
+
parser = get_parser()
|
| 39 |
+
args = parser.parse_args()
|
| 40 |
+
print(args)
|
| 41 |
+
|
| 42 |
+
assert args.sys == "-" or os.path.exists(
|
| 43 |
+
args.sys
|
| 44 |
+
), "System output file {} does not exist".format(args.sys)
|
| 45 |
+
assert os.path.exists(args.ref), "Reference file {} does not exist".format(args.ref)
|
| 46 |
+
|
| 47 |
+
dict = dictionary.Dictionary()
|
| 48 |
+
|
| 49 |
+
def readlines(fd):
|
| 50 |
+
for line in fd.readlines():
|
| 51 |
+
if args.ignore_case:
|
| 52 |
+
yield line.lower()
|
| 53 |
+
else:
|
| 54 |
+
yield line
|
| 55 |
+
|
| 56 |
+
if args.sacrebleu:
|
| 57 |
+
import sacrebleu
|
| 58 |
+
|
| 59 |
+
def score(fdsys):
|
| 60 |
+
with open(args.ref) as fdref:
|
| 61 |
+
print(sacrebleu.corpus_bleu(fdsys, [fdref]).format())
|
| 62 |
+
|
| 63 |
+
elif args.sentence_bleu:
|
| 64 |
+
|
| 65 |
+
def score(fdsys):
|
| 66 |
+
with open(args.ref) as fdref:
|
| 67 |
+
scorer = bleu.Scorer(dict.pad(), dict.eos(), dict.unk())
|
| 68 |
+
for i, (sys_tok, ref_tok) in enumerate(
|
| 69 |
+
zip(readlines(fdsys), readlines(fdref))
|
| 70 |
+
):
|
| 71 |
+
scorer.reset(one_init=True)
|
| 72 |
+
sys_tok = dict.encode_line(sys_tok)
|
| 73 |
+
ref_tok = dict.encode_line(ref_tok)
|
| 74 |
+
scorer.add(ref_tok, sys_tok)
|
| 75 |
+
print(i, scorer.result_string(args.order))
|
| 76 |
+
|
| 77 |
+
else:
|
| 78 |
+
|
| 79 |
+
def score(fdsys):
|
| 80 |
+
with open(args.ref) as fdref:
|
| 81 |
+
scorer = bleu.Scorer(
|
| 82 |
+
bleu.BleuConfig(
|
| 83 |
+
pad=dict.pad(),
|
| 84 |
+
eos=dict.eos(),
|
| 85 |
+
unk=dict.unk(),
|
| 86 |
+
)
|
| 87 |
+
)
|
| 88 |
+
for sys_tok, ref_tok in zip(readlines(fdsys), readlines(fdref)):
|
| 89 |
+
sys_tok = dict.encode_line(sys_tok)
|
| 90 |
+
ref_tok = dict.encode_line(ref_tok)
|
| 91 |
+
scorer.add(ref_tok, sys_tok)
|
| 92 |
+
print(scorer.result_string(args.order))
|
| 93 |
+
|
| 94 |
+
if args.sys == "-":
|
| 95 |
+
score(sys.stdin)
|
| 96 |
+
else:
|
| 97 |
+
with open(args.sys, "r") as f:
|
| 98 |
+
score(f)
|
| 99 |
+
|
| 100 |
+
|
| 101 |
+
if __name__ == "__main__":
|
| 102 |
+
cli_main()
|
data/fairseq/fairseq_cli/train.py
ADDED
|
@@ -0,0 +1,581 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python3 -u
|
| 2 |
+
# Copyright (c) Facebook, Inc. and its affiliates.
|
| 3 |
+
#
|
| 4 |
+
# This source code is licensed under the MIT license found in the
|
| 5 |
+
# LICENSE file in the root directory of this source tree.
|
| 6 |
+
"""
|
| 7 |
+
Train a new model on one or across multiple GPUs.
|
| 8 |
+
"""
|
| 9 |
+
|
| 10 |
+
import argparse
|
| 11 |
+
import logging
|
| 12 |
+
import math
|
| 13 |
+
import os
|
| 14 |
+
import sys
|
| 15 |
+
from typing import Any, Callable, Dict, List, Optional, Tuple
|
| 16 |
+
|
| 17 |
+
# We need to setup root logger before importing any fairseq libraries.
|
| 18 |
+
logging.basicConfig(
|
| 19 |
+
format="%(asctime)s | %(levelname)s | %(name)s | %(message)s",
|
| 20 |
+
datefmt="%Y-%m-%d %H:%M:%S",
|
| 21 |
+
level=os.environ.get("LOGLEVEL", "INFO").upper(),
|
| 22 |
+
stream=sys.stdout,
|
| 23 |
+
)
|
| 24 |
+
logger = logging.getLogger("fairseq_cli.train")
|
| 25 |
+
|
| 26 |
+
import numpy as np
|
| 27 |
+
import torch
|
| 28 |
+
from omegaconf import DictConfig, OmegaConf
|
| 29 |
+
|
| 30 |
+
from fairseq import checkpoint_utils, options, quantization_utils, tasks, utils
|
| 31 |
+
from fairseq.data import data_utils, iterators
|
| 32 |
+
from fairseq.data.plasma_utils import PlasmaStore
|
| 33 |
+
from fairseq.dataclass.configs import FairseqConfig
|
| 34 |
+
from fairseq.dataclass.initialize import add_defaults
|
| 35 |
+
from fairseq.dataclass.utils import convert_namespace_to_omegaconf
|
| 36 |
+
from fairseq.distributed import fsdp_enable_wrap, fsdp_wrap
|
| 37 |
+
from fairseq.distributed import utils as distributed_utils
|
| 38 |
+
from fairseq.file_io import PathManager
|
| 39 |
+
from fairseq.logging import meters, metrics, progress_bar
|
| 40 |
+
from fairseq.model_parallel.megatron_trainer import MegatronTrainer
|
| 41 |
+
from fairseq.trainer import Trainer
|
| 42 |
+
|
| 43 |
+
|
| 44 |
+
def main(cfg: FairseqConfig) -> None:
|
| 45 |
+
if isinstance(cfg, argparse.Namespace):
|
| 46 |
+
cfg = convert_namespace_to_omegaconf(cfg)
|
| 47 |
+
|
| 48 |
+
utils.import_user_module(cfg.common)
|
| 49 |
+
add_defaults(cfg)
|
| 50 |
+
|
| 51 |
+
if (
|
| 52 |
+
distributed_utils.is_master(cfg.distributed_training)
|
| 53 |
+
and "job_logging_cfg" in cfg
|
| 54 |
+
):
|
| 55 |
+
# make hydra logging work with ddp (see # see https://github.com/facebookresearch/hydra/issues/1126)
|
| 56 |
+
logging.config.dictConfig(OmegaConf.to_container(cfg.job_logging_cfg))
|
| 57 |
+
|
| 58 |
+
assert (
|
| 59 |
+
cfg.dataset.max_tokens is not None or cfg.dataset.batch_size is not None
|
| 60 |
+
), "Must specify batch size either with --max-tokens or --batch-size"
|
| 61 |
+
metrics.reset()
|
| 62 |
+
|
| 63 |
+
if cfg.common.log_file is not None:
|
| 64 |
+
handler = logging.FileHandler(filename=cfg.common.log_file)
|
| 65 |
+
logger.addHandler(handler)
|
| 66 |
+
|
| 67 |
+
np.random.seed(cfg.common.seed)
|
| 68 |
+
utils.set_torch_seed(cfg.common.seed)
|
| 69 |
+
|
| 70 |
+
if distributed_utils.is_master(cfg.distributed_training):
|
| 71 |
+
checkpoint_utils.verify_checkpoint_directory(cfg.checkpoint.save_dir)
|
| 72 |
+
|
| 73 |
+
# Print args
|
| 74 |
+
logger.info(cfg)
|
| 75 |
+
|
| 76 |
+
if cfg.checkpoint.write_checkpoints_asynchronously:
|
| 77 |
+
try:
|
| 78 |
+
import iopath # noqa: F401
|
| 79 |
+
except ImportError:
|
| 80 |
+
logging.exception(
|
| 81 |
+
"Asynchronous checkpoint writing is specified but iopath is "
|
| 82 |
+
"not installed: `pip install iopath`"
|
| 83 |
+
)
|
| 84 |
+
return
|
| 85 |
+
|
| 86 |
+
# Setup task, e.g., translation, language modeling, etc.
|
| 87 |
+
task = tasks.setup_task(cfg.task)
|
| 88 |
+
|
| 89 |
+
assert cfg.criterion, "Please specify criterion to train a model"
|
| 90 |
+
|
| 91 |
+
# Build model and criterion
|
| 92 |
+
if cfg.distributed_training.ddp_backend == "fully_sharded":
|
| 93 |
+
with fsdp_enable_wrap(cfg.distributed_training):
|
| 94 |
+
model = fsdp_wrap(task.build_model(cfg.model))
|
| 95 |
+
else:
|
| 96 |
+
model = task.build_model(cfg.model)
|
| 97 |
+
criterion = task.build_criterion(cfg.criterion)
|
| 98 |
+
logger.info(model)
|
| 99 |
+
logger.info("task: {}".format(task.__class__.__name__))
|
| 100 |
+
logger.info("model: {}".format(model.__class__.__name__))
|
| 101 |
+
logger.info("criterion: {}".format(criterion.__class__.__name__))
|
| 102 |
+
logger.info(
|
| 103 |
+
"num. shared model params: {:,} (num. trained: {:,})".format(
|
| 104 |
+
sum(
|
| 105 |
+
p.numel() for p in model.parameters() if not getattr(p, "expert", False)
|
| 106 |
+
),
|
| 107 |
+
sum(
|
| 108 |
+
p.numel()
|
| 109 |
+
for p in model.parameters()
|
| 110 |
+
if not getattr(p, "expert", False) and p.requires_grad
|
| 111 |
+
),
|
| 112 |
+
)
|
| 113 |
+
)
|
| 114 |
+
|
| 115 |
+
logger.info(
|
| 116 |
+
"num. expert model params: {} (num. trained: {})".format(
|
| 117 |
+
sum(p.numel() for p in model.parameters() if getattr(p, "expert", False)),
|
| 118 |
+
sum(
|
| 119 |
+
p.numel()
|
| 120 |
+
for p in model.parameters()
|
| 121 |
+
if getattr(p, "expert", False) and p.requires_grad
|
| 122 |
+
),
|
| 123 |
+
)
|
| 124 |
+
)
|
| 125 |
+
|
| 126 |
+
# Load valid dataset (we load training data below, based on the latest checkpoint)
|
| 127 |
+
# We load the valid dataset AFTER building the model
|
| 128 |
+
if not cfg.dataset.disable_validation:
|
| 129 |
+
data_utils.raise_if_valid_subsets_unintentionally_ignored(cfg)
|
| 130 |
+
if cfg.dataset.combine_valid_subsets:
|
| 131 |
+
task.load_dataset("valid", combine=True, epoch=1)
|
| 132 |
+
else:
|
| 133 |
+
for valid_sub_split in cfg.dataset.valid_subset.split(","):
|
| 134 |
+
task.load_dataset(valid_sub_split, combine=False, epoch=1)
|
| 135 |
+
|
| 136 |
+
# (optionally) Configure quantization
|
| 137 |
+
if cfg.common.quantization_config_path is not None:
|
| 138 |
+
quantizer = quantization_utils.Quantizer(
|
| 139 |
+
config_path=cfg.common.quantization_config_path,
|
| 140 |
+
max_epoch=cfg.optimization.max_epoch,
|
| 141 |
+
max_update=cfg.optimization.max_update,
|
| 142 |
+
)
|
| 143 |
+
else:
|
| 144 |
+
quantizer = None
|
| 145 |
+
|
| 146 |
+
# Build trainer
|
| 147 |
+
if cfg.common.model_parallel_size == 1:
|
| 148 |
+
trainer = Trainer(cfg, task, model, criterion, quantizer)
|
| 149 |
+
else:
|
| 150 |
+
trainer = MegatronTrainer(cfg, task, model, criterion)
|
| 151 |
+
logger.info(
|
| 152 |
+
"training on {} devices (GPUs/TPUs)".format(
|
| 153 |
+
cfg.distributed_training.distributed_world_size
|
| 154 |
+
)
|
| 155 |
+
)
|
| 156 |
+
logger.info(
|
| 157 |
+
"max tokens per device = {} and max sentences per device = {}".format(
|
| 158 |
+
cfg.dataset.max_tokens,
|
| 159 |
+
cfg.dataset.batch_size,
|
| 160 |
+
)
|
| 161 |
+
)
|
| 162 |
+
|
| 163 |
+
# Load the latest checkpoint if one is available and restore the
|
| 164 |
+
# corresponding train iterator
|
| 165 |
+
extra_state, epoch_itr = checkpoint_utils.load_checkpoint(
|
| 166 |
+
cfg.checkpoint,
|
| 167 |
+
trainer,
|
| 168 |
+
# don't cache epoch iterators for sharded datasets
|
| 169 |
+
disable_iterator_cache=task.has_sharded_data("train"),
|
| 170 |
+
)
|
| 171 |
+
if cfg.common.tpu:
|
| 172 |
+
import torch_xla.core.xla_model as xm
|
| 173 |
+
|
| 174 |
+
xm.rendezvous("load_checkpoint") # wait for all workers
|
| 175 |
+
|
| 176 |
+
max_epoch = cfg.optimization.max_epoch or math.inf
|
| 177 |
+
lr = trainer.get_lr()
|
| 178 |
+
|
| 179 |
+
# TODO: a dry run on validation set to pin the memory
|
| 180 |
+
valid_subsets = cfg.dataset.valid_subset.split(",")
|
| 181 |
+
if not cfg.dataset.disable_validation:
|
| 182 |
+
for subset in valid_subsets:
|
| 183 |
+
logger.info('begin dry-run validation on "{}" subset'.format(subset))
|
| 184 |
+
itr = trainer.get_valid_iterator(subset).next_epoch_itr(
|
| 185 |
+
shuffle=False, set_dataset_epoch=False # use a fixed valid set
|
| 186 |
+
)
|
| 187 |
+
if cfg.common.tpu:
|
| 188 |
+
itr = utils.tpu_data_loader(itr)
|
| 189 |
+
for _ in itr:
|
| 190 |
+
pass
|
| 191 |
+
# TODO: end of dry run section
|
| 192 |
+
|
| 193 |
+
train_meter = meters.StopwatchMeter()
|
| 194 |
+
train_meter.start()
|
| 195 |
+
while epoch_itr.next_epoch_idx <= max_epoch:
|
| 196 |
+
if lr <= cfg.optimization.stop_min_lr:
|
| 197 |
+
logger.info(
|
| 198 |
+
f"stopping training because current learning rate ({lr}) is smaller "
|
| 199 |
+
"than or equal to minimum learning rate "
|
| 200 |
+
f"(--stop-min-lr={cfg.optimization.stop_min_lr})"
|
| 201 |
+
)
|
| 202 |
+
break
|
| 203 |
+
|
| 204 |
+
# train for one epoch
|
| 205 |
+
valid_losses, should_stop = train(cfg, trainer, task, epoch_itr)
|
| 206 |
+
if should_stop:
|
| 207 |
+
break
|
| 208 |
+
|
| 209 |
+
# only use first validation loss to update the learning rate
|
| 210 |
+
lr = trainer.lr_step(epoch_itr.epoch, valid_losses[0])
|
| 211 |
+
|
| 212 |
+
epoch_itr = trainer.get_train_iterator(
|
| 213 |
+
epoch_itr.next_epoch_idx,
|
| 214 |
+
# sharded data: get train iterator for next epoch
|
| 215 |
+
load_dataset=task.has_sharded_data("train"),
|
| 216 |
+
# don't cache epoch iterators for sharded datasets
|
| 217 |
+
disable_iterator_cache=task.has_sharded_data("train"),
|
| 218 |
+
)
|
| 219 |
+
train_meter.stop()
|
| 220 |
+
logger.info("done training in {:.1f} seconds".format(train_meter.sum))
|
| 221 |
+
|
| 222 |
+
# ioPath implementation to wait for all asynchronous file writes to complete.
|
| 223 |
+
if cfg.checkpoint.write_checkpoints_asynchronously:
|
| 224 |
+
logger.info(
|
| 225 |
+
"ioPath PathManager waiting for all asynchronous checkpoint "
|
| 226 |
+
"writes to finish."
|
| 227 |
+
)
|
| 228 |
+
PathManager.async_close()
|
| 229 |
+
logger.info("ioPath PathManager finished waiting.")
|
| 230 |
+
|
| 231 |
+
|
| 232 |
+
def should_stop_early(cfg: DictConfig, valid_loss: float) -> bool:
|
| 233 |
+
# skip check if no validation was done in the current epoch
|
| 234 |
+
if valid_loss is None:
|
| 235 |
+
return False
|
| 236 |
+
if cfg.checkpoint.patience <= 0:
|
| 237 |
+
return False
|
| 238 |
+
|
| 239 |
+
def is_better(a, b):
|
| 240 |
+
return a > b if cfg.checkpoint.maximize_best_checkpoint_metric else a < b
|
| 241 |
+
|
| 242 |
+
prev_best = getattr(should_stop_early, "best", None)
|
| 243 |
+
if prev_best is None or is_better(valid_loss, prev_best):
|
| 244 |
+
should_stop_early.best = valid_loss
|
| 245 |
+
should_stop_early.num_runs = 0
|
| 246 |
+
return False
|
| 247 |
+
else:
|
| 248 |
+
should_stop_early.num_runs += 1
|
| 249 |
+
if should_stop_early.num_runs >= cfg.checkpoint.patience:
|
| 250 |
+
logger.info(
|
| 251 |
+
"early stop since valid performance hasn't improved for last {} runs".format(
|
| 252 |
+
cfg.checkpoint.patience
|
| 253 |
+
)
|
| 254 |
+
)
|
| 255 |
+
return True
|
| 256 |
+
else:
|
| 257 |
+
return False
|
| 258 |
+
|
| 259 |
+
|
| 260 |
+
@metrics.aggregate("train")
|
| 261 |
+
def train(
|
| 262 |
+
cfg: DictConfig, trainer: Trainer, task: tasks.FairseqTask, epoch_itr
|
| 263 |
+
) -> Tuple[List[Optional[float]], bool]:
|
| 264 |
+
"""Train the model for one epoch and return validation losses."""
|
| 265 |
+
# Initialize data iterator
|
| 266 |
+
itr = epoch_itr.next_epoch_itr(
|
| 267 |
+
fix_batches_to_gpus=cfg.distributed_training.fix_batches_to_gpus,
|
| 268 |
+
shuffle=(epoch_itr.next_epoch_idx > cfg.dataset.curriculum),
|
| 269 |
+
)
|
| 270 |
+
update_freq = (
|
| 271 |
+
cfg.optimization.update_freq[epoch_itr.epoch - 1]
|
| 272 |
+
if epoch_itr.epoch <= len(cfg.optimization.update_freq)
|
| 273 |
+
else cfg.optimization.update_freq[-1]
|
| 274 |
+
)
|
| 275 |
+
itr = iterators.GroupedIterator(
|
| 276 |
+
itr,
|
| 277 |
+
update_freq,
|
| 278 |
+
skip_remainder_batch=cfg.optimization.skip_remainder_batch,
|
| 279 |
+
)
|
| 280 |
+
if cfg.common.tpu:
|
| 281 |
+
itr = utils.tpu_data_loader(itr)
|
| 282 |
+
progress = progress_bar.progress_bar(
|
| 283 |
+
itr,
|
| 284 |
+
log_format=cfg.common.log_format,
|
| 285 |
+
log_file=cfg.common.log_file,
|
| 286 |
+
log_interval=cfg.common.log_interval,
|
| 287 |
+
epoch=epoch_itr.epoch,
|
| 288 |
+
aim_repo=(
|
| 289 |
+
cfg.common.aim_repo
|
| 290 |
+
if distributed_utils.is_master(cfg.distributed_training)
|
| 291 |
+
else None
|
| 292 |
+
),
|
| 293 |
+
aim_run_hash=(
|
| 294 |
+
cfg.common.aim_run_hash
|
| 295 |
+
if distributed_utils.is_master(cfg.distributed_training)
|
| 296 |
+
else None
|
| 297 |
+
),
|
| 298 |
+
aim_param_checkpoint_dir=cfg.checkpoint.save_dir,
|
| 299 |
+
tensorboard_logdir=(
|
| 300 |
+
cfg.common.tensorboard_logdir
|
| 301 |
+
if distributed_utils.is_master(cfg.distributed_training)
|
| 302 |
+
else None
|
| 303 |
+
),
|
| 304 |
+
default_log_format=("tqdm" if not cfg.common.no_progress_bar else "simple"),
|
| 305 |
+
wandb_project=(
|
| 306 |
+
cfg.common.wandb_project
|
| 307 |
+
if distributed_utils.is_master(cfg.distributed_training)
|
| 308 |
+
else None
|
| 309 |
+
),
|
| 310 |
+
wandb_run_name=os.environ.get(
|
| 311 |
+
"WANDB_NAME", os.path.basename(cfg.checkpoint.save_dir)
|
| 312 |
+
),
|
| 313 |
+
azureml_logging=(
|
| 314 |
+
cfg.common.azureml_logging
|
| 315 |
+
if distributed_utils.is_master(cfg.distributed_training)
|
| 316 |
+
else False
|
| 317 |
+
),
|
| 318 |
+
)
|
| 319 |
+
progress.update_config(_flatten_config(cfg))
|
| 320 |
+
|
| 321 |
+
trainer.begin_epoch(epoch_itr.epoch)
|
| 322 |
+
|
| 323 |
+
valid_subsets = cfg.dataset.valid_subset.split(",")
|
| 324 |
+
should_stop = False
|
| 325 |
+
num_updates = trainer.get_num_updates()
|
| 326 |
+
logger.info("Start iterating over samples")
|
| 327 |
+
for i, samples in enumerate(progress):
|
| 328 |
+
with metrics.aggregate("train_inner"), torch.autograd.profiler.record_function(
|
| 329 |
+
"train_step-%d" % i
|
| 330 |
+
):
|
| 331 |
+
log_output = trainer.train_step(samples)
|
| 332 |
+
|
| 333 |
+
if log_output is not None: # not OOM, overflow, ...
|
| 334 |
+
# log mid-epoch stats
|
| 335 |
+
num_updates = trainer.get_num_updates()
|
| 336 |
+
if num_updates % cfg.common.log_interval == 0:
|
| 337 |
+
stats = get_training_stats(metrics.get_smoothed_values("train_inner"))
|
| 338 |
+
progress.log(stats, tag="train_inner", step=num_updates)
|
| 339 |
+
|
| 340 |
+
# reset mid-epoch stats after each log interval
|
| 341 |
+
# the end-of-epoch stats will still be preserved
|
| 342 |
+
metrics.reset_meters("train_inner")
|
| 343 |
+
|
| 344 |
+
end_of_epoch = not itr.has_next()
|
| 345 |
+
valid_losses, should_stop = validate_and_save(
|
| 346 |
+
cfg, trainer, task, epoch_itr, valid_subsets, end_of_epoch
|
| 347 |
+
)
|
| 348 |
+
|
| 349 |
+
if should_stop:
|
| 350 |
+
break
|
| 351 |
+
|
| 352 |
+
# log end-of-epoch stats
|
| 353 |
+
logger.info("end of epoch {} (average epoch stats below)".format(epoch_itr.epoch))
|
| 354 |
+
stats = get_training_stats(metrics.get_smoothed_values("train"))
|
| 355 |
+
progress.print(stats, tag="train", step=num_updates)
|
| 356 |
+
|
| 357 |
+
# reset epoch-level meters
|
| 358 |
+
metrics.reset_meters("train")
|
| 359 |
+
return valid_losses, should_stop
|
| 360 |
+
|
| 361 |
+
|
| 362 |
+
def _flatten_config(cfg: DictConfig):
|
| 363 |
+
config = OmegaConf.to_container(cfg)
|
| 364 |
+
# remove any legacy Namespaces and replace with a single "args"
|
| 365 |
+
namespace = None
|
| 366 |
+
for k, v in list(config.items()):
|
| 367 |
+
if isinstance(v, argparse.Namespace):
|
| 368 |
+
namespace = v
|
| 369 |
+
del config[k]
|
| 370 |
+
if namespace is not None:
|
| 371 |
+
config["args"] = vars(namespace)
|
| 372 |
+
return config
|
| 373 |
+
|
| 374 |
+
|
| 375 |
+
def validate_and_save(
|
| 376 |
+
cfg: DictConfig,
|
| 377 |
+
trainer: Trainer,
|
| 378 |
+
task: tasks.FairseqTask,
|
| 379 |
+
epoch_itr,
|
| 380 |
+
valid_subsets: List[str],
|
| 381 |
+
end_of_epoch: bool,
|
| 382 |
+
) -> Tuple[List[Optional[float]], bool]:
|
| 383 |
+
num_updates = trainer.get_num_updates()
|
| 384 |
+
max_update = cfg.optimization.max_update or math.inf
|
| 385 |
+
|
| 386 |
+
# Stopping conditions (and an additional one based on validation loss later
|
| 387 |
+
# on)
|
| 388 |
+
should_stop = False
|
| 389 |
+
if num_updates >= max_update:
|
| 390 |
+
should_stop = True
|
| 391 |
+
logger.info(
|
| 392 |
+
f"Stopping training due to "
|
| 393 |
+
f"num_updates: {num_updates} >= max_update: {max_update}"
|
| 394 |
+
)
|
| 395 |
+
|
| 396 |
+
training_time_hours = trainer.cumulative_training_time() / (60 * 60)
|
| 397 |
+
if (
|
| 398 |
+
cfg.optimization.stop_time_hours > 0
|
| 399 |
+
and training_time_hours > cfg.optimization.stop_time_hours
|
| 400 |
+
):
|
| 401 |
+
should_stop = True
|
| 402 |
+
logger.info(
|
| 403 |
+
f"Stopping training due to "
|
| 404 |
+
f"cumulative_training_time: {training_time_hours} > "
|
| 405 |
+
f"stop_time_hours: {cfg.optimization.stop_time_hours} hour(s)"
|
| 406 |
+
)
|
| 407 |
+
|
| 408 |
+
do_save = (
|
| 409 |
+
(end_of_epoch and epoch_itr.epoch % cfg.checkpoint.save_interval == 0)
|
| 410 |
+
or should_stop
|
| 411 |
+
or (
|
| 412 |
+
cfg.checkpoint.save_interval_updates > 0
|
| 413 |
+
and num_updates > 0
|
| 414 |
+
and num_updates % cfg.checkpoint.save_interval_updates == 0
|
| 415 |
+
and num_updates >= cfg.dataset.validate_after_updates
|
| 416 |
+
)
|
| 417 |
+
)
|
| 418 |
+
do_validate = (
|
| 419 |
+
(
|
| 420 |
+
(not end_of_epoch and do_save) # validate during mid-epoch saves
|
| 421 |
+
or (end_of_epoch and epoch_itr.epoch % cfg.dataset.validate_interval == 0)
|
| 422 |
+
or should_stop
|
| 423 |
+
or (
|
| 424 |
+
cfg.dataset.validate_interval_updates > 0
|
| 425 |
+
and num_updates > 0
|
| 426 |
+
and num_updates % cfg.dataset.validate_interval_updates == 0
|
| 427 |
+
)
|
| 428 |
+
)
|
| 429 |
+
and not cfg.dataset.disable_validation
|
| 430 |
+
and num_updates >= cfg.dataset.validate_after_updates
|
| 431 |
+
)
|
| 432 |
+
|
| 433 |
+
# Validate
|
| 434 |
+
valid_losses = [None]
|
| 435 |
+
if do_validate:
|
| 436 |
+
valid_losses = validate(cfg, trainer, task, epoch_itr, valid_subsets)
|
| 437 |
+
|
| 438 |
+
should_stop |= should_stop_early(cfg, valid_losses[0])
|
| 439 |
+
|
| 440 |
+
# Save checkpoint
|
| 441 |
+
if do_save or should_stop:
|
| 442 |
+
cp_path = checkpoint_utils.save_checkpoint(
|
| 443 |
+
cfg.checkpoint, trainer, epoch_itr, valid_losses[0]
|
| 444 |
+
)
|
| 445 |
+
if cp_path is not None and hasattr(task, "post_save"):
|
| 446 |
+
task.post_save(cp_path, num_updates)
|
| 447 |
+
|
| 448 |
+
return valid_losses, should_stop
|
| 449 |
+
|
| 450 |
+
|
| 451 |
+
def get_training_stats(stats: Dict[str, Any]) -> Dict[str, Any]:
|
| 452 |
+
stats["wall"] = round(metrics.get_meter("default", "wall").elapsed_time, 0)
|
| 453 |
+
return stats
|
| 454 |
+
|
| 455 |
+
|
| 456 |
+
def validate(
|
| 457 |
+
cfg: DictConfig,
|
| 458 |
+
trainer: Trainer,
|
| 459 |
+
task: tasks.FairseqTask,
|
| 460 |
+
epoch_itr,
|
| 461 |
+
subsets: List[str],
|
| 462 |
+
) -> List[Optional[float]]:
|
| 463 |
+
"""Evaluate the model on the validation set(s) and return the losses."""
|
| 464 |
+
|
| 465 |
+
if cfg.dataset.fixed_validation_seed is not None:
|
| 466 |
+
# set fixed seed for every validation
|
| 467 |
+
utils.set_torch_seed(cfg.dataset.fixed_validation_seed)
|
| 468 |
+
|
| 469 |
+
trainer.begin_valid_epoch(epoch_itr.epoch)
|
| 470 |
+
valid_losses = []
|
| 471 |
+
for subset_idx, subset in enumerate(subsets):
|
| 472 |
+
logger.info('begin validation on "{}" subset'.format(subset))
|
| 473 |
+
|
| 474 |
+
# Initialize data iterator
|
| 475 |
+
itr = trainer.get_valid_iterator(subset).next_epoch_itr(
|
| 476 |
+
shuffle=False, set_dataset_epoch=False # use a fixed valid set
|
| 477 |
+
)
|
| 478 |
+
if cfg.common.tpu:
|
| 479 |
+
itr = utils.tpu_data_loader(itr)
|
| 480 |
+
progress = progress_bar.progress_bar(
|
| 481 |
+
itr,
|
| 482 |
+
log_format=cfg.common.log_format,
|
| 483 |
+
log_interval=cfg.common.log_interval,
|
| 484 |
+
epoch=epoch_itr.epoch,
|
| 485 |
+
prefix=f"valid on '{subset}' subset",
|
| 486 |
+
aim_repo=(
|
| 487 |
+
cfg.common.aim_repo
|
| 488 |
+
if distributed_utils.is_master(cfg.distributed_training)
|
| 489 |
+
else None
|
| 490 |
+
),
|
| 491 |
+
aim_run_hash=(
|
| 492 |
+
cfg.common.aim_run_hash
|
| 493 |
+
if distributed_utils.is_master(cfg.distributed_training)
|
| 494 |
+
else None
|
| 495 |
+
),
|
| 496 |
+
aim_param_checkpoint_dir=cfg.checkpoint.save_dir,
|
| 497 |
+
tensorboard_logdir=(
|
| 498 |
+
cfg.common.tensorboard_logdir
|
| 499 |
+
if distributed_utils.is_master(cfg.distributed_training)
|
| 500 |
+
else None
|
| 501 |
+
),
|
| 502 |
+
default_log_format=("tqdm" if not cfg.common.no_progress_bar else "simple"),
|
| 503 |
+
wandb_project=(
|
| 504 |
+
cfg.common.wandb_project
|
| 505 |
+
if distributed_utils.is_master(cfg.distributed_training)
|
| 506 |
+
else None
|
| 507 |
+
),
|
| 508 |
+
wandb_run_name=os.environ.get(
|
| 509 |
+
"WANDB_NAME", os.path.basename(cfg.checkpoint.save_dir)
|
| 510 |
+
),
|
| 511 |
+
)
|
| 512 |
+
|
| 513 |
+
# create a new root metrics aggregator so validation metrics
|
| 514 |
+
# don't pollute other aggregators (e.g., train meters)
|
| 515 |
+
with metrics.aggregate(new_root=True) as agg:
|
| 516 |
+
for i, sample in enumerate(progress):
|
| 517 |
+
if (
|
| 518 |
+
cfg.dataset.max_valid_steps is not None
|
| 519 |
+
and i > cfg.dataset.max_valid_steps
|
| 520 |
+
):
|
| 521 |
+
break
|
| 522 |
+
trainer.valid_step(sample)
|
| 523 |
+
|
| 524 |
+
# log validation stats
|
| 525 |
+
# only tracking the best metric on the 1st validation subset
|
| 526 |
+
tracking_best = subset_idx == 0
|
| 527 |
+
stats = get_valid_stats(cfg, trainer, agg.get_smoothed_values(), tracking_best)
|
| 528 |
+
|
| 529 |
+
if hasattr(task, "post_validate"):
|
| 530 |
+
task.post_validate(trainer.get_model(), stats, agg)
|
| 531 |
+
|
| 532 |
+
progress.print(stats, tag=subset, step=trainer.get_num_updates())
|
| 533 |
+
|
| 534 |
+
valid_losses.append(stats[cfg.checkpoint.best_checkpoint_metric])
|
| 535 |
+
return valid_losses
|
| 536 |
+
|
| 537 |
+
|
| 538 |
+
def get_valid_stats(
|
| 539 |
+
cfg: DictConfig,
|
| 540 |
+
trainer: Trainer,
|
| 541 |
+
stats: Dict[str, Any],
|
| 542 |
+
tracking_best: bool,
|
| 543 |
+
) -> Dict[str, Any]:
|
| 544 |
+
stats["num_updates"] = trainer.get_num_updates()
|
| 545 |
+
if tracking_best and hasattr(checkpoint_utils.save_checkpoint, "best"):
|
| 546 |
+
key = "best_{0}".format(cfg.checkpoint.best_checkpoint_metric)
|
| 547 |
+
best_function = max if cfg.checkpoint.maximize_best_checkpoint_metric else min
|
| 548 |
+
stats[key] = best_function(
|
| 549 |
+
checkpoint_utils.save_checkpoint.best,
|
| 550 |
+
stats[cfg.checkpoint.best_checkpoint_metric],
|
| 551 |
+
)
|
| 552 |
+
return stats
|
| 553 |
+
|
| 554 |
+
|
| 555 |
+
def cli_main(
|
| 556 |
+
modify_parser: Optional[Callable[[argparse.ArgumentParser], None]] = None
|
| 557 |
+
) -> None:
|
| 558 |
+
parser = options.get_training_parser()
|
| 559 |
+
args = options.parse_args_and_arch(parser, modify_parser=modify_parser)
|
| 560 |
+
|
| 561 |
+
cfg = convert_namespace_to_omegaconf(args)
|
| 562 |
+
|
| 563 |
+
if cfg.common.use_plasma_view:
|
| 564 |
+
server = PlasmaStore(path=cfg.common.plasma_path)
|
| 565 |
+
logger.info(
|
| 566 |
+
f"Started plasma server pid {server.server.pid} {cfg.common.plasma_path}"
|
| 567 |
+
)
|
| 568 |
+
|
| 569 |
+
if args.profile:
|
| 570 |
+
with torch.cuda.profiler.profile():
|
| 571 |
+
with torch.autograd.profiler.emit_nvtx():
|
| 572 |
+
distributed_utils.call_main(cfg, main)
|
| 573 |
+
else:
|
| 574 |
+
distributed_utils.call_main(cfg, main)
|
| 575 |
+
|
| 576 |
+
# if cfg.common.use_plasma_view:
|
| 577 |
+
# server.server.kill()
|
| 578 |
+
|
| 579 |
+
|
| 580 |
+
if __name__ == "__main__":
|
| 581 |
+
cli_main()
|
data/fairseq/fairseq_cli/validate.py
ADDED
|
@@ -0,0 +1,153 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python3 -u
|
| 2 |
+
# Copyright (c) Facebook, Inc. and its affiliates.
|
| 3 |
+
#
|
| 4 |
+
# This source code is licensed under the MIT license found in the
|
| 5 |
+
# LICENSE file in the root directory of this source tree.
|
| 6 |
+
|
| 7 |
+
import logging
|
| 8 |
+
import os
|
| 9 |
+
import sys
|
| 10 |
+
from argparse import Namespace
|
| 11 |
+
from itertools import chain
|
| 12 |
+
|
| 13 |
+
import torch
|
| 14 |
+
from omegaconf import DictConfig
|
| 15 |
+
|
| 16 |
+
from fairseq import checkpoint_utils, distributed_utils, options, utils
|
| 17 |
+
from fairseq.dataclass.utils import convert_namespace_to_omegaconf
|
| 18 |
+
from fairseq.logging import metrics, progress_bar
|
| 19 |
+
from fairseq.utils import reset_logging
|
| 20 |
+
|
| 21 |
+
logging.basicConfig(
|
| 22 |
+
format="%(asctime)s | %(levelname)s | %(name)s | %(message)s",
|
| 23 |
+
datefmt="%Y-%m-%d %H:%M:%S",
|
| 24 |
+
level=os.environ.get("LOGLEVEL", "INFO").upper(),
|
| 25 |
+
stream=sys.stdout,
|
| 26 |
+
)
|
| 27 |
+
logger = logging.getLogger("fairseq_cli.validate")
|
| 28 |
+
|
| 29 |
+
|
| 30 |
+
def main(cfg: DictConfig, override_args=None):
|
| 31 |
+
if isinstance(cfg, Namespace):
|
| 32 |
+
cfg = convert_namespace_to_omegaconf(cfg)
|
| 33 |
+
|
| 34 |
+
utils.import_user_module(cfg.common)
|
| 35 |
+
|
| 36 |
+
reset_logging()
|
| 37 |
+
|
| 38 |
+
assert (
|
| 39 |
+
cfg.dataset.max_tokens is not None or cfg.dataset.batch_size is not None
|
| 40 |
+
), "Must specify batch size either with --max-tokens or --batch-size"
|
| 41 |
+
|
| 42 |
+
use_fp16 = cfg.common.fp16
|
| 43 |
+
use_cuda = torch.cuda.is_available() and not cfg.common.cpu
|
| 44 |
+
|
| 45 |
+
if use_cuda:
|
| 46 |
+
torch.cuda.set_device(cfg.distributed_training.device_id)
|
| 47 |
+
|
| 48 |
+
if cfg.distributed_training.distributed_world_size > 1:
|
| 49 |
+
data_parallel_world_size = distributed_utils.get_data_parallel_world_size()
|
| 50 |
+
data_parallel_rank = distributed_utils.get_data_parallel_rank()
|
| 51 |
+
else:
|
| 52 |
+
data_parallel_world_size = 1
|
| 53 |
+
data_parallel_rank = 0
|
| 54 |
+
|
| 55 |
+
if override_args is not None:
|
| 56 |
+
overrides = vars(override_args)
|
| 57 |
+
overrides.update(eval(getattr(override_args, "model_overrides", "{}")))
|
| 58 |
+
else:
|
| 59 |
+
overrides = None
|
| 60 |
+
|
| 61 |
+
# Load ensemble
|
| 62 |
+
logger.info("loading model(s) from {}".format(cfg.common_eval.path))
|
| 63 |
+
models, saved_cfg, task = checkpoint_utils.load_model_ensemble_and_task(
|
| 64 |
+
[cfg.common_eval.path],
|
| 65 |
+
arg_overrides=overrides,
|
| 66 |
+
suffix=cfg.checkpoint.checkpoint_suffix,
|
| 67 |
+
)
|
| 68 |
+
model = models[0]
|
| 69 |
+
|
| 70 |
+
# Move models to GPU
|
| 71 |
+
for model in models:
|
| 72 |
+
model.eval()
|
| 73 |
+
if use_fp16:
|
| 74 |
+
model.half()
|
| 75 |
+
if use_cuda:
|
| 76 |
+
model.cuda()
|
| 77 |
+
|
| 78 |
+
# Print args
|
| 79 |
+
logger.info(saved_cfg)
|
| 80 |
+
|
| 81 |
+
# Build criterion
|
| 82 |
+
criterion = task.build_criterion(saved_cfg.criterion)
|
| 83 |
+
criterion.eval()
|
| 84 |
+
|
| 85 |
+
for subset in cfg.dataset.valid_subset.split(","):
|
| 86 |
+
try:
|
| 87 |
+
task.load_dataset(subset, combine=False, epoch=1, task_cfg=saved_cfg.task)
|
| 88 |
+
dataset = task.dataset(subset)
|
| 89 |
+
except KeyError:
|
| 90 |
+
raise Exception("Cannot find dataset: " + subset)
|
| 91 |
+
|
| 92 |
+
# Initialize data iterator
|
| 93 |
+
itr = task.get_batch_iterator(
|
| 94 |
+
dataset=dataset,
|
| 95 |
+
max_tokens=cfg.dataset.max_tokens,
|
| 96 |
+
max_sentences=cfg.dataset.batch_size,
|
| 97 |
+
max_positions=utils.resolve_max_positions(
|
| 98 |
+
task.max_positions(),
|
| 99 |
+
*[m.max_positions() for m in models],
|
| 100 |
+
),
|
| 101 |
+
ignore_invalid_inputs=cfg.dataset.skip_invalid_size_inputs_valid_test,
|
| 102 |
+
required_batch_size_multiple=cfg.dataset.required_batch_size_multiple,
|
| 103 |
+
seed=cfg.common.seed,
|
| 104 |
+
num_shards=data_parallel_world_size,
|
| 105 |
+
shard_id=data_parallel_rank,
|
| 106 |
+
num_workers=cfg.dataset.num_workers,
|
| 107 |
+
data_buffer_size=cfg.dataset.data_buffer_size,
|
| 108 |
+
).next_epoch_itr(shuffle=False)
|
| 109 |
+
progress = progress_bar.progress_bar(
|
| 110 |
+
itr,
|
| 111 |
+
log_format=cfg.common.log_format,
|
| 112 |
+
log_interval=cfg.common.log_interval,
|
| 113 |
+
prefix=f"valid on '{subset}' subset",
|
| 114 |
+
default_log_format=("tqdm" if not cfg.common.no_progress_bar else "simple"),
|
| 115 |
+
)
|
| 116 |
+
|
| 117 |
+
log_outputs = []
|
| 118 |
+
for i, sample in enumerate(progress):
|
| 119 |
+
sample = utils.move_to_cuda(sample) if use_cuda else sample
|
| 120 |
+
_loss, _sample_size, log_output = task.valid_step(sample, model, criterion)
|
| 121 |
+
progress.log(log_output, step=i)
|
| 122 |
+
log_outputs.append(log_output)
|
| 123 |
+
|
| 124 |
+
if data_parallel_world_size > 1:
|
| 125 |
+
log_outputs = distributed_utils.all_gather_list(
|
| 126 |
+
log_outputs,
|
| 127 |
+
max_size=cfg.common.all_gather_list_size,
|
| 128 |
+
group=distributed_utils.get_data_parallel_group(),
|
| 129 |
+
)
|
| 130 |
+
log_outputs = list(chain.from_iterable(log_outputs))
|
| 131 |
+
|
| 132 |
+
with metrics.aggregate() as agg:
|
| 133 |
+
task.reduce_metrics(log_outputs, criterion)
|
| 134 |
+
log_output = agg.get_smoothed_values()
|
| 135 |
+
|
| 136 |
+
progress.print(log_output, tag=subset, step=i)
|
| 137 |
+
|
| 138 |
+
|
| 139 |
+
def cli_main():
|
| 140 |
+
parser = options.get_validation_parser()
|
| 141 |
+
args = options.parse_args_and_arch(parser)
|
| 142 |
+
|
| 143 |
+
# only override args that are explicitly given on the command line
|
| 144 |
+
override_parser = options.get_validation_parser()
|
| 145 |
+
override_args = options.parse_args_and_arch(override_parser, suppress_defaults=True)
|
| 146 |
+
|
| 147 |
+
distributed_utils.call_main(
|
| 148 |
+
convert_namespace_to_omegaconf(args), main, override_args=override_args
|
| 149 |
+
)
|
| 150 |
+
|
| 151 |
+
|
| 152 |
+
if __name__ == "__main__":
|
| 153 |
+
cli_main()
|
data/fairseq/tests/distributed/__init__.py
ADDED
|
File without changes
|
data/fairseq/tests/distributed/test_bmuf.py
ADDED
|
@@ -0,0 +1,204 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) Facebook, Inc. and its affiliates.
|
| 2 |
+
#
|
| 3 |
+
# This source code is licensed under the MIT license found in the
|
| 4 |
+
# LICENSE file in the root directory of this source tree.
|
| 5 |
+
|
| 6 |
+
import argparse
|
| 7 |
+
import functools
|
| 8 |
+
import random
|
| 9 |
+
import unittest
|
| 10 |
+
from multiprocessing import Manager
|
| 11 |
+
|
| 12 |
+
import torch
|
| 13 |
+
import torch.nn as nn
|
| 14 |
+
from omegaconf import OmegaConf
|
| 15 |
+
|
| 16 |
+
from fairseq import optim
|
| 17 |
+
from fairseq.distributed import utils as distributed_utils
|
| 18 |
+
|
| 19 |
+
|
| 20 |
+
class Model(nn.Module):
|
| 21 |
+
def __init__(self, input_size, output_size):
|
| 22 |
+
super(Model, self).__init__()
|
| 23 |
+
self.fc = nn.Linear(input_size, output_size)
|
| 24 |
+
|
| 25 |
+
def forward(self, input):
|
| 26 |
+
output = self.fc(input)
|
| 27 |
+
return output
|
| 28 |
+
|
| 29 |
+
|
| 30 |
+
def setup_model_loss_criterion(cfg, args, rank, is_cuda):
|
| 31 |
+
"""
|
| 32 |
+
setup model, criterion and optimizer based on input args
|
| 33 |
+
"""
|
| 34 |
+
args.distributed_rank = rank
|
| 35 |
+
cfg.distributed_training.distributed_rank = args.distributed_rank
|
| 36 |
+
if cfg.distributed_training.distributed_world_size > 1:
|
| 37 |
+
distributed_utils.distributed_init(cfg)
|
| 38 |
+
torch.manual_seed(1)
|
| 39 |
+
model = Model(args.input_size, args.nb_classes)
|
| 40 |
+
loss_fn = nn.CrossEntropyLoss()
|
| 41 |
+
if is_cuda:
|
| 42 |
+
model = model.cuda()
|
| 43 |
+
loss_fn = loss_fn.cuda()
|
| 44 |
+
|
| 45 |
+
optimizer = optim.sgd.SGD(args, model.parameters())
|
| 46 |
+
optimizer = optim.FairseqBMUF(cfg=cfg.bmuf, optimizer=optimizer)
|
| 47 |
+
|
| 48 |
+
return model, loss_fn, optimizer
|
| 49 |
+
|
| 50 |
+
|
| 51 |
+
def train_step(input, target, model, loss_fn, optimizer, **unused):
|
| 52 |
+
"""Do forward, backward and parameter update."""
|
| 53 |
+
model.train()
|
| 54 |
+
output = model(input)
|
| 55 |
+
loss = loss_fn(output, target)
|
| 56 |
+
optimizer.backward(loss)
|
| 57 |
+
optimizer.step()
|
| 58 |
+
|
| 59 |
+
|
| 60 |
+
def single_gpu_training(cfg, args, rank, iterations, shared_results):
|
| 61 |
+
|
| 62 |
+
is_cuda = torch.cuda.is_available()
|
| 63 |
+
if is_cuda:
|
| 64 |
+
torch.cuda.set_device(rank)
|
| 65 |
+
|
| 66 |
+
model, loss_fn, optimizer = setup_model_loss_criterion(cfg, args, rank, is_cuda)
|
| 67 |
+
|
| 68 |
+
for _ in range(iterations):
|
| 69 |
+
input = torch.randn(1, args.input_size)
|
| 70 |
+
target = torch.empty(args.batch_size, dtype=torch.long).random_(args.nb_classes)
|
| 71 |
+
|
| 72 |
+
if is_cuda:
|
| 73 |
+
input = input.cuda()
|
| 74 |
+
target = target.cuda()
|
| 75 |
+
train_step(input, target, model, loss_fn, optimizer)
|
| 76 |
+
|
| 77 |
+
results = []
|
| 78 |
+
for param in model.parameters():
|
| 79 |
+
if len(results) == 0:
|
| 80 |
+
results = param.flatten().cpu().data
|
| 81 |
+
else:
|
| 82 |
+
results = torch.cat((results, param.flatten().cpu().data), 0)
|
| 83 |
+
|
| 84 |
+
shared_results[rank] = results
|
| 85 |
+
|
| 86 |
+
|
| 87 |
+
def setup_args():
|
| 88 |
+
args = argparse.Namespace()
|
| 89 |
+
args.global_sync_iter = 20
|
| 90 |
+
args.block_momentum = 0.875
|
| 91 |
+
args.block_lr = 0.5
|
| 92 |
+
args.input_size = 5
|
| 93 |
+
args.nb_classes = 2
|
| 94 |
+
args.batch_size = 1
|
| 95 |
+
args.lr = [1e-3]
|
| 96 |
+
args.momentum = 0
|
| 97 |
+
args.weight_decay = 0
|
| 98 |
+
args.warmup_iterations = 0
|
| 99 |
+
args.use_nbm = True
|
| 100 |
+
args.average_sync = True
|
| 101 |
+
args.global_sync_iter = 1
|
| 102 |
+
args.model_parallel_size = 1
|
| 103 |
+
args.distributed_backend = "gloo"
|
| 104 |
+
|
| 105 |
+
args.distributed_world_size = 2
|
| 106 |
+
port = random.randint(10000, 20000)
|
| 107 |
+
args.distributed_init_method = "tcp://localhost:{port}".format(port=port)
|
| 108 |
+
args.distributed_init_host = "localhost"
|
| 109 |
+
args.distributed_port = port + 1
|
| 110 |
+
args.local_world_size = args.distributed_world_size
|
| 111 |
+
|
| 112 |
+
cfg = OmegaConf.create()
|
| 113 |
+
cfg.optimization = OmegaConf.create()
|
| 114 |
+
cfg.common = OmegaConf.create()
|
| 115 |
+
cfg.distributed_training = OmegaConf.create()
|
| 116 |
+
cfg.dataset = OmegaConf.create()
|
| 117 |
+
cfg.bmuf = OmegaConf.create()
|
| 118 |
+
cfg.optimizer = OmegaConf.create()
|
| 119 |
+
|
| 120 |
+
cfg.bmuf.global_sync_iter = args.global_sync_iter
|
| 121 |
+
cfg.bmuf.block_momentum = args.block_momentum
|
| 122 |
+
cfg.bmuf.block_lr = args.block_lr
|
| 123 |
+
cfg.dataset.batch_size = args.batch_size
|
| 124 |
+
cfg.optimization.lr = args.lr
|
| 125 |
+
cfg.optimizer.momentum = args.momentum
|
| 126 |
+
cfg.optimizer.weight_decay = args.weight_decay
|
| 127 |
+
cfg.bmuf.warmup_iterations = args.warmup_iterations
|
| 128 |
+
cfg.bmuf.use_nbm = args.use_nbm
|
| 129 |
+
cfg.bmuf.average_sync = args.average_sync
|
| 130 |
+
cfg.common.model_parallel_size = args.model_parallel_size
|
| 131 |
+
cfg.distributed_training.distributed_backend = args.distributed_backend
|
| 132 |
+
cfg.distributed_training.distributed_world_size = args.distributed_world_size
|
| 133 |
+
cfg.bmuf.distributed_world_size = args.distributed_world_size
|
| 134 |
+
cfg.distributed_training.distributed_init_method = args.distributed_init_method
|
| 135 |
+
cfg.distributed_training.distributed_port = args.distributed_port
|
| 136 |
+
|
| 137 |
+
return cfg, args
|
| 138 |
+
|
| 139 |
+
|
| 140 |
+
@unittest.skipIf(torch.cuda.device_count() < 2, "test requires 2 GPUs")
|
| 141 |
+
class TestBMUF(unittest.TestCase):
|
| 142 |
+
def bmuf_process(self, cfg, args, iterations):
|
| 143 |
+
results = Manager().dict()
|
| 144 |
+
torch.multiprocessing.spawn(
|
| 145 |
+
fn=functools.partial(single_gpu_training, cfg, args),
|
| 146 |
+
args=(iterations, results),
|
| 147 |
+
nprocs=args.distributed_world_size,
|
| 148 |
+
join=True,
|
| 149 |
+
)
|
| 150 |
+
return results
|
| 151 |
+
|
| 152 |
+
def test_bmuf_sync(self):
|
| 153 |
+
# Train model for 1 iteration and do bmuf sync without doing warmup
|
| 154 |
+
cfg, args = setup_args()
|
| 155 |
+
iterations = 1
|
| 156 |
+
results = self.bmuf_process(cfg, args, iterations)
|
| 157 |
+
# Make sure params in both machines are same
|
| 158 |
+
assert len(results) == 2
|
| 159 |
+
self.assertAlmostEqual(results[0], results[1])
|
| 160 |
+
|
| 161 |
+
def test_warmup_sync(self):
|
| 162 |
+
# Train model for 20 iteration and do warmup sync without doing bmuf sync
|
| 163 |
+
cfg, args = setup_args()
|
| 164 |
+
args.warmup_iterations = 20
|
| 165 |
+
cfg.bmuf.warmup_iterations = args.warmup_iterations
|
| 166 |
+
iterations = 20
|
| 167 |
+
results = self.bmuf_process(cfg, args, iterations)
|
| 168 |
+
# Make sure params in both machines are same
|
| 169 |
+
assert len(results) == 2
|
| 170 |
+
self.assertAlmostEqual(results[0], results[1])
|
| 171 |
+
|
| 172 |
+
def test_warmup_sync_bmuf_sync(self):
|
| 173 |
+
# Train model for 25 iteration and do warmup sync after 20 iteration
|
| 174 |
+
# and bmuf sync after 25 iteration
|
| 175 |
+
cfg, args = setup_args()
|
| 176 |
+
args.warmup_iterations = 20
|
| 177 |
+
args.global_sync_iter = 5
|
| 178 |
+
cfg.bmuf.warmup_iterations = args.warmup_iterations
|
| 179 |
+
cfg.bmuf.global_sync_iter = args.global_sync_iter
|
| 180 |
+
iterations = 25
|
| 181 |
+
results = self.bmuf_process(cfg, args, iterations)
|
| 182 |
+
# Make sure params in both machines are same
|
| 183 |
+
assert len(results) == 2
|
| 184 |
+
self.assertAlmostEqual(results[0], results[1])
|
| 185 |
+
|
| 186 |
+
def test_single_gpu_bmuf(self):
|
| 187 |
+
# Train model for 5 iterations and use GPU 1
|
| 188 |
+
cfg, args = setup_args()
|
| 189 |
+
args.distributed_world_size = 1
|
| 190 |
+
args.warmup_iterations = 5
|
| 191 |
+
cfg.distributed_training.distributed_world_size = args.distributed_world_size
|
| 192 |
+
cfg.bmuf.distributed_world_size = args.distributed_world_size
|
| 193 |
+
cfg.bmuf.warmup_iterations = args.warmup_iterations
|
| 194 |
+
iterations = 20
|
| 195 |
+
results = self.bmuf_process(cfg, args, iterations)
|
| 196 |
+
assert len(results) == 1
|
| 197 |
+
|
| 198 |
+
def assertAlmostEqual(self, t1, t2):
|
| 199 |
+
self.assertEqual(t1.size(), t2.size(), "size mismatch")
|
| 200 |
+
self.assertLess((t1 - t2).abs().max(), 1e-4)
|
| 201 |
+
|
| 202 |
+
|
| 203 |
+
if __name__ == "__main__":
|
| 204 |
+
unittest.main()
|
data/fairseq/tests/distributed/test_distributed_timeout_wrapper.py
ADDED
|
@@ -0,0 +1,52 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) Facebook, Inc. and its affiliates.
|
| 2 |
+
#
|
| 3 |
+
# This source code is licensed under the MIT license found in the
|
| 4 |
+
# LICENSE file in the root directory of this source tree.
|
| 5 |
+
|
| 6 |
+
import logging
|
| 7 |
+
import signal
|
| 8 |
+
import time
|
| 9 |
+
import unittest
|
| 10 |
+
|
| 11 |
+
import torch
|
| 12 |
+
from torch import nn
|
| 13 |
+
|
| 14 |
+
from fairseq.distributed import DistributedTimeoutWrapper
|
| 15 |
+
|
| 16 |
+
|
| 17 |
+
class ModuleWithDelay(nn.Module):
|
| 18 |
+
def __init__(self, delay):
|
| 19 |
+
super().__init__()
|
| 20 |
+
self.delay = delay
|
| 21 |
+
|
| 22 |
+
def forward(self, x):
|
| 23 |
+
time.sleep(self.delay)
|
| 24 |
+
return x
|
| 25 |
+
|
| 26 |
+
|
| 27 |
+
class TestDistributedTimeoutWrapper(unittest.TestCase):
|
| 28 |
+
def setUp(self):
|
| 29 |
+
logging.disable(logging.CRITICAL)
|
| 30 |
+
|
| 31 |
+
def tearDown(self):
|
| 32 |
+
logging.disable(logging.NOTSET)
|
| 33 |
+
|
| 34 |
+
def test_no_timeout(self):
|
| 35 |
+
module = DistributedTimeoutWrapper(ModuleWithDelay(1), 0, signal.SIGINT)
|
| 36 |
+
module(torch.rand(5))
|
| 37 |
+
module.stop_timeout()
|
| 38 |
+
|
| 39 |
+
def test_timeout_safe(self):
|
| 40 |
+
module = DistributedTimeoutWrapper(ModuleWithDelay(1), 10, signal.SIGINT)
|
| 41 |
+
module(torch.rand(5))
|
| 42 |
+
module.stop_timeout()
|
| 43 |
+
|
| 44 |
+
def test_timeout_killed(self):
|
| 45 |
+
with self.assertRaises(KeyboardInterrupt):
|
| 46 |
+
module = DistributedTimeoutWrapper(ModuleWithDelay(5), 1, signal.SIGINT)
|
| 47 |
+
module(torch.rand(5))
|
| 48 |
+
module.stop_timeout()
|
| 49 |
+
|
| 50 |
+
|
| 51 |
+
if __name__ == "__main__":
|
| 52 |
+
unittest.main()
|
data/fairseq/tests/distributed/test_module_proxy_wrapper.py
ADDED
|
@@ -0,0 +1,74 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) Facebook, Inc. and its affiliates.
|
| 2 |
+
#
|
| 3 |
+
# This source code is licensed under the MIT license found in the
|
| 4 |
+
# LICENSE file in the root directory of this source tree.
|
| 5 |
+
|
| 6 |
+
import unittest
|
| 7 |
+
|
| 8 |
+
import torch
|
| 9 |
+
from torch import nn
|
| 10 |
+
|
| 11 |
+
from fairseq.distributed import ModuleProxyWrapper
|
| 12 |
+
|
| 13 |
+
from .utils import objects_are_equal
|
| 14 |
+
|
| 15 |
+
|
| 16 |
+
class MockDDPWrapper(nn.Module):
|
| 17 |
+
"""A simple wrapper with an interface similar to DistributedDataParallel."""
|
| 18 |
+
|
| 19 |
+
def __init__(self, module):
|
| 20 |
+
super().__init__()
|
| 21 |
+
self.module = module
|
| 22 |
+
|
| 23 |
+
def forward(self, x):
|
| 24 |
+
return self.module(x)
|
| 25 |
+
|
| 26 |
+
|
| 27 |
+
class Model(nn.Module):
|
| 28 |
+
def __init__(self):
|
| 29 |
+
super().__init__()
|
| 30 |
+
self.linear = nn.Linear(5, 10)
|
| 31 |
+
self.xyz = "hello"
|
| 32 |
+
|
| 33 |
+
def forward(self, x):
|
| 34 |
+
return self.linear(x)
|
| 35 |
+
|
| 36 |
+
def get_xyz(self):
|
| 37 |
+
return self.xyz
|
| 38 |
+
|
| 39 |
+
|
| 40 |
+
class TestModuleProxyWrapper(unittest.TestCase):
|
| 41 |
+
def _get_module(self):
|
| 42 |
+
module = Model()
|
| 43 |
+
wrapped_module = MockDDPWrapper(module)
|
| 44 |
+
wrapped_module = ModuleProxyWrapper(wrapped_module)
|
| 45 |
+
return wrapped_module, module
|
| 46 |
+
|
| 47 |
+
def test_getattr_forwarding(self):
|
| 48 |
+
wrapped_module, module = self._get_module()
|
| 49 |
+
assert module.xyz == "hello"
|
| 50 |
+
assert module.get_xyz() == "hello"
|
| 51 |
+
assert wrapped_module.xyz == "hello"
|
| 52 |
+
|
| 53 |
+
wrapped_module.xyz = "world"
|
| 54 |
+
assert wrapped_module.xyz == "world"
|
| 55 |
+
assert module.get_xyz() == "hello"
|
| 56 |
+
|
| 57 |
+
def test_state_dict(self):
|
| 58 |
+
wrapped_module, module = self._get_module()
|
| 59 |
+
assert objects_are_equal(wrapped_module.state_dict(), module.state_dict())
|
| 60 |
+
|
| 61 |
+
def test_load_state_dict(self):
|
| 62 |
+
wrapped_module, module = self._get_module()
|
| 63 |
+
wrapped_module.load_state_dict(module.state_dict())
|
| 64 |
+
input = torch.rand(4, 5)
|
| 65 |
+
torch.testing.assert_allclose(wrapped_module(input), module(input))
|
| 66 |
+
|
| 67 |
+
def test_forward(self):
|
| 68 |
+
wrapped_module, module = self._get_module()
|
| 69 |
+
input = torch.rand(4, 5)
|
| 70 |
+
torch.testing.assert_allclose(wrapped_module(input), module(input))
|
| 71 |
+
|
| 72 |
+
|
| 73 |
+
if __name__ == "__main__":
|
| 74 |
+
unittest.main()
|
data/fairseq/tests/distributed/test_utils.py
ADDED
|
@@ -0,0 +1,124 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) Facebook, Inc. and its affiliates.
|
| 2 |
+
#
|
| 3 |
+
# This source code is licensed under the MIT license found in the
|
| 4 |
+
# LICENSE file in the root directory of this source tree.
|
| 5 |
+
|
| 6 |
+
import functools
|
| 7 |
+
import sys
|
| 8 |
+
import unittest
|
| 9 |
+
|
| 10 |
+
import torch
|
| 11 |
+
|
| 12 |
+
from fairseq.distributed import utils as dist_utils
|
| 13 |
+
|
| 14 |
+
from .utils import objects_are_equal, spawn_and_init
|
| 15 |
+
|
| 16 |
+
|
| 17 |
+
class DistributedTest(unittest.TestCase):
|
| 18 |
+
def setUp(self):
|
| 19 |
+
if not torch.cuda.is_available():
|
| 20 |
+
raise unittest.SkipTest("CUDA not available, skipping test")
|
| 21 |
+
if sys.platform == "win32":
|
| 22 |
+
raise unittest.SkipTest("NCCL doesn't support Windows, skipping test")
|
| 23 |
+
if torch.cuda.device_count() < 2:
|
| 24 |
+
raise unittest.SkipTest("distributed tests require 2+ GPUs, skipping")
|
| 25 |
+
|
| 26 |
+
|
| 27 |
+
class TestBroadcastObject(DistributedTest):
|
| 28 |
+
def test_str(self):
|
| 29 |
+
spawn_and_init(
|
| 30 |
+
functools.partial(
|
| 31 |
+
TestBroadcastObject._test_broadcast_object, "hello world"
|
| 32 |
+
),
|
| 33 |
+
world_size=2,
|
| 34 |
+
)
|
| 35 |
+
|
| 36 |
+
def test_tensor(self):
|
| 37 |
+
spawn_and_init(
|
| 38 |
+
functools.partial(
|
| 39 |
+
TestBroadcastObject._test_broadcast_object,
|
| 40 |
+
torch.rand(5),
|
| 41 |
+
),
|
| 42 |
+
world_size=2,
|
| 43 |
+
)
|
| 44 |
+
|
| 45 |
+
def test_complex(self):
|
| 46 |
+
spawn_and_init(
|
| 47 |
+
functools.partial(
|
| 48 |
+
TestBroadcastObject._test_broadcast_object,
|
| 49 |
+
{
|
| 50 |
+
"a": "1",
|
| 51 |
+
"b": [2, torch.rand(2, 3), 3],
|
| 52 |
+
"c": (torch.rand(2, 3), 4),
|
| 53 |
+
"d": {5, torch.rand(5)},
|
| 54 |
+
"e": torch.rand(5),
|
| 55 |
+
"f": torch.rand(5).int().cuda(),
|
| 56 |
+
},
|
| 57 |
+
),
|
| 58 |
+
world_size=2,
|
| 59 |
+
)
|
| 60 |
+
|
| 61 |
+
@staticmethod
|
| 62 |
+
def _test_broadcast_object(ref_obj, rank, group):
|
| 63 |
+
obj = dist_utils.broadcast_object(
|
| 64 |
+
ref_obj if rank == 0 else None, src_rank=0, group=group
|
| 65 |
+
)
|
| 66 |
+
assert objects_are_equal(ref_obj, obj)
|
| 67 |
+
|
| 68 |
+
|
| 69 |
+
class TestAllGatherList(DistributedTest):
|
| 70 |
+
def test_str_equality(self):
|
| 71 |
+
spawn_and_init(
|
| 72 |
+
functools.partial(
|
| 73 |
+
TestAllGatherList._test_all_gather_list_equality,
|
| 74 |
+
"hello world",
|
| 75 |
+
),
|
| 76 |
+
world_size=2,
|
| 77 |
+
)
|
| 78 |
+
|
| 79 |
+
def test_tensor_equality(self):
|
| 80 |
+
spawn_and_init(
|
| 81 |
+
functools.partial(
|
| 82 |
+
TestAllGatherList._test_all_gather_list_equality,
|
| 83 |
+
torch.rand(5),
|
| 84 |
+
),
|
| 85 |
+
world_size=2,
|
| 86 |
+
)
|
| 87 |
+
|
| 88 |
+
def test_complex_equality(self):
|
| 89 |
+
spawn_and_init(
|
| 90 |
+
functools.partial(
|
| 91 |
+
TestAllGatherList._test_all_gather_list_equality,
|
| 92 |
+
{
|
| 93 |
+
"a": "1",
|
| 94 |
+
"b": [2, torch.rand(2, 3), 3],
|
| 95 |
+
"c": (torch.rand(2, 3), 4),
|
| 96 |
+
"d": {5, torch.rand(5)},
|
| 97 |
+
"e": torch.rand(5),
|
| 98 |
+
"f": torch.rand(5).int(),
|
| 99 |
+
},
|
| 100 |
+
),
|
| 101 |
+
world_size=2,
|
| 102 |
+
)
|
| 103 |
+
|
| 104 |
+
@staticmethod
|
| 105 |
+
def _test_all_gather_list_equality(ref_obj, rank, group):
|
| 106 |
+
objs = dist_utils.all_gather_list(ref_obj, group)
|
| 107 |
+
for obj in objs:
|
| 108 |
+
assert objects_are_equal(ref_obj, obj)
|
| 109 |
+
|
| 110 |
+
def test_rank_tensor(self):
|
| 111 |
+
spawn_and_init(
|
| 112 |
+
TestAllGatherList._test_all_gather_list_rank_tensor, world_size=2
|
| 113 |
+
)
|
| 114 |
+
|
| 115 |
+
@staticmethod
|
| 116 |
+
def _test_all_gather_list_rank_tensor(rank, group):
|
| 117 |
+
obj = torch.tensor([rank])
|
| 118 |
+
objs = dist_utils.all_gather_list(obj, group)
|
| 119 |
+
for i, obj in enumerate(objs):
|
| 120 |
+
assert obj.item() == i
|
| 121 |
+
|
| 122 |
+
|
| 123 |
+
if __name__ == "__main__":
|
| 124 |
+
unittest.main()
|
data/fairseq/tests/distributed/utils.py
ADDED
|
@@ -0,0 +1,65 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) Facebook, Inc. and its affiliates.
|
| 2 |
+
#
|
| 3 |
+
# This source code is licensed under the MIT license found in the
|
| 4 |
+
# LICENSE file in the root directory of this source tree.
|
| 5 |
+
|
| 6 |
+
import functools
|
| 7 |
+
import tempfile
|
| 8 |
+
|
| 9 |
+
import torch
|
| 10 |
+
|
| 11 |
+
|
| 12 |
+
def spawn_and_init(fn, world_size, args=None):
|
| 13 |
+
if args is None:
|
| 14 |
+
args = ()
|
| 15 |
+
with tempfile.NamedTemporaryFile(delete=False) as tmp_file:
|
| 16 |
+
torch.multiprocessing.spawn(
|
| 17 |
+
fn=functools.partial(init_and_run, fn, args),
|
| 18 |
+
args=(
|
| 19 |
+
world_size,
|
| 20 |
+
tmp_file.name,
|
| 21 |
+
),
|
| 22 |
+
nprocs=world_size,
|
| 23 |
+
join=True,
|
| 24 |
+
)
|
| 25 |
+
|
| 26 |
+
|
| 27 |
+
def distributed_init(rank, world_size, tmp_file):
|
| 28 |
+
torch.distributed.init_process_group(
|
| 29 |
+
backend="nccl",
|
| 30 |
+
init_method="file://{}".format(tmp_file),
|
| 31 |
+
world_size=world_size,
|
| 32 |
+
rank=rank,
|
| 33 |
+
)
|
| 34 |
+
torch.cuda.set_device(rank)
|
| 35 |
+
|
| 36 |
+
|
| 37 |
+
def init_and_run(fn, args, rank, world_size, tmp_file):
|
| 38 |
+
distributed_init(rank, world_size, tmp_file)
|
| 39 |
+
group = torch.distributed.new_group()
|
| 40 |
+
fn(rank, group, *args)
|
| 41 |
+
|
| 42 |
+
|
| 43 |
+
def objects_are_equal(a, b) -> bool:
|
| 44 |
+
if type(a) is not type(b):
|
| 45 |
+
return False
|
| 46 |
+
if isinstance(a, dict):
|
| 47 |
+
if set(a.keys()) != set(b.keys()):
|
| 48 |
+
return False
|
| 49 |
+
for k in a.keys():
|
| 50 |
+
if not objects_are_equal(a[k], b[k]):
|
| 51 |
+
return False
|
| 52 |
+
return True
|
| 53 |
+
elif isinstance(a, (list, tuple, set)):
|
| 54 |
+
if len(a) != len(b):
|
| 55 |
+
return False
|
| 56 |
+
return all(objects_are_equal(x, y) for x, y in zip(a, b))
|
| 57 |
+
elif torch.is_tensor(a):
|
| 58 |
+
return (
|
| 59 |
+
a.size() == b.size()
|
| 60 |
+
and a.dtype == b.dtype
|
| 61 |
+
and a.device == b.device
|
| 62 |
+
and torch.all(a == b)
|
| 63 |
+
)
|
| 64 |
+
else:
|
| 65 |
+
return a == b
|
data/fairseq/tests/gpu/__init__.py
ADDED
|
File without changes
|
data/fairseq/tests/gpu/test_binaries_gpu.py
ADDED
|
@@ -0,0 +1,590 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) Facebook, Inc. and its affiliates.
|
| 2 |
+
#
|
| 3 |
+
# This source code is licensed under the MIT license found in the
|
| 4 |
+
# LICENSE file in the root directory of this source tree.
|
| 5 |
+
|
| 6 |
+
import contextlib
|
| 7 |
+
import json
|
| 8 |
+
import logging
|
| 9 |
+
import os
|
| 10 |
+
import tempfile
|
| 11 |
+
import unittest
|
| 12 |
+
from io import StringIO
|
| 13 |
+
|
| 14 |
+
import torch
|
| 15 |
+
|
| 16 |
+
from fairseq import options
|
| 17 |
+
from fairseq_cli import train
|
| 18 |
+
from tests.utils import (
|
| 19 |
+
create_dummy_data,
|
| 20 |
+
generate_main,
|
| 21 |
+
preprocess_lm_data,
|
| 22 |
+
preprocess_translation_data,
|
| 23 |
+
train_language_model,
|
| 24 |
+
train_translation_model,
|
| 25 |
+
)
|
| 26 |
+
|
| 27 |
+
|
| 28 |
+
@unittest.skipIf(not torch.cuda.is_available(), "test requires a GPU")
|
| 29 |
+
class TestMultiGPU(unittest.TestCase):
|
| 30 |
+
@staticmethod
|
| 31 |
+
def parse_logs(logfile):
|
| 32 |
+
logs = []
|
| 33 |
+
for ln in open(logfile, "r").readlines():
|
| 34 |
+
try:
|
| 35 |
+
logs.append(json.loads(ln))
|
| 36 |
+
except json.JSONDecodeError:
|
| 37 |
+
continue
|
| 38 |
+
return logs
|
| 39 |
+
|
| 40 |
+
@property
|
| 41 |
+
def world_size(self):
|
| 42 |
+
return torch.cuda.device_count()
|
| 43 |
+
|
| 44 |
+
def train_flags(self, mu):
|
| 45 |
+
return [
|
| 46 |
+
"--memory-efficient-fp16",
|
| 47 |
+
"--update-freq",
|
| 48 |
+
"1",
|
| 49 |
+
"--seed",
|
| 50 |
+
"1",
|
| 51 |
+
"--log-format",
|
| 52 |
+
"json",
|
| 53 |
+
"--max-update",
|
| 54 |
+
str(mu),
|
| 55 |
+
"--tokens-per-sample",
|
| 56 |
+
"20",
|
| 57 |
+
"--batch-size",
|
| 58 |
+
"2",
|
| 59 |
+
"--share-decoder-input-output-embed",
|
| 60 |
+
"--optimizer",
|
| 61 |
+
"adam",
|
| 62 |
+
"--max-valid-steps",
|
| 63 |
+
"1",
|
| 64 |
+
"--pad-to-fixed-length",
|
| 65 |
+
"--sample-break-mode",
|
| 66 |
+
"none",
|
| 67 |
+
]
|
| 68 |
+
|
| 69 |
+
def _test_resume_multilingual_training(
|
| 70 |
+
self, extra_clargs, arch="transformer_lm_gpt2_tiny"
|
| 71 |
+
):
|
| 72 |
+
languages = ["en_XX", "fr_XX", "zh_CN"]
|
| 73 |
+
save_interval = 5
|
| 74 |
+
mu = 10
|
| 75 |
+
flags = (
|
| 76 |
+
self.train_flags(mu)
|
| 77 |
+
+ ["--save-interval-updates", str(save_interval), "--log-interval", "1"]
|
| 78 |
+
+ extra_clargs
|
| 79 |
+
)
|
| 80 |
+
with contextlib.redirect_stdout(StringIO()):
|
| 81 |
+
with tempfile.TemporaryDirectory("test_fp16") as data_dir:
|
| 82 |
+
log = os.path.join(data_dir, "train.log")
|
| 83 |
+
create_dummy_data(
|
| 84 |
+
data_dir,
|
| 85 |
+
num_examples=int(
|
| 86 |
+
mu * 20 * self.world_size * 1.5
|
| 87 |
+
), # make sure enough data for max updates
|
| 88 |
+
languages=languages,
|
| 89 |
+
)
|
| 90 |
+
preprocess_lm_data(data_dir, languages)
|
| 91 |
+
train_language_model(
|
| 92 |
+
data_dir,
|
| 93 |
+
arch,
|
| 94 |
+
flags + ["--log-file", log],
|
| 95 |
+
task="multilingual_language_modeling",
|
| 96 |
+
world_size=self.world_size,
|
| 97 |
+
)
|
| 98 |
+
log2 = os.path.join(data_dir, "resume.log")
|
| 99 |
+
ckpt_name = f"checkpoint_1_{save_interval}.pt"
|
| 100 |
+
restore_file = os.path.join(data_dir, ckpt_name)
|
| 101 |
+
train_language_model(
|
| 102 |
+
data_dir,
|
| 103 |
+
arch,
|
| 104 |
+
flags
|
| 105 |
+
+ ["--log-file", log2, "--restore-file", restore_file, "--no-save"],
|
| 106 |
+
task="multilingual_language_modeling",
|
| 107 |
+
world_size=self.world_size,
|
| 108 |
+
)
|
| 109 |
+
|
| 110 |
+
l1 = self.parse_logs(log)
|
| 111 |
+
assert (
|
| 112 |
+
int(l1[-1]["train_num_updates"]) == mu
|
| 113 |
+
), f"The first run did not complete {mu} updates. Add more data"
|
| 114 |
+
l2 = self.parse_logs(log2)
|
| 115 |
+
|
| 116 |
+
if int(l2[0]["num_updates"]) != save_interval + 1:
|
| 117 |
+
all_ckpt_files = [
|
| 118 |
+
x for x in os.listdir(data_dir) if x.endswith(".pt")
|
| 119 |
+
]
|
| 120 |
+
import shutil
|
| 121 |
+
|
| 122 |
+
shutil.move(data_dir, "last_failed_resume")
|
| 123 |
+
raise AssertionError(
|
| 124 |
+
f"Likely failed to load {ckpt_name}. {all_ckpt_files} \n LOGS: {l1} \n\n {l2}. "
|
| 125 |
+
)
|
| 126 |
+
for k in [
|
| 127 |
+
"train_loss",
|
| 128 |
+
"train_num_updates",
|
| 129 |
+
"train_ppl",
|
| 130 |
+
"train_gnorm",
|
| 131 |
+
]:
|
| 132 |
+
from_scratch, resumed = float(l1[-1][k]), float(l2[-1][k])
|
| 133 |
+
# This fails without rounding!
|
| 134 |
+
assert (
|
| 135 |
+
from_scratch == resumed
|
| 136 |
+
), f"difference at {k} {from_scratch} != {resumed}"
|
| 137 |
+
|
| 138 |
+
|
| 139 |
+
@unittest.skipIf(not torch.cuda.is_available(), "test requires a GPU")
|
| 140 |
+
class TestTranslationGPU(unittest.TestCase):
|
| 141 |
+
def setUp(self):
|
| 142 |
+
logging.disable(logging.CRITICAL)
|
| 143 |
+
|
| 144 |
+
def tearDown(self):
|
| 145 |
+
logging.disable(logging.NOTSET)
|
| 146 |
+
|
| 147 |
+
def test_fp16_multigpu(self):
|
| 148 |
+
self._test_multigpu("test_fp16", ["--fp16"])
|
| 149 |
+
|
| 150 |
+
def test_slowmo_multigpu(self):
|
| 151 |
+
self._test_multigpu(
|
| 152 |
+
"test_slowmo", ["--ddp-backend", "slowmo", "--nprocs-per-node", "1"]
|
| 153 |
+
)
|
| 154 |
+
|
| 155 |
+
def test_slowmo_single_node_multigpu(self):
|
| 156 |
+
self._test_multigpu(
|
| 157 |
+
"test_slowmo_single_node",
|
| 158 |
+
["--ddp-backend", "slowmo", "--nprocs-per-node", "2"],
|
| 159 |
+
)
|
| 160 |
+
|
| 161 |
+
def _test_multigpu(self, test_name, test_args):
|
| 162 |
+
with contextlib.redirect_stdout(StringIO()):
|
| 163 |
+
with tempfile.TemporaryDirectory(test_name) as data_dir:
|
| 164 |
+
log = os.path.join(data_dir, "train.log")
|
| 165 |
+
create_dummy_data(data_dir)
|
| 166 |
+
preprocess_translation_data(data_dir)
|
| 167 |
+
train_translation_model(
|
| 168 |
+
data_dir,
|
| 169 |
+
"fconv_iwslt_de_en",
|
| 170 |
+
test_args + ["--log-file", log],
|
| 171 |
+
world_size=min(torch.cuda.device_count(), 2),
|
| 172 |
+
)
|
| 173 |
+
generate_main(data_dir)
|
| 174 |
+
assert os.path.exists(log)
|
| 175 |
+
|
| 176 |
+
@staticmethod
|
| 177 |
+
def parse_logs(logfile):
|
| 178 |
+
logs = []
|
| 179 |
+
for ln in open(logfile, "r").readlines():
|
| 180 |
+
try:
|
| 181 |
+
logs.append(json.loads(ln))
|
| 182 |
+
except json.JSONDecodeError:
|
| 183 |
+
continue
|
| 184 |
+
return logs
|
| 185 |
+
|
| 186 |
+
def test_resume_training_fsdp(self):
|
| 187 |
+
self._test_resume_training(["--ddp-backend", "fully_sharded"])
|
| 188 |
+
|
| 189 |
+
def test_resume_training_fsdp_sharded_state(self):
|
| 190 |
+
self._test_resume_training(
|
| 191 |
+
["--ddp-backend", "fully_sharded", "--use-sharded-state"]
|
| 192 |
+
)
|
| 193 |
+
|
| 194 |
+
def test_resume_training_noc10d(self):
|
| 195 |
+
self._test_resume_training([])
|
| 196 |
+
|
| 197 |
+
def _test_resume_training(self, extra_clargs, arch="fconv_iwslt_de_en"):
|
| 198 |
+
flags = [
|
| 199 |
+
"--fp16",
|
| 200 |
+
"--log-format",
|
| 201 |
+
"json",
|
| 202 |
+
"--max-update",
|
| 203 |
+
"10",
|
| 204 |
+
"--save-interval-updates",
|
| 205 |
+
"2",
|
| 206 |
+
"--log-interval",
|
| 207 |
+
"1",
|
| 208 |
+
] + extra_clargs
|
| 209 |
+
world_size = min(torch.cuda.device_count(), 2)
|
| 210 |
+
with contextlib.redirect_stdout(StringIO()):
|
| 211 |
+
with tempfile.TemporaryDirectory("test_fp16") as data_dir:
|
| 212 |
+
log = os.path.join(data_dir, "train.log")
|
| 213 |
+
create_dummy_data(data_dir)
|
| 214 |
+
preprocess_translation_data(data_dir)
|
| 215 |
+
train_translation_model(
|
| 216 |
+
data_dir,
|
| 217 |
+
arch,
|
| 218 |
+
flags + ["--log-file", log],
|
| 219 |
+
world_size=world_size,
|
| 220 |
+
)
|
| 221 |
+
log2 = os.path.join(data_dir, "resume.log")
|
| 222 |
+
restore_file = os.path.join(data_dir, "checkpoint_1_2.pt")
|
| 223 |
+
train_translation_model(
|
| 224 |
+
data_dir,
|
| 225 |
+
arch,
|
| 226 |
+
flags + ["--log-file", log2, "--restore-file", restore_file],
|
| 227 |
+
world_size=world_size,
|
| 228 |
+
)
|
| 229 |
+
|
| 230 |
+
l1 = self.parse_logs(log)
|
| 231 |
+
l2 = self.parse_logs(log2)
|
| 232 |
+
assert int(l2[0]["num_updates"]) == 3, f"{l1}\n\n {l2}"
|
| 233 |
+
for k in [
|
| 234 |
+
"train_loss",
|
| 235 |
+
"train_num_updates",
|
| 236 |
+
"train_ppl",
|
| 237 |
+
"train_gnorm",
|
| 238 |
+
]:
|
| 239 |
+
from_scratch, resumed = l1[-1][k], l2[-1][k]
|
| 240 |
+
assert (
|
| 241 |
+
from_scratch == resumed
|
| 242 |
+
), f"difference at {k} {from_scratch} != {resumed}"
|
| 243 |
+
|
| 244 |
+
def test_memory_efficient_fp16(self):
|
| 245 |
+
with contextlib.redirect_stdout(StringIO()):
|
| 246 |
+
with tempfile.TemporaryDirectory("test_memory_efficient_fp16") as data_dir:
|
| 247 |
+
create_dummy_data(data_dir)
|
| 248 |
+
preprocess_translation_data(data_dir)
|
| 249 |
+
train_translation_model(
|
| 250 |
+
data_dir, "fconv_iwslt_de_en", ["--memory-efficient-fp16"]
|
| 251 |
+
)
|
| 252 |
+
generate_main(data_dir)
|
| 253 |
+
|
| 254 |
+
def test_transformer_fp16(self):
|
| 255 |
+
with contextlib.redirect_stdout(StringIO()):
|
| 256 |
+
with tempfile.TemporaryDirectory("test_transformer") as data_dir:
|
| 257 |
+
create_dummy_data(data_dir)
|
| 258 |
+
preprocess_translation_data(data_dir)
|
| 259 |
+
train_translation_model(
|
| 260 |
+
data_dir,
|
| 261 |
+
"transformer_iwslt_de_en",
|
| 262 |
+
[
|
| 263 |
+
"--encoder-layers",
|
| 264 |
+
"2",
|
| 265 |
+
"--decoder-layers",
|
| 266 |
+
"2",
|
| 267 |
+
"--encoder-embed-dim",
|
| 268 |
+
"64",
|
| 269 |
+
"--decoder-embed-dim",
|
| 270 |
+
"64",
|
| 271 |
+
"--fp16",
|
| 272 |
+
],
|
| 273 |
+
run_validation=True,
|
| 274 |
+
)
|
| 275 |
+
generate_main(data_dir)
|
| 276 |
+
|
| 277 |
+
@unittest.skipIf(not torch.cuda.is_available(), "test requires a GPU")
|
| 278 |
+
def test_amp(self):
|
| 279 |
+
with contextlib.redirect_stdout(StringIO()):
|
| 280 |
+
with tempfile.TemporaryDirectory("test_amp") as data_dir:
|
| 281 |
+
create_dummy_data(data_dir)
|
| 282 |
+
preprocess_translation_data(data_dir)
|
| 283 |
+
train_translation_model(data_dir, "fconv_iwslt_de_en", ["--amp"])
|
| 284 |
+
generate_main(data_dir)
|
| 285 |
+
|
| 286 |
+
@unittest.skipIf(not torch.cuda.is_available(), "test requires a GPU")
|
| 287 |
+
def test_transformer_amp(self):
|
| 288 |
+
with contextlib.redirect_stdout(StringIO()):
|
| 289 |
+
with tempfile.TemporaryDirectory("test_transformer") as data_dir:
|
| 290 |
+
create_dummy_data(data_dir)
|
| 291 |
+
preprocess_translation_data(data_dir)
|
| 292 |
+
train_translation_model(
|
| 293 |
+
data_dir,
|
| 294 |
+
"transformer_iwslt_de_en",
|
| 295 |
+
[
|
| 296 |
+
"--encoder-layers",
|
| 297 |
+
"2",
|
| 298 |
+
"--decoder-layers",
|
| 299 |
+
"2",
|
| 300 |
+
"--encoder-embed-dim",
|
| 301 |
+
"64",
|
| 302 |
+
"--decoder-embed-dim",
|
| 303 |
+
"64",
|
| 304 |
+
"--amp",
|
| 305 |
+
],
|
| 306 |
+
run_validation=True,
|
| 307 |
+
)
|
| 308 |
+
generate_main(data_dir)
|
| 309 |
+
|
| 310 |
+
@unittest.skipIf(not torch.cuda.is_available(), "test requires a GPU")
|
| 311 |
+
def test_levenshtein_transformer(self):
|
| 312 |
+
with contextlib.redirect_stdout(StringIO()):
|
| 313 |
+
with tempfile.TemporaryDirectory(
|
| 314 |
+
"test_levenshtein_transformer"
|
| 315 |
+
) as data_dir:
|
| 316 |
+
create_dummy_data(data_dir)
|
| 317 |
+
preprocess_translation_data(data_dir, ["--joined-dictionary"])
|
| 318 |
+
train_translation_model(
|
| 319 |
+
data_dir,
|
| 320 |
+
"levenshtein_transformer",
|
| 321 |
+
[
|
| 322 |
+
"--apply-bert-init",
|
| 323 |
+
"--early-exit",
|
| 324 |
+
"6,6,6",
|
| 325 |
+
"--criterion",
|
| 326 |
+
"nat_loss",
|
| 327 |
+
],
|
| 328 |
+
task="translation_lev",
|
| 329 |
+
)
|
| 330 |
+
gen_config = [
|
| 331 |
+
"--task",
|
| 332 |
+
"translation_lev",
|
| 333 |
+
"--iter-decode-max-iter",
|
| 334 |
+
"9",
|
| 335 |
+
"--iter-decode-eos-penalty",
|
| 336 |
+
"0",
|
| 337 |
+
"--print-step",
|
| 338 |
+
]
|
| 339 |
+
# non-ensemble generation
|
| 340 |
+
generate_main(data_dir, gen_config)
|
| 341 |
+
# ensemble generation
|
| 342 |
+
generate_main(
|
| 343 |
+
data_dir,
|
| 344 |
+
gen_config,
|
| 345 |
+
path=os.pathsep.join(
|
| 346 |
+
[
|
| 347 |
+
os.path.join(data_dir, "checkpoint_last.pt"),
|
| 348 |
+
os.path.join(data_dir, "checkpoint_last.pt"),
|
| 349 |
+
]
|
| 350 |
+
),
|
| 351 |
+
)
|
| 352 |
+
|
| 353 |
+
def test_fsdp_checkpoint_generate(self):
|
| 354 |
+
with contextlib.redirect_stdout(StringIO()):
|
| 355 |
+
with tempfile.TemporaryDirectory("test_fsdp_sharded") as data_dir:
|
| 356 |
+
log = os.path.join(data_dir, "train.log")
|
| 357 |
+
create_dummy_data(data_dir)
|
| 358 |
+
preprocess_translation_data(data_dir)
|
| 359 |
+
world_size = min(torch.cuda.device_count(), 2)
|
| 360 |
+
train_translation_model(
|
| 361 |
+
data_dir,
|
| 362 |
+
"fconv_iwslt_de_en",
|
| 363 |
+
["--log-file", log, "--ddp-backend", "fully_sharded"],
|
| 364 |
+
world_size=world_size,
|
| 365 |
+
)
|
| 366 |
+
generate_main(data_dir)
|
| 367 |
+
assert os.path.exists(log)
|
| 368 |
+
|
| 369 |
+
def test_fsdp_sharded_checkpoint_generate(self):
|
| 370 |
+
with contextlib.redirect_stdout(StringIO()):
|
| 371 |
+
with tempfile.TemporaryDirectory("test_fsdp_sharded") as data_dir:
|
| 372 |
+
log = os.path.join(data_dir, "train.log")
|
| 373 |
+
create_dummy_data(data_dir)
|
| 374 |
+
preprocess_translation_data(data_dir)
|
| 375 |
+
world_size = min(torch.cuda.device_count(), 2)
|
| 376 |
+
train_translation_model(
|
| 377 |
+
data_dir,
|
| 378 |
+
"fconv_iwslt_de_en",
|
| 379 |
+
[
|
| 380 |
+
"--log-file",
|
| 381 |
+
log,
|
| 382 |
+
"--ddp-backend",
|
| 383 |
+
"fully_sharded",
|
| 384 |
+
"--use-sharded-state",
|
| 385 |
+
],
|
| 386 |
+
world_size=world_size,
|
| 387 |
+
)
|
| 388 |
+
generate_main(data_dir, ["--checkpoint-shard-count", str(world_size)])
|
| 389 |
+
assert os.path.exists(log)
|
| 390 |
+
|
| 391 |
+
|
| 392 |
+
def _quantize_language_model(data_dir, arch, extra_flags=None, run_validation=False):
|
| 393 |
+
train_parser = options.get_training_parser()
|
| 394 |
+
train_args = options.parse_args_and_arch(
|
| 395 |
+
train_parser,
|
| 396 |
+
[
|
| 397 |
+
"--task",
|
| 398 |
+
"language_modeling",
|
| 399 |
+
data_dir,
|
| 400 |
+
"--arch",
|
| 401 |
+
arch,
|
| 402 |
+
"--optimizer",
|
| 403 |
+
"adam",
|
| 404 |
+
"--lr",
|
| 405 |
+
"0.0001",
|
| 406 |
+
"--criterion",
|
| 407 |
+
"adaptive_loss",
|
| 408 |
+
"--adaptive-softmax-cutoff",
|
| 409 |
+
"5,10,15",
|
| 410 |
+
"--max-tokens",
|
| 411 |
+
"500",
|
| 412 |
+
"--tokens-per-sample",
|
| 413 |
+
"500",
|
| 414 |
+
"--save-dir",
|
| 415 |
+
data_dir,
|
| 416 |
+
"--max-epoch",
|
| 417 |
+
"1",
|
| 418 |
+
"--no-progress-bar",
|
| 419 |
+
"--distributed-world-size",
|
| 420 |
+
"1",
|
| 421 |
+
"--ddp-backend",
|
| 422 |
+
"no_c10d",
|
| 423 |
+
"--num-workers",
|
| 424 |
+
"0",
|
| 425 |
+
]
|
| 426 |
+
+ (extra_flags or []),
|
| 427 |
+
)
|
| 428 |
+
train.main(train_args)
|
| 429 |
+
|
| 430 |
+
# try scalar quantization
|
| 431 |
+
scalar_quant_train_parser = options.get_training_parser()
|
| 432 |
+
scalar_quant_train_args = options.parse_args_and_arch(
|
| 433 |
+
scalar_quant_train_parser,
|
| 434 |
+
[
|
| 435 |
+
"--task",
|
| 436 |
+
"language_modeling",
|
| 437 |
+
data_dir,
|
| 438 |
+
"--arch",
|
| 439 |
+
arch,
|
| 440 |
+
"--optimizer",
|
| 441 |
+
"adam",
|
| 442 |
+
"--lr",
|
| 443 |
+
"0.0001",
|
| 444 |
+
"--criterion",
|
| 445 |
+
"adaptive_loss",
|
| 446 |
+
"--adaptive-softmax-cutoff",
|
| 447 |
+
"5,10,15",
|
| 448 |
+
"--max-tokens",
|
| 449 |
+
"500",
|
| 450 |
+
"--tokens-per-sample",
|
| 451 |
+
"500",
|
| 452 |
+
"--save-dir",
|
| 453 |
+
data_dir,
|
| 454 |
+
"--max-update",
|
| 455 |
+
"3",
|
| 456 |
+
"--no-progress-bar",
|
| 457 |
+
"--distributed-world-size",
|
| 458 |
+
"1",
|
| 459 |
+
"--ddp-backend",
|
| 460 |
+
"no_c10d",
|
| 461 |
+
"--num-workers",
|
| 462 |
+
"0",
|
| 463 |
+
"--quant-noise-scalar",
|
| 464 |
+
"0.5",
|
| 465 |
+
]
|
| 466 |
+
+ (extra_flags or []),
|
| 467 |
+
)
|
| 468 |
+
train.main(scalar_quant_train_args)
|
| 469 |
+
|
| 470 |
+
# try iterative PQ quantization
|
| 471 |
+
quantize_parser = options.get_training_parser()
|
| 472 |
+
quantize_args = options.parse_args_and_arch(
|
| 473 |
+
quantize_parser,
|
| 474 |
+
[
|
| 475 |
+
"--task",
|
| 476 |
+
"language_modeling",
|
| 477 |
+
data_dir,
|
| 478 |
+
"--arch",
|
| 479 |
+
arch,
|
| 480 |
+
"--optimizer",
|
| 481 |
+
"adam",
|
| 482 |
+
"--lr",
|
| 483 |
+
"0.0001",
|
| 484 |
+
"--criterion",
|
| 485 |
+
"adaptive_loss",
|
| 486 |
+
"--adaptive-softmax-cutoff",
|
| 487 |
+
"5,10,15",
|
| 488 |
+
"--max-tokens",
|
| 489 |
+
"50",
|
| 490 |
+
"--tokens-per-sample",
|
| 491 |
+
"50",
|
| 492 |
+
"--max-update",
|
| 493 |
+
"6",
|
| 494 |
+
"--no-progress-bar",
|
| 495 |
+
"--distributed-world-size",
|
| 496 |
+
"1",
|
| 497 |
+
"--ddp-backend",
|
| 498 |
+
"no_c10d",
|
| 499 |
+
"--num-workers",
|
| 500 |
+
"0",
|
| 501 |
+
"--restore-file",
|
| 502 |
+
os.path.join(data_dir, "checkpoint_last.pt"),
|
| 503 |
+
"--reset-optimizer",
|
| 504 |
+
"--quantization-config-path",
|
| 505 |
+
os.path.join(
|
| 506 |
+
os.path.dirname(__file__), "transformer_quantization_config.yaml"
|
| 507 |
+
),
|
| 508 |
+
]
|
| 509 |
+
+ (extra_flags or []),
|
| 510 |
+
)
|
| 511 |
+
train.main(quantize_args)
|
| 512 |
+
|
| 513 |
+
|
| 514 |
+
@unittest.skipIf(
|
| 515 |
+
int(torch.__version__[2]) < 10, reason="quantized kernels are only supported on CPU"
|
| 516 |
+
)
|
| 517 |
+
@unittest.skipIf(not torch.cuda.is_available(), "test requires a GPU")
|
| 518 |
+
class TestQuantization(unittest.TestCase):
|
| 519 |
+
def setUp(self):
|
| 520 |
+
logging.disable(logging.CRITICAL)
|
| 521 |
+
|
| 522 |
+
def tearDown(self):
|
| 523 |
+
logging.disable(logging.NOTSET)
|
| 524 |
+
|
| 525 |
+
def test_quantization(self):
|
| 526 |
+
with contextlib.redirect_stdout(StringIO()):
|
| 527 |
+
with tempfile.TemporaryDirectory("test_quantization") as data_dir:
|
| 528 |
+
create_dummy_data(data_dir)
|
| 529 |
+
preprocess_lm_data(data_dir)
|
| 530 |
+
# tests both scalar and iterative PQ quantization
|
| 531 |
+
_quantize_language_model(data_dir, "transformer_lm")
|
| 532 |
+
|
| 533 |
+
|
| 534 |
+
@unittest.skipIf(not torch.cuda.is_available(), "test requires a GPU")
|
| 535 |
+
class TestOptimizersGPU(unittest.TestCase):
|
| 536 |
+
def setUp(self):
|
| 537 |
+
logging.disable(logging.CRITICAL)
|
| 538 |
+
|
| 539 |
+
def tearDown(self):
|
| 540 |
+
logging.disable(logging.NOTSET)
|
| 541 |
+
|
| 542 |
+
def test_flat_grads(self):
|
| 543 |
+
with contextlib.redirect_stdout(StringIO()):
|
| 544 |
+
with tempfile.TemporaryDirectory("test_flat_grads") as data_dir:
|
| 545 |
+
# Use just a bit of data and tiny model to keep this test runtime reasonable
|
| 546 |
+
create_dummy_data(data_dir, num_examples=10, maxlen=5)
|
| 547 |
+
preprocess_translation_data(data_dir)
|
| 548 |
+
with self.assertRaises(RuntimeError):
|
| 549 |
+
# adafactor isn't compatible with flat grads, which
|
| 550 |
+
# are used by default with --fp16
|
| 551 |
+
train_translation_model(
|
| 552 |
+
data_dir,
|
| 553 |
+
"lstm",
|
| 554 |
+
[
|
| 555 |
+
"--required-batch-size-multiple",
|
| 556 |
+
"1",
|
| 557 |
+
"--encoder-layers",
|
| 558 |
+
"1",
|
| 559 |
+
"--encoder-hidden-size",
|
| 560 |
+
"32",
|
| 561 |
+
"--decoder-layers",
|
| 562 |
+
"1",
|
| 563 |
+
"--optimizer",
|
| 564 |
+
"adafactor",
|
| 565 |
+
"--fp16",
|
| 566 |
+
],
|
| 567 |
+
)
|
| 568 |
+
# but it should pass once we set --fp16-no-flatten-grads
|
| 569 |
+
train_translation_model(
|
| 570 |
+
data_dir,
|
| 571 |
+
"lstm",
|
| 572 |
+
[
|
| 573 |
+
"--required-batch-size-multiple",
|
| 574 |
+
"1",
|
| 575 |
+
"--encoder-layers",
|
| 576 |
+
"1",
|
| 577 |
+
"--encoder-hidden-size",
|
| 578 |
+
"32",
|
| 579 |
+
"--decoder-layers",
|
| 580 |
+
"1",
|
| 581 |
+
"--optimizer",
|
| 582 |
+
"adafactor",
|
| 583 |
+
"--fp16",
|
| 584 |
+
"--fp16-no-flatten-grads",
|
| 585 |
+
],
|
| 586 |
+
)
|
| 587 |
+
|
| 588 |
+
|
| 589 |
+
if __name__ == "__main__":
|
| 590 |
+
unittest.main()
|
data/fairseq/tests/gpu/test_ema_gpu.py
ADDED
|
@@ -0,0 +1,215 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) Facebook, Inc. and its affiliates.
|
| 2 |
+
#
|
| 3 |
+
# This source code is licensed under the MIT license found in the
|
| 4 |
+
# LICENSE file in the root directory of this source tree.
|
| 5 |
+
|
| 6 |
+
import unittest
|
| 7 |
+
from copy import deepcopy
|
| 8 |
+
from dataclasses import dataclass
|
| 9 |
+
from typing import Optional
|
| 10 |
+
|
| 11 |
+
import torch
|
| 12 |
+
|
| 13 |
+
from fairseq.models.ema import EMA
|
| 14 |
+
|
| 15 |
+
|
| 16 |
+
class DummyModule(torch.nn.Module):
|
| 17 |
+
def __init__(self) -> None:
|
| 18 |
+
"""LightningModule for testing purposes
|
| 19 |
+
|
| 20 |
+
Args:
|
| 21 |
+
epoch_min_loss_override (int, optional): Pass in an epoch that will be set to the minimum
|
| 22 |
+
validation loss for testing purposes (zero based). If None this is ignored. Defaults to None.
|
| 23 |
+
"""
|
| 24 |
+
super().__init__()
|
| 25 |
+
self.layer = torch.nn.Linear(in_features=32, out_features=2)
|
| 26 |
+
self.another_layer = torch.nn.Linear(in_features=2, out_features=2)
|
| 27 |
+
|
| 28 |
+
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
| 29 |
+
x = self.layer(x)
|
| 30 |
+
return self.another_layer(x)
|
| 31 |
+
|
| 32 |
+
|
| 33 |
+
@dataclass
|
| 34 |
+
class EMAConfig(object):
|
| 35 |
+
ema_decay: float = 0.99
|
| 36 |
+
ema_start_update: int = 0
|
| 37 |
+
ema_fp32: bool = False
|
| 38 |
+
ema_seed_model: Optional[str] = None
|
| 39 |
+
ema_update_freq: int = 1
|
| 40 |
+
|
| 41 |
+
|
| 42 |
+
@unittest.skipIf(not torch.cuda.is_available(), "test requires a GPU")
|
| 43 |
+
class TestEMAGPU(unittest.TestCase):
|
| 44 |
+
def assertTorchAllClose(self, x, y, atol=1e-8, rtol=1e-5, msg=None):
|
| 45 |
+
diff = x.float() - y.float()
|
| 46 |
+
diff_norm = torch.norm(diff)
|
| 47 |
+
other_norm = torch.norm(y.float())
|
| 48 |
+
|
| 49 |
+
if msg is None:
|
| 50 |
+
msg = "|input - other| > {} + {} * |other|".format(atol, rtol)
|
| 51 |
+
|
| 52 |
+
self.assertLessEqual(
|
| 53 |
+
diff_norm,
|
| 54 |
+
atol + rtol * other_norm,
|
| 55 |
+
msg=msg,
|
| 56 |
+
)
|
| 57 |
+
|
| 58 |
+
def test_ema(self):
|
| 59 |
+
model = DummyModule().cuda()
|
| 60 |
+
optimizer = torch.optim.SGD(model.parameters(), lr=0.01)
|
| 61 |
+
state = deepcopy(model.state_dict())
|
| 62 |
+
config = EMAConfig()
|
| 63 |
+
ema = EMA(model, config)
|
| 64 |
+
|
| 65 |
+
# set decay
|
| 66 |
+
ema._set_decay(config.ema_decay)
|
| 67 |
+
self.assertEqual(ema.get_decay(), config.ema_decay)
|
| 68 |
+
|
| 69 |
+
# get model
|
| 70 |
+
self.assertEqual(ema.get_model(), ema.model)
|
| 71 |
+
|
| 72 |
+
# Since fp32 params is not used, it should be of size 0
|
| 73 |
+
self.assertEqual(len(ema.fp32_params), 0)
|
| 74 |
+
|
| 75 |
+
# EMA step
|
| 76 |
+
x = torch.randn(32).cuda()
|
| 77 |
+
y = model(x)
|
| 78 |
+
loss = y.sum()
|
| 79 |
+
loss.backward()
|
| 80 |
+
optimizer.step()
|
| 81 |
+
|
| 82 |
+
ema.step(model)
|
| 83 |
+
|
| 84 |
+
ema_state_dict = ema.get_model().state_dict()
|
| 85 |
+
|
| 86 |
+
for key, param in model.state_dict().items():
|
| 87 |
+
prev_param = state[key]
|
| 88 |
+
ema_param = ema_state_dict[key]
|
| 89 |
+
|
| 90 |
+
if "version" in key:
|
| 91 |
+
# Do not decay a model.version pytorch param
|
| 92 |
+
continue
|
| 93 |
+
self.assertTorchAllClose(
|
| 94 |
+
ema_param,
|
| 95 |
+
config.ema_decay * prev_param + (1 - config.ema_decay) * param,
|
| 96 |
+
)
|
| 97 |
+
|
| 98 |
+
# Since fp32 params is not used, it should be of size 0
|
| 99 |
+
self.assertEqual(len(ema.fp32_params), 0)
|
| 100 |
+
|
| 101 |
+
# Load EMA into model
|
| 102 |
+
model2 = DummyModule().cuda()
|
| 103 |
+
ema.reverse(model2)
|
| 104 |
+
|
| 105 |
+
for key, param in model2.state_dict().items():
|
| 106 |
+
ema_param = ema_state_dict[key]
|
| 107 |
+
self.assertTrue(torch.allclose(ema_param, param))
|
| 108 |
+
|
| 109 |
+
def test_ema_fp32(self):
|
| 110 |
+
model = DummyModule().cuda().half()
|
| 111 |
+
optimizer = torch.optim.SGD(model.parameters(), lr=0.01)
|
| 112 |
+
state = deepcopy(model.state_dict())
|
| 113 |
+
config = EMAConfig(ema_fp32=True)
|
| 114 |
+
ema = EMA(model, config)
|
| 115 |
+
|
| 116 |
+
x = torch.randn(32).cuda()
|
| 117 |
+
y = model(x.half())
|
| 118 |
+
loss = y.sum()
|
| 119 |
+
loss.backward()
|
| 120 |
+
optimizer.step()
|
| 121 |
+
|
| 122 |
+
ema.step(model)
|
| 123 |
+
|
| 124 |
+
for key, param in model.state_dict().items():
|
| 125 |
+
prev_param = state[key]
|
| 126 |
+
ema_param = ema.get_model().state_dict()[key]
|
| 127 |
+
|
| 128 |
+
if "version" in key:
|
| 129 |
+
# Do not decay a model.version pytorch param
|
| 130 |
+
continue
|
| 131 |
+
self.assertIn(key, ema.fp32_params)
|
| 132 |
+
|
| 133 |
+
# EMA update is done in fp32, and hence the EMA param must be
|
| 134 |
+
# closer to the EMA update done in fp32 than in fp16.
|
| 135 |
+
self.assertLessEqual(
|
| 136 |
+
torch.norm(
|
| 137 |
+
ema_param.float()
|
| 138 |
+
- (
|
| 139 |
+
config.ema_decay * prev_param.float()
|
| 140 |
+
+ (1 - config.ema_decay) * param.float()
|
| 141 |
+
)
|
| 142 |
+
.half()
|
| 143 |
+
.float()
|
| 144 |
+
),
|
| 145 |
+
torch.norm(
|
| 146 |
+
ema_param.float()
|
| 147 |
+
- (
|
| 148 |
+
config.ema_decay * prev_param + (1 - config.ema_decay) * param
|
| 149 |
+
).float()
|
| 150 |
+
),
|
| 151 |
+
)
|
| 152 |
+
self.assertTorchAllClose(
|
| 153 |
+
ema_param,
|
| 154 |
+
(
|
| 155 |
+
config.ema_decay * prev_param.float()
|
| 156 |
+
+ (1 - config.ema_decay) * param.float()
|
| 157 |
+
).half(),
|
| 158 |
+
)
|
| 159 |
+
|
| 160 |
+
def test_ema_fp16(self):
|
| 161 |
+
model = DummyModule().cuda().half()
|
| 162 |
+
optimizer = torch.optim.SGD(model.parameters(), lr=0.01)
|
| 163 |
+
state = deepcopy(model.state_dict())
|
| 164 |
+
config = EMAConfig(ema_fp32=False)
|
| 165 |
+
ema = EMA(model, config)
|
| 166 |
+
|
| 167 |
+
# Since fp32 params is not used, it should be of size 0
|
| 168 |
+
self.assertEqual(len(ema.fp32_params), 0)
|
| 169 |
+
|
| 170 |
+
x = torch.randn(32).cuda()
|
| 171 |
+
y = model(x.half())
|
| 172 |
+
loss = y.sum()
|
| 173 |
+
loss.backward()
|
| 174 |
+
optimizer.step()
|
| 175 |
+
|
| 176 |
+
ema.step(model)
|
| 177 |
+
|
| 178 |
+
for key, param in model.state_dict().items():
|
| 179 |
+
prev_param = state[key]
|
| 180 |
+
ema_param = ema.get_model().state_dict()[key]
|
| 181 |
+
|
| 182 |
+
if "version" in key:
|
| 183 |
+
# Do not decay a model.version pytorch param
|
| 184 |
+
continue
|
| 185 |
+
|
| 186 |
+
# EMA update is done in fp16, and hence the EMA param must be
|
| 187 |
+
# closer to the EMA update done in fp16 than in fp32.
|
| 188 |
+
self.assertLessEqual(
|
| 189 |
+
torch.norm(
|
| 190 |
+
ema_param.float()
|
| 191 |
+
- (
|
| 192 |
+
config.ema_decay * prev_param + (1 - config.ema_decay) * param
|
| 193 |
+
).float()
|
| 194 |
+
),
|
| 195 |
+
torch.norm(
|
| 196 |
+
ema_param.float()
|
| 197 |
+
- (
|
| 198 |
+
config.ema_decay * prev_param.float()
|
| 199 |
+
+ (1 - config.ema_decay) * param.float()
|
| 200 |
+
)
|
| 201 |
+
.half()
|
| 202 |
+
.float()
|
| 203 |
+
),
|
| 204 |
+
)
|
| 205 |
+
self.assertTorchAllClose(
|
| 206 |
+
ema_param,
|
| 207 |
+
config.ema_decay * prev_param + (1 - config.ema_decay) * param,
|
| 208 |
+
)
|
| 209 |
+
|
| 210 |
+
# Since fp32 params is not used, it should be of size 0
|
| 211 |
+
self.assertEqual(len(ema.fp32_params), 0)
|
| 212 |
+
|
| 213 |
+
|
| 214 |
+
if __name__ == "__main__":
|
| 215 |
+
unittest.main()
|
data/fairseq/tests/gpu/transformer_quantization_config.yaml
ADDED
|
@@ -0,0 +1,28 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) Facebook, Inc. and its affiliates.
|
| 2 |
+
#
|
| 3 |
+
# This source code is licensed under the MIT license found in the
|
| 4 |
+
# LICENSE file in the root directory of this source tree.
|
| 5 |
+
|
| 6 |
+
# This file defines example configuration arguments for quantizing
|
| 7 |
+
# a transformer model with product quantization
|
| 8 |
+
|
| 9 |
+
n_centroids:
|
| 10 |
+
Linear:
|
| 11 |
+
key: in_features
|
| 12 |
+
value: {"*": 8}
|
| 13 |
+
Embedding:
|
| 14 |
+
key: embedding_dim
|
| 15 |
+
value: {"*": 8}
|
| 16 |
+
|
| 17 |
+
block_sizes:
|
| 18 |
+
Linear:
|
| 19 |
+
key: fuzzy_name
|
| 20 |
+
value: {fc: 8, attn: 4, emb: 4}
|
| 21 |
+
Embedding:
|
| 22 |
+
key: fuzzy_name
|
| 23 |
+
value: {emb: 8}
|
| 24 |
+
|
| 25 |
+
layers_to_quantize:
|
| 26 |
+
- decoder\\.layers\\.\d+\\.fc[12]
|
| 27 |
+
- decoder\\.embed_tokens\\.embeddings\\.[012]\\.[01]
|
| 28 |
+
- decoder\\.layers\\.\d+\\.self_attn\\.(k_proj|v_proj|q_proj|out_proj)
|
data/fairseq/tests/speech/__init__.py
ADDED
|
@@ -0,0 +1,210 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) Facebook, Inc. and its affiliates.
|
| 2 |
+
#
|
| 3 |
+
# This source code is licensed under the MIT license found in the
|
| 4 |
+
# LICENSE file in the root directory of this source tree.
|
| 5 |
+
|
| 6 |
+
from argparse import Namespace
|
| 7 |
+
import os
|
| 8 |
+
import re
|
| 9 |
+
import unittest
|
| 10 |
+
from pathlib import Path
|
| 11 |
+
from tqdm import tqdm
|
| 12 |
+
from typing import List, Dict, Optional
|
| 13 |
+
import torch
|
| 14 |
+
from fairseq.checkpoint_utils import load_model_ensemble_and_task
|
| 15 |
+
from fairseq.scoring.wer import WerScorer
|
| 16 |
+
from fairseq.scoring.bleu import SacrebleuScorer
|
| 17 |
+
from fairseq import utils
|
| 18 |
+
import zipfile
|
| 19 |
+
|
| 20 |
+
S3_BASE_URL = "https://dl.fbaipublicfiles.com/fairseq"
|
| 21 |
+
|
| 22 |
+
|
| 23 |
+
class TestFairseqSpeech(unittest.TestCase):
|
| 24 |
+
@classmethod
|
| 25 |
+
def download(cls, base_url: str, out_root: Path, filename: str):
|
| 26 |
+
url = f"{base_url}/{filename}"
|
| 27 |
+
path = out_root / filename
|
| 28 |
+
if not path.exists():
|
| 29 |
+
torch.hub.download_url_to_file(url, path.as_posix(), progress=True)
|
| 30 |
+
return path
|
| 31 |
+
|
| 32 |
+
def _set_up(self, dataset_id: str, s3_dir: str, data_filenames: List[str]):
|
| 33 |
+
self.use_cuda = torch.cuda.is_available()
|
| 34 |
+
self.root = Path.home() / ".cache" / "fairseq" / dataset_id
|
| 35 |
+
self.root.mkdir(exist_ok=True, parents=True)
|
| 36 |
+
os.chdir(self.root)
|
| 37 |
+
self.base_url = (
|
| 38 |
+
s3_dir if re.search("^https:", s3_dir) else f"{S3_BASE_URL}/{s3_dir}"
|
| 39 |
+
)
|
| 40 |
+
for filename in data_filenames:
|
| 41 |
+
self.download(self.base_url, self.root, filename)
|
| 42 |
+
|
| 43 |
+
def set_up_librispeech(self):
|
| 44 |
+
self._set_up(
|
| 45 |
+
"librispeech",
|
| 46 |
+
"s2t/librispeech",
|
| 47 |
+
[
|
| 48 |
+
"cfg_librispeech.yaml",
|
| 49 |
+
"spm_librispeech_unigram10000.model",
|
| 50 |
+
"spm_librispeech_unigram10000.txt",
|
| 51 |
+
"librispeech_test-other.tsv",
|
| 52 |
+
"librispeech_test-other.zip",
|
| 53 |
+
],
|
| 54 |
+
)
|
| 55 |
+
|
| 56 |
+
def set_up_ljspeech(self):
|
| 57 |
+
self._set_up(
|
| 58 |
+
"ljspeech",
|
| 59 |
+
"s2/ljspeech",
|
| 60 |
+
[
|
| 61 |
+
"cfg_ljspeech_g2p.yaml",
|
| 62 |
+
"ljspeech_g2p_gcmvn_stats.npz",
|
| 63 |
+
"ljspeech_g2p.txt",
|
| 64 |
+
"ljspeech_test.tsv",
|
| 65 |
+
"ljspeech_test.zip",
|
| 66 |
+
],
|
| 67 |
+
)
|
| 68 |
+
|
| 69 |
+
def set_up_sotasty_es_en(self):
|
| 70 |
+
self._set_up(
|
| 71 |
+
"sotasty_es_en",
|
| 72 |
+
"s2t/big/es-en",
|
| 73 |
+
[
|
| 74 |
+
"cfg_es_en.yaml",
|
| 75 |
+
"spm_bpe32768_es_en.model",
|
| 76 |
+
"spm_bpe32768_es_en.txt",
|
| 77 |
+
"sotasty_es_en_test_ted.tsv",
|
| 78 |
+
"sotasty_es_en_test_ted.zip",
|
| 79 |
+
],
|
| 80 |
+
)
|
| 81 |
+
|
| 82 |
+
def set_up_mustc_de_fbank(self):
|
| 83 |
+
self._set_up(
|
| 84 |
+
"mustc_de_fbank",
|
| 85 |
+
"https://dl.fbaipublicfiles.com/joint_speech_text_4_s2t/must_c/en_de",
|
| 86 |
+
[
|
| 87 |
+
"config.yaml",
|
| 88 |
+
"spm.model",
|
| 89 |
+
"dict.txt",
|
| 90 |
+
"src_dict.txt",
|
| 91 |
+
"tgt_dict.txt",
|
| 92 |
+
"tst-COMMON.tsv",
|
| 93 |
+
"tst-COMMON.zip",
|
| 94 |
+
],
|
| 95 |
+
)
|
| 96 |
+
|
| 97 |
+
def download_and_load_checkpoint(
|
| 98 |
+
self,
|
| 99 |
+
checkpoint_filename: str,
|
| 100 |
+
arg_overrides: Optional[Dict[str, str]] = None,
|
| 101 |
+
strict: bool = True,
|
| 102 |
+
):
|
| 103 |
+
path = self.download(self.base_url, self.root, checkpoint_filename)
|
| 104 |
+
_arg_overrides = arg_overrides or {}
|
| 105 |
+
_arg_overrides["data"] = self.root.as_posix()
|
| 106 |
+
models, cfg, task = load_model_ensemble_and_task(
|
| 107 |
+
[path.as_posix()], arg_overrides=_arg_overrides, strict=strict
|
| 108 |
+
)
|
| 109 |
+
if self.use_cuda:
|
| 110 |
+
for model in models:
|
| 111 |
+
model.cuda()
|
| 112 |
+
|
| 113 |
+
return models, cfg, task, self.build_generator(task, models, cfg)
|
| 114 |
+
|
| 115 |
+
def build_generator(
|
| 116 |
+
self,
|
| 117 |
+
task,
|
| 118 |
+
models,
|
| 119 |
+
cfg,
|
| 120 |
+
):
|
| 121 |
+
return task.build_generator(models, cfg)
|
| 122 |
+
|
| 123 |
+
@classmethod
|
| 124 |
+
def get_batch_iterator(cls, task, test_split, max_tokens, max_positions):
|
| 125 |
+
task.load_dataset(test_split)
|
| 126 |
+
return task.get_batch_iterator(
|
| 127 |
+
dataset=task.dataset(test_split),
|
| 128 |
+
max_tokens=max_tokens,
|
| 129 |
+
max_positions=max_positions,
|
| 130 |
+
num_workers=1,
|
| 131 |
+
).next_epoch_itr(shuffle=False)
|
| 132 |
+
|
| 133 |
+
@classmethod
|
| 134 |
+
def get_wer_scorer(
|
| 135 |
+
cls, tokenizer="none", lowercase=False, remove_punct=False, char_level=False
|
| 136 |
+
):
|
| 137 |
+
scorer_args = {
|
| 138 |
+
"wer_tokenizer": tokenizer,
|
| 139 |
+
"wer_lowercase": lowercase,
|
| 140 |
+
"wer_remove_punct": remove_punct,
|
| 141 |
+
"wer_char_level": char_level,
|
| 142 |
+
}
|
| 143 |
+
return WerScorer(Namespace(**scorer_args))
|
| 144 |
+
|
| 145 |
+
@classmethod
|
| 146 |
+
def get_bleu_scorer(cls, tokenizer="13a", lowercase=False, char_level=False):
|
| 147 |
+
scorer_args = {
|
| 148 |
+
"sacrebleu_tokenizer": tokenizer,
|
| 149 |
+
"sacrebleu_lowercase": lowercase,
|
| 150 |
+
"sacrebleu_char_level": char_level,
|
| 151 |
+
}
|
| 152 |
+
return SacrebleuScorer(Namespace(**scorer_args))
|
| 153 |
+
|
| 154 |
+
@torch.no_grad()
|
| 155 |
+
def base_test(
|
| 156 |
+
self,
|
| 157 |
+
ckpt_name,
|
| 158 |
+
reference_score,
|
| 159 |
+
score_delta=0.3,
|
| 160 |
+
dataset="librispeech_test-other",
|
| 161 |
+
max_tokens=65_536,
|
| 162 |
+
max_positions=(4_096, 1_024),
|
| 163 |
+
arg_overrides=None,
|
| 164 |
+
strict=True,
|
| 165 |
+
score_type="wer",
|
| 166 |
+
):
|
| 167 |
+
models, _, task, generator = self.download_and_load_checkpoint(
|
| 168 |
+
ckpt_name, arg_overrides=arg_overrides, strict=strict
|
| 169 |
+
)
|
| 170 |
+
if not self.use_cuda:
|
| 171 |
+
return
|
| 172 |
+
|
| 173 |
+
batch_iterator = self.get_batch_iterator(
|
| 174 |
+
task, dataset, max_tokens, max_positions
|
| 175 |
+
)
|
| 176 |
+
if score_type == "bleu":
|
| 177 |
+
scorer = self.get_bleu_scorer()
|
| 178 |
+
elif score_type == "wer":
|
| 179 |
+
scorer = self.get_wer_scorer()
|
| 180 |
+
else:
|
| 181 |
+
raise Exception(f"Unsupported score type {score_type}")
|
| 182 |
+
|
| 183 |
+
progress = tqdm(enumerate(batch_iterator), total=len(batch_iterator))
|
| 184 |
+
for batch_idx, sample in progress:
|
| 185 |
+
sample = utils.move_to_cuda(sample) if self.use_cuda else sample
|
| 186 |
+
hypo = task.inference_step(generator, models, sample)
|
| 187 |
+
for i, sample_id in enumerate(sample["id"].tolist()):
|
| 188 |
+
tgt_str, hypo_str = self.postprocess_tokens(
|
| 189 |
+
task,
|
| 190 |
+
sample["target"][i, :],
|
| 191 |
+
hypo[i][0]["tokens"].int().cpu(),
|
| 192 |
+
)
|
| 193 |
+
if batch_idx == 0 and i < 3:
|
| 194 |
+
print(f"T-{sample_id} {tgt_str}")
|
| 195 |
+
print(f"H-{sample_id} {hypo_str}")
|
| 196 |
+
scorer.add_string(tgt_str, hypo_str)
|
| 197 |
+
|
| 198 |
+
print(scorer.result_string() + f" (reference: {reference_score})")
|
| 199 |
+
self.assertAlmostEqual(scorer.score(), reference_score, delta=score_delta)
|
| 200 |
+
|
| 201 |
+
def postprocess_tokens(self, task, target, hypo_tokens):
|
| 202 |
+
tgt_tokens = utils.strip_pad(target, task.tgt_dict.pad()).int().cpu()
|
| 203 |
+
tgt_str = task.tgt_dict.string(tgt_tokens, "sentencepiece")
|
| 204 |
+
hypo_str = task.tgt_dict.string(hypo_tokens, "sentencepiece")
|
| 205 |
+
return tgt_str, hypo_str
|
| 206 |
+
|
| 207 |
+
def unzip_files(self, zip_file_name):
|
| 208 |
+
zip_file_path = self.root / zip_file_name
|
| 209 |
+
with zipfile.ZipFile(zip_file_path, "r") as zip_ref:
|
| 210 |
+
zip_ref.extractall(self.root / zip_file_name.strip(".zip"))
|
data/fairseq/tests/speech/test_convtransformer_simul_trans.py
ADDED
|
@@ -0,0 +1,33 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) Facebook, Inc. and its affiliates.
|
| 2 |
+
#
|
| 3 |
+
# This source code is licensed under the MIT license found in the
|
| 4 |
+
# LICENSE file in the root directory of this source tree.
|
| 5 |
+
|
| 6 |
+
import unittest
|
| 7 |
+
from tests.speech import TestFairseqSpeech
|
| 8 |
+
|
| 9 |
+
S3_BASE_URL = "https://dl.fbaipublicfiles.com/fairseq/"
|
| 10 |
+
|
| 11 |
+
|
| 12 |
+
class TestConvtransformerSimulTrans(TestFairseqSpeech):
|
| 13 |
+
def setUp(self):
|
| 14 |
+
self._set_up(
|
| 15 |
+
"simul",
|
| 16 |
+
"speech_tests/simul",
|
| 17 |
+
["config_gcmvn_specaug.yaml", "dict.txt", "dev.tsv"],
|
| 18 |
+
)
|
| 19 |
+
|
| 20 |
+
def test_waitk_checkpoint(self):
|
| 21 |
+
"""Only test model loading since fairseq currently doesn't support inference of simultaneous models"""
|
| 22 |
+
_, _, _, _ = self.download_and_load_checkpoint(
|
| 23 |
+
"checkpoint_best.pt",
|
| 24 |
+
arg_overrides={
|
| 25 |
+
"config_yaml": "config_gcmvn_specaug.yaml",
|
| 26 |
+
"load_pretrained_encoder_from": None,
|
| 27 |
+
},
|
| 28 |
+
)
|
| 29 |
+
return
|
| 30 |
+
|
| 31 |
+
|
| 32 |
+
if __name__ == "__main__":
|
| 33 |
+
unittest.main()
|
data/fairseq/tests/speech/test_dual_input_wav_transformer.py
ADDED
|
@@ -0,0 +1,76 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) Facebook, Inc. and its affiliates.
|
| 2 |
+
#
|
| 3 |
+
# This source code is licensed under the MIT license found in the
|
| 4 |
+
# LICENSE file in the root directory of this source tree.
|
| 5 |
+
|
| 6 |
+
import unittest
|
| 7 |
+
from collections import namedtuple
|
| 8 |
+
from pathlib import Path
|
| 9 |
+
|
| 10 |
+
import torch
|
| 11 |
+
from tqdm import tqdm
|
| 12 |
+
|
| 13 |
+
import fairseq
|
| 14 |
+
from fairseq import utils
|
| 15 |
+
from fairseq.checkpoint_utils import load_model_ensemble_and_task
|
| 16 |
+
from fairseq.scoring.bleu import SacrebleuScorer
|
| 17 |
+
from fairseq.tasks import import_tasks
|
| 18 |
+
from tests.speech import S3_BASE_URL, TestFairseqSpeech
|
| 19 |
+
|
| 20 |
+
|
| 21 |
+
@unittest.skipIf(not torch.cuda.is_available(), "test requires a GPU")
|
| 22 |
+
class TestLibrispeechDualInputWavTransformer(TestFairseqSpeech):
|
| 23 |
+
def setUp(self):
|
| 24 |
+
dataset_id = "librispeech_wvtrasnformer"
|
| 25 |
+
base_url = "https://dl.fbaipublicfiles.com/joint_speech_text_4_s2t/acl2022/librispeech/finetuned"
|
| 26 |
+
data_filenames = [
|
| 27 |
+
"checkpoint_ave_10.pt",
|
| 28 |
+
"spm.model",
|
| 29 |
+
"src_dict.txt",
|
| 30 |
+
"tgt_dict.txt",
|
| 31 |
+
"config.yaml",
|
| 32 |
+
]
|
| 33 |
+
self._set_up(
|
| 34 |
+
dataset_id,
|
| 35 |
+
"s2t",
|
| 36 |
+
[
|
| 37 |
+
"librispeech_flac_test-other.tsv",
|
| 38 |
+
"librispeech_flac_test-other.zip",
|
| 39 |
+
],
|
| 40 |
+
)
|
| 41 |
+
for filename in data_filenames:
|
| 42 |
+
self.download(base_url, self.root, filename)
|
| 43 |
+
|
| 44 |
+
def import_user_module(self):
|
| 45 |
+
user_dir = (
|
| 46 |
+
Path(fairseq.__file__).parent.parent / "examples/speech_text_joint_to_text"
|
| 47 |
+
)
|
| 48 |
+
Arg = namedtuple("Arg", ["user_dir"])
|
| 49 |
+
arg = Arg(user_dir.__str__())
|
| 50 |
+
utils.import_user_module(arg)
|
| 51 |
+
|
| 52 |
+
@torch.no_grad()
|
| 53 |
+
def test_librispeech_dualinput_wav_transformer_checkpoint(self):
|
| 54 |
+
self.import_user_module()
|
| 55 |
+
checkpoint_filename = "checkpoint_ave_10.pt"
|
| 56 |
+
arg_overrides = {
|
| 57 |
+
"config_yaml": "config.yaml",
|
| 58 |
+
"load_pretrained_speech_text_encoder": "",
|
| 59 |
+
"load_pretrained_speech_text_decoder": "",
|
| 60 |
+
"beam": 10,
|
| 61 |
+
"nbest": 1,
|
| 62 |
+
"lenpen": 1.0,
|
| 63 |
+
"load_speech_only": True,
|
| 64 |
+
}
|
| 65 |
+
self.base_test(
|
| 66 |
+
checkpoint_filename,
|
| 67 |
+
4.6,
|
| 68 |
+
dataset="librispeech_flac_test-other",
|
| 69 |
+
max_tokens=800000,
|
| 70 |
+
max_positions=(800000, 1024),
|
| 71 |
+
arg_overrides=arg_overrides,
|
| 72 |
+
)
|
| 73 |
+
|
| 74 |
+
|
| 75 |
+
if __name__ == "__main__":
|
| 76 |
+
unittest.main()
|
data/fairseq/tests/speech/test_dualinput_s2t_transformer.py
ADDED
|
@@ -0,0 +1,110 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) Facebook, Inc. and its affiliates.
|
| 2 |
+
#
|
| 3 |
+
# This source code is licensed under the MIT license found in the
|
| 4 |
+
# LICENSE file in the root directory of this source tree.
|
| 5 |
+
|
| 6 |
+
import unittest
|
| 7 |
+
from argparse import Namespace
|
| 8 |
+
from collections import namedtuple
|
| 9 |
+
from pathlib import Path
|
| 10 |
+
|
| 11 |
+
import torch
|
| 12 |
+
from tqdm import tqdm
|
| 13 |
+
|
| 14 |
+
import fairseq
|
| 15 |
+
from fairseq import utils
|
| 16 |
+
from fairseq.checkpoint_utils import load_model_ensemble_and_task
|
| 17 |
+
from fairseq.scoring.bleu import SacrebleuScorer
|
| 18 |
+
from fairseq.tasks import import_tasks
|
| 19 |
+
from tests.speech import TestFairseqSpeech
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
@unittest.skipIf(not torch.cuda.is_available(), "test requires a GPU")
|
| 23 |
+
class TestDualInputS2TTransformer(TestFairseqSpeech):
|
| 24 |
+
def setUp(self):
|
| 25 |
+
self.set_up_mustc_de_fbank()
|
| 26 |
+
|
| 27 |
+
def import_user_module(self):
|
| 28 |
+
user_dir = (
|
| 29 |
+
Path(fairseq.__file__).parent.parent / "examples/speech_text_joint_to_text"
|
| 30 |
+
)
|
| 31 |
+
Arg = namedtuple("Arg", ["user_dir"])
|
| 32 |
+
arg = Arg(user_dir.__str__())
|
| 33 |
+
utils.import_user_module(arg)
|
| 34 |
+
|
| 35 |
+
@torch.no_grad()
|
| 36 |
+
def test_mustc_de_fbank_dualinput_s2t_transformer_checkpoint(self):
|
| 37 |
+
self.import_user_module()
|
| 38 |
+
checkpoint_filename = "checkpoint_ave_10.pt"
|
| 39 |
+
path = self.download(self.base_url, self.root, checkpoint_filename)
|
| 40 |
+
models, cfg, task = load_model_ensemble_and_task(
|
| 41 |
+
[path.as_posix()],
|
| 42 |
+
arg_overrides={
|
| 43 |
+
"data": self.root.as_posix(),
|
| 44 |
+
"config_yaml": "config.yaml",
|
| 45 |
+
"load_pretrain_speech_encoder": "",
|
| 46 |
+
"load_pretrain_text_encoder_last": "",
|
| 47 |
+
"load_pretrain_decoder": "",
|
| 48 |
+
"beam": 10,
|
| 49 |
+
"nbest": 1,
|
| 50 |
+
"lenpen": 1.0,
|
| 51 |
+
"load_speech_only": True,
|
| 52 |
+
},
|
| 53 |
+
)
|
| 54 |
+
if self.use_cuda:
|
| 55 |
+
for model in models:
|
| 56 |
+
model.cuda()
|
| 57 |
+
generator = task.build_generator(models, cfg)
|
| 58 |
+
test_split = "tst-COMMON"
|
| 59 |
+
task.load_dataset(test_split)
|
| 60 |
+
batch_iterator = task.get_batch_iterator(
|
| 61 |
+
dataset=task.dataset(test_split),
|
| 62 |
+
max_tokens=250_000,
|
| 63 |
+
max_positions=(10_000, 1_024),
|
| 64 |
+
num_workers=1,
|
| 65 |
+
).next_epoch_itr(shuffle=False)
|
| 66 |
+
|
| 67 |
+
tokenizer = task.build_tokenizer(cfg.tokenizer)
|
| 68 |
+
bpe = task.build_bpe(cfg.bpe)
|
| 69 |
+
|
| 70 |
+
def decode_fn(x):
|
| 71 |
+
if bpe is not None:
|
| 72 |
+
x = bpe.decode(x)
|
| 73 |
+
if tokenizer is not None:
|
| 74 |
+
x = tokenizer.decode(x)
|
| 75 |
+
return x
|
| 76 |
+
|
| 77 |
+
scorer_args = {
|
| 78 |
+
"sacrebleu_tokenizer": "13a",
|
| 79 |
+
"sacrebleu_lowercase": False,
|
| 80 |
+
"sacrebleu_char_level": False,
|
| 81 |
+
}
|
| 82 |
+
scorer = SacrebleuScorer(Namespace(**scorer_args))
|
| 83 |
+
progress = tqdm(enumerate(batch_iterator), total=len(batch_iterator))
|
| 84 |
+
for batch_idx, sample in progress:
|
| 85 |
+
sample = utils.move_to_cuda(sample) if self.use_cuda else sample
|
| 86 |
+
hypo = task.inference_step(generator, models, sample)
|
| 87 |
+
for i, sample_id in enumerate(sample["id"].tolist()):
|
| 88 |
+
tgt_tokens = (
|
| 89 |
+
utils.strip_pad(sample["target"][i, :], task.tgt_dict.pad())
|
| 90 |
+
.int()
|
| 91 |
+
.cpu()
|
| 92 |
+
)
|
| 93 |
+
|
| 94 |
+
tgt_str = task.tgt_dict.string(tgt_tokens, "sentencepiece")
|
| 95 |
+
hypo_str = task.tgt_dict.string(
|
| 96 |
+
hypo[i][0]["tokens"].int().cpu(), "sentencepiece"
|
| 97 |
+
)
|
| 98 |
+
if batch_idx == 0 and i < 3:
|
| 99 |
+
print(f"T-{sample_id} {tgt_str}")
|
| 100 |
+
print(f"D-{sample_id} {hypo_str}")
|
| 101 |
+
scorer.add_string(tgt_str, hypo_str)
|
| 102 |
+
reference_bleu = 27.3
|
| 103 |
+
result = scorer.result_string()
|
| 104 |
+
print(result + f" (reference: {reference_bleu})")
|
| 105 |
+
res_bleu = float(result.split()[2])
|
| 106 |
+
self.assertAlmostEqual(res_bleu, reference_bleu, delta=0.3)
|
| 107 |
+
|
| 108 |
+
|
| 109 |
+
if __name__ == "__main__":
|
| 110 |
+
unittest.main()
|
data/fairseq/tests/speech/test_fastspeech2.py
ADDED
|
@@ -0,0 +1,53 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) Facebook, Inc. and its affiliates.
|
| 2 |
+
#
|
| 3 |
+
# This source code is licensed under the MIT license found in the
|
| 4 |
+
# LICENSE file in the root directory of this source tree.
|
| 5 |
+
|
| 6 |
+
import unittest
|
| 7 |
+
|
| 8 |
+
import torch
|
| 9 |
+
from tqdm import tqdm
|
| 10 |
+
|
| 11 |
+
from fairseq import utils
|
| 12 |
+
from fairseq.tasks.text_to_speech import batch_mel_cepstral_distortion
|
| 13 |
+
from tests.speech import TestFairseqSpeech
|
| 14 |
+
|
| 15 |
+
|
| 16 |
+
@unittest.skipIf(not torch.cuda.is_available(), "test requires a GPU")
|
| 17 |
+
class TestFastSpeech2(TestFairseqSpeech):
|
| 18 |
+
def setUp(self):
|
| 19 |
+
self.set_up_ljspeech()
|
| 20 |
+
|
| 21 |
+
@torch.no_grad()
|
| 22 |
+
def test_ljspeech_fastspeech2_checkpoint(self):
|
| 23 |
+
models, cfg, task, generator = self.download_and_load_checkpoint(
|
| 24 |
+
"ljspeech_fastspeech2_g2p.pt",
|
| 25 |
+
arg_overrides={
|
| 26 |
+
"config_yaml": "cfg_ljspeech_g2p.yaml",
|
| 27 |
+
"vocoder": "griffin_lim",
|
| 28 |
+
"fp16": False,
|
| 29 |
+
},
|
| 30 |
+
)
|
| 31 |
+
|
| 32 |
+
batch_iterator = self.get_batch_iterator(task, "ljspeech_test", 65_536, 4_096)
|
| 33 |
+
progress = tqdm(batch_iterator, total=len(batch_iterator))
|
| 34 |
+
mcd, n_samples = 0.0, 0
|
| 35 |
+
for sample in progress:
|
| 36 |
+
sample = utils.move_to_cuda(sample) if self.use_cuda else sample
|
| 37 |
+
hypos = generator.generate(models[0], sample, has_targ=True)
|
| 38 |
+
rets = batch_mel_cepstral_distortion(
|
| 39 |
+
[hypo["targ_waveform"] for hypo in hypos],
|
| 40 |
+
[hypo["waveform"] for hypo in hypos],
|
| 41 |
+
sr=task.sr,
|
| 42 |
+
)
|
| 43 |
+
mcd += sum(d.item() for d, _ in rets)
|
| 44 |
+
n_samples += len(sample["id"].tolist())
|
| 45 |
+
|
| 46 |
+
mcd = round(mcd / n_samples, 1)
|
| 47 |
+
reference_mcd = 3.2
|
| 48 |
+
print(f"MCD: {mcd} (reference: {reference_mcd})")
|
| 49 |
+
self.assertAlmostEqual(mcd, reference_mcd, delta=0.1)
|
| 50 |
+
|
| 51 |
+
|
| 52 |
+
if __name__ == "__main__":
|
| 53 |
+
unittest.main()
|
data/fairseq/tests/speech/test_s2s_transformer.py
ADDED
|
@@ -0,0 +1,51 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) Facebook, Inc. and its affiliates.
|
| 2 |
+
#
|
| 3 |
+
# This source code is licensed under the MIT license found in the
|
| 4 |
+
# LICENSE file in the root directory of this source tree.
|
| 5 |
+
|
| 6 |
+
import unittest
|
| 7 |
+
from tests.speech import TestFairseqSpeech
|
| 8 |
+
from fairseq import utils
|
| 9 |
+
|
| 10 |
+
S3_BASE_URL = "https://dl.fbaipublicfiles.com/fairseq/"
|
| 11 |
+
|
| 12 |
+
|
| 13 |
+
class TestS2STransformer(TestFairseqSpeech):
|
| 14 |
+
def setUp(self):
|
| 15 |
+
self._set_up(
|
| 16 |
+
"s2s",
|
| 17 |
+
"speech_tests/s2s",
|
| 18 |
+
[
|
| 19 |
+
"dev_shuf200.tsv",
|
| 20 |
+
"src_feat.zip",
|
| 21 |
+
"config_specaug_lb.yaml",
|
| 22 |
+
"vocoder",
|
| 23 |
+
"vocoder_config.json",
|
| 24 |
+
],
|
| 25 |
+
)
|
| 26 |
+
|
| 27 |
+
def test_s2s_transformer_checkpoint(self):
|
| 28 |
+
self.base_test(
|
| 29 |
+
ckpt_name="s2u_transformer_reduced_fisher.pt",
|
| 30 |
+
reference_score=38.3,
|
| 31 |
+
dataset="dev_shuf200",
|
| 32 |
+
arg_overrides={
|
| 33 |
+
"config_yaml": "config_specaug_lb.yaml",
|
| 34 |
+
"multitask_config_yaml": None,
|
| 35 |
+
"target_is_code": True,
|
| 36 |
+
"target_code_size": 100,
|
| 37 |
+
"eval_inference": False,
|
| 38 |
+
},
|
| 39 |
+
score_type="bleu",
|
| 40 |
+
strict=False,
|
| 41 |
+
)
|
| 42 |
+
|
| 43 |
+
def postprocess_tokens(self, task, target, hypo_tokens):
|
| 44 |
+
tgt_tokens = utils.strip_pad(target, task.tgt_dict.pad()).int().cpu()
|
| 45 |
+
tgt_str = task.tgt_dict.string(tgt_tokens)
|
| 46 |
+
hypo_str = task.tgt_dict.string(hypo_tokens)
|
| 47 |
+
return tgt_str, hypo_str
|
| 48 |
+
|
| 49 |
+
|
| 50 |
+
if __name__ == "__main__":
|
| 51 |
+
unittest.main()
|
data/fairseq/tests/speech/test_s2t_conformer.py
ADDED
|
@@ -0,0 +1,23 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) Facebook, Inc. and its affiliates.
|
| 2 |
+
#
|
| 3 |
+
# This source code is licensed under the MIT license found in the
|
| 4 |
+
# LICENSE file in the root directory of this source tree.
|
| 5 |
+
|
| 6 |
+
import unittest
|
| 7 |
+
from tests.speech import TestFairseqSpeech
|
| 8 |
+
|
| 9 |
+
|
| 10 |
+
class TestS2TConformer(TestFairseqSpeech):
|
| 11 |
+
def setUp(self):
|
| 12 |
+
self.set_up_librispeech()
|
| 13 |
+
|
| 14 |
+
def test_librispeech_s2t_conformer_s_checkpoint(self):
|
| 15 |
+
self.base_test(
|
| 16 |
+
ckpt_name="librispeech_conformer_rel_pos_s.pt",
|
| 17 |
+
reference_score=12,
|
| 18 |
+
arg_overrides={"config_yaml": "cfg_librispeech.yaml"},
|
| 19 |
+
)
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
if __name__ == "__main__":
|
| 23 |
+
unittest.main()
|
data/fairseq/tests/speech/test_s2t_transformer.py
ADDED
|
@@ -0,0 +1,23 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) Facebook, Inc. and its affiliates.
|
| 2 |
+
#
|
| 3 |
+
# This source code is licensed under the MIT license found in the
|
| 4 |
+
# LICENSE file in the root directory of this source tree.
|
| 5 |
+
|
| 6 |
+
import unittest
|
| 7 |
+
from tests.speech import TestFairseqSpeech
|
| 8 |
+
|
| 9 |
+
|
| 10 |
+
class TestS2TTransformer(TestFairseqSpeech):
|
| 11 |
+
def setUp(self):
|
| 12 |
+
self.set_up_librispeech()
|
| 13 |
+
|
| 14 |
+
def test_librispeech_s2t_transformer_s_checkpoint(self):
|
| 15 |
+
self.base_test(
|
| 16 |
+
ckpt_name="librispeech_transformer_s.pt",
|
| 17 |
+
reference_score=9,
|
| 18 |
+
arg_overrides={"config_yaml": "cfg_librispeech.yaml"},
|
| 19 |
+
)
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
if __name__ == "__main__":
|
| 23 |
+
unittest.main()
|
data/fairseq/tests/speech/test_tts_transformer.py
ADDED
|
@@ -0,0 +1,53 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) Facebook, Inc. and its affiliates.
|
| 2 |
+
#
|
| 3 |
+
# This source code is licensed under the MIT license found in the
|
| 4 |
+
# LICENSE file in the root directory of this source tree.
|
| 5 |
+
|
| 6 |
+
import unittest
|
| 7 |
+
|
| 8 |
+
import torch
|
| 9 |
+
from tqdm import tqdm
|
| 10 |
+
|
| 11 |
+
from fairseq import utils
|
| 12 |
+
from fairseq.tasks.text_to_speech import batch_mel_cepstral_distortion
|
| 13 |
+
from tests.speech import TestFairseqSpeech
|
| 14 |
+
|
| 15 |
+
|
| 16 |
+
@unittest.skipIf(not torch.cuda.is_available(), "test requires a GPU")
|
| 17 |
+
class TestTTSTransformer(TestFairseqSpeech):
|
| 18 |
+
def setUp(self):
|
| 19 |
+
self.set_up_ljspeech()
|
| 20 |
+
|
| 21 |
+
@torch.no_grad()
|
| 22 |
+
def test_ljspeech_tts_transformer_checkpoint(self):
|
| 23 |
+
models, cfg, task, generator = self.download_and_load_checkpoint(
|
| 24 |
+
"ljspeech_transformer_g2p.pt",
|
| 25 |
+
arg_overrides={
|
| 26 |
+
"config_yaml": "cfg_ljspeech_g2p.yaml",
|
| 27 |
+
"vocoder": "griffin_lim",
|
| 28 |
+
"fp16": False,
|
| 29 |
+
},
|
| 30 |
+
)
|
| 31 |
+
|
| 32 |
+
batch_iterator = self.get_batch_iterator(task, "ljspeech_test", 65_536, 1024)
|
| 33 |
+
progress = tqdm(batch_iterator, total=len(batch_iterator))
|
| 34 |
+
mcd, n_samples = 0.0, 0
|
| 35 |
+
for sample in progress:
|
| 36 |
+
sample = utils.move_to_cuda(sample) if self.use_cuda else sample
|
| 37 |
+
hypos = generator.generate(models[0], sample, has_targ=True)
|
| 38 |
+
rets = batch_mel_cepstral_distortion(
|
| 39 |
+
[hypo["targ_waveform"] for hypo in hypos],
|
| 40 |
+
[hypo["waveform"] for hypo in hypos],
|
| 41 |
+
sr=task.sr,
|
| 42 |
+
)
|
| 43 |
+
mcd += sum(d.item() for d, _ in rets)
|
| 44 |
+
n_samples += len(sample["id"].tolist())
|
| 45 |
+
|
| 46 |
+
mcd = round(mcd / n_samples, 1)
|
| 47 |
+
reference_mcd = 3.3
|
| 48 |
+
print(f"MCD: {mcd} (reference: {reference_mcd})")
|
| 49 |
+
self.assertAlmostEqual(mcd, reference_mcd, delta=0.1)
|
| 50 |
+
|
| 51 |
+
|
| 52 |
+
if __name__ == "__main__":
|
| 53 |
+
unittest.main()
|
data/fairseq/tests/speech/test_wav2vec2.py
ADDED
|
@@ -0,0 +1,90 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) Facebook, Inc. and its affiliates.
|
| 2 |
+
#
|
| 3 |
+
# This source code is licensed under the MIT license found in the
|
| 4 |
+
# LICENSE file in the root directory of this source tree.
|
| 5 |
+
|
| 6 |
+
import unittest
|
| 7 |
+
import torch
|
| 8 |
+
from tests.speech import TestFairseqSpeech
|
| 9 |
+
from fairseq.data.data_utils import post_process
|
| 10 |
+
from fairseq import utils
|
| 11 |
+
from omegaconf import open_dict
|
| 12 |
+
|
| 13 |
+
S3_BASE_URL = "https://dl.fbaipublicfiles.com/fairseq"
|
| 14 |
+
|
| 15 |
+
|
| 16 |
+
@unittest.skipIf(not torch.cuda.is_available(), "test requires a GPU")
|
| 17 |
+
class TestWav2Vec2(TestFairseqSpeech):
|
| 18 |
+
def setUp(self):
|
| 19 |
+
self._set_up(
|
| 20 |
+
"librispeech_w2v2",
|
| 21 |
+
"conformer/wav2vec2/librispeech",
|
| 22 |
+
[
|
| 23 |
+
"test_librispeech-other.ltr",
|
| 24 |
+
"test_librispeech-other.tsv",
|
| 25 |
+
"test_librispeech-other_small.ltr_100",
|
| 26 |
+
"test_librispeech-other_small.tsv",
|
| 27 |
+
"test-other.zip",
|
| 28 |
+
"dict.ltr.txt",
|
| 29 |
+
"dict.ltr_100.txt",
|
| 30 |
+
],
|
| 31 |
+
)
|
| 32 |
+
self.unzip_files(
|
| 33 |
+
"test-other.zip",
|
| 34 |
+
)
|
| 35 |
+
|
| 36 |
+
def test_transformer_w2v2(self):
|
| 37 |
+
self.base_test(
|
| 38 |
+
ckpt_name="transformer_oss_small_100h.pt",
|
| 39 |
+
reference_score=38,
|
| 40 |
+
score_delta=1,
|
| 41 |
+
dataset="test_librispeech-other",
|
| 42 |
+
max_tokens=1000000,
|
| 43 |
+
max_positions=(700000, 1000),
|
| 44 |
+
arg_overrides={
|
| 45 |
+
"task": "audio_finetuning",
|
| 46 |
+
"labels": "ltr",
|
| 47 |
+
"nbest": 1,
|
| 48 |
+
"tpu": False,
|
| 49 |
+
},
|
| 50 |
+
strict=False,
|
| 51 |
+
)
|
| 52 |
+
|
| 53 |
+
def test_conformer_w2v2(self):
|
| 54 |
+
self.base_test(
|
| 55 |
+
ckpt_name="conformer_LS_PT_LS_FT_rope.pt",
|
| 56 |
+
reference_score=4.5,
|
| 57 |
+
score_delta=1,
|
| 58 |
+
dataset="test_librispeech-other_small",
|
| 59 |
+
max_tokens=1000000,
|
| 60 |
+
max_positions=(700000, 1000),
|
| 61 |
+
arg_overrides={
|
| 62 |
+
"task": "audio_finetuning",
|
| 63 |
+
"labels": "ltr_100",
|
| 64 |
+
"nbest": 1,
|
| 65 |
+
"tpu": False,
|
| 66 |
+
},
|
| 67 |
+
strict=True,
|
| 68 |
+
)
|
| 69 |
+
|
| 70 |
+
def build_generator(self, task, models, cfg):
|
| 71 |
+
try:
|
| 72 |
+
from examples.speech_recognition.w2l_decoder import W2lViterbiDecoder
|
| 73 |
+
except Exception:
|
| 74 |
+
raise Exception("Cannot run this test without flashlight dependency")
|
| 75 |
+
with open_dict(cfg):
|
| 76 |
+
cfg.nbest = 1
|
| 77 |
+
return W2lViterbiDecoder(cfg, task.target_dictionary)
|
| 78 |
+
|
| 79 |
+
def postprocess_tokens(self, task, target, hypo_tokens):
|
| 80 |
+
tgt_tokens = utils.strip_pad(target, task.target_dictionary.pad()).int().cpu()
|
| 81 |
+
tgt_str = task.target_dictionary.string(tgt_tokens)
|
| 82 |
+
tgt_str = post_process(tgt_str, "letter")
|
| 83 |
+
|
| 84 |
+
hypo_pieces = task.target_dictionary.string(hypo_tokens)
|
| 85 |
+
hypo_str = post_process(hypo_pieces, "letter")
|
| 86 |
+
return tgt_str, hypo_str
|
| 87 |
+
|
| 88 |
+
|
| 89 |
+
if __name__ == "__main__":
|
| 90 |
+
unittest.main()
|
data/fairseq/tests/speech/test_xm_transformer.py
ADDED
|
@@ -0,0 +1,29 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) Facebook, Inc. and its affiliates.
|
| 2 |
+
#
|
| 3 |
+
# This source code is licensed under the MIT license found in the
|
| 4 |
+
# LICENSE file in the root directory of this source tree.
|
| 5 |
+
|
| 6 |
+
import unittest
|
| 7 |
+
from tests.speech import TestFairseqSpeech
|
| 8 |
+
|
| 9 |
+
|
| 10 |
+
class TestXMTransformer(TestFairseqSpeech):
|
| 11 |
+
def setUp(self):
|
| 12 |
+
self.set_up_sotasty_es_en()
|
| 13 |
+
|
| 14 |
+
# TODO: investigate increases BLEU score (30.42 -> 31.74)
|
| 15 |
+
def test_sotasty_es_en_600m_checkpoint(self):
|
| 16 |
+
self.base_test(
|
| 17 |
+
ckpt_name="xm_transformer_600m_es_en_md.pt",
|
| 18 |
+
reference_score=31.74,
|
| 19 |
+
score_delta=0.2,
|
| 20 |
+
max_tokens=3_000_000,
|
| 21 |
+
max_positions=(1_000_000, 1_024),
|
| 22 |
+
dataset="sotasty_es_en_test_ted",
|
| 23 |
+
arg_overrides={"config_yaml": "cfg_es_en.yaml"},
|
| 24 |
+
score_type="bleu",
|
| 25 |
+
)
|
| 26 |
+
|
| 27 |
+
|
| 28 |
+
if __name__ == "__main__":
|
| 29 |
+
unittest.main()
|
data/fairseq/tests/speech_recognition/__init__.py
ADDED
|
File without changes
|
data/fairseq/tests/speech_recognition/asr_test_base.py
ADDED
|
@@ -0,0 +1,557 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python3
|
| 2 |
+
|
| 3 |
+
import argparse
|
| 4 |
+
import os
|
| 5 |
+
import unittest
|
| 6 |
+
from inspect import currentframe, getframeinfo
|
| 7 |
+
|
| 8 |
+
import numpy as np
|
| 9 |
+
import torch
|
| 10 |
+
from examples.speech_recognition.data.data_utils import lengths_to_encoder_padding_mask
|
| 11 |
+
from fairseq.data import data_utils as fairseq_data_utils
|
| 12 |
+
from fairseq.data.dictionary import Dictionary
|
| 13 |
+
from fairseq.models import (
|
| 14 |
+
BaseFairseqModel,
|
| 15 |
+
FairseqDecoder,
|
| 16 |
+
FairseqEncoder,
|
| 17 |
+
FairseqEncoderDecoderModel,
|
| 18 |
+
FairseqEncoderModel,
|
| 19 |
+
FairseqModel,
|
| 20 |
+
)
|
| 21 |
+
from fairseq.tasks.fairseq_task import LegacyFairseqTask
|
| 22 |
+
|
| 23 |
+
|
| 24 |
+
DEFAULT_TEST_VOCAB_SIZE = 100
|
| 25 |
+
|
| 26 |
+
|
| 27 |
+
# ///////////////////////////////////////////////////////////////////////////
|
| 28 |
+
# utility function to setup dummy dict/task/input
|
| 29 |
+
# ///////////////////////////////////////////////////////////////////////////
|
| 30 |
+
|
| 31 |
+
|
| 32 |
+
def get_dummy_dictionary(vocab_size=DEFAULT_TEST_VOCAB_SIZE):
|
| 33 |
+
dummy_dict = Dictionary()
|
| 34 |
+
# add dummy symbol to satisfy vocab size
|
| 35 |
+
for id, _ in enumerate(range(vocab_size)):
|
| 36 |
+
dummy_dict.add_symbol("{}".format(id), 1000)
|
| 37 |
+
return dummy_dict
|
| 38 |
+
|
| 39 |
+
|
| 40 |
+
class DummyTask(LegacyFairseqTask):
|
| 41 |
+
def __init__(self, args):
|
| 42 |
+
super().__init__(args)
|
| 43 |
+
self.dictionary = get_dummy_dictionary()
|
| 44 |
+
if getattr(self.args, "ctc", False):
|
| 45 |
+
self.dictionary.add_symbol("<ctc_blank>")
|
| 46 |
+
self.tgt_dict = self.dictionary
|
| 47 |
+
|
| 48 |
+
@property
|
| 49 |
+
def target_dictionary(self):
|
| 50 |
+
return self.dictionary
|
| 51 |
+
|
| 52 |
+
|
| 53 |
+
def get_dummy_task_and_parser():
|
| 54 |
+
"""
|
| 55 |
+
to build a fariseq model, we need some dummy parse and task. This function
|
| 56 |
+
is used to create dummy task and parser to faciliate model/criterion test
|
| 57 |
+
|
| 58 |
+
Note: we use FbSpeechRecognitionTask as the dummy task. You may want
|
| 59 |
+
to use other task by providing another function
|
| 60 |
+
"""
|
| 61 |
+
parser = argparse.ArgumentParser(
|
| 62 |
+
description="test_dummy_s2s_task", argument_default=argparse.SUPPRESS
|
| 63 |
+
)
|
| 64 |
+
DummyTask.add_args(parser)
|
| 65 |
+
args = parser.parse_args([])
|
| 66 |
+
task = DummyTask.setup_task(args)
|
| 67 |
+
return task, parser
|
| 68 |
+
|
| 69 |
+
|
| 70 |
+
def get_dummy_input(T=100, D=80, B=5, K=100):
|
| 71 |
+
forward_input = {}
|
| 72 |
+
# T max sequence length
|
| 73 |
+
# D feature vector dimension
|
| 74 |
+
# B batch size
|
| 75 |
+
# K target dimension size
|
| 76 |
+
feature = torch.randn(B, T, D)
|
| 77 |
+
# this (B, T, D) layout is just a convention, you can override it by
|
| 78 |
+
# write your own _prepare_forward_input function
|
| 79 |
+
src_lengths = torch.from_numpy(
|
| 80 |
+
np.random.randint(low=1, high=T, size=B, dtype=np.int64)
|
| 81 |
+
)
|
| 82 |
+
src_lengths[0] = T # make sure the maximum length matches
|
| 83 |
+
prev_output_tokens = []
|
| 84 |
+
for b in range(B):
|
| 85 |
+
token_length = np.random.randint(low=1, high=src_lengths[b].item() + 1)
|
| 86 |
+
tokens = np.random.randint(low=0, high=K, size=token_length, dtype=np.int64)
|
| 87 |
+
prev_output_tokens.append(torch.from_numpy(tokens))
|
| 88 |
+
|
| 89 |
+
prev_output_tokens = fairseq_data_utils.collate_tokens(
|
| 90 |
+
prev_output_tokens,
|
| 91 |
+
pad_idx=1,
|
| 92 |
+
eos_idx=2,
|
| 93 |
+
left_pad=False,
|
| 94 |
+
move_eos_to_beginning=False,
|
| 95 |
+
)
|
| 96 |
+
src_lengths, sorted_order = src_lengths.sort(descending=True)
|
| 97 |
+
forward_input["src_tokens"] = feature.index_select(0, sorted_order)
|
| 98 |
+
forward_input["src_lengths"] = src_lengths
|
| 99 |
+
forward_input["prev_output_tokens"] = prev_output_tokens
|
| 100 |
+
|
| 101 |
+
return forward_input
|
| 102 |
+
|
| 103 |
+
|
| 104 |
+
def get_dummy_encoder_output(encoder_out_shape=(100, 80, 5)):
|
| 105 |
+
"""
|
| 106 |
+
This only provides an example to generate dummy encoder output
|
| 107 |
+
"""
|
| 108 |
+
(T, B, D) = encoder_out_shape
|
| 109 |
+
encoder_out = {}
|
| 110 |
+
|
| 111 |
+
encoder_out["encoder_out"] = torch.from_numpy(
|
| 112 |
+
np.random.randn(*encoder_out_shape).astype(np.float32)
|
| 113 |
+
)
|
| 114 |
+
seq_lengths = torch.from_numpy(np.random.randint(low=1, high=T, size=B))
|
| 115 |
+
# some dummy mask
|
| 116 |
+
encoder_out["encoder_padding_mask"] = torch.arange(T).view(1, T).expand(
|
| 117 |
+
B, -1
|
| 118 |
+
) >= seq_lengths.view(B, 1).expand(-1, T)
|
| 119 |
+
encoder_out["encoder_padding_mask"].t_()
|
| 120 |
+
|
| 121 |
+
# encoer_padding_mask is (T, B) tensor, with (t, b)-th element indicate
|
| 122 |
+
# whether encoder_out[t, b] is valid (=0) or not (=1)
|
| 123 |
+
return encoder_out
|
| 124 |
+
|
| 125 |
+
|
| 126 |
+
def _current_postion_info():
|
| 127 |
+
cf = currentframe()
|
| 128 |
+
frameinfo = " (at {}:{})".format(
|
| 129 |
+
os.path.basename(getframeinfo(cf).filename), cf.f_back.f_lineno
|
| 130 |
+
)
|
| 131 |
+
return frameinfo
|
| 132 |
+
|
| 133 |
+
|
| 134 |
+
def check_encoder_output(encoder_output, batch_size=None):
|
| 135 |
+
"""we expect encoder_output to be a dict with the following
|
| 136 |
+
key/value pairs:
|
| 137 |
+
- encoder_out: a Torch.Tensor
|
| 138 |
+
- encoder_padding_mask: a binary Torch.Tensor
|
| 139 |
+
"""
|
| 140 |
+
if not isinstance(encoder_output, dict):
|
| 141 |
+
msg = (
|
| 142 |
+
"FairseqEncoderModel.forward(...) must be a dict" + _current_postion_info()
|
| 143 |
+
)
|
| 144 |
+
return False, msg
|
| 145 |
+
|
| 146 |
+
if "encoder_out" not in encoder_output:
|
| 147 |
+
msg = (
|
| 148 |
+
"FairseqEncoderModel.forward(...) must contain encoder_out"
|
| 149 |
+
+ _current_postion_info()
|
| 150 |
+
)
|
| 151 |
+
return False, msg
|
| 152 |
+
|
| 153 |
+
if "encoder_padding_mask" not in encoder_output:
|
| 154 |
+
msg = (
|
| 155 |
+
"FairseqEncoderModel.forward(...) must contain encoder_padding_mask"
|
| 156 |
+
+ _current_postion_info()
|
| 157 |
+
)
|
| 158 |
+
return False, msg
|
| 159 |
+
|
| 160 |
+
if not isinstance(encoder_output["encoder_out"], torch.Tensor):
|
| 161 |
+
msg = "encoder_out must be a torch.Tensor" + _current_postion_info()
|
| 162 |
+
return False, msg
|
| 163 |
+
|
| 164 |
+
if encoder_output["encoder_out"].dtype != torch.float32:
|
| 165 |
+
msg = "encoder_out must have float32 dtype" + _current_postion_info()
|
| 166 |
+
return False, msg
|
| 167 |
+
|
| 168 |
+
mask = encoder_output["encoder_padding_mask"]
|
| 169 |
+
if mask is not None:
|
| 170 |
+
if not isinstance(mask, torch.Tensor):
|
| 171 |
+
msg = (
|
| 172 |
+
"encoder_padding_mask must be a torch.Tensor" + _current_postion_info()
|
| 173 |
+
)
|
| 174 |
+
return False, msg
|
| 175 |
+
if mask.dtype != torch.uint8 and (
|
| 176 |
+
not hasattr(torch, "bool") or mask.dtype != torch.bool
|
| 177 |
+
):
|
| 178 |
+
msg = (
|
| 179 |
+
"encoder_padding_mask must have dtype of uint8"
|
| 180 |
+
+ _current_postion_info()
|
| 181 |
+
)
|
| 182 |
+
return False, msg
|
| 183 |
+
|
| 184 |
+
if mask.dim() != 2:
|
| 185 |
+
msg = (
|
| 186 |
+
"we expect encoder_padding_mask to be a 2-d tensor, in shape (T, B)"
|
| 187 |
+
+ _current_postion_info()
|
| 188 |
+
)
|
| 189 |
+
return False, msg
|
| 190 |
+
|
| 191 |
+
if batch_size is not None and mask.size(1) != batch_size:
|
| 192 |
+
msg = (
|
| 193 |
+
"we expect encoder_padding_mask to be a 2-d tensor, with size(1)"
|
| 194 |
+
+ " being the batch size"
|
| 195 |
+
+ _current_postion_info()
|
| 196 |
+
)
|
| 197 |
+
return False, msg
|
| 198 |
+
return True, None
|
| 199 |
+
|
| 200 |
+
|
| 201 |
+
def check_decoder_output(decoder_output):
|
| 202 |
+
"""we expect output from a decoder is a tuple with the following constraint:
|
| 203 |
+
- the first element is a torch.Tensor
|
| 204 |
+
- the second element can be anything (reserved for future use)
|
| 205 |
+
"""
|
| 206 |
+
if not isinstance(decoder_output, tuple):
|
| 207 |
+
msg = "FariseqDecoder output must be a tuple" + _current_postion_info()
|
| 208 |
+
return False, msg
|
| 209 |
+
|
| 210 |
+
if len(decoder_output) != 2:
|
| 211 |
+
msg = "FairseqDecoder output must be 2-elem tuple" + _current_postion_info()
|
| 212 |
+
return False, msg
|
| 213 |
+
|
| 214 |
+
if not isinstance(decoder_output[0], torch.Tensor):
|
| 215 |
+
msg = (
|
| 216 |
+
"FariseqDecoder output[0] must be a torch.Tensor" + _current_postion_info()
|
| 217 |
+
)
|
| 218 |
+
return False, msg
|
| 219 |
+
|
| 220 |
+
return True, None
|
| 221 |
+
|
| 222 |
+
|
| 223 |
+
# ///////////////////////////////////////////////////////////////////////////
|
| 224 |
+
# Base Test class
|
| 225 |
+
# ///////////////////////////////////////////////////////////////////////////
|
| 226 |
+
|
| 227 |
+
|
| 228 |
+
class TestBaseFairseqModelBase(unittest.TestCase):
|
| 229 |
+
"""
|
| 230 |
+
This class is used to facilitate writing unittest for any class derived from
|
| 231 |
+
`BaseFairseqModel`.
|
| 232 |
+
"""
|
| 233 |
+
|
| 234 |
+
@classmethod
|
| 235 |
+
def setUpClass(cls):
|
| 236 |
+
if cls is TestBaseFairseqModelBase:
|
| 237 |
+
raise unittest.SkipTest("Skipping test case in base")
|
| 238 |
+
super().setUpClass()
|
| 239 |
+
|
| 240 |
+
def setUpModel(self, model):
|
| 241 |
+
self.assertTrue(isinstance(model, BaseFairseqModel))
|
| 242 |
+
self.model = model
|
| 243 |
+
|
| 244 |
+
def setupInput(self):
|
| 245 |
+
pass
|
| 246 |
+
|
| 247 |
+
def setUp(self):
|
| 248 |
+
self.model = None
|
| 249 |
+
self.forward_input = None
|
| 250 |
+
pass
|
| 251 |
+
|
| 252 |
+
|
| 253 |
+
class TestFairseqEncoderDecoderModelBase(TestBaseFairseqModelBase):
|
| 254 |
+
"""
|
| 255 |
+
base code to test FairseqEncoderDecoderModel (formally known as
|
| 256 |
+
`FairseqModel`) must be derived from this base class
|
| 257 |
+
"""
|
| 258 |
+
|
| 259 |
+
@classmethod
|
| 260 |
+
def setUpClass(cls):
|
| 261 |
+
if cls is TestFairseqEncoderDecoderModelBase:
|
| 262 |
+
raise unittest.SkipTest("Skipping test case in base")
|
| 263 |
+
super().setUpClass()
|
| 264 |
+
|
| 265 |
+
def setUpModel(self, model_cls, extra_args_setters=None):
|
| 266 |
+
self.assertTrue(
|
| 267 |
+
issubclass(model_cls, (FairseqEncoderDecoderModel, FairseqModel)),
|
| 268 |
+
msg="This class only tests for FairseqModel subclasses",
|
| 269 |
+
)
|
| 270 |
+
|
| 271 |
+
task, parser = get_dummy_task_and_parser()
|
| 272 |
+
model_cls.add_args(parser)
|
| 273 |
+
|
| 274 |
+
args = parser.parse_args([])
|
| 275 |
+
|
| 276 |
+
if extra_args_setters is not None:
|
| 277 |
+
for args_setter in extra_args_setters:
|
| 278 |
+
args_setter(args)
|
| 279 |
+
model = model_cls.build_model(args, task)
|
| 280 |
+
self.model = model
|
| 281 |
+
|
| 282 |
+
def setUpInput(self, input=None):
|
| 283 |
+
self.forward_input = get_dummy_input() if input is None else input
|
| 284 |
+
|
| 285 |
+
def setUp(self):
|
| 286 |
+
super().setUp()
|
| 287 |
+
|
| 288 |
+
def test_forward(self):
|
| 289 |
+
if self.model and self.forward_input:
|
| 290 |
+
forward_output = self.model.forward(**self.forward_input)
|
| 291 |
+
# for FairseqEncoderDecoderModel, forward returns a tuple of two
|
| 292 |
+
# elements, the first one is a Torch.Tensor
|
| 293 |
+
succ, msg = check_decoder_output(forward_output)
|
| 294 |
+
if not succ:
|
| 295 |
+
self.assertTrue(succ, msg=msg)
|
| 296 |
+
self.forward_output = forward_output
|
| 297 |
+
|
| 298 |
+
def test_get_normalized_probs(self):
|
| 299 |
+
if self.model and self.forward_input:
|
| 300 |
+
forward_output = self.model.forward(**self.forward_input)
|
| 301 |
+
logprob = self.model.get_normalized_probs(forward_output, log_probs=True)
|
| 302 |
+
prob = self.model.get_normalized_probs(forward_output, log_probs=False)
|
| 303 |
+
|
| 304 |
+
# in order for different models/criterion to play with each other
|
| 305 |
+
# we need to know whether the logprob or prob output is batch_first
|
| 306 |
+
# or not. We assume an additional attribute will be attached to logprob
|
| 307 |
+
# or prob. If you find your code failed here, simply override
|
| 308 |
+
# FairseqModel.get_normalized_probs, see example at
|
| 309 |
+
# https://fburl.com/batch_first_example
|
| 310 |
+
self.assertTrue(hasattr(logprob, "batch_first"))
|
| 311 |
+
self.assertTrue(hasattr(prob, "batch_first"))
|
| 312 |
+
|
| 313 |
+
self.assertTrue(torch.is_tensor(logprob))
|
| 314 |
+
self.assertTrue(torch.is_tensor(prob))
|
| 315 |
+
|
| 316 |
+
|
| 317 |
+
class TestFairseqEncoderModelBase(TestBaseFairseqModelBase):
|
| 318 |
+
"""
|
| 319 |
+
base class to test FairseqEncoderModel
|
| 320 |
+
"""
|
| 321 |
+
|
| 322 |
+
@classmethod
|
| 323 |
+
def setUpClass(cls):
|
| 324 |
+
if cls is TestFairseqEncoderModelBase:
|
| 325 |
+
raise unittest.SkipTest("Skipping test case in base")
|
| 326 |
+
super().setUpClass()
|
| 327 |
+
|
| 328 |
+
def setUpModel(self, model_cls, extra_args_setters=None):
|
| 329 |
+
self.assertTrue(
|
| 330 |
+
issubclass(model_cls, FairseqEncoderModel),
|
| 331 |
+
msg="This class is only used for testing FairseqEncoderModel",
|
| 332 |
+
)
|
| 333 |
+
task, parser = get_dummy_task_and_parser()
|
| 334 |
+
model_cls.add_args(parser)
|
| 335 |
+
args = parser.parse_args([])
|
| 336 |
+
if extra_args_setters is not None:
|
| 337 |
+
for args_setter in extra_args_setters:
|
| 338 |
+
args_setter(args)
|
| 339 |
+
|
| 340 |
+
model = model_cls.build_model(args, task)
|
| 341 |
+
self.model = model
|
| 342 |
+
|
| 343 |
+
def setUpInput(self, input=None):
|
| 344 |
+
self.forward_input = get_dummy_input() if input is None else input
|
| 345 |
+
# get_dummy_input() is originally for s2s, here we delete extra dict
|
| 346 |
+
# items, so it can be used for EncoderModel / Encoder as well
|
| 347 |
+
self.forward_input.pop("prev_output_tokens", None)
|
| 348 |
+
|
| 349 |
+
def setUp(self):
|
| 350 |
+
super().setUp()
|
| 351 |
+
|
| 352 |
+
def test_forward(self):
|
| 353 |
+
if self.forward_input and self.model:
|
| 354 |
+
bsz = self.forward_input["src_tokens"].size(0)
|
| 355 |
+
forward_output = self.model.forward(**self.forward_input)
|
| 356 |
+
|
| 357 |
+
# we expect forward_output to be a dict with the following
|
| 358 |
+
# key/value pairs:
|
| 359 |
+
# - encoder_out: a Torch.Tensor
|
| 360 |
+
# - encoder_padding_mask: a binary Torch.Tensor
|
| 361 |
+
succ, msg = check_encoder_output(forward_output, batch_size=bsz)
|
| 362 |
+
if not succ:
|
| 363 |
+
self.assertTrue(succ, msg=msg)
|
| 364 |
+
self.forward_output = forward_output
|
| 365 |
+
|
| 366 |
+
def test_get_normalized_probs(self):
|
| 367 |
+
if self.model and self.forward_input:
|
| 368 |
+
forward_output = self.model.forward(**self.forward_input)
|
| 369 |
+
logprob = self.model.get_normalized_probs(forward_output, log_probs=True)
|
| 370 |
+
prob = self.model.get_normalized_probs(forward_output, log_probs=False)
|
| 371 |
+
|
| 372 |
+
# in order for different models/criterion to play with each other
|
| 373 |
+
# we need to know whether the logprob or prob output is batch_first
|
| 374 |
+
# or not. We assume an additional attribute will be attached to logprob
|
| 375 |
+
# or prob. If you find your code failed here, simply override
|
| 376 |
+
# FairseqModel.get_normalized_probs, see example at
|
| 377 |
+
# https://fburl.com/batch_first_example
|
| 378 |
+
self.assertTrue(hasattr(logprob, "batch_first"))
|
| 379 |
+
self.assertTrue(hasattr(prob, "batch_first"))
|
| 380 |
+
|
| 381 |
+
self.assertTrue(torch.is_tensor(logprob))
|
| 382 |
+
self.assertTrue(torch.is_tensor(prob))
|
| 383 |
+
|
| 384 |
+
|
| 385 |
+
class TestFairseqEncoderBase(unittest.TestCase):
|
| 386 |
+
"""
|
| 387 |
+
base class to test FairseqEncoder
|
| 388 |
+
"""
|
| 389 |
+
|
| 390 |
+
@classmethod
|
| 391 |
+
def setUpClass(cls):
|
| 392 |
+
if cls is TestFairseqEncoderBase:
|
| 393 |
+
raise unittest.SkipTest("Skipping test case in base")
|
| 394 |
+
super().setUpClass()
|
| 395 |
+
|
| 396 |
+
def setUpEncoder(self, encoder):
|
| 397 |
+
self.assertTrue(
|
| 398 |
+
isinstance(encoder, FairseqEncoder),
|
| 399 |
+
msg="This class is only used for test FairseqEncoder",
|
| 400 |
+
)
|
| 401 |
+
self.encoder = encoder
|
| 402 |
+
|
| 403 |
+
def setUpInput(self, input=None):
|
| 404 |
+
self.forward_input = get_dummy_input() if input is None else input
|
| 405 |
+
# get_dummy_input() is originally for s2s, here we delete extra dict
|
| 406 |
+
# items, so it can be used for EncoderModel / Encoder as well
|
| 407 |
+
self.forward_input.pop("prev_output_tokens", None)
|
| 408 |
+
|
| 409 |
+
def setUp(self):
|
| 410 |
+
self.encoder = None
|
| 411 |
+
self.forward_input = None
|
| 412 |
+
|
| 413 |
+
def test_forward(self):
|
| 414 |
+
if self.encoder and self.forward_input:
|
| 415 |
+
bsz = self.forward_input["src_tokens"].size(0)
|
| 416 |
+
|
| 417 |
+
forward_output = self.encoder.forward(**self.forward_input)
|
| 418 |
+
succ, msg = check_encoder_output(forward_output, batch_size=bsz)
|
| 419 |
+
if not succ:
|
| 420 |
+
self.assertTrue(succ, msg=msg)
|
| 421 |
+
self.forward_output = forward_output
|
| 422 |
+
|
| 423 |
+
|
| 424 |
+
class TestFairseqDecoderBase(unittest.TestCase):
|
| 425 |
+
"""
|
| 426 |
+
base class to test FairseqDecoder
|
| 427 |
+
"""
|
| 428 |
+
|
| 429 |
+
@classmethod
|
| 430 |
+
def setUpClass(cls):
|
| 431 |
+
if cls is TestFairseqDecoderBase:
|
| 432 |
+
raise unittest.SkipTest("Skipping test case in base")
|
| 433 |
+
super().setUpClass()
|
| 434 |
+
|
| 435 |
+
def setUpDecoder(self, decoder):
|
| 436 |
+
self.assertTrue(
|
| 437 |
+
isinstance(decoder, FairseqDecoder),
|
| 438 |
+
msg="This class is only used for test FairseqDecoder",
|
| 439 |
+
)
|
| 440 |
+
self.decoder = decoder
|
| 441 |
+
|
| 442 |
+
def setUpInput(self, input=None):
|
| 443 |
+
self.forward_input = get_dummy_encoder_output() if input is None else input
|
| 444 |
+
|
| 445 |
+
def setUpPrevOutputTokens(self, tokens=None):
|
| 446 |
+
if tokens is None:
|
| 447 |
+
self.encoder_input = get_dummy_input()
|
| 448 |
+
self.prev_output_tokens = self.encoder_input["prev_output_tokens"]
|
| 449 |
+
else:
|
| 450 |
+
self.prev_output_tokens = tokens
|
| 451 |
+
|
| 452 |
+
def setUp(self):
|
| 453 |
+
self.decoder = None
|
| 454 |
+
self.forward_input = None
|
| 455 |
+
self.prev_output_tokens = None
|
| 456 |
+
|
| 457 |
+
def test_forward(self):
|
| 458 |
+
if (
|
| 459 |
+
self.decoder is not None
|
| 460 |
+
and self.forward_input is not None
|
| 461 |
+
and self.prev_output_tokens is not None
|
| 462 |
+
):
|
| 463 |
+
forward_output = self.decoder.forward(
|
| 464 |
+
prev_output_tokens=self.prev_output_tokens,
|
| 465 |
+
encoder_out=self.forward_input,
|
| 466 |
+
)
|
| 467 |
+
succ, msg = check_decoder_output(forward_output)
|
| 468 |
+
if not succ:
|
| 469 |
+
self.assertTrue(succ, msg=msg)
|
| 470 |
+
self.forward_input = forward_output
|
| 471 |
+
|
| 472 |
+
|
| 473 |
+
class DummyEncoderModel(FairseqEncoderModel):
|
| 474 |
+
def __init__(self, encoder):
|
| 475 |
+
super().__init__(encoder)
|
| 476 |
+
|
| 477 |
+
@classmethod
|
| 478 |
+
def build_model(cls, args, task):
|
| 479 |
+
return cls(DummyEncoder())
|
| 480 |
+
|
| 481 |
+
def get_logits(self, net_output):
|
| 482 |
+
# Inverse of sigmoid to use with BinaryCrossEntropyWithLogitsCriterion as
|
| 483 |
+
# F.binary_cross_entropy_with_logits combines sigmoid and CE
|
| 484 |
+
return torch.log(
|
| 485 |
+
torch.div(net_output["encoder_out"], 1 - net_output["encoder_out"])
|
| 486 |
+
)
|
| 487 |
+
|
| 488 |
+
def get_normalized_probs(self, net_output, log_probs, sample=None):
|
| 489 |
+
lprobs = super().get_normalized_probs(net_output, log_probs, sample=sample)
|
| 490 |
+
lprobs.batch_first = True
|
| 491 |
+
return lprobs
|
| 492 |
+
|
| 493 |
+
|
| 494 |
+
class DummyEncoder(FairseqEncoder):
|
| 495 |
+
def __init__(self):
|
| 496 |
+
super().__init__(None)
|
| 497 |
+
|
| 498 |
+
def forward(self, src_tokens, src_lengths):
|
| 499 |
+
mask, max_len = lengths_to_encoder_padding_mask(src_lengths)
|
| 500 |
+
return {"encoder_out": src_tokens, "encoder_padding_mask": mask}
|
| 501 |
+
|
| 502 |
+
|
| 503 |
+
class CrossEntropyCriterionTestBase(unittest.TestCase):
|
| 504 |
+
@classmethod
|
| 505 |
+
def setUpClass(cls):
|
| 506 |
+
if cls is CrossEntropyCriterionTestBase:
|
| 507 |
+
raise unittest.SkipTest("Skipping base class test case")
|
| 508 |
+
super().setUpClass()
|
| 509 |
+
|
| 510 |
+
def setUpArgs(self):
|
| 511 |
+
args = argparse.Namespace()
|
| 512 |
+
args.sentence_avg = False
|
| 513 |
+
args.threshold = 0.1 # to use with BinaryCrossEntropyWithLogitsCriterion
|
| 514 |
+
return args
|
| 515 |
+
|
| 516 |
+
def setUp(self):
|
| 517 |
+
args = self.setUpArgs()
|
| 518 |
+
self.model = DummyEncoderModel(encoder=DummyEncoder())
|
| 519 |
+
self.criterion = self.criterion_cls.build_criterion(args, task=DummyTask(args))
|
| 520 |
+
|
| 521 |
+
def get_src_tokens(self, correct_prediction, aggregate):
|
| 522 |
+
"""
|
| 523 |
+
correct_prediction: True if the net_output (src_tokens) should
|
| 524 |
+
predict the correct target
|
| 525 |
+
aggregate: True if the criterion expects net_output (src_tokens)
|
| 526 |
+
aggregated across time axis
|
| 527 |
+
"""
|
| 528 |
+
predicted_idx = 0 if correct_prediction else 1
|
| 529 |
+
if aggregate:
|
| 530 |
+
src_tokens = torch.zeros((2, 2), dtype=torch.float)
|
| 531 |
+
for b in range(2):
|
| 532 |
+
src_tokens[b][predicted_idx] = 1.0
|
| 533 |
+
else:
|
| 534 |
+
src_tokens = torch.zeros((2, 10, 2), dtype=torch.float)
|
| 535 |
+
for b in range(2):
|
| 536 |
+
for t in range(10):
|
| 537 |
+
src_tokens[b][t][predicted_idx] = 1.0
|
| 538 |
+
return src_tokens
|
| 539 |
+
|
| 540 |
+
def get_target(self, soft_target):
|
| 541 |
+
if soft_target:
|
| 542 |
+
target = torch.zeros((2, 2), dtype=torch.float)
|
| 543 |
+
for b in range(2):
|
| 544 |
+
target[b][0] = 1.0
|
| 545 |
+
else:
|
| 546 |
+
target = torch.zeros((2, 10), dtype=torch.long)
|
| 547 |
+
return target
|
| 548 |
+
|
| 549 |
+
def get_test_sample(self, correct, soft_target, aggregate):
|
| 550 |
+
src_tokens = self.get_src_tokens(correct, aggregate)
|
| 551 |
+
target = self.get_target(soft_target)
|
| 552 |
+
L = src_tokens.size(1)
|
| 553 |
+
return {
|
| 554 |
+
"net_input": {"src_tokens": src_tokens, "src_lengths": torch.tensor([L])},
|
| 555 |
+
"target": target,
|
| 556 |
+
"ntokens": src_tokens.size(0) * src_tokens.size(1),
|
| 557 |
+
}
|
data/fairseq/tests/speech_recognition/test_collaters.py
ADDED
|
@@ -0,0 +1,58 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python3
|
| 2 |
+
# Copyright (c) Facebook, Inc. and its affiliates.
|
| 3 |
+
#
|
| 4 |
+
# This source code is licensed under the MIT license found in the
|
| 5 |
+
# LICENSE file in the root directory of this source tree.
|
| 6 |
+
|
| 7 |
+
import unittest
|
| 8 |
+
|
| 9 |
+
import numpy as np
|
| 10 |
+
import torch
|
| 11 |
+
from examples.speech_recognition.data.collaters import Seq2SeqCollater
|
| 12 |
+
|
| 13 |
+
|
| 14 |
+
class TestSeq2SeqCollator(unittest.TestCase):
|
| 15 |
+
def test_collate(self):
|
| 16 |
+
|
| 17 |
+
eos_idx = 1
|
| 18 |
+
pad_idx = 0
|
| 19 |
+
collater = Seq2SeqCollater(
|
| 20 |
+
feature_index=0, label_index=1, pad_index=pad_idx, eos_index=eos_idx
|
| 21 |
+
)
|
| 22 |
+
|
| 23 |
+
# 2 frames in the first sample and 3 frames in the second one
|
| 24 |
+
frames1 = np.array([[7, 8], [9, 10]])
|
| 25 |
+
frames2 = np.array([[1, 2], [3, 4], [5, 6]])
|
| 26 |
+
target1 = np.array([4, 2, 3, eos_idx])
|
| 27 |
+
target2 = np.array([3, 2, eos_idx])
|
| 28 |
+
sample1 = {"id": 0, "data": [frames1, target1]}
|
| 29 |
+
sample2 = {"id": 1, "data": [frames2, target2]}
|
| 30 |
+
batch = collater.collate([sample1, sample2])
|
| 31 |
+
|
| 32 |
+
# collate sort inputs by frame's length before creating the batch
|
| 33 |
+
self.assertTensorEqual(batch["id"], torch.tensor([1, 0]))
|
| 34 |
+
self.assertEqual(batch["ntokens"], 7)
|
| 35 |
+
self.assertTensorEqual(
|
| 36 |
+
batch["net_input"]["src_tokens"],
|
| 37 |
+
torch.tensor(
|
| 38 |
+
[[[1, 2], [3, 4], [5, 6]], [[7, 8], [9, 10], [pad_idx, pad_idx]]]
|
| 39 |
+
),
|
| 40 |
+
)
|
| 41 |
+
self.assertTensorEqual(
|
| 42 |
+
batch["net_input"]["prev_output_tokens"],
|
| 43 |
+
torch.tensor([[eos_idx, 3, 2, pad_idx], [eos_idx, 4, 2, 3]]),
|
| 44 |
+
)
|
| 45 |
+
self.assertTensorEqual(batch["net_input"]["src_lengths"], torch.tensor([3, 2]))
|
| 46 |
+
self.assertTensorEqual(
|
| 47 |
+
batch["target"],
|
| 48 |
+
torch.tensor([[3, 2, eos_idx, pad_idx], [4, 2, 3, eos_idx]]),
|
| 49 |
+
)
|
| 50 |
+
self.assertEqual(batch["nsentences"], 2)
|
| 51 |
+
|
| 52 |
+
def assertTensorEqual(self, t1, t2):
|
| 53 |
+
self.assertEqual(t1.size(), t2.size(), "size mismatch")
|
| 54 |
+
self.assertEqual(t1.ne(t2).long().sum(), 0)
|
| 55 |
+
|
| 56 |
+
|
| 57 |
+
if __name__ == "__main__":
|
| 58 |
+
unittest.main()
|
data/fairseq/tests/speech_recognition/test_cross_entropy.py
ADDED
|
@@ -0,0 +1,37 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python3
|
| 2 |
+
# Copyright (c) Facebook, Inc. and its affiliates.
|
| 3 |
+
#
|
| 4 |
+
# This source code is licensed under the MIT license found in the
|
| 5 |
+
# LICENSE file in the root directory of this source tree.
|
| 6 |
+
|
| 7 |
+
from examples.speech_recognition.criterions.cross_entropy_acc import (
|
| 8 |
+
CrossEntropyWithAccCriterion,
|
| 9 |
+
)
|
| 10 |
+
|
| 11 |
+
from .asr_test_base import CrossEntropyCriterionTestBase
|
| 12 |
+
|
| 13 |
+
|
| 14 |
+
class CrossEntropyWithAccCriterionTest(CrossEntropyCriterionTestBase):
|
| 15 |
+
def setUp(self):
|
| 16 |
+
self.criterion_cls = CrossEntropyWithAccCriterion
|
| 17 |
+
super().setUp()
|
| 18 |
+
|
| 19 |
+
def test_cross_entropy_all_correct(self):
|
| 20 |
+
sample = self.get_test_sample(correct=True, soft_target=False, aggregate=False)
|
| 21 |
+
loss, sample_size, logging_output = self.criterion(
|
| 22 |
+
self.model, sample, "sum", log_probs=True
|
| 23 |
+
)
|
| 24 |
+
assert logging_output["correct"] == 20
|
| 25 |
+
assert logging_output["total"] == 20
|
| 26 |
+
assert logging_output["sample_size"] == 20
|
| 27 |
+
assert logging_output["ntokens"] == 20
|
| 28 |
+
|
| 29 |
+
def test_cross_entropy_all_wrong(self):
|
| 30 |
+
sample = self.get_test_sample(correct=False, soft_target=False, aggregate=False)
|
| 31 |
+
loss, sample_size, logging_output = self.criterion(
|
| 32 |
+
self.model, sample, "sum", log_probs=True
|
| 33 |
+
)
|
| 34 |
+
assert logging_output["correct"] == 0
|
| 35 |
+
assert logging_output["total"] == 20
|
| 36 |
+
assert logging_output["sample_size"] == 20
|
| 37 |
+
assert logging_output["ntokens"] == 20
|
data/fairseq/tests/speech_recognition/test_data_utils.py
ADDED
|
@@ -0,0 +1,62 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python3
|
| 2 |
+
# Copyright (c) Facebook, Inc. and its affiliates.
|
| 3 |
+
#
|
| 4 |
+
# This source code is licensed under the MIT license found in the
|
| 5 |
+
# LICENSE file in the root directory of this source tree.
|
| 6 |
+
import unittest
|
| 7 |
+
|
| 8 |
+
import torch
|
| 9 |
+
from examples.speech_recognition.data import data_utils
|
| 10 |
+
|
| 11 |
+
|
| 12 |
+
class DataUtilsTest(unittest.TestCase):
|
| 13 |
+
def test_normalization(self):
|
| 14 |
+
sample_len1 = torch.tensor(
|
| 15 |
+
[
|
| 16 |
+
[
|
| 17 |
+
-0.7661,
|
| 18 |
+
-1.3889,
|
| 19 |
+
-2.0972,
|
| 20 |
+
-0.9134,
|
| 21 |
+
-0.7071,
|
| 22 |
+
-0.9765,
|
| 23 |
+
-0.8700,
|
| 24 |
+
-0.8283,
|
| 25 |
+
0.7512,
|
| 26 |
+
1.3211,
|
| 27 |
+
2.1532,
|
| 28 |
+
2.1174,
|
| 29 |
+
1.2800,
|
| 30 |
+
1.2633,
|
| 31 |
+
1.6147,
|
| 32 |
+
1.6322,
|
| 33 |
+
2.0723,
|
| 34 |
+
3.1522,
|
| 35 |
+
3.2852,
|
| 36 |
+
2.2309,
|
| 37 |
+
2.5569,
|
| 38 |
+
2.2183,
|
| 39 |
+
2.2862,
|
| 40 |
+
1.5886,
|
| 41 |
+
0.8773,
|
| 42 |
+
0.8725,
|
| 43 |
+
1.2662,
|
| 44 |
+
0.9899,
|
| 45 |
+
1.1069,
|
| 46 |
+
1.3926,
|
| 47 |
+
1.2795,
|
| 48 |
+
1.1199,
|
| 49 |
+
1.1477,
|
| 50 |
+
1.2687,
|
| 51 |
+
1.3843,
|
| 52 |
+
1.1903,
|
| 53 |
+
0.8355,
|
| 54 |
+
1.1367,
|
| 55 |
+
1.2639,
|
| 56 |
+
1.4707,
|
| 57 |
+
]
|
| 58 |
+
]
|
| 59 |
+
)
|
| 60 |
+
out = data_utils.apply_mv_norm(sample_len1)
|
| 61 |
+
assert not torch.isnan(out).any()
|
| 62 |
+
assert (out == sample_len1).all()
|
data/fairseq/tests/speech_recognition/test_vggtransformer.py
ADDED
|
@@ -0,0 +1,135 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python3
|
| 2 |
+
|
| 3 |
+
# import models/encoder/decoder to be tested
|
| 4 |
+
from examples.speech_recognition.models.vggtransformer import (
|
| 5 |
+
TransformerDecoder,
|
| 6 |
+
VGGTransformerEncoder,
|
| 7 |
+
VGGTransformerModel,
|
| 8 |
+
vggtransformer_1,
|
| 9 |
+
vggtransformer_2,
|
| 10 |
+
vggtransformer_base,
|
| 11 |
+
)
|
| 12 |
+
|
| 13 |
+
# import base test class
|
| 14 |
+
from .asr_test_base import (
|
| 15 |
+
DEFAULT_TEST_VOCAB_SIZE,
|
| 16 |
+
TestFairseqDecoderBase,
|
| 17 |
+
TestFairseqEncoderBase,
|
| 18 |
+
TestFairseqEncoderDecoderModelBase,
|
| 19 |
+
get_dummy_dictionary,
|
| 20 |
+
get_dummy_encoder_output,
|
| 21 |
+
get_dummy_input,
|
| 22 |
+
)
|
| 23 |
+
|
| 24 |
+
|
| 25 |
+
class VGGTransformerModelTest_mid(TestFairseqEncoderDecoderModelBase):
|
| 26 |
+
def setUp(self):
|
| 27 |
+
def override_config(args):
|
| 28 |
+
"""
|
| 29 |
+
vggtrasformer_1 use 14 layers of transformer,
|
| 30 |
+
for testing purpose, it is too expensive. For fast turn-around
|
| 31 |
+
test, reduce the number of layers to 3.
|
| 32 |
+
"""
|
| 33 |
+
args.transformer_enc_config = (
|
| 34 |
+
"((1024, 16, 4096, True, 0.15, 0.15, 0.15),) * 3"
|
| 35 |
+
)
|
| 36 |
+
|
| 37 |
+
super().setUp()
|
| 38 |
+
extra_args_setter = [vggtransformer_1, override_config]
|
| 39 |
+
|
| 40 |
+
self.setUpModel(VGGTransformerModel, extra_args_setter)
|
| 41 |
+
self.setUpInput(get_dummy_input(T=50, D=80, B=5, K=DEFAULT_TEST_VOCAB_SIZE))
|
| 42 |
+
|
| 43 |
+
|
| 44 |
+
class VGGTransformerModelTest_big(TestFairseqEncoderDecoderModelBase):
|
| 45 |
+
def setUp(self):
|
| 46 |
+
def override_config(args):
|
| 47 |
+
"""
|
| 48 |
+
vggtrasformer_2 use 16 layers of transformer,
|
| 49 |
+
for testing purpose, it is too expensive. For fast turn-around
|
| 50 |
+
test, reduce the number of layers to 3.
|
| 51 |
+
"""
|
| 52 |
+
args.transformer_enc_config = (
|
| 53 |
+
"((1024, 16, 4096, True, 0.15, 0.15, 0.15),) * 3"
|
| 54 |
+
)
|
| 55 |
+
|
| 56 |
+
super().setUp()
|
| 57 |
+
extra_args_setter = [vggtransformer_2, override_config]
|
| 58 |
+
|
| 59 |
+
self.setUpModel(VGGTransformerModel, extra_args_setter)
|
| 60 |
+
self.setUpInput(get_dummy_input(T=50, D=80, B=5, K=DEFAULT_TEST_VOCAB_SIZE))
|
| 61 |
+
|
| 62 |
+
|
| 63 |
+
class VGGTransformerModelTest_base(TestFairseqEncoderDecoderModelBase):
|
| 64 |
+
def setUp(self):
|
| 65 |
+
def override_config(args):
|
| 66 |
+
"""
|
| 67 |
+
vggtrasformer_base use 12 layers of transformer,
|
| 68 |
+
for testing purpose, it is too expensive. For fast turn-around
|
| 69 |
+
test, reduce the number of layers to 3.
|
| 70 |
+
"""
|
| 71 |
+
args.transformer_enc_config = (
|
| 72 |
+
"((512, 8, 2048, True, 0.15, 0.15, 0.15),) * 3"
|
| 73 |
+
)
|
| 74 |
+
|
| 75 |
+
super().setUp()
|
| 76 |
+
extra_args_setter = [vggtransformer_base, override_config]
|
| 77 |
+
|
| 78 |
+
self.setUpModel(VGGTransformerModel, extra_args_setter)
|
| 79 |
+
self.setUpInput(get_dummy_input(T=50, D=80, B=5, K=DEFAULT_TEST_VOCAB_SIZE))
|
| 80 |
+
|
| 81 |
+
|
| 82 |
+
class VGGTransformerEncoderTest(TestFairseqEncoderBase):
|
| 83 |
+
def setUp(self):
|
| 84 |
+
super().setUp()
|
| 85 |
+
|
| 86 |
+
self.setUpInput(get_dummy_input(T=50, D=80, B=5))
|
| 87 |
+
|
| 88 |
+
def test_forward(self):
|
| 89 |
+
print("1. test standard vggtransformer")
|
| 90 |
+
self.setUpEncoder(VGGTransformerEncoder(input_feat_per_channel=80))
|
| 91 |
+
super().test_forward()
|
| 92 |
+
print("2. test vggtransformer with limited right context")
|
| 93 |
+
self.setUpEncoder(
|
| 94 |
+
VGGTransformerEncoder(
|
| 95 |
+
input_feat_per_channel=80, transformer_context=(-1, 5)
|
| 96 |
+
)
|
| 97 |
+
)
|
| 98 |
+
super().test_forward()
|
| 99 |
+
print("3. test vggtransformer with limited left context")
|
| 100 |
+
self.setUpEncoder(
|
| 101 |
+
VGGTransformerEncoder(
|
| 102 |
+
input_feat_per_channel=80, transformer_context=(5, -1)
|
| 103 |
+
)
|
| 104 |
+
)
|
| 105 |
+
super().test_forward()
|
| 106 |
+
print("4. test vggtransformer with limited right context and sampling")
|
| 107 |
+
self.setUpEncoder(
|
| 108 |
+
VGGTransformerEncoder(
|
| 109 |
+
input_feat_per_channel=80,
|
| 110 |
+
transformer_context=(-1, 12),
|
| 111 |
+
transformer_sampling=(2, 2),
|
| 112 |
+
)
|
| 113 |
+
)
|
| 114 |
+
super().test_forward()
|
| 115 |
+
print("5. test vggtransformer with windowed context and sampling")
|
| 116 |
+
self.setUpEncoder(
|
| 117 |
+
VGGTransformerEncoder(
|
| 118 |
+
input_feat_per_channel=80,
|
| 119 |
+
transformer_context=(12, 12),
|
| 120 |
+
transformer_sampling=(2, 2),
|
| 121 |
+
)
|
| 122 |
+
)
|
| 123 |
+
|
| 124 |
+
|
| 125 |
+
class TransformerDecoderTest(TestFairseqDecoderBase):
|
| 126 |
+
def setUp(self):
|
| 127 |
+
super().setUp()
|
| 128 |
+
|
| 129 |
+
dict = get_dummy_dictionary(vocab_size=DEFAULT_TEST_VOCAB_SIZE)
|
| 130 |
+
decoder = TransformerDecoder(dict)
|
| 131 |
+
dummy_encoder_output = get_dummy_encoder_output(encoder_out_shape=(50, 5, 256))
|
| 132 |
+
|
| 133 |
+
self.setUpDecoder(decoder)
|
| 134 |
+
self.setUpInput(dummy_encoder_output)
|
| 135 |
+
self.setUpPrevOutputTokens()
|
data/fairseq/tests/tasks/test_denoising.py
ADDED
|
@@ -0,0 +1,96 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) Facebook, Inc. and its affiliates.
|
| 2 |
+
#
|
| 3 |
+
# This source code is licensed under the MIT license found in the
|
| 4 |
+
# LICENSE file in the root directory of this source tree.
|
| 5 |
+
|
| 6 |
+
import os
|
| 7 |
+
import unittest
|
| 8 |
+
from tempfile import TemporaryDirectory
|
| 9 |
+
|
| 10 |
+
from fairseq import options
|
| 11 |
+
from fairseq.binarizer import FileBinarizer, VocabularyDatasetBinarizer
|
| 12 |
+
from fairseq.dataclass.utils import convert_namespace_to_omegaconf
|
| 13 |
+
from fairseq.tasks.denoising import DenoisingTask
|
| 14 |
+
from tests.utils import build_vocab, make_data
|
| 15 |
+
|
| 16 |
+
|
| 17 |
+
class TestDenoising(unittest.TestCase):
|
| 18 |
+
def test_denoising(self):
|
| 19 |
+
with TemporaryDirectory() as dirname:
|
| 20 |
+
|
| 21 |
+
# prep input file
|
| 22 |
+
raw_file = os.path.join(dirname, "raw")
|
| 23 |
+
data = make_data(out_file=raw_file)
|
| 24 |
+
vocab = build_vocab(data)
|
| 25 |
+
|
| 26 |
+
# binarize
|
| 27 |
+
binarizer = VocabularyDatasetBinarizer(vocab, append_eos=False)
|
| 28 |
+
split = "train"
|
| 29 |
+
bin_file = os.path.join(dirname, split)
|
| 30 |
+
dataset_impl = "mmap"
|
| 31 |
+
FileBinarizer.multiprocess_dataset(
|
| 32 |
+
input_file=raw_file,
|
| 33 |
+
binarizer=binarizer,
|
| 34 |
+
dataset_impl=dataset_impl,
|
| 35 |
+
vocab_size=len(vocab),
|
| 36 |
+
output_prefix=bin_file,
|
| 37 |
+
)
|
| 38 |
+
|
| 39 |
+
# setup task
|
| 40 |
+
train_args = options.parse_args_and_arch(
|
| 41 |
+
options.get_training_parser(),
|
| 42 |
+
[
|
| 43 |
+
"--task",
|
| 44 |
+
"denoising",
|
| 45 |
+
"--arch",
|
| 46 |
+
"bart_base",
|
| 47 |
+
"--seed",
|
| 48 |
+
"42",
|
| 49 |
+
"--mask-length",
|
| 50 |
+
"word",
|
| 51 |
+
"--permute-sentences",
|
| 52 |
+
"1",
|
| 53 |
+
"--rotate",
|
| 54 |
+
"0",
|
| 55 |
+
"--replace-length",
|
| 56 |
+
"-1",
|
| 57 |
+
"--mask",
|
| 58 |
+
"0.2",
|
| 59 |
+
dirname,
|
| 60 |
+
],
|
| 61 |
+
)
|
| 62 |
+
cfg = convert_namespace_to_omegaconf(train_args)
|
| 63 |
+
task = DenoisingTask(cfg.task, binarizer.dict)
|
| 64 |
+
|
| 65 |
+
# load datasets
|
| 66 |
+
original_dataset = task._load_dataset_split(bin_file, 1, False)
|
| 67 |
+
task.load_dataset(split)
|
| 68 |
+
masked_dataset = task.dataset(split)
|
| 69 |
+
|
| 70 |
+
iterator = task.get_batch_iterator(
|
| 71 |
+
dataset=masked_dataset,
|
| 72 |
+
max_tokens=65_536,
|
| 73 |
+
max_positions=4_096,
|
| 74 |
+
).next_epoch_itr(shuffle=False)
|
| 75 |
+
mask_index = task.source_dictionary.index("<mask>")
|
| 76 |
+
for batch in iterator:
|
| 77 |
+
for sample in range(len(batch)):
|
| 78 |
+
net_input = batch["net_input"]
|
| 79 |
+
masked_src_tokens = net_input["src_tokens"][sample]
|
| 80 |
+
masked_src_length = net_input["src_lengths"][sample]
|
| 81 |
+
masked_tgt_tokens = batch["target"][sample]
|
| 82 |
+
|
| 83 |
+
sample_id = batch["id"][sample]
|
| 84 |
+
original_tokens = original_dataset[sample_id]
|
| 85 |
+
original_tokens = original_tokens.masked_select(
|
| 86 |
+
masked_src_tokens[:masked_src_length] == mask_index
|
| 87 |
+
)
|
| 88 |
+
masked_tokens = masked_tgt_tokens.masked_select(
|
| 89 |
+
masked_src_tokens == mask_index
|
| 90 |
+
)
|
| 91 |
+
|
| 92 |
+
assert masked_tokens.equal(original_tokens)
|
| 93 |
+
|
| 94 |
+
|
| 95 |
+
if __name__ == "__main__":
|
| 96 |
+
unittest.main()
|
data/fairseq/tests/tasks/test_masked_lm.py
ADDED
|
@@ -0,0 +1,78 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) Facebook, Inc. and its affiliates.
|
| 2 |
+
#
|
| 3 |
+
# This source code is licensed under the MIT license found in the
|
| 4 |
+
# LICENSE file in the root directory of this source tree.
|
| 5 |
+
|
| 6 |
+
import os
|
| 7 |
+
import unittest
|
| 8 |
+
from tempfile import TemporaryDirectory
|
| 9 |
+
|
| 10 |
+
from fairseq.binarizer import FileBinarizer, VocabularyDatasetBinarizer
|
| 11 |
+
from fairseq.tasks.masked_lm import MaskedLMConfig, MaskedLMTask
|
| 12 |
+
from tests.utils import build_vocab, make_data
|
| 13 |
+
|
| 14 |
+
|
| 15 |
+
class TestMaskedLM(unittest.TestCase):
|
| 16 |
+
def test_masks_tokens(self):
|
| 17 |
+
with TemporaryDirectory() as dirname:
|
| 18 |
+
|
| 19 |
+
# prep input file
|
| 20 |
+
raw_file = os.path.join(dirname, "raw")
|
| 21 |
+
data = make_data(out_file=raw_file)
|
| 22 |
+
vocab = build_vocab(data)
|
| 23 |
+
|
| 24 |
+
# binarize
|
| 25 |
+
binarizer = VocabularyDatasetBinarizer(vocab, append_eos=False)
|
| 26 |
+
split = "train"
|
| 27 |
+
bin_file = os.path.join(dirname, split)
|
| 28 |
+
FileBinarizer.multiprocess_dataset(
|
| 29 |
+
input_file=raw_file,
|
| 30 |
+
binarizer=binarizer,
|
| 31 |
+
dataset_impl="mmap",
|
| 32 |
+
vocab_size=len(vocab),
|
| 33 |
+
output_prefix=bin_file,
|
| 34 |
+
)
|
| 35 |
+
|
| 36 |
+
# setup task
|
| 37 |
+
cfg = MaskedLMConfig(
|
| 38 |
+
data=dirname,
|
| 39 |
+
seed=42,
|
| 40 |
+
mask_prob=0.5, # increasing the odds of masking
|
| 41 |
+
random_token_prob=0, # avoiding random tokens for exact match
|
| 42 |
+
leave_unmasked_prob=0, # always masking for exact match
|
| 43 |
+
)
|
| 44 |
+
task = MaskedLMTask(cfg, binarizer.dict)
|
| 45 |
+
|
| 46 |
+
original_dataset = task._load_dataset_split(bin_file, 1, False)
|
| 47 |
+
|
| 48 |
+
# load datasets
|
| 49 |
+
task.load_dataset(split)
|
| 50 |
+
masked_dataset = task.dataset(split)
|
| 51 |
+
|
| 52 |
+
mask_index = task.source_dictionary.index("<mask>")
|
| 53 |
+
iterator = task.get_batch_iterator(
|
| 54 |
+
dataset=masked_dataset,
|
| 55 |
+
max_tokens=65_536,
|
| 56 |
+
max_positions=4_096,
|
| 57 |
+
).next_epoch_itr(shuffle=False)
|
| 58 |
+
for batch in iterator:
|
| 59 |
+
for sample in range(len(batch)):
|
| 60 |
+
net_input = batch["net_input"]
|
| 61 |
+
masked_src_tokens = net_input["src_tokens"][sample]
|
| 62 |
+
masked_src_length = net_input["src_lengths"][sample]
|
| 63 |
+
masked_tgt_tokens = batch["target"][sample]
|
| 64 |
+
|
| 65 |
+
sample_id = batch["id"][sample]
|
| 66 |
+
original_tokens = original_dataset[sample_id]
|
| 67 |
+
original_tokens = original_tokens.masked_select(
|
| 68 |
+
masked_src_tokens[:masked_src_length] == mask_index
|
| 69 |
+
)
|
| 70 |
+
masked_tokens = masked_tgt_tokens.masked_select(
|
| 71 |
+
masked_tgt_tokens != task.source_dictionary.pad()
|
| 72 |
+
)
|
| 73 |
+
|
| 74 |
+
assert masked_tokens.equal(original_tokens)
|
| 75 |
+
|
| 76 |
+
|
| 77 |
+
if __name__ == "__main__":
|
| 78 |
+
unittest.main()
|
data/fairseq/tests/tasks/test_multilingual_denoising.py
ADDED
|
@@ -0,0 +1,98 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) Facebook, Inc. and its affiliates.
|
| 2 |
+
#
|
| 3 |
+
# This source code is licensed under the MIT license found in the
|
| 4 |
+
# LICENSE file in the root directory of this source tree.
|
| 5 |
+
|
| 6 |
+
import os
|
| 7 |
+
import unittest
|
| 8 |
+
from tempfile import TemporaryDirectory
|
| 9 |
+
|
| 10 |
+
from fairseq import options
|
| 11 |
+
from fairseq.binarizer import FileBinarizer, VocabularyDatasetBinarizer
|
| 12 |
+
from fairseq.dataclass.utils import convert_namespace_to_omegaconf
|
| 13 |
+
from fairseq.tasks.multilingual_denoising import MultilingualDenoisingTask
|
| 14 |
+
from tests.utils import build_vocab, make_data
|
| 15 |
+
|
| 16 |
+
|
| 17 |
+
class TestMultilingualDenoising(unittest.TestCase):
|
| 18 |
+
def test_multilingual_denoising(self):
|
| 19 |
+
with TemporaryDirectory() as dirname:
|
| 20 |
+
|
| 21 |
+
# prep input file
|
| 22 |
+
lang_dir = os.path.join(dirname, "en")
|
| 23 |
+
os.mkdir(lang_dir)
|
| 24 |
+
raw_file = os.path.join(lang_dir, "raw")
|
| 25 |
+
data = make_data(out_file=raw_file)
|
| 26 |
+
vocab = build_vocab(data)
|
| 27 |
+
|
| 28 |
+
# binarize
|
| 29 |
+
binarizer = VocabularyDatasetBinarizer(vocab, append_eos=False)
|
| 30 |
+
split = "train"
|
| 31 |
+
bin_file = os.path.join(lang_dir, split)
|
| 32 |
+
dataset_impl = "mmap"
|
| 33 |
+
FileBinarizer.multiprocess_dataset(
|
| 34 |
+
input_file=raw_file,
|
| 35 |
+
binarizer=binarizer,
|
| 36 |
+
dataset_impl=dataset_impl,
|
| 37 |
+
vocab_size=len(vocab),
|
| 38 |
+
output_prefix=bin_file,
|
| 39 |
+
)
|
| 40 |
+
|
| 41 |
+
# setup task
|
| 42 |
+
train_args = options.parse_args_and_arch(
|
| 43 |
+
options.get_training_parser(),
|
| 44 |
+
[
|
| 45 |
+
"--task",
|
| 46 |
+
"multilingual_denoising",
|
| 47 |
+
"--arch",
|
| 48 |
+
"bart_base",
|
| 49 |
+
"--seed",
|
| 50 |
+
"42",
|
| 51 |
+
"--mask-length",
|
| 52 |
+
"word",
|
| 53 |
+
"--permute-sentences",
|
| 54 |
+
"1",
|
| 55 |
+
"--rotate",
|
| 56 |
+
"0",
|
| 57 |
+
"--replace-length",
|
| 58 |
+
"-1",
|
| 59 |
+
"--mask",
|
| 60 |
+
"0.2",
|
| 61 |
+
dirname,
|
| 62 |
+
],
|
| 63 |
+
)
|
| 64 |
+
cfg = convert_namespace_to_omegaconf(train_args)
|
| 65 |
+
task = MultilingualDenoisingTask(cfg.task, binarizer.dict)
|
| 66 |
+
|
| 67 |
+
# load datasets
|
| 68 |
+
original_dataset = task._load_dataset_split(bin_file, 1, False)
|
| 69 |
+
task.load_dataset(split)
|
| 70 |
+
masked_dataset = task.dataset(split)
|
| 71 |
+
|
| 72 |
+
iterator = task.get_batch_iterator(
|
| 73 |
+
dataset=masked_dataset,
|
| 74 |
+
max_tokens=65_536,
|
| 75 |
+
max_positions=4_096,
|
| 76 |
+
).next_epoch_itr(shuffle=False)
|
| 77 |
+
mask_index = task.source_dictionary.index("<mask>")
|
| 78 |
+
for batch in iterator:
|
| 79 |
+
for sample in range(len(batch)):
|
| 80 |
+
net_input = batch["net_input"]
|
| 81 |
+
masked_src_tokens = net_input["src_tokens"][sample]
|
| 82 |
+
masked_src_length = net_input["src_lengths"][sample]
|
| 83 |
+
masked_tgt_tokens = batch["target"][sample]
|
| 84 |
+
|
| 85 |
+
sample_id = batch["id"][sample]
|
| 86 |
+
original_tokens = original_dataset[sample_id]
|
| 87 |
+
original_tokens = original_tokens.masked_select(
|
| 88 |
+
masked_src_tokens[:masked_src_length] == mask_index
|
| 89 |
+
)
|
| 90 |
+
masked_tokens = masked_tgt_tokens.masked_select(
|
| 91 |
+
masked_src_tokens == mask_index
|
| 92 |
+
)
|
| 93 |
+
|
| 94 |
+
assert masked_tokens.equal(original_tokens)
|
| 95 |
+
|
| 96 |
+
|
| 97 |
+
if __name__ == "__main__":
|
| 98 |
+
unittest.main()
|