Add files using upload-large-folder tool
Browse filesThis view is limited to 50 files because it contains too many changes. See raw diff
- data/fairseq/fairseq/criterions/__init__.py +36 -0
- data/fairseq/fairseq/criterions/__pycache__/__init__.cpython-38.pyc +0 -0
- data/fairseq/fairseq/criterions/__pycache__/adaptive_loss.cpython-38.pyc +0 -0
- data/fairseq/fairseq/criterions/__pycache__/composite_loss.cpython-38.pyc +0 -0
- data/fairseq/fairseq/criterions/__pycache__/cross_entropy.cpython-38.pyc +0 -0
- data/fairseq/fairseq/criterions/__pycache__/ctc.cpython-38.pyc +0 -0
- data/fairseq/fairseq/criterions/__pycache__/fairseq_criterion.cpython-38.pyc +0 -0
- data/fairseq/fairseq/criterions/__pycache__/fastspeech2_loss.cpython-38.pyc +0 -0
- data/fairseq/fairseq/criterions/__pycache__/hubert_criterion.cpython-38.pyc +0 -0
- data/fairseq/fairseq/criterions/__pycache__/label_smoothed_cross_entropy.cpython-38.pyc +0 -0
- data/fairseq/fairseq/criterions/__pycache__/label_smoothed_cross_entropy_latency_augmented.cpython-38.pyc +0 -0
- data/fairseq/fairseq/criterions/__pycache__/label_smoothed_cross_entropy_with_alignment.cpython-38.pyc +0 -0
- data/fairseq/fairseq/criterions/__pycache__/label_smoothed_cross_entropy_with_ctc.cpython-38.pyc +0 -0
- data/fairseq/fairseq/criterions/__pycache__/label_smoothed_cross_entropy_with_rdrop.cpython-38.pyc +0 -0
- data/fairseq/fairseq/criterions/__pycache__/legacy_masked_lm.cpython-38.pyc +0 -0
- data/fairseq/fairseq/criterions/__pycache__/masked_lm.cpython-38.pyc +0 -0
- data/fairseq/fairseq/criterions/__pycache__/model_criterion.cpython-38.pyc +0 -0
- data/fairseq/fairseq/criterions/__pycache__/nat_loss.cpython-38.pyc +0 -0
- data/fairseq/fairseq/criterions/__pycache__/sentence_prediction.cpython-38.pyc +0 -0
- data/fairseq/fairseq/criterions/__pycache__/sentence_prediction_adapters.cpython-38.pyc +0 -0
- data/fairseq/fairseq/criterions/__pycache__/sentence_ranking.cpython-38.pyc +0 -0
- data/fairseq/fairseq/criterions/__pycache__/speech_dlm_criterion.cpython-38.pyc +0 -0
- data/fairseq/fairseq/criterions/__pycache__/speech_to_speech_criterion.cpython-38.pyc +0 -0
- data/fairseq/fairseq/criterions/__pycache__/speech_ulm_criterion.cpython-38.pyc +0 -0
- data/fairseq/fairseq/criterions/__pycache__/tacotron2_loss.cpython-38.pyc +0 -0
- data/fairseq/fairseq/criterions/__pycache__/wav2vec_criterion.cpython-38.pyc +0 -0
- data/fairseq/fairseq/criterions/adaptive_loss.py +124 -0
- data/fairseq/fairseq/criterions/composite_loss.py +100 -0
- data/fairseq/fairseq/criterions/cross_entropy.py +91 -0
- data/fairseq/fairseq/criterions/ctc.py +325 -0
- data/fairseq/fairseq/criterions/fairseq_criterion.py +121 -0
- data/fairseq/fairseq/criterions/fastspeech2_loss.py +137 -0
- data/fairseq/fairseq/criterions/hubert_criterion.py +195 -0
- data/fairseq/fairseq/criterions/label_smoothed_cross_entropy.py +168 -0
- data/fairseq/fairseq/criterions/label_smoothed_cross_entropy_latency_augmented.py +221 -0
- data/fairseq/fairseq/criterions/label_smoothed_cross_entropy_with_alignment.py +131 -0
- data/fairseq/fairseq/criterions/label_smoothed_cross_entropy_with_ctc.py +97 -0
- data/fairseq/fairseq/criterions/label_smoothed_cross_entropy_with_rdrop.py +177 -0
- data/fairseq/fairseq/criterions/legacy_masked_lm.py +178 -0
- data/fairseq/fairseq/criterions/masked_lm.py +99 -0
- data/fairseq/fairseq/criterions/model_criterion.py +177 -0
- data/fairseq/fairseq/criterions/nat_loss.py +181 -0
- data/fairseq/fairseq/criterions/sentence_prediction.py +288 -0
- data/fairseq/fairseq/criterions/sentence_prediction_adapters.py +63 -0
- data/fairseq/fairseq/criterions/sentence_ranking.py +121 -0
- data/fairseq/fairseq/criterions/speech_dlm_criterion.py +335 -0
- data/fairseq/fairseq/criterions/speech_to_speech_criterion.py +517 -0
- data/fairseq/fairseq/criterions/speech_ulm_criterion.py +126 -0
- data/fairseq/fairseq/criterions/tacotron2_loss.py +227 -0
- data/fairseq/fairseq/criterions/wav2vec_criterion.py +231 -0
data/fairseq/fairseq/criterions/__init__.py
ADDED
|
@@ -0,0 +1,36 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) Facebook, Inc. and its affiliates.
|
| 2 |
+
#
|
| 3 |
+
# This source code is licensed under the MIT license found in the
|
| 4 |
+
# LICENSE file in the root directory of this source tree.
|
| 5 |
+
"""isort:skip_file"""
|
| 6 |
+
|
| 7 |
+
import importlib
|
| 8 |
+
import os
|
| 9 |
+
|
| 10 |
+
from fairseq import registry
|
| 11 |
+
from fairseq.criterions.fairseq_criterion import ( # noqa
|
| 12 |
+
FairseqCriterion,
|
| 13 |
+
LegacyFairseqCriterion,
|
| 14 |
+
)
|
| 15 |
+
from omegaconf import DictConfig
|
| 16 |
+
|
| 17 |
+
|
| 18 |
+
(
|
| 19 |
+
build_criterion_,
|
| 20 |
+
register_criterion,
|
| 21 |
+
CRITERION_REGISTRY,
|
| 22 |
+
CRITERION_DATACLASS_REGISTRY,
|
| 23 |
+
) = registry.setup_registry(
|
| 24 |
+
"--criterion", base_class=FairseqCriterion, default="cross_entropy"
|
| 25 |
+
)
|
| 26 |
+
|
| 27 |
+
|
| 28 |
+
def build_criterion(cfg: DictConfig, task, from_checkpoint=False):
|
| 29 |
+
return build_criterion_(cfg, task, from_checkpoint=from_checkpoint)
|
| 30 |
+
|
| 31 |
+
|
| 32 |
+
# automatically import any Python files in the criterions/ directory
|
| 33 |
+
for file in sorted(os.listdir(os.path.dirname(__file__))):
|
| 34 |
+
if file.endswith(".py") and not file.startswith("_"):
|
| 35 |
+
file_name = file[: file.find(".py")]
|
| 36 |
+
importlib.import_module("fairseq.criterions." + file_name)
|
data/fairseq/fairseq/criterions/__pycache__/__init__.cpython-38.pyc
ADDED
|
Binary file (1.02 kB). View file
|
|
|
data/fairseq/fairseq/criterions/__pycache__/adaptive_loss.cpython-38.pyc
ADDED
|
Binary file (4.81 kB). View file
|
|
|
data/fairseq/fairseq/criterions/__pycache__/composite_loss.cpython-38.pyc
ADDED
|
Binary file (4.45 kB). View file
|
|
|
data/fairseq/fairseq/criterions/__pycache__/cross_entropy.cpython-38.pyc
ADDED
|
Binary file (3.92 kB). View file
|
|
|
data/fairseq/fairseq/criterions/__pycache__/ctc.cpython-38.pyc
ADDED
|
Binary file (8.68 kB). View file
|
|
|
data/fairseq/fairseq/criterions/__pycache__/fairseq_criterion.cpython-38.pyc
ADDED
|
Binary file (4.49 kB). View file
|
|
|
data/fairseq/fairseq/criterions/__pycache__/fastspeech2_loss.cpython-38.pyc
ADDED
|
Binary file (4.73 kB). View file
|
|
|
data/fairseq/fairseq/criterions/__pycache__/hubert_criterion.cpython-38.pyc
ADDED
|
Binary file (6.85 kB). View file
|
|
|
data/fairseq/fairseq/criterions/__pycache__/label_smoothed_cross_entropy.cpython-38.pyc
ADDED
|
Binary file (6.36 kB). View file
|
|
|
data/fairseq/fairseq/criterions/__pycache__/label_smoothed_cross_entropy_latency_augmented.cpython-38.pyc
ADDED
|
Binary file (6 kB). View file
|
|
|
data/fairseq/fairseq/criterions/__pycache__/label_smoothed_cross_entropy_with_alignment.cpython-38.pyc
ADDED
|
Binary file (4.85 kB). View file
|
|
|
data/fairseq/fairseq/criterions/__pycache__/label_smoothed_cross_entropy_with_ctc.cpython-38.pyc
ADDED
|
Binary file (3.31 kB). View file
|
|
|
data/fairseq/fairseq/criterions/__pycache__/label_smoothed_cross_entropy_with_rdrop.cpython-38.pyc
ADDED
|
Binary file (5.26 kB). View file
|
|
|
data/fairseq/fairseq/criterions/__pycache__/legacy_masked_lm.cpython-38.pyc
ADDED
|
Binary file (5.64 kB). View file
|
|
|
data/fairseq/fairseq/criterions/__pycache__/masked_lm.cpython-38.pyc
ADDED
|
Binary file (3.5 kB). View file
|
|
|
data/fairseq/fairseq/criterions/__pycache__/model_criterion.cpython-38.pyc
ADDED
|
Binary file (5.84 kB). View file
|
|
|
data/fairseq/fairseq/criterions/__pycache__/nat_loss.cpython-38.pyc
ADDED
|
Binary file (6.11 kB). View file
|
|
|
data/fairseq/fairseq/criterions/__pycache__/sentence_prediction.cpython-38.pyc
ADDED
|
Binary file (8.7 kB). View file
|
|
|
data/fairseq/fairseq/criterions/__pycache__/sentence_prediction_adapters.cpython-38.pyc
ADDED
|
Binary file (1.95 kB). View file
|
|
|
data/fairseq/fairseq/criterions/__pycache__/sentence_ranking.cpython-38.pyc
ADDED
|
Binary file (4.69 kB). View file
|
|
|
data/fairseq/fairseq/criterions/__pycache__/speech_dlm_criterion.cpython-38.pyc
ADDED
|
Binary file (9.17 kB). View file
|
|
|
data/fairseq/fairseq/criterions/__pycache__/speech_to_speech_criterion.cpython-38.pyc
ADDED
|
Binary file (11.7 kB). View file
|
|
|
data/fairseq/fairseq/criterions/__pycache__/speech_ulm_criterion.cpython-38.pyc
ADDED
|
Binary file (5.02 kB). View file
|
|
|
data/fairseq/fairseq/criterions/__pycache__/tacotron2_loss.cpython-38.pyc
ADDED
|
Binary file (7.43 kB). View file
|
|
|
data/fairseq/fairseq/criterions/__pycache__/wav2vec_criterion.cpython-38.pyc
ADDED
|
Binary file (6.61 kB). View file
|
|
|
data/fairseq/fairseq/criterions/adaptive_loss.py
ADDED
|
@@ -0,0 +1,124 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) Facebook, Inc. and its affiliates.
|
| 2 |
+
#
|
| 3 |
+
# This source code is licensed under the MIT license found in the
|
| 4 |
+
# LICENSE file in the root directory of this source tree.
|
| 5 |
+
|
| 6 |
+
import math
|
| 7 |
+
from dataclasses import dataclass
|
| 8 |
+
|
| 9 |
+
import torch.nn.functional as F
|
| 10 |
+
from fairseq import utils
|
| 11 |
+
from fairseq.logging import metrics
|
| 12 |
+
from fairseq.criterions import FairseqCriterion, register_criterion
|
| 13 |
+
from fairseq.dataclass import FairseqDataclass
|
| 14 |
+
from fairseq.dataclass.constants import DDP_BACKEND_CHOICES
|
| 15 |
+
from omegaconf import II
|
| 16 |
+
|
| 17 |
+
|
| 18 |
+
@dataclass
|
| 19 |
+
class AdaptiveLossConfig(FairseqDataclass):
|
| 20 |
+
sentence_avg: bool = II("optimization.sentence_avg")
|
| 21 |
+
ddp_backend: DDP_BACKEND_CHOICES = II("distributed_training.ddp_backend")
|
| 22 |
+
|
| 23 |
+
|
| 24 |
+
@register_criterion("adaptive_loss", dataclass=AdaptiveLossConfig)
|
| 25 |
+
class AdaptiveLoss(FairseqCriterion):
|
| 26 |
+
"""This is an implementation of the loss function accompanying the adaptive softmax approximation for
|
| 27 |
+
graphical processing units (GPU), described in the paper "Efficient softmax approximation for GPUs"
|
| 28 |
+
(http://arxiv.org/abs/1609.04309)."""
|
| 29 |
+
|
| 30 |
+
def __init__(self, task, sentence_avg):
|
| 31 |
+
super().__init__(task)
|
| 32 |
+
self.sentence_avg = sentence_avg
|
| 33 |
+
|
| 34 |
+
@classmethod
|
| 35 |
+
def build_criterion(cls, cfg: AdaptiveLossConfig, task):
|
| 36 |
+
if cfg.ddp_backend in {"c10d", "pytorch_ddp"}:
|
| 37 |
+
raise Exception(
|
| 38 |
+
"AdaptiveLoss is not compatible with the PyTorch "
|
| 39 |
+
"version of DistributedDataParallel. Please use "
|
| 40 |
+
"`--ddp-backend=legacy_ddp` instead."
|
| 41 |
+
)
|
| 42 |
+
return cls(task, cfg.sentence_avg)
|
| 43 |
+
|
| 44 |
+
def forward(self, model, sample, reduce=True):
|
| 45 |
+
"""Compute the loss for the given sample.
|
| 46 |
+
|
| 47 |
+
Returns a tuple with three elements:
|
| 48 |
+
1) the loss
|
| 49 |
+
2) the sample size, which is used as the denominator for the gradient
|
| 50 |
+
3) logging outputs to display while training
|
| 51 |
+
"""
|
| 52 |
+
|
| 53 |
+
assert (
|
| 54 |
+
hasattr(model.decoder, "adaptive_softmax")
|
| 55 |
+
and model.decoder.adaptive_softmax is not None
|
| 56 |
+
)
|
| 57 |
+
adaptive_softmax = model.decoder.adaptive_softmax
|
| 58 |
+
|
| 59 |
+
net_output = model(**sample["net_input"])
|
| 60 |
+
orig_target = model.get_targets(sample, net_output)
|
| 61 |
+
|
| 62 |
+
nsentences = orig_target.size(0)
|
| 63 |
+
orig_target = orig_target.view(-1)
|
| 64 |
+
|
| 65 |
+
bsz = orig_target.size(0)
|
| 66 |
+
|
| 67 |
+
logits, target = adaptive_softmax(net_output[0], orig_target)
|
| 68 |
+
assert len(target) == len(logits)
|
| 69 |
+
|
| 70 |
+
loss = net_output[0].new(1 if reduce else bsz).zero_()
|
| 71 |
+
|
| 72 |
+
for i in range(len(target)):
|
| 73 |
+
if target[i] is not None:
|
| 74 |
+
assert target[i].min() >= 0 and target[i].max() <= logits[i].size(1)
|
| 75 |
+
loss += F.cross_entropy(
|
| 76 |
+
logits[i],
|
| 77 |
+
target[i],
|
| 78 |
+
ignore_index=self.padding_idx,
|
| 79 |
+
reduction="sum" if reduce else "none",
|
| 80 |
+
)
|
| 81 |
+
|
| 82 |
+
orig = utils.strip_pad(orig_target, self.padding_idx)
|
| 83 |
+
ntokens = orig.numel()
|
| 84 |
+
sample_size = sample["target"].size(0) if self.sentence_avg else ntokens
|
| 85 |
+
logging_output = {
|
| 86 |
+
"loss": loss.data,
|
| 87 |
+
"ntokens": ntokens,
|
| 88 |
+
"nsentences": nsentences,
|
| 89 |
+
"sample_size": sample_size,
|
| 90 |
+
}
|
| 91 |
+
return loss, sample_size, logging_output
|
| 92 |
+
|
| 93 |
+
@staticmethod
|
| 94 |
+
def reduce_metrics(logging_outputs) -> None:
|
| 95 |
+
"""Aggregate logging outputs from data parallel training."""
|
| 96 |
+
loss_sum = utils.item(sum(log.get("loss", 0) for log in logging_outputs))
|
| 97 |
+
ntokens = utils.item(sum(log.get("ntokens", 0) for log in logging_outputs))
|
| 98 |
+
sample_size = utils.item(
|
| 99 |
+
sum(log.get("sample_size", 0) for log in logging_outputs)
|
| 100 |
+
)
|
| 101 |
+
|
| 102 |
+
metrics.log_scalar(
|
| 103 |
+
"loss", loss_sum / sample_size / math.log(2), sample_size, round=3
|
| 104 |
+
)
|
| 105 |
+
if sample_size != ntokens:
|
| 106 |
+
metrics.log_scalar(
|
| 107 |
+
"nll_loss", loss_sum / ntokens / math.log(2), ntokens, round=3
|
| 108 |
+
)
|
| 109 |
+
metrics.log_derived(
|
| 110 |
+
"ppl", lambda meters: utils.get_perplexity(meters["nll_loss"].avg)
|
| 111 |
+
)
|
| 112 |
+
else:
|
| 113 |
+
metrics.log_derived(
|
| 114 |
+
"ppl", lambda meters: utils.get_perplexity(meters["loss"].avg)
|
| 115 |
+
)
|
| 116 |
+
|
| 117 |
+
@staticmethod
|
| 118 |
+
def logging_outputs_can_be_summed() -> bool:
|
| 119 |
+
"""
|
| 120 |
+
Whether the logging outputs returned by `forward` can be summed
|
| 121 |
+
across workers prior to calling `reduce_metrics`. Setting this
|
| 122 |
+
to True will improves distributed training speed.
|
| 123 |
+
"""
|
| 124 |
+
return True
|
data/fairseq/fairseq/criterions/composite_loss.py
ADDED
|
@@ -0,0 +1,100 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) Facebook, Inc. and its affiliates.
|
| 2 |
+
#
|
| 3 |
+
# This source code is licensed under the MIT license found in the
|
| 4 |
+
# LICENSE file in the root directory of this source tree.
|
| 5 |
+
|
| 6 |
+
from fairseq import utils
|
| 7 |
+
from fairseq.criterions import LegacyFairseqCriterion, register_criterion
|
| 8 |
+
from torch import nn
|
| 9 |
+
|
| 10 |
+
|
| 11 |
+
@register_criterion("composite_loss")
|
| 12 |
+
class CompositeLoss(LegacyFairseqCriterion):
|
| 13 |
+
"""This is a composite loss that, given a list of model outputs and a list of targets,
|
| 14 |
+
computes an average of losses for each output-target pair"""
|
| 15 |
+
|
| 16 |
+
def __init__(self, args, task):
|
| 17 |
+
super().__init__(args, task)
|
| 18 |
+
self.underlying_criterion = args.underlying_criterion
|
| 19 |
+
|
| 20 |
+
@staticmethod
|
| 21 |
+
def add_args(parser):
|
| 22 |
+
"""Add criterion-specific arguments to the parser."""
|
| 23 |
+
# fmt: off
|
| 24 |
+
parser.add_argument('--underlying-criterion', type=str, metavar='VAL', required=True,
|
| 25 |
+
help='underlying criterion to use for the composite loss')
|
| 26 |
+
# fmt: on
|
| 27 |
+
|
| 28 |
+
@staticmethod
|
| 29 |
+
def build_underlying_criterion(args, task):
|
| 30 |
+
saved_criterion = args.criterion
|
| 31 |
+
args.criterion = args.underlying_criterion
|
| 32 |
+
assert saved_criterion != args.underlying_criterion
|
| 33 |
+
underlying_criterion = task.build_criterion(args)
|
| 34 |
+
args.criterion = saved_criterion
|
| 35 |
+
return underlying_criterion
|
| 36 |
+
|
| 37 |
+
@classmethod
|
| 38 |
+
def build_criterion(cls, args, task):
|
| 39 |
+
underlying_criterion = CompositeLoss.build_underlying_criterion(args, task)
|
| 40 |
+
|
| 41 |
+
class FakeModel(nn.Module):
|
| 42 |
+
def __init__(self, model, net_out, target):
|
| 43 |
+
super().__init__()
|
| 44 |
+
self.model = model
|
| 45 |
+
self.net_out = net_out
|
| 46 |
+
self.target = target
|
| 47 |
+
|
| 48 |
+
def forward(self, **unused):
|
| 49 |
+
return self.net_out
|
| 50 |
+
|
| 51 |
+
def get_normalized_probs(self, net_output, log_probs, sample=None):
|
| 52 |
+
return self.model.get_normalized_probs(
|
| 53 |
+
net_output, log_probs, sample=sample
|
| 54 |
+
)
|
| 55 |
+
|
| 56 |
+
def get_targets(self, *unused):
|
| 57 |
+
return self.target
|
| 58 |
+
|
| 59 |
+
@property
|
| 60 |
+
def decoder(self):
|
| 61 |
+
return self.model.decoder
|
| 62 |
+
|
| 63 |
+
class _CompositeLoss(LegacyFairseqCriterion):
|
| 64 |
+
def __init__(self, args, task, underlying_criterion):
|
| 65 |
+
super().__init__(args, task)
|
| 66 |
+
self.underlying_criterion = underlying_criterion
|
| 67 |
+
|
| 68 |
+
def forward(self, model, sample, reduce=True):
|
| 69 |
+
net_outputs = model(**sample["net_input"])
|
| 70 |
+
targets = sample["target"]
|
| 71 |
+
|
| 72 |
+
bsz = targets[0].size(0)
|
| 73 |
+
loss = net_outputs[0][0].new(1 if reduce else bsz).float().zero_()
|
| 74 |
+
|
| 75 |
+
sample_size = 0
|
| 76 |
+
logging_output = {}
|
| 77 |
+
for o, t in zip(net_outputs[0], targets):
|
| 78 |
+
m = FakeModel(model, (o, net_outputs[1]), t)
|
| 79 |
+
sample["target"] = t
|
| 80 |
+
l, ss, logging_output = self.underlying_criterion(m, sample, reduce)
|
| 81 |
+
loss += l
|
| 82 |
+
sample_size += ss
|
| 83 |
+
|
| 84 |
+
loss.div_(len(targets))
|
| 85 |
+
sample_size /= len(targets)
|
| 86 |
+
|
| 87 |
+
logging_output["loss"] = utils.item(loss.data) if reduce else loss.data
|
| 88 |
+
return loss, sample_size, logging_output
|
| 89 |
+
|
| 90 |
+
@staticmethod
|
| 91 |
+
def aggregate_logging_outputs(logging_outputs):
|
| 92 |
+
return underlying_criterion.__class__.aggregate_logging_outputs(
|
| 93 |
+
logging_outputs
|
| 94 |
+
)
|
| 95 |
+
|
| 96 |
+
@staticmethod
|
| 97 |
+
def reduce_metrics(logging_outputs) -> None:
|
| 98 |
+
underlying_criterion.__class__.reduce_metrics(logging_outputs)
|
| 99 |
+
|
| 100 |
+
return _CompositeLoss(args, task, underlying_criterion)
|
data/fairseq/fairseq/criterions/cross_entropy.py
ADDED
|
@@ -0,0 +1,91 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) Facebook, Inc. and its affiliates.
|
| 2 |
+
#
|
| 3 |
+
# This source code is licensed under the MIT license found in the
|
| 4 |
+
# LICENSE file in the root directory of this source tree.
|
| 5 |
+
|
| 6 |
+
import math
|
| 7 |
+
from dataclasses import dataclass
|
| 8 |
+
|
| 9 |
+
import torch.nn.functional as F
|
| 10 |
+
from fairseq import utils
|
| 11 |
+
from fairseq.logging import metrics
|
| 12 |
+
from fairseq.criterions import FairseqCriterion, register_criterion
|
| 13 |
+
from fairseq.dataclass import FairseqDataclass
|
| 14 |
+
from omegaconf import II
|
| 15 |
+
|
| 16 |
+
|
| 17 |
+
@dataclass
|
| 18 |
+
class CrossEntropyCriterionConfig(FairseqDataclass):
|
| 19 |
+
sentence_avg: bool = II("optimization.sentence_avg")
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
@register_criterion("cross_entropy", dataclass=CrossEntropyCriterionConfig)
|
| 23 |
+
class CrossEntropyCriterion(FairseqCriterion):
|
| 24 |
+
def __init__(self, task, sentence_avg):
|
| 25 |
+
super().__init__(task)
|
| 26 |
+
self.sentence_avg = sentence_avg
|
| 27 |
+
|
| 28 |
+
def forward(self, model, sample, reduce=True):
|
| 29 |
+
"""Compute the loss for the given sample.
|
| 30 |
+
|
| 31 |
+
Returns a tuple with three elements:
|
| 32 |
+
1) the loss
|
| 33 |
+
2) the sample size, which is used as the denominator for the gradient
|
| 34 |
+
3) logging outputs to display while training
|
| 35 |
+
"""
|
| 36 |
+
net_output = model(**sample["net_input"])
|
| 37 |
+
loss, _ = self.compute_loss(model, net_output, sample, reduce=reduce)
|
| 38 |
+
sample_size = (
|
| 39 |
+
sample["target"].size(0) if self.sentence_avg else sample["ntokens"]
|
| 40 |
+
)
|
| 41 |
+
logging_output = {
|
| 42 |
+
"loss": loss.data,
|
| 43 |
+
"ntokens": sample["ntokens"],
|
| 44 |
+
"nsentences": sample["target"].size(0),
|
| 45 |
+
"sample_size": sample_size,
|
| 46 |
+
}
|
| 47 |
+
return loss, sample_size, logging_output
|
| 48 |
+
|
| 49 |
+
def compute_loss(self, model, net_output, sample, reduce=True):
|
| 50 |
+
lprobs = model.get_normalized_probs(net_output, log_probs=True)
|
| 51 |
+
lprobs = lprobs.view(-1, lprobs.size(-1))
|
| 52 |
+
target = model.get_targets(sample, net_output).view(-1)
|
| 53 |
+
loss = F.nll_loss(
|
| 54 |
+
lprobs,
|
| 55 |
+
target,
|
| 56 |
+
ignore_index=self.padding_idx,
|
| 57 |
+
reduction="sum" if reduce else "none",
|
| 58 |
+
)
|
| 59 |
+
return loss, loss
|
| 60 |
+
|
| 61 |
+
@staticmethod
|
| 62 |
+
def reduce_metrics(logging_outputs) -> None:
|
| 63 |
+
"""Aggregate logging outputs from data parallel training."""
|
| 64 |
+
loss_sum = sum(log.get("loss", 0) for log in logging_outputs)
|
| 65 |
+
ntokens = sum(log.get("ntokens", 0) for log in logging_outputs)
|
| 66 |
+
sample_size = sum(log.get("sample_size", 0) for log in logging_outputs)
|
| 67 |
+
|
| 68 |
+
# we divide by log(2) to convert the loss from base e to base 2
|
| 69 |
+
metrics.log_scalar(
|
| 70 |
+
"loss", loss_sum / sample_size / math.log(2), sample_size, round=3
|
| 71 |
+
)
|
| 72 |
+
if sample_size != ntokens:
|
| 73 |
+
metrics.log_scalar(
|
| 74 |
+
"nll_loss", loss_sum / ntokens / math.log(2), ntokens, round=3
|
| 75 |
+
)
|
| 76 |
+
metrics.log_derived(
|
| 77 |
+
"ppl", lambda meters: utils.get_perplexity(meters["nll_loss"].avg)
|
| 78 |
+
)
|
| 79 |
+
else:
|
| 80 |
+
metrics.log_derived(
|
| 81 |
+
"ppl", lambda meters: utils.get_perplexity(meters["loss"].avg)
|
| 82 |
+
)
|
| 83 |
+
|
| 84 |
+
@staticmethod
|
| 85 |
+
def logging_outputs_can_be_summed() -> bool:
|
| 86 |
+
"""
|
| 87 |
+
Whether the logging outputs returned by `forward` can be summed
|
| 88 |
+
across workers prior to calling `reduce_metrics`. Setting this
|
| 89 |
+
to True will improves distributed training speed.
|
| 90 |
+
"""
|
| 91 |
+
return True
|
data/fairseq/fairseq/criterions/ctc.py
ADDED
|
@@ -0,0 +1,325 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# All rights reserved.
|
| 2 |
+
#
|
| 3 |
+
# This source code is licensed under the license found in the LICENSE file in
|
| 4 |
+
# the root directory of this source tree. An additional grant of patent rights
|
| 5 |
+
# can be found in the PATENTS file in the same directory.
|
| 6 |
+
|
| 7 |
+
import math
|
| 8 |
+
from argparse import Namespace
|
| 9 |
+
from dataclasses import dataclass, field
|
| 10 |
+
from omegaconf import II
|
| 11 |
+
from typing import Optional
|
| 12 |
+
|
| 13 |
+
import torch
|
| 14 |
+
import torch.nn.functional as F
|
| 15 |
+
|
| 16 |
+
from fairseq import utils
|
| 17 |
+
from fairseq.logging import metrics
|
| 18 |
+
from fairseq.criterions import FairseqCriterion, register_criterion
|
| 19 |
+
from fairseq.dataclass import FairseqDataclass
|
| 20 |
+
from fairseq.data.data_utils import post_process
|
| 21 |
+
from fairseq.tasks import FairseqTask
|
| 22 |
+
from fairseq.logging.meters import safe_round
|
| 23 |
+
|
| 24 |
+
|
| 25 |
+
@dataclass
|
| 26 |
+
class CtcCriterionConfig(FairseqDataclass):
|
| 27 |
+
zero_infinity: bool = field(
|
| 28 |
+
default=False,
|
| 29 |
+
metadata={"help": "zero inf loss when source length <= target length"},
|
| 30 |
+
)
|
| 31 |
+
sentence_avg: bool = II("optimization.sentence_avg")
|
| 32 |
+
post_process: str = field(
|
| 33 |
+
default="letter",
|
| 34 |
+
metadata={
|
| 35 |
+
"help": "how to post process predictions into words. can be letter, "
|
| 36 |
+
"wordpiece, BPE symbols, etc. "
|
| 37 |
+
"See fairseq.data.data_utils.post_process() for full list of options"
|
| 38 |
+
},
|
| 39 |
+
)
|
| 40 |
+
wer_kenlm_model: Optional[str] = field(
|
| 41 |
+
default=None,
|
| 42 |
+
metadata={
|
| 43 |
+
"help": "if this is provided, use kenlm to compute wer (along with other wer_* args)"
|
| 44 |
+
},
|
| 45 |
+
)
|
| 46 |
+
wer_lexicon: Optional[str] = field(
|
| 47 |
+
default=None,
|
| 48 |
+
metadata={"help": "lexicon to use with wer_kenlm_model"},
|
| 49 |
+
)
|
| 50 |
+
wer_lm_weight: float = field(
|
| 51 |
+
default=2.0,
|
| 52 |
+
metadata={"help": "lm weight to use with wer_kenlm_model"},
|
| 53 |
+
)
|
| 54 |
+
wer_word_score: float = field(
|
| 55 |
+
default=-1.0,
|
| 56 |
+
metadata={"help": "lm word score to use with wer_kenlm_model"},
|
| 57 |
+
)
|
| 58 |
+
wer_sil_weight: float = field(
|
| 59 |
+
default=0,
|
| 60 |
+
metadata={"help": "lm word score to use with wer_kenlm_model"},
|
| 61 |
+
)
|
| 62 |
+
|
| 63 |
+
wer_args: Optional[str] = field(
|
| 64 |
+
default=None,
|
| 65 |
+
metadata={
|
| 66 |
+
"help": "DEPRECATED: tuple of (wer_kenlm_model, wer_lexicon, wer_lm_weight, wer_word_score)"
|
| 67 |
+
},
|
| 68 |
+
)
|
| 69 |
+
|
| 70 |
+
|
| 71 |
+
@register_criterion("ctc", dataclass=CtcCriterionConfig)
|
| 72 |
+
class CtcCriterion(FairseqCriterion):
|
| 73 |
+
def __init__(
|
| 74 |
+
self, cfg: CtcCriterionConfig, task: FairseqTask, rdrop_alpha: int = 0.0
|
| 75 |
+
):
|
| 76 |
+
super().__init__(task)
|
| 77 |
+
self.blank_idx = (
|
| 78 |
+
task.target_dictionary.index(task.blank_symbol)
|
| 79 |
+
if hasattr(task, "blank_symbol")
|
| 80 |
+
else 0
|
| 81 |
+
)
|
| 82 |
+
self.pad_idx = task.target_dictionary.pad()
|
| 83 |
+
self.eos_idx = task.target_dictionary.eos()
|
| 84 |
+
self.post_process = cfg.post_process
|
| 85 |
+
|
| 86 |
+
self.rdrop_alpha = rdrop_alpha
|
| 87 |
+
|
| 88 |
+
if cfg.wer_args is not None:
|
| 89 |
+
(
|
| 90 |
+
cfg.wer_kenlm_model,
|
| 91 |
+
cfg.wer_lexicon,
|
| 92 |
+
cfg.wer_lm_weight,
|
| 93 |
+
cfg.wer_word_score,
|
| 94 |
+
) = eval(cfg.wer_args)
|
| 95 |
+
|
| 96 |
+
if cfg.wer_kenlm_model is not None and cfg.wer_kenlm_model != "":
|
| 97 |
+
from examples.speech_recognition.w2l_decoder import W2lKenLMDecoder
|
| 98 |
+
|
| 99 |
+
dec_args = Namespace()
|
| 100 |
+
dec_args.nbest = 1
|
| 101 |
+
dec_args.criterion = "ctc"
|
| 102 |
+
dec_args.kenlm_model = cfg.wer_kenlm_model
|
| 103 |
+
dec_args.lexicon = cfg.wer_lexicon
|
| 104 |
+
dec_args.beam = 50
|
| 105 |
+
dec_args.beam_size_token = min(50, len(task.target_dictionary))
|
| 106 |
+
dec_args.beam_threshold = min(50, len(task.target_dictionary))
|
| 107 |
+
dec_args.lm_weight = cfg.wer_lm_weight
|
| 108 |
+
dec_args.word_score = cfg.wer_word_score
|
| 109 |
+
dec_args.sil_weight = cfg.wer_sil_weight
|
| 110 |
+
dec_args.unk_weight = -math.inf
|
| 111 |
+
dec_args.sil_weight = 0
|
| 112 |
+
|
| 113 |
+
self.w2l_decoder = W2lKenLMDecoder(dec_args, task.target_dictionary)
|
| 114 |
+
else:
|
| 115 |
+
self.w2l_decoder = None
|
| 116 |
+
|
| 117 |
+
self.zero_infinity = cfg.zero_infinity
|
| 118 |
+
self.sentence_avg = cfg.sentence_avg
|
| 119 |
+
|
| 120 |
+
def forward(self, model, sample, reduce=True, **kwargs):
|
| 121 |
+
net_output = model(**sample["net_input"])
|
| 122 |
+
lprobs = model.get_normalized_probs(
|
| 123 |
+
net_output, log_probs=True
|
| 124 |
+
).contiguous() # (T, B, C) from the encoder
|
| 125 |
+
|
| 126 |
+
# CTC loss is calculated over duplicated inputs
|
| 127 |
+
# sample is already duplicated for R-Drop
|
| 128 |
+
if self.rdrop_alpha > 0:
|
| 129 |
+
for k, v in sample.items():
|
| 130 |
+
if k in ["target", "target_lengths"]:
|
| 131 |
+
sample[k] = torch.cat([v, v.clone()], dim=0)
|
| 132 |
+
elif k == "net_input":
|
| 133 |
+
if sample[k]["src_tokens"].size(1) != sample[k]["src_lengths"].size(
|
| 134 |
+
0
|
| 135 |
+
):
|
| 136 |
+
# for decoder CTC loss
|
| 137 |
+
sample[k]["src_lengths"] = torch.cat(
|
| 138 |
+
[
|
| 139 |
+
sample[k]["src_lengths"],
|
| 140 |
+
sample[k]["src_lengths"].clone(),
|
| 141 |
+
],
|
| 142 |
+
dim=0,
|
| 143 |
+
)
|
| 144 |
+
|
| 145 |
+
if "src_lengths" in sample["net_input"]:
|
| 146 |
+
input_lengths = sample["net_input"]["src_lengths"]
|
| 147 |
+
else:
|
| 148 |
+
if net_output["padding_mask"] is not None:
|
| 149 |
+
non_padding_mask = ~net_output["padding_mask"]
|
| 150 |
+
input_lengths = non_padding_mask.long().sum(-1)
|
| 151 |
+
else:
|
| 152 |
+
input_lengths = lprobs.new_full(
|
| 153 |
+
(lprobs.size(1),), lprobs.size(0), dtype=torch.long
|
| 154 |
+
)
|
| 155 |
+
|
| 156 |
+
pad_mask = (sample["target"] != self.pad_idx) & (
|
| 157 |
+
sample["target"] != self.eos_idx
|
| 158 |
+
)
|
| 159 |
+
targets_flat = sample["target"].masked_select(pad_mask)
|
| 160 |
+
if "target_lengths" in sample:
|
| 161 |
+
target_lengths = sample["target_lengths"]
|
| 162 |
+
else:
|
| 163 |
+
target_lengths = pad_mask.sum(-1)
|
| 164 |
+
|
| 165 |
+
with torch.backends.cudnn.flags(enabled=False):
|
| 166 |
+
loss = F.ctc_loss(
|
| 167 |
+
lprobs,
|
| 168 |
+
targets_flat,
|
| 169 |
+
input_lengths,
|
| 170 |
+
target_lengths,
|
| 171 |
+
blank=self.blank_idx,
|
| 172 |
+
reduction="sum",
|
| 173 |
+
zero_infinity=self.zero_infinity,
|
| 174 |
+
)
|
| 175 |
+
|
| 176 |
+
ntokens = (
|
| 177 |
+
sample["ntokens"] if "ntokens" in sample else target_lengths.sum().item()
|
| 178 |
+
)
|
| 179 |
+
|
| 180 |
+
sample_size = sample["target"].size(0) if self.sentence_avg else ntokens
|
| 181 |
+
logging_output = {
|
| 182 |
+
"loss": utils.item(loss.data), # * sample['ntokens'],
|
| 183 |
+
"ntokens": ntokens,
|
| 184 |
+
"nsentences": sample["id"].numel(),
|
| 185 |
+
"sample_size": sample_size,
|
| 186 |
+
}
|
| 187 |
+
|
| 188 |
+
if not model.training:
|
| 189 |
+
import editdistance
|
| 190 |
+
|
| 191 |
+
with torch.no_grad():
|
| 192 |
+
lprobs_t = lprobs.transpose(0, 1).float().contiguous().cpu()
|
| 193 |
+
|
| 194 |
+
c_err = 0
|
| 195 |
+
c_len = 0
|
| 196 |
+
w_errs = 0
|
| 197 |
+
w_len = 0
|
| 198 |
+
wv_errs = 0
|
| 199 |
+
for lp, t, inp_l in zip(
|
| 200 |
+
lprobs_t,
|
| 201 |
+
sample["target_label"]
|
| 202 |
+
if "target_label" in sample
|
| 203 |
+
else sample["target"],
|
| 204 |
+
input_lengths,
|
| 205 |
+
):
|
| 206 |
+
lp = lp[:inp_l].unsqueeze(0)
|
| 207 |
+
|
| 208 |
+
decoded = None
|
| 209 |
+
if self.w2l_decoder is not None:
|
| 210 |
+
decoded = self.w2l_decoder.decode(lp)
|
| 211 |
+
if len(decoded) < 1:
|
| 212 |
+
decoded = None
|
| 213 |
+
else:
|
| 214 |
+
decoded = decoded[0]
|
| 215 |
+
if len(decoded) < 1:
|
| 216 |
+
decoded = None
|
| 217 |
+
else:
|
| 218 |
+
decoded = decoded[0]
|
| 219 |
+
|
| 220 |
+
p = (t != self.task.target_dictionary.pad()) & (
|
| 221 |
+
t != self.task.target_dictionary.eos()
|
| 222 |
+
)
|
| 223 |
+
targ = t[p]
|
| 224 |
+
targ_units = self.task.target_dictionary.string(targ)
|
| 225 |
+
targ_units_arr = targ.tolist()
|
| 226 |
+
|
| 227 |
+
toks = lp.argmax(dim=-1).unique_consecutive()
|
| 228 |
+
pred_units_arr = toks[toks != self.blank_idx].tolist()
|
| 229 |
+
|
| 230 |
+
c_err += editdistance.eval(pred_units_arr, targ_units_arr)
|
| 231 |
+
c_len += len(targ_units_arr)
|
| 232 |
+
|
| 233 |
+
targ_words = post_process(targ_units, self.post_process).split()
|
| 234 |
+
|
| 235 |
+
pred_units = self.task.target_dictionary.string(pred_units_arr)
|
| 236 |
+
pred_words_raw = post_process(pred_units, self.post_process).split()
|
| 237 |
+
|
| 238 |
+
if decoded is not None and "words" in decoded:
|
| 239 |
+
pred_words = decoded["words"]
|
| 240 |
+
w_errs += editdistance.eval(pred_words, targ_words)
|
| 241 |
+
wv_errs += editdistance.eval(pred_words_raw, targ_words)
|
| 242 |
+
else:
|
| 243 |
+
dist = editdistance.eval(pred_words_raw, targ_words)
|
| 244 |
+
w_errs += dist
|
| 245 |
+
wv_errs += dist
|
| 246 |
+
|
| 247 |
+
w_len += len(targ_words)
|
| 248 |
+
|
| 249 |
+
logging_output["wv_errors"] = wv_errs
|
| 250 |
+
logging_output["w_errors"] = w_errs
|
| 251 |
+
logging_output["w_total"] = w_len
|
| 252 |
+
logging_output["c_errors"] = c_err
|
| 253 |
+
logging_output["c_total"] = c_len
|
| 254 |
+
|
| 255 |
+
return loss, sample_size, logging_output
|
| 256 |
+
|
| 257 |
+
@staticmethod
|
| 258 |
+
def reduce_metrics(logging_outputs) -> None:
|
| 259 |
+
"""Aggregate logging outputs from data parallel training."""
|
| 260 |
+
|
| 261 |
+
loss_sum = utils.item(sum(log.get("loss", 0) for log in logging_outputs))
|
| 262 |
+
ntokens = utils.item(sum(log.get("ntokens", 0) for log in logging_outputs))
|
| 263 |
+
nsentences = utils.item(
|
| 264 |
+
sum(log.get("nsentences", 0) for log in logging_outputs)
|
| 265 |
+
)
|
| 266 |
+
sample_size = utils.item(
|
| 267 |
+
sum(log.get("sample_size", 0) for log in logging_outputs)
|
| 268 |
+
)
|
| 269 |
+
|
| 270 |
+
metrics.log_scalar(
|
| 271 |
+
"loss", loss_sum / sample_size / math.log(2), sample_size, round=3
|
| 272 |
+
)
|
| 273 |
+
metrics.log_scalar("ntokens", ntokens)
|
| 274 |
+
metrics.log_scalar("nsentences", nsentences)
|
| 275 |
+
if sample_size != ntokens:
|
| 276 |
+
metrics.log_scalar(
|
| 277 |
+
"nll_loss", loss_sum / ntokens / math.log(2), ntokens, round=3
|
| 278 |
+
)
|
| 279 |
+
|
| 280 |
+
c_errors = sum(log.get("c_errors", 0) for log in logging_outputs)
|
| 281 |
+
metrics.log_scalar("_c_errors", c_errors)
|
| 282 |
+
c_total = sum(log.get("c_total", 0) for log in logging_outputs)
|
| 283 |
+
metrics.log_scalar("_c_total", c_total)
|
| 284 |
+
w_errors = sum(log.get("w_errors", 0) for log in logging_outputs)
|
| 285 |
+
metrics.log_scalar("_w_errors", w_errors)
|
| 286 |
+
wv_errors = sum(log.get("wv_errors", 0) for log in logging_outputs)
|
| 287 |
+
metrics.log_scalar("_wv_errors", wv_errors)
|
| 288 |
+
w_total = sum(log.get("w_total", 0) for log in logging_outputs)
|
| 289 |
+
metrics.log_scalar("_w_total", w_total)
|
| 290 |
+
|
| 291 |
+
if c_total > 0:
|
| 292 |
+
metrics.log_derived(
|
| 293 |
+
"uer",
|
| 294 |
+
lambda meters: safe_round(
|
| 295 |
+
meters["_c_errors"].sum * 100.0 / meters["_c_total"].sum, 3
|
| 296 |
+
)
|
| 297 |
+
if meters["_c_total"].sum > 0
|
| 298 |
+
else float("nan"),
|
| 299 |
+
)
|
| 300 |
+
if w_total > 0:
|
| 301 |
+
metrics.log_derived(
|
| 302 |
+
"wer",
|
| 303 |
+
lambda meters: safe_round(
|
| 304 |
+
meters["_w_errors"].sum * 100.0 / meters["_w_total"].sum, 3
|
| 305 |
+
)
|
| 306 |
+
if meters["_w_total"].sum > 0
|
| 307 |
+
else float("nan"),
|
| 308 |
+
)
|
| 309 |
+
metrics.log_derived(
|
| 310 |
+
"raw_wer",
|
| 311 |
+
lambda meters: safe_round(
|
| 312 |
+
meters["_wv_errors"].sum * 100.0 / meters["_w_total"].sum, 3
|
| 313 |
+
)
|
| 314 |
+
if meters["_w_total"].sum > 0
|
| 315 |
+
else float("nan"),
|
| 316 |
+
)
|
| 317 |
+
|
| 318 |
+
@staticmethod
|
| 319 |
+
def logging_outputs_can_be_summed() -> bool:
|
| 320 |
+
"""
|
| 321 |
+
Whether the logging outputs returned by `forward` can be summed
|
| 322 |
+
across workers prior to calling `reduce_metrics`. Setting this
|
| 323 |
+
to True will improves distributed training speed.
|
| 324 |
+
"""
|
| 325 |
+
return True
|
data/fairseq/fairseq/criterions/fairseq_criterion.py
ADDED
|
@@ -0,0 +1,121 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) Facebook, Inc. and its affiliates.
|
| 2 |
+
#
|
| 3 |
+
# This source code is licensed under the MIT license found in the
|
| 4 |
+
# LICENSE file in the root directory of this source tree.
|
| 5 |
+
|
| 6 |
+
import inspect
|
| 7 |
+
from typing import Any, Dict, List
|
| 8 |
+
|
| 9 |
+
from fairseq import utils
|
| 10 |
+
from fairseq.logging import metrics
|
| 11 |
+
from fairseq.dataclass import FairseqDataclass
|
| 12 |
+
from fairseq.dataclass.utils import gen_parser_from_dataclass
|
| 13 |
+
from torch.nn.modules.loss import _Loss
|
| 14 |
+
|
| 15 |
+
|
| 16 |
+
class FairseqCriterion(_Loss):
|
| 17 |
+
def __init__(self, task):
|
| 18 |
+
super().__init__()
|
| 19 |
+
self.task = task
|
| 20 |
+
if hasattr(task, "target_dictionary"):
|
| 21 |
+
tgt_dict = task.target_dictionary
|
| 22 |
+
self.padding_idx = tgt_dict.pad() if tgt_dict is not None else -100
|
| 23 |
+
|
| 24 |
+
@classmethod
|
| 25 |
+
def add_args(cls, parser):
|
| 26 |
+
"""Add criterion-specific arguments to the parser."""
|
| 27 |
+
dc = getattr(cls, "__dataclass", None)
|
| 28 |
+
if dc is not None:
|
| 29 |
+
gen_parser_from_dataclass(parser, dc())
|
| 30 |
+
|
| 31 |
+
@classmethod
|
| 32 |
+
def build_criterion(cls, cfg: FairseqDataclass, task):
|
| 33 |
+
"""Construct a criterion from command-line args."""
|
| 34 |
+
# arguments in the __init__.
|
| 35 |
+
init_args = {}
|
| 36 |
+
for p in inspect.signature(cls).parameters.values():
|
| 37 |
+
if (
|
| 38 |
+
p.kind == p.POSITIONAL_ONLY
|
| 39 |
+
or p.kind == p.VAR_POSITIONAL
|
| 40 |
+
or p.kind == p.VAR_KEYWORD
|
| 41 |
+
):
|
| 42 |
+
# we haven't implemented inference for these argument types,
|
| 43 |
+
# but PRs welcome :)
|
| 44 |
+
raise NotImplementedError("{} not supported".format(p.kind))
|
| 45 |
+
|
| 46 |
+
assert p.kind in {p.POSITIONAL_OR_KEYWORD, p.KEYWORD_ONLY}
|
| 47 |
+
|
| 48 |
+
if p.name == "task":
|
| 49 |
+
init_args["task"] = task
|
| 50 |
+
elif p.name == "cfg":
|
| 51 |
+
init_args["cfg"] = cfg
|
| 52 |
+
elif hasattr(cfg, p.name):
|
| 53 |
+
init_args[p.name] = getattr(cfg, p.name)
|
| 54 |
+
elif p.default != p.empty:
|
| 55 |
+
pass # we'll use the default value
|
| 56 |
+
else:
|
| 57 |
+
raise NotImplementedError(
|
| 58 |
+
"Unable to infer Criterion arguments, please implement "
|
| 59 |
+
"{}.build_criterion".format(cls.__name__)
|
| 60 |
+
)
|
| 61 |
+
return cls(**init_args)
|
| 62 |
+
|
| 63 |
+
def forward(self, model, sample, reduce=True):
|
| 64 |
+
"""Compute the loss for the given sample.
|
| 65 |
+
|
| 66 |
+
Returns a tuple with three elements:
|
| 67 |
+
1) the loss
|
| 68 |
+
2) the sample size, which is used as the denominator for the gradient
|
| 69 |
+
3) logging outputs to display while training
|
| 70 |
+
"""
|
| 71 |
+
raise NotImplementedError
|
| 72 |
+
|
| 73 |
+
@staticmethod
|
| 74 |
+
def aggregate_logging_outputs(
|
| 75 |
+
logging_outputs: List[Dict[str, Any]]
|
| 76 |
+
) -> Dict[str, Any]:
|
| 77 |
+
"""Aggregate logging outputs from data parallel training."""
|
| 78 |
+
utils.deprecation_warning(
|
| 79 |
+
"The aggregate_logging_outputs API is deprecated. "
|
| 80 |
+
"Please use the reduce_metrics API instead."
|
| 81 |
+
)
|
| 82 |
+
raise NotImplementedError
|
| 83 |
+
|
| 84 |
+
@classmethod
|
| 85 |
+
def reduce_metrics(cls, logging_outputs: List[Dict[str, Any]]) -> None:
|
| 86 |
+
"""Aggregate logging outputs from data parallel training."""
|
| 87 |
+
utils.deprecation_warning(
|
| 88 |
+
"Criterions should implement the reduce_metrics API. "
|
| 89 |
+
"Falling back to deprecated aggregate_logging_outputs API."
|
| 90 |
+
)
|
| 91 |
+
agg_logging_outputs = cls.aggregate_logging_outputs(logging_outputs)
|
| 92 |
+
for k, v in agg_logging_outputs.items():
|
| 93 |
+
if k in {"nsentences", "ntokens", "sample_size"}:
|
| 94 |
+
continue
|
| 95 |
+
metrics.log_scalar(k, v)
|
| 96 |
+
|
| 97 |
+
@staticmethod
|
| 98 |
+
def logging_outputs_can_be_summed() -> bool:
|
| 99 |
+
"""
|
| 100 |
+
Whether the logging outputs returned by `forward` can be summed
|
| 101 |
+
across workers prior to calling `reduce_metrics`. Setting this
|
| 102 |
+
to True will improves distributed training speed.
|
| 103 |
+
"""
|
| 104 |
+
return False
|
| 105 |
+
|
| 106 |
+
|
| 107 |
+
class LegacyFairseqCriterion(FairseqCriterion):
|
| 108 |
+
def __init__(self, args, task):
|
| 109 |
+
super().__init__(task=task)
|
| 110 |
+
self.args = args
|
| 111 |
+
|
| 112 |
+
utils.deprecation_warning(
|
| 113 |
+
"Criterions should take explicit arguments instead of an "
|
| 114 |
+
"argparse.Namespace object, please update your criterion by "
|
| 115 |
+
"extending FairseqCriterion instead of LegacyFairseqCriterion."
|
| 116 |
+
)
|
| 117 |
+
|
| 118 |
+
@classmethod
|
| 119 |
+
def build_criterion(cls, args, task):
|
| 120 |
+
"""Construct a criterion from command-line args."""
|
| 121 |
+
return cls(args, task)
|
data/fairseq/fairseq/criterions/fastspeech2_loss.py
ADDED
|
@@ -0,0 +1,137 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) 2017-present, Facebook, Inc.
|
| 2 |
+
# All rights reserved.
|
| 3 |
+
#
|
| 4 |
+
# This source code is licensed under the license found in the LICENSE file in
|
| 5 |
+
# the root directory of this source tree. An additional grant of patent rights
|
| 6 |
+
# can be found in the PATENTS file in the same directory.
|
| 7 |
+
|
| 8 |
+
from typing import List, Dict, Any
|
| 9 |
+
from dataclasses import dataclass, field
|
| 10 |
+
|
| 11 |
+
import torch
|
| 12 |
+
import torch.nn.functional as F
|
| 13 |
+
|
| 14 |
+
from fairseq import utils
|
| 15 |
+
from fairseq.logging import metrics
|
| 16 |
+
from fairseq.criterions import FairseqCriterion, register_criterion
|
| 17 |
+
from fairseq.dataclass import FairseqDataclass
|
| 18 |
+
from fairseq.data.data_utils import lengths_to_mask
|
| 19 |
+
from fairseq.models.fairseq_model import FairseqEncoderModel
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
@dataclass
|
| 23 |
+
class FastSpeech2CriterionConfig(FairseqDataclass):
|
| 24 |
+
ctc_weight: float = field(default=0.0, metadata={"help": "weight for CTC loss"})
|
| 25 |
+
|
| 26 |
+
|
| 27 |
+
@register_criterion("fastspeech2", dataclass=FastSpeech2CriterionConfig)
|
| 28 |
+
class FastSpeech2Loss(FairseqCriterion):
|
| 29 |
+
def __init__(self, task, ctc_weight):
|
| 30 |
+
super().__init__(task)
|
| 31 |
+
self.ctc_weight = ctc_weight
|
| 32 |
+
|
| 33 |
+
def forward(self, model: FairseqEncoderModel, sample, reduction="mean"):
|
| 34 |
+
src_tokens = sample["net_input"]["src_tokens"]
|
| 35 |
+
src_lens = sample["net_input"]["src_lengths"]
|
| 36 |
+
tgt_lens = sample["target_lengths"]
|
| 37 |
+
_feat_out, _feat_out_post, _, log_dur_out, pitch_out, energy_out = model(
|
| 38 |
+
src_tokens=src_tokens,
|
| 39 |
+
src_lengths=src_lens,
|
| 40 |
+
prev_output_tokens=sample["net_input"]["prev_output_tokens"],
|
| 41 |
+
incremental_state=None,
|
| 42 |
+
target_lengths=tgt_lens,
|
| 43 |
+
speaker=sample["speaker"],
|
| 44 |
+
durations=sample["durations"],
|
| 45 |
+
pitches=sample["pitches"],
|
| 46 |
+
energies=sample["energies"],
|
| 47 |
+
)
|
| 48 |
+
|
| 49 |
+
src_mask = lengths_to_mask(sample["net_input"]["src_lengths"])
|
| 50 |
+
tgt_mask = lengths_to_mask(sample["target_lengths"])
|
| 51 |
+
|
| 52 |
+
pitches, energies = sample["pitches"], sample["energies"]
|
| 53 |
+
pitch_out, pitches = pitch_out[src_mask], pitches[src_mask]
|
| 54 |
+
energy_out, energies = energy_out[src_mask], energies[src_mask]
|
| 55 |
+
|
| 56 |
+
feat_out, feat = _feat_out[tgt_mask], sample["target"][tgt_mask]
|
| 57 |
+
l1_loss = F.l1_loss(feat_out, feat, reduction=reduction)
|
| 58 |
+
if _feat_out_post is not None:
|
| 59 |
+
l1_loss += F.l1_loss(_feat_out_post[tgt_mask], feat, reduction=reduction)
|
| 60 |
+
|
| 61 |
+
pitch_loss = F.mse_loss(pitch_out, pitches, reduction=reduction)
|
| 62 |
+
energy_loss = F.mse_loss(energy_out, energies, reduction=reduction)
|
| 63 |
+
|
| 64 |
+
log_dur_out = log_dur_out[src_mask]
|
| 65 |
+
dur = sample["durations"].float()
|
| 66 |
+
dur = dur.half() if log_dur_out.type().endswith(".HalfTensor") else dur
|
| 67 |
+
log_dur = torch.log(dur + 1)[src_mask]
|
| 68 |
+
dur_loss = F.mse_loss(log_dur_out, log_dur, reduction=reduction)
|
| 69 |
+
|
| 70 |
+
ctc_loss = torch.tensor(0.0).type_as(l1_loss)
|
| 71 |
+
if self.ctc_weight > 0.0:
|
| 72 |
+
lprobs = model.get_normalized_probs((_feat_out,), log_probs=True)
|
| 73 |
+
lprobs = lprobs.transpose(0, 1) # T x B x C
|
| 74 |
+
src_mask = lengths_to_mask(src_lens)
|
| 75 |
+
src_tokens_flat = src_tokens.masked_select(src_mask)
|
| 76 |
+
ctc_loss = (
|
| 77 |
+
F.ctc_loss(
|
| 78 |
+
lprobs,
|
| 79 |
+
src_tokens_flat,
|
| 80 |
+
tgt_lens,
|
| 81 |
+
src_lens,
|
| 82 |
+
reduction=reduction,
|
| 83 |
+
zero_infinity=True,
|
| 84 |
+
)
|
| 85 |
+
* self.ctc_weight
|
| 86 |
+
)
|
| 87 |
+
|
| 88 |
+
loss = l1_loss + dur_loss + pitch_loss + energy_loss + ctc_loss
|
| 89 |
+
|
| 90 |
+
sample_size = sample["nsentences"]
|
| 91 |
+
logging_output = {
|
| 92 |
+
"loss": utils.item(loss.data),
|
| 93 |
+
"ntokens": sample["ntokens"],
|
| 94 |
+
"nsentences": sample["nsentences"],
|
| 95 |
+
"sample_size": sample_size,
|
| 96 |
+
"l1_loss": utils.item(l1_loss.data),
|
| 97 |
+
"dur_loss": utils.item(dur_loss.data),
|
| 98 |
+
"pitch_loss": utils.item(pitch_loss.data),
|
| 99 |
+
"energy_loss": utils.item(energy_loss.data),
|
| 100 |
+
"ctc_loss": utils.item(ctc_loss.data),
|
| 101 |
+
}
|
| 102 |
+
return loss, sample_size, logging_output
|
| 103 |
+
|
| 104 |
+
@classmethod
|
| 105 |
+
def reduce_metrics(cls, logging_outputs: List[Dict[str, Any]]) -> None:
|
| 106 |
+
ns = [log.get("sample_size", 0) for log in logging_outputs]
|
| 107 |
+
ntot = sum(ns)
|
| 108 |
+
ws = [n / (ntot + 1e-8) for n in ns]
|
| 109 |
+
for key in [
|
| 110 |
+
"loss",
|
| 111 |
+
"l1_loss",
|
| 112 |
+
"dur_loss",
|
| 113 |
+
"pitch_loss",
|
| 114 |
+
"energy_loss",
|
| 115 |
+
"ctc_loss",
|
| 116 |
+
]:
|
| 117 |
+
vals = [log.get(key, 0) for log in logging_outputs]
|
| 118 |
+
val = sum(val * w for val, w in zip(vals, ws))
|
| 119 |
+
metrics.log_scalar(key, val, ntot, round=3)
|
| 120 |
+
metrics.log_scalar("sample_size", ntot, len(logging_outputs))
|
| 121 |
+
|
| 122 |
+
# inference metrics
|
| 123 |
+
if "targ_frames" not in logging_outputs[0]:
|
| 124 |
+
return
|
| 125 |
+
n = sum(log.get("targ_frames", 0) for log in logging_outputs)
|
| 126 |
+
for key, new_key in [
|
| 127 |
+
("mcd_loss", "mcd_loss"),
|
| 128 |
+
("pred_frames", "pred_ratio"),
|
| 129 |
+
("nins", "ins_rate"),
|
| 130 |
+
("ndel", "del_rate"),
|
| 131 |
+
]:
|
| 132 |
+
val = sum(log.get(key, 0) for log in logging_outputs)
|
| 133 |
+
metrics.log_scalar(new_key, val / n, n, round=3)
|
| 134 |
+
|
| 135 |
+
@staticmethod
|
| 136 |
+
def logging_outputs_can_be_summed() -> bool:
|
| 137 |
+
return False
|
data/fairseq/fairseq/criterions/hubert_criterion.py
ADDED
|
@@ -0,0 +1,195 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) Facebook, Inc. and its affiliates.
|
| 2 |
+
#
|
| 3 |
+
# This source code is licensed under the MIT license found in the
|
| 4 |
+
# LICENSE file in the root directory of this source tree.
|
| 5 |
+
|
| 6 |
+
import math
|
| 7 |
+
import re
|
| 8 |
+
from dataclasses import dataclass, field
|
| 9 |
+
from typing import List, Optional
|
| 10 |
+
|
| 11 |
+
import torch
|
| 12 |
+
import torch.nn.functional as F
|
| 13 |
+
from fairseq import utils
|
| 14 |
+
from fairseq.logging import metrics
|
| 15 |
+
from fairseq.criterions import FairseqCriterion, register_criterion
|
| 16 |
+
from fairseq.dataclass import FairseqDataclass
|
| 17 |
+
|
| 18 |
+
|
| 19 |
+
@dataclass
|
| 20 |
+
class HubertCriterionConfig(FairseqDataclass):
|
| 21 |
+
pred_masked_weight: float = field(
|
| 22 |
+
default=1.0,
|
| 23 |
+
metadata={"help": "weight for predictive loss for masked frames"},
|
| 24 |
+
)
|
| 25 |
+
pred_nomask_weight: float = field(
|
| 26 |
+
default=0.0,
|
| 27 |
+
metadata={"help": "weight for predictive loss for unmasked frames"},
|
| 28 |
+
)
|
| 29 |
+
loss_weights: Optional[List[float]] = field(
|
| 30 |
+
default=None,
|
| 31 |
+
metadata={"help": "weights for additional loss terms (not first one)"},
|
| 32 |
+
)
|
| 33 |
+
log_keys: List[str] = field(
|
| 34 |
+
default_factory=lambda: [],
|
| 35 |
+
metadata={"help": "output keys to log"},
|
| 36 |
+
)
|
| 37 |
+
|
| 38 |
+
|
| 39 |
+
@register_criterion("hubert", dataclass=HubertCriterionConfig)
|
| 40 |
+
class HubertCriterion(FairseqCriterion):
|
| 41 |
+
def __init__(
|
| 42 |
+
self,
|
| 43 |
+
task,
|
| 44 |
+
pred_masked_weight,
|
| 45 |
+
pred_nomask_weight,
|
| 46 |
+
loss_weights=None,
|
| 47 |
+
log_keys=None,
|
| 48 |
+
):
|
| 49 |
+
super().__init__(task)
|
| 50 |
+
self.pred_masked_weight = pred_masked_weight
|
| 51 |
+
self.pred_nomask_weight = pred_nomask_weight
|
| 52 |
+
self.loss_weights = loss_weights
|
| 53 |
+
self.log_keys = [] if log_keys is None else log_keys
|
| 54 |
+
|
| 55 |
+
def forward(self, model, sample, reduce=True, log_pred=False):
|
| 56 |
+
"""Compute the loss for the given sample.
|
| 57 |
+
Returns a tuple with three elements:
|
| 58 |
+
1) the loss
|
| 59 |
+
2) the sample size, which is used as the denominator for the gradient
|
| 60 |
+
3) logging outputs to display while training
|
| 61 |
+
"""
|
| 62 |
+
net_output = model(target_list=sample["target_list"], **sample["net_input"])
|
| 63 |
+
loss = 0.0
|
| 64 |
+
sample_size = 0
|
| 65 |
+
logging_output = {}
|
| 66 |
+
reduction = "sum" if reduce else "none"
|
| 67 |
+
|
| 68 |
+
loss_m_list = []
|
| 69 |
+
logp_m_list = model.get_logits(net_output, True)
|
| 70 |
+
targ_m_list = model.get_targets(net_output, True)
|
| 71 |
+
assert self.pred_masked_weight == 0 or len(logp_m_list) > 0
|
| 72 |
+
for i, (logp_m, targ_m) in enumerate(zip(logp_m_list, targ_m_list)):
|
| 73 |
+
loss_m = F.cross_entropy(logp_m, targ_m, reduction=reduction)
|
| 74 |
+
loss_m_list.append(loss_m)
|
| 75 |
+
logging_output[f"loss_m_{i}"] = loss_m.detach().item()
|
| 76 |
+
if self.pred_masked_weight > 0:
|
| 77 |
+
loss += self.pred_masked_weight * sum(loss_m_list)
|
| 78 |
+
sample_size += targ_m_list[0].numel()
|
| 79 |
+
|
| 80 |
+
loss_u_list = []
|
| 81 |
+
logp_u_list = model.get_logits(net_output, False)
|
| 82 |
+
targ_u_list = model.get_targets(net_output, False)
|
| 83 |
+
assert self.pred_nomask_weight == 0 or len(logp_u_list) > 0
|
| 84 |
+
for i, (logp_u, targ_u) in enumerate(zip(logp_u_list, targ_u_list)):
|
| 85 |
+
loss_u = F.cross_entropy(logp_u, targ_u, reduction=reduction)
|
| 86 |
+
loss_u_list.append(loss_u)
|
| 87 |
+
logging_output[f"loss_u_{i}"] = loss_u.detach().item()
|
| 88 |
+
if self.pred_nomask_weight > 0:
|
| 89 |
+
loss += self.pred_nomask_weight * sum(loss_u_list)
|
| 90 |
+
sample_size += targ_u_list[0].numel()
|
| 91 |
+
|
| 92 |
+
if self.loss_weights is not None:
|
| 93 |
+
assert hasattr(model, "get_extra_losses")
|
| 94 |
+
extra_losses, names = model.get_extra_losses(net_output)
|
| 95 |
+
if torch.is_tensor(extra_losses):
|
| 96 |
+
extra_losses = [extra_losses]
|
| 97 |
+
names = [names]
|
| 98 |
+
if len(self.loss_weights) == 1 and len(extra_losses) != 1:
|
| 99 |
+
self.loss_weights = [self.loss_weights[0]] * len(extra_losses)
|
| 100 |
+
assert len(extra_losses) == len(
|
| 101 |
+
self.loss_weights
|
| 102 |
+
), f"{len(extra_losses)}, {len(self.loss_weights)}"
|
| 103 |
+
for p, n, coef in zip(extra_losses, names, self.loss_weights):
|
| 104 |
+
if coef != 0 and p is not None:
|
| 105 |
+
p = coef * p.float() * sample_size
|
| 106 |
+
loss += p
|
| 107 |
+
logging_output[f"loss_{n}"] = p.item()
|
| 108 |
+
|
| 109 |
+
logging_output = {
|
| 110 |
+
"loss": loss.item() if reduce else loss,
|
| 111 |
+
"ntokens": sample_size,
|
| 112 |
+
"nsentences": sample["id"].numel(),
|
| 113 |
+
"sample_size": sample_size,
|
| 114 |
+
**logging_output,
|
| 115 |
+
}
|
| 116 |
+
|
| 117 |
+
for lk in self.log_keys:
|
| 118 |
+
if lk in net_output:
|
| 119 |
+
logging_output[lk] = float((net_output[lk]))
|
| 120 |
+
|
| 121 |
+
def compute_correct(logits):
|
| 122 |
+
if logits.numel() == 0:
|
| 123 |
+
return 0, 0
|
| 124 |
+
else:
|
| 125 |
+
assert logits.dim() > 1, logits.shape
|
| 126 |
+
max = logits.argmax(-1) == 0
|
| 127 |
+
min = logits.argmin(-1) == 0
|
| 128 |
+
both = max & min
|
| 129 |
+
corr = max.long().sum().item() - both.long().sum().item()
|
| 130 |
+
count = max.numel()
|
| 131 |
+
return corr, count
|
| 132 |
+
|
| 133 |
+
with torch.no_grad():
|
| 134 |
+
for i, logp_m in enumerate(logp_m_list):
|
| 135 |
+
corr_m, count_m = compute_correct(logp_m)
|
| 136 |
+
logging_output[f"correct_m_{i}"] = corr_m
|
| 137 |
+
logging_output[f"count_m_{i}"] = count_m
|
| 138 |
+
|
| 139 |
+
for i, logp_u in enumerate(logp_u_list):
|
| 140 |
+
corr_u, count_u = compute_correct(logp_u)
|
| 141 |
+
logging_output[f"correct_u_{i}"] = corr_u
|
| 142 |
+
logging_output[f"count_u_{i}"] = count_u
|
| 143 |
+
|
| 144 |
+
return loss, sample_size, logging_output
|
| 145 |
+
|
| 146 |
+
@staticmethod
|
| 147 |
+
def reduce_metrics(logging_outputs) -> None:
|
| 148 |
+
"""Aggregate logging outputs from data parallel training (copied from normal cross entropy)."""
|
| 149 |
+
loss_sum = sum(log.get("loss", 0) for log in logging_outputs)
|
| 150 |
+
ntokens = sum(log.get("ntokens", 0) for log in logging_outputs)
|
| 151 |
+
sample_size = sum(log.get("sample_size", 0) for log in logging_outputs)
|
| 152 |
+
|
| 153 |
+
metrics.log_scalar(
|
| 154 |
+
"loss", loss_sum / sample_size / math.log(2), sample_size, round=3
|
| 155 |
+
)
|
| 156 |
+
if sample_size != ntokens:
|
| 157 |
+
metrics.log_scalar(
|
| 158 |
+
"nll_loss", loss_sum / ntokens / math.log(2), ntokens, round=3
|
| 159 |
+
)
|
| 160 |
+
metrics.log_derived(
|
| 161 |
+
"ppl", lambda meters: utils.get_perplexity(meters["nll_loss"].avg)
|
| 162 |
+
)
|
| 163 |
+
else:
|
| 164 |
+
metrics.log_derived(
|
| 165 |
+
"ppl", lambda meters: utils.get_perplexity(meters["loss"].avg)
|
| 166 |
+
)
|
| 167 |
+
|
| 168 |
+
counts = {}
|
| 169 |
+
for lk in logging_outputs[0].keys():
|
| 170 |
+
if lk.startswith("count_"):
|
| 171 |
+
val = sum(log[lk] for log in logging_outputs)
|
| 172 |
+
metrics.log_scalar(lk, val)
|
| 173 |
+
counts[lk] = val
|
| 174 |
+
|
| 175 |
+
for lk in logging_outputs[0].keys():
|
| 176 |
+
if lk.startswith("loss_"):
|
| 177 |
+
val = sum(log[lk] for log in logging_outputs)
|
| 178 |
+
metrics.log_scalar(lk, val / sample_size / math.log(2), round=3)
|
| 179 |
+
elif lk.startswith("correct_"):
|
| 180 |
+
val = sum(log[lk] for log in logging_outputs)
|
| 181 |
+
metrics.log_scalar(lk, val / counts[re.sub("correct", "count", lk)])
|
| 182 |
+
|
| 183 |
+
@staticmethod
|
| 184 |
+
def aggregate_logging_outputs(logging_outputs):
|
| 185 |
+
"""Aggregate logging outputs from data parallel training."""
|
| 186 |
+
raise NotImplementedError()
|
| 187 |
+
|
| 188 |
+
@staticmethod
|
| 189 |
+
def logging_outputs_can_be_summed() -> bool:
|
| 190 |
+
"""
|
| 191 |
+
Whether the logging outputs returned by `forward` can be summed
|
| 192 |
+
across workers prior to calling `reduce_metrics`. Setting this
|
| 193 |
+
to True will improves distributed training speed.
|
| 194 |
+
"""
|
| 195 |
+
return False
|
data/fairseq/fairseq/criterions/label_smoothed_cross_entropy.py
ADDED
|
@@ -0,0 +1,168 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) Facebook, Inc. and its affiliates.
|
| 2 |
+
#
|
| 3 |
+
# This source code is licensed under the MIT license found in the
|
| 4 |
+
# LICENSE file in the root directory of this source tree.
|
| 5 |
+
|
| 6 |
+
import math
|
| 7 |
+
from dataclasses import dataclass, field
|
| 8 |
+
|
| 9 |
+
import torch
|
| 10 |
+
from fairseq import utils
|
| 11 |
+
from fairseq.logging import metrics
|
| 12 |
+
from fairseq.criterions import FairseqCriterion, register_criterion
|
| 13 |
+
from fairseq.dataclass import FairseqDataclass
|
| 14 |
+
from omegaconf import II
|
| 15 |
+
|
| 16 |
+
|
| 17 |
+
@dataclass
|
| 18 |
+
class LabelSmoothedCrossEntropyCriterionConfig(FairseqDataclass):
|
| 19 |
+
label_smoothing: float = field(
|
| 20 |
+
default=0.0,
|
| 21 |
+
metadata={"help": "epsilon for label smoothing, 0 means no label smoothing"},
|
| 22 |
+
)
|
| 23 |
+
report_accuracy: bool = field(
|
| 24 |
+
default=False,
|
| 25 |
+
metadata={"help": "report accuracy metric"},
|
| 26 |
+
)
|
| 27 |
+
ignore_prefix_size: int = field(
|
| 28 |
+
default=0,
|
| 29 |
+
metadata={"help": "Ignore first N tokens"},
|
| 30 |
+
)
|
| 31 |
+
sentence_avg: bool = II("optimization.sentence_avg")
|
| 32 |
+
|
| 33 |
+
|
| 34 |
+
def label_smoothed_nll_loss(lprobs, target, epsilon, ignore_index=None, reduce=True):
|
| 35 |
+
if target.dim() == lprobs.dim() - 1:
|
| 36 |
+
target = target.unsqueeze(-1)
|
| 37 |
+
nll_loss = -lprobs.gather(dim=-1, index=target)
|
| 38 |
+
smooth_loss = -lprobs.sum(dim=-1, keepdim=True)
|
| 39 |
+
if ignore_index is not None:
|
| 40 |
+
pad_mask = target.eq(ignore_index)
|
| 41 |
+
nll_loss.masked_fill_(pad_mask, 0.0)
|
| 42 |
+
smooth_loss.masked_fill_(pad_mask, 0.0)
|
| 43 |
+
else:
|
| 44 |
+
nll_loss = nll_loss.squeeze(-1)
|
| 45 |
+
smooth_loss = smooth_loss.squeeze(-1)
|
| 46 |
+
if reduce:
|
| 47 |
+
nll_loss = nll_loss.sum()
|
| 48 |
+
smooth_loss = smooth_loss.sum()
|
| 49 |
+
eps_i = epsilon / (lprobs.size(-1) - 1)
|
| 50 |
+
loss = (1.0 - epsilon - eps_i) * nll_loss + eps_i * smooth_loss
|
| 51 |
+
return loss, nll_loss
|
| 52 |
+
|
| 53 |
+
|
| 54 |
+
@register_criterion(
|
| 55 |
+
"label_smoothed_cross_entropy", dataclass=LabelSmoothedCrossEntropyCriterionConfig
|
| 56 |
+
)
|
| 57 |
+
class LabelSmoothedCrossEntropyCriterion(FairseqCriterion):
|
| 58 |
+
def __init__(
|
| 59 |
+
self,
|
| 60 |
+
task,
|
| 61 |
+
sentence_avg,
|
| 62 |
+
label_smoothing,
|
| 63 |
+
ignore_prefix_size=0,
|
| 64 |
+
report_accuracy=False,
|
| 65 |
+
):
|
| 66 |
+
super().__init__(task)
|
| 67 |
+
self.sentence_avg = sentence_avg
|
| 68 |
+
self.eps = label_smoothing
|
| 69 |
+
self.ignore_prefix_size = ignore_prefix_size
|
| 70 |
+
self.report_accuracy = report_accuracy
|
| 71 |
+
|
| 72 |
+
def forward(self, model, sample, reduce=True):
|
| 73 |
+
"""Compute the loss for the given sample.
|
| 74 |
+
|
| 75 |
+
Returns a tuple with three elements:
|
| 76 |
+
1) the loss
|
| 77 |
+
2) the sample size, which is used as the denominator for the gradient
|
| 78 |
+
3) logging outputs to display while training
|
| 79 |
+
"""
|
| 80 |
+
net_output = model(**sample["net_input"])
|
| 81 |
+
loss, nll_loss = self.compute_loss(model, net_output, sample, reduce=reduce)
|
| 82 |
+
sample_size = (
|
| 83 |
+
sample["target"].size(0) if self.sentence_avg else sample["ntokens"]
|
| 84 |
+
)
|
| 85 |
+
logging_output = {
|
| 86 |
+
"loss": loss.data,
|
| 87 |
+
"nll_loss": nll_loss.data,
|
| 88 |
+
"ntokens": sample["ntokens"],
|
| 89 |
+
"nsentences": sample["target"].size(0),
|
| 90 |
+
"sample_size": sample_size,
|
| 91 |
+
}
|
| 92 |
+
if self.report_accuracy:
|
| 93 |
+
n_correct, total = self.compute_accuracy(model, net_output, sample)
|
| 94 |
+
logging_output["n_correct"] = utils.item(n_correct.data)
|
| 95 |
+
logging_output["total"] = utils.item(total.data)
|
| 96 |
+
return loss, sample_size, logging_output
|
| 97 |
+
|
| 98 |
+
def get_lprobs_and_target(self, model, net_output, sample):
|
| 99 |
+
lprobs = model.get_normalized_probs(net_output, log_probs=True)
|
| 100 |
+
target = model.get_targets(sample, net_output)
|
| 101 |
+
if self.ignore_prefix_size > 0:
|
| 102 |
+
# lprobs: B x T x C
|
| 103 |
+
lprobs = lprobs[:, self.ignore_prefix_size :, :].contiguous()
|
| 104 |
+
target = target[:, self.ignore_prefix_size :].contiguous()
|
| 105 |
+
return lprobs.view(-1, lprobs.size(-1)), target.view(-1)
|
| 106 |
+
|
| 107 |
+
def compute_loss(self, model, net_output, sample, reduce=True):
|
| 108 |
+
lprobs, target = self.get_lprobs_and_target(model, net_output, sample)
|
| 109 |
+
loss, nll_loss = label_smoothed_nll_loss(
|
| 110 |
+
lprobs,
|
| 111 |
+
target,
|
| 112 |
+
self.eps,
|
| 113 |
+
ignore_index=self.padding_idx,
|
| 114 |
+
reduce=reduce,
|
| 115 |
+
)
|
| 116 |
+
return loss, nll_loss
|
| 117 |
+
|
| 118 |
+
def compute_accuracy(self, model, net_output, sample):
|
| 119 |
+
lprobs, target = self.get_lprobs_and_target(model, net_output, sample)
|
| 120 |
+
mask = target.ne(self.padding_idx)
|
| 121 |
+
n_correct = torch.sum(
|
| 122 |
+
lprobs.argmax(1).masked_select(mask).eq(target.masked_select(mask))
|
| 123 |
+
)
|
| 124 |
+
total = torch.sum(mask)
|
| 125 |
+
return n_correct, total
|
| 126 |
+
|
| 127 |
+
@classmethod
|
| 128 |
+
def reduce_metrics(cls, logging_outputs) -> None:
|
| 129 |
+
"""Aggregate logging outputs from data parallel training."""
|
| 130 |
+
loss_sum = sum(log.get("loss", 0) for log in logging_outputs)
|
| 131 |
+
nll_loss_sum = sum(log.get("nll_loss", 0) for log in logging_outputs)
|
| 132 |
+
ntokens = sum(log.get("ntokens", 0) for log in logging_outputs)
|
| 133 |
+
sample_size = sum(log.get("sample_size", 0) for log in logging_outputs)
|
| 134 |
+
|
| 135 |
+
metrics.log_scalar(
|
| 136 |
+
"loss", loss_sum / sample_size / math.log(2), sample_size, round=3
|
| 137 |
+
)
|
| 138 |
+
metrics.log_scalar(
|
| 139 |
+
"nll_loss", nll_loss_sum / ntokens / math.log(2), ntokens, round=3
|
| 140 |
+
)
|
| 141 |
+
metrics.log_derived(
|
| 142 |
+
"ppl", lambda meters: utils.get_perplexity(meters["nll_loss"].avg)
|
| 143 |
+
)
|
| 144 |
+
|
| 145 |
+
total = utils.item(sum(log.get("total", 0) for log in logging_outputs))
|
| 146 |
+
if total > 0:
|
| 147 |
+
metrics.log_scalar("total", total)
|
| 148 |
+
n_correct = utils.item(
|
| 149 |
+
sum(log.get("n_correct", 0) for log in logging_outputs)
|
| 150 |
+
)
|
| 151 |
+
metrics.log_scalar("n_correct", n_correct)
|
| 152 |
+
metrics.log_derived(
|
| 153 |
+
"accuracy",
|
| 154 |
+
lambda meters: round(
|
| 155 |
+
meters["n_correct"].sum * 100.0 / meters["total"].sum, 3
|
| 156 |
+
)
|
| 157 |
+
if meters["total"].sum > 0
|
| 158 |
+
else float("nan"),
|
| 159 |
+
)
|
| 160 |
+
|
| 161 |
+
@staticmethod
|
| 162 |
+
def logging_outputs_can_be_summed() -> bool:
|
| 163 |
+
"""
|
| 164 |
+
Whether the logging outputs returned by `forward` can be summed
|
| 165 |
+
across workers prior to calling `reduce_metrics`. Setting this
|
| 166 |
+
to True will improves distributed training speed.
|
| 167 |
+
"""
|
| 168 |
+
return True
|
data/fairseq/fairseq/criterions/label_smoothed_cross_entropy_latency_augmented.py
ADDED
|
@@ -0,0 +1,221 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) Facebook, Inc. and its affiliates.
|
| 2 |
+
#
|
| 3 |
+
# This source code is licensed under the MIT license found in the
|
| 4 |
+
# LICENSE file in the root directory of this source tree.
|
| 5 |
+
|
| 6 |
+
from dataclasses import dataclass, field
|
| 7 |
+
import torch
|
| 8 |
+
from fairseq import utils
|
| 9 |
+
from fairseq.logging import metrics
|
| 10 |
+
from fairseq.criterions import register_criterion
|
| 11 |
+
from fairseq.criterions.label_smoothed_cross_entropy import (
|
| 12 |
+
LabelSmoothedCrossEntropyCriterion,
|
| 13 |
+
LabelSmoothedCrossEntropyCriterionConfig,
|
| 14 |
+
)
|
| 15 |
+
|
| 16 |
+
try:
|
| 17 |
+
from simuleval.metrics.latency import (
|
| 18 |
+
AverageLagging,
|
| 19 |
+
AverageProportion,
|
| 20 |
+
DifferentiableAverageLagging,
|
| 21 |
+
)
|
| 22 |
+
|
| 23 |
+
LATENCY_METRICS = {
|
| 24 |
+
"average_lagging": AverageLagging,
|
| 25 |
+
"average_proportion": AverageProportion,
|
| 26 |
+
"differentiable_average_lagging": DifferentiableAverageLagging,
|
| 27 |
+
}
|
| 28 |
+
except ImportError:
|
| 29 |
+
LATENCY_METRICS = None
|
| 30 |
+
|
| 31 |
+
|
| 32 |
+
@dataclass
|
| 33 |
+
class LabelSmoothedCrossEntropyCriterionLatencyAugmentConfig(
|
| 34 |
+
LabelSmoothedCrossEntropyCriterionConfig
|
| 35 |
+
):
|
| 36 |
+
latency_avg_weight: float = field(
|
| 37 |
+
default=0.0,
|
| 38 |
+
metadata={"help": "weight fot average latency loss."},
|
| 39 |
+
)
|
| 40 |
+
latency_var_weight: float = field(
|
| 41 |
+
default=0.0,
|
| 42 |
+
metadata={"help": "weight fot variance latency loss."},
|
| 43 |
+
)
|
| 44 |
+
latency_avg_type: str = field(
|
| 45 |
+
default="differentiable_average_lagging",
|
| 46 |
+
metadata={"help": "latency type for average loss"},
|
| 47 |
+
)
|
| 48 |
+
latency_var_type: str = field(
|
| 49 |
+
default="variance_delay",
|
| 50 |
+
metadata={"help": "latency typ for variance loss"},
|
| 51 |
+
)
|
| 52 |
+
latency_gather_method: str = field(
|
| 53 |
+
default="weighted_average",
|
| 54 |
+
metadata={"help": "method to gather latency loss for all heads"},
|
| 55 |
+
)
|
| 56 |
+
latency_update_after: int = field(
|
| 57 |
+
default=0,
|
| 58 |
+
metadata={"help": "Add latency loss after certain steps"},
|
| 59 |
+
)
|
| 60 |
+
|
| 61 |
+
|
| 62 |
+
@register_criterion(
|
| 63 |
+
"latency_augmented_label_smoothed_cross_entropy",
|
| 64 |
+
dataclass=LabelSmoothedCrossEntropyCriterionLatencyAugmentConfig,
|
| 65 |
+
)
|
| 66 |
+
class LatencyAugmentedLabelSmoothedCrossEntropyCriterion(
|
| 67 |
+
LabelSmoothedCrossEntropyCriterion
|
| 68 |
+
):
|
| 69 |
+
def __init__(
|
| 70 |
+
self,
|
| 71 |
+
task,
|
| 72 |
+
sentence_avg,
|
| 73 |
+
label_smoothing,
|
| 74 |
+
ignore_prefix_size,
|
| 75 |
+
report_accuracy,
|
| 76 |
+
latency_avg_weight,
|
| 77 |
+
latency_var_weight,
|
| 78 |
+
latency_avg_type,
|
| 79 |
+
latency_var_type,
|
| 80 |
+
latency_gather_method,
|
| 81 |
+
latency_update_after,
|
| 82 |
+
):
|
| 83 |
+
super().__init__(
|
| 84 |
+
task, sentence_avg, label_smoothing, ignore_prefix_size, report_accuracy
|
| 85 |
+
)
|
| 86 |
+
assert LATENCY_METRICS is not None, "Please make sure SimulEval is installed."
|
| 87 |
+
|
| 88 |
+
self.latency_avg_weight = latency_avg_weight
|
| 89 |
+
self.latency_var_weight = latency_var_weight
|
| 90 |
+
self.latency_avg_type = latency_avg_type
|
| 91 |
+
self.latency_var_type = latency_var_type
|
| 92 |
+
self.latency_gather_method = latency_gather_method
|
| 93 |
+
self.latency_update_after = latency_update_after
|
| 94 |
+
|
| 95 |
+
def forward(self, model, sample, reduce=True):
|
| 96 |
+
net_output = model(**sample["net_input"])
|
| 97 |
+
# 1. Compute cross entropy loss
|
| 98 |
+
loss, nll_loss = self.compute_loss(model, net_output, sample, reduce=reduce)
|
| 99 |
+
|
| 100 |
+
# 2. Compute cross latency loss
|
| 101 |
+
latency_loss, expected_latency, expected_delays_var = self.compute_latency_loss(
|
| 102 |
+
model, sample, net_output
|
| 103 |
+
)
|
| 104 |
+
|
| 105 |
+
if self.latency_update_after > 0:
|
| 106 |
+
num_updates = getattr(model.decoder, "num_updates", None)
|
| 107 |
+
assert (
|
| 108 |
+
num_updates is not None
|
| 109 |
+
), "model.decoder doesn't have attribute 'num_updates'"
|
| 110 |
+
if num_updates <= self.latency_update_after:
|
| 111 |
+
latency_loss = 0
|
| 112 |
+
|
| 113 |
+
loss += latency_loss
|
| 114 |
+
|
| 115 |
+
sample_size = (
|
| 116 |
+
sample["target"].size(0) if self.sentence_avg else sample["ntokens"]
|
| 117 |
+
)
|
| 118 |
+
|
| 119 |
+
logging_output = {
|
| 120 |
+
"loss": loss.data,
|
| 121 |
+
"nll_loss": nll_loss.data,
|
| 122 |
+
"ntokens": sample["ntokens"],
|
| 123 |
+
"nsentences": sample["target"].size(0),
|
| 124 |
+
"sample_size": sample_size,
|
| 125 |
+
"latency": expected_latency,
|
| 126 |
+
"delays_var": expected_delays_var,
|
| 127 |
+
"latency_loss": latency_loss,
|
| 128 |
+
}
|
| 129 |
+
|
| 130 |
+
if self.report_accuracy:
|
| 131 |
+
n_correct, total = self.compute_accuracy(model, net_output, sample)
|
| 132 |
+
logging_output["n_correct"] = utils.item(n_correct.data)
|
| 133 |
+
logging_output["total"] = utils.item(total.data)
|
| 134 |
+
return loss, sample_size, logging_output
|
| 135 |
+
|
| 136 |
+
def compute_latency_loss(self, model, sample, net_output):
|
| 137 |
+
assert (
|
| 138 |
+
net_output[-1].encoder_padding_mask is None
|
| 139 |
+
or not net_output[-1].encoder_padding_mask[:, 0].any()
|
| 140 |
+
), "Only right padding on source is supported."
|
| 141 |
+
# 1. Obtain the expected alignment
|
| 142 |
+
alpha_list = [item["alpha"] for item in net_output[1].attn_list]
|
| 143 |
+
num_layers = len(alpha_list)
|
| 144 |
+
bsz, num_heads, tgt_len, src_len = alpha_list[0].size()
|
| 145 |
+
|
| 146 |
+
# bsz * num_layers * num_heads, tgt_len, src_len
|
| 147 |
+
alpha_all = torch.cat(alpha_list, dim=1).view(-1, tgt_len, src_len)
|
| 148 |
+
|
| 149 |
+
# 2 compute expected delays
|
| 150 |
+
# bsz * num_heads * num_layers, tgt_len, src_len for MMA
|
| 151 |
+
steps = (
|
| 152 |
+
torch.arange(1, 1 + src_len)
|
| 153 |
+
.unsqueeze(0)
|
| 154 |
+
.unsqueeze(1)
|
| 155 |
+
.expand_as(alpha_all)
|
| 156 |
+
.type_as(alpha_all)
|
| 157 |
+
)
|
| 158 |
+
|
| 159 |
+
expected_delays = torch.sum(steps * alpha_all, dim=-1)
|
| 160 |
+
|
| 161 |
+
target_padding_mask = (
|
| 162 |
+
model.get_targets(sample, net_output)
|
| 163 |
+
.eq(self.padding_idx)
|
| 164 |
+
.unsqueeze(1)
|
| 165 |
+
.expand(bsz, num_layers * num_heads, tgt_len)
|
| 166 |
+
.contiguous()
|
| 167 |
+
.view(-1, tgt_len)
|
| 168 |
+
)
|
| 169 |
+
|
| 170 |
+
src_lengths = (
|
| 171 |
+
sample["net_input"]["src_lengths"]
|
| 172 |
+
.unsqueeze(1)
|
| 173 |
+
.expand(bsz, num_layers * num_heads)
|
| 174 |
+
.contiguous()
|
| 175 |
+
.view(-1)
|
| 176 |
+
)
|
| 177 |
+
expected_latency = LATENCY_METRICS[self.latency_avg_type](
|
| 178 |
+
expected_delays, src_lengths, None, target_padding_mask=target_padding_mask
|
| 179 |
+
)
|
| 180 |
+
|
| 181 |
+
# 2.1 average expected latency of heads
|
| 182 |
+
# bsz, num_layers * num_heads
|
| 183 |
+
expected_latency = expected_latency.view(bsz, -1)
|
| 184 |
+
if self.latency_gather_method == "average":
|
| 185 |
+
# bsz * tgt_len
|
| 186 |
+
expected_latency = expected_delays.mean(dim=1)
|
| 187 |
+
elif self.latency_gather_method == "weighted_average":
|
| 188 |
+
weights = torch.nn.functional.softmax(expected_latency, dim=1)
|
| 189 |
+
expected_latency = torch.sum(expected_latency * weights, dim=1)
|
| 190 |
+
elif self.latency_gather_method == "max":
|
| 191 |
+
expected_latency = expected_latency.max(dim=1)[0]
|
| 192 |
+
else:
|
| 193 |
+
raise NotImplementedError
|
| 194 |
+
|
| 195 |
+
expected_latency = expected_latency.sum()
|
| 196 |
+
avg_loss = self.latency_avg_weight * expected_latency
|
| 197 |
+
|
| 198 |
+
# 2.2 variance of expected delays
|
| 199 |
+
expected_delays_var = (
|
| 200 |
+
expected_delays.view(bsz, -1, tgt_len).var(dim=1).mean(dim=1)
|
| 201 |
+
)
|
| 202 |
+
expected_delays_var = expected_delays_var.sum()
|
| 203 |
+
var_loss = self.latency_avg_weight * expected_delays_var
|
| 204 |
+
|
| 205 |
+
# 3. Final loss
|
| 206 |
+
latency_loss = avg_loss + var_loss
|
| 207 |
+
|
| 208 |
+
return latency_loss, expected_latency, expected_delays_var
|
| 209 |
+
|
| 210 |
+
@classmethod
|
| 211 |
+
def reduce_metrics(cls, logging_outputs) -> None:
|
| 212 |
+
super().reduce_metrics(logging_outputs)
|
| 213 |
+
latency = sum(log.get("latency", 0) for log in logging_outputs)
|
| 214 |
+
delays_var = sum(log.get("delays_var", 0) for log in logging_outputs)
|
| 215 |
+
latency_loss = sum(log.get("latency_loss", 0) for log in logging_outputs)
|
| 216 |
+
nsentences = sum(log.get("nsentences", 0) for log in logging_outputs)
|
| 217 |
+
metrics.log_scalar("latency", latency.float() / nsentences, nsentences, round=3)
|
| 218 |
+
metrics.log_scalar("delays_var", delays_var / nsentences, nsentences, round=3)
|
| 219 |
+
metrics.log_scalar(
|
| 220 |
+
"latency_loss", latency_loss / nsentences, nsentences, round=3
|
| 221 |
+
)
|
data/fairseq/fairseq/criterions/label_smoothed_cross_entropy_with_alignment.py
ADDED
|
@@ -0,0 +1,131 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) Facebook, Inc. and its affiliates.
|
| 2 |
+
#
|
| 3 |
+
# This source code is licensed under the MIT license found in the
|
| 4 |
+
# LICENSE file in the root directory of this source tree.
|
| 5 |
+
|
| 6 |
+
import math
|
| 7 |
+
|
| 8 |
+
from fairseq import utils
|
| 9 |
+
from fairseq.logging import metrics
|
| 10 |
+
from fairseq.criterions import register_criterion
|
| 11 |
+
|
| 12 |
+
from .label_smoothed_cross_entropy import (
|
| 13 |
+
LabelSmoothedCrossEntropyCriterion,
|
| 14 |
+
LabelSmoothedCrossEntropyCriterionConfig,
|
| 15 |
+
)
|
| 16 |
+
|
| 17 |
+
from dataclasses import dataclass, field
|
| 18 |
+
|
| 19 |
+
|
| 20 |
+
@dataclass
|
| 21 |
+
class LabelSmoothedCrossEntropyCriterionWithAlignmentConfig(
|
| 22 |
+
LabelSmoothedCrossEntropyCriterionConfig
|
| 23 |
+
):
|
| 24 |
+
alignment_lambda: float = field(
|
| 25 |
+
default=0.05, metadata={"help": "weight for the alignment loss"}
|
| 26 |
+
)
|
| 27 |
+
|
| 28 |
+
|
| 29 |
+
@register_criterion(
|
| 30 |
+
"label_smoothed_cross_entropy_with_alignment",
|
| 31 |
+
dataclass=LabelSmoothedCrossEntropyCriterionWithAlignmentConfig,
|
| 32 |
+
)
|
| 33 |
+
class LabelSmoothedCrossEntropyCriterionWithAlignment(
|
| 34 |
+
LabelSmoothedCrossEntropyCriterion
|
| 35 |
+
):
|
| 36 |
+
def __init__(self, task, sentence_avg, label_smoothing, alignment_lambda):
|
| 37 |
+
super().__init__(task, sentence_avg, label_smoothing)
|
| 38 |
+
self.alignment_lambda = alignment_lambda
|
| 39 |
+
|
| 40 |
+
def forward(self, model, sample, reduce=True):
|
| 41 |
+
"""Compute the loss for the given sample.
|
| 42 |
+
|
| 43 |
+
Returns a tuple with three elements:
|
| 44 |
+
1) the loss
|
| 45 |
+
2) the sample size, which is used as the denominator for the gradient
|
| 46 |
+
3) logging outputs to display while training
|
| 47 |
+
"""
|
| 48 |
+
net_output = model(**sample["net_input"])
|
| 49 |
+
loss, nll_loss = self.compute_loss(model, net_output, sample, reduce=reduce)
|
| 50 |
+
sample_size = (
|
| 51 |
+
sample["target"].size(0) if self.sentence_avg else sample["ntokens"]
|
| 52 |
+
)
|
| 53 |
+
logging_output = {
|
| 54 |
+
"loss": utils.item(loss.data) if reduce else loss.data,
|
| 55 |
+
"nll_loss": utils.item(nll_loss.data) if reduce else nll_loss.data,
|
| 56 |
+
"ntokens": sample["ntokens"],
|
| 57 |
+
"nsentences": sample["target"].size(0),
|
| 58 |
+
"sample_size": sample_size,
|
| 59 |
+
}
|
| 60 |
+
|
| 61 |
+
alignment_loss = None
|
| 62 |
+
|
| 63 |
+
# Compute alignment loss only for training set and non dummy batches.
|
| 64 |
+
if "alignments" in sample and sample["alignments"] is not None:
|
| 65 |
+
alignment_loss = self.compute_alignment_loss(sample, net_output)
|
| 66 |
+
|
| 67 |
+
if alignment_loss is not None:
|
| 68 |
+
logging_output["alignment_loss"] = utils.item(alignment_loss.data)
|
| 69 |
+
loss += self.alignment_lambda * alignment_loss
|
| 70 |
+
|
| 71 |
+
return loss, sample_size, logging_output
|
| 72 |
+
|
| 73 |
+
def compute_alignment_loss(self, sample, net_output):
|
| 74 |
+
attn_prob = net_output[1]["attn"][0]
|
| 75 |
+
bsz, tgt_sz, src_sz = attn_prob.shape
|
| 76 |
+
attn = attn_prob.view(bsz * tgt_sz, src_sz)
|
| 77 |
+
|
| 78 |
+
align = sample["alignments"]
|
| 79 |
+
align_weights = sample["align_weights"].float()
|
| 80 |
+
|
| 81 |
+
if len(align) > 0:
|
| 82 |
+
# Alignment loss computation. align (shape [:, 2]) contains the src-tgt index pairs corresponding to
|
| 83 |
+
# the alignments. align_weights (shape [:]) contains the 1 / frequency of a tgt index for normalizing.
|
| 84 |
+
loss = -(
|
| 85 |
+
(attn[align[:, 1][:, None], align[:, 0][:, None]]).log()
|
| 86 |
+
* align_weights[:, None]
|
| 87 |
+
).sum()
|
| 88 |
+
else:
|
| 89 |
+
return None
|
| 90 |
+
|
| 91 |
+
return loss
|
| 92 |
+
|
| 93 |
+
@staticmethod
|
| 94 |
+
def reduce_metrics(logging_outputs) -> None:
|
| 95 |
+
"""Aggregate logging outputs from data parallel training."""
|
| 96 |
+
loss_sum = utils.item(sum(log.get("loss", 0) for log in logging_outputs))
|
| 97 |
+
nll_loss_sum = utils.item(
|
| 98 |
+
sum(log.get("nll_loss", 0) for log in logging_outputs)
|
| 99 |
+
)
|
| 100 |
+
alignment_loss_sum = utils.item(
|
| 101 |
+
sum(log.get("alignment_loss", 0) for log in logging_outputs)
|
| 102 |
+
)
|
| 103 |
+
ntokens = utils.item(sum(log.get("ntokens", 0) for log in logging_outputs))
|
| 104 |
+
sample_size = utils.item(
|
| 105 |
+
sum(log.get("sample_size", 0) for log in logging_outputs)
|
| 106 |
+
)
|
| 107 |
+
|
| 108 |
+
metrics.log_scalar(
|
| 109 |
+
"loss", loss_sum / sample_size / math.log(2), sample_size, round=3
|
| 110 |
+
)
|
| 111 |
+
metrics.log_scalar(
|
| 112 |
+
"nll_loss", nll_loss_sum / ntokens / math.log(2), ntokens, round=3
|
| 113 |
+
)
|
| 114 |
+
metrics.log_scalar(
|
| 115 |
+
"alignment_loss",
|
| 116 |
+
alignment_loss_sum / sample_size / math.log(2),
|
| 117 |
+
sample_size,
|
| 118 |
+
round=3,
|
| 119 |
+
)
|
| 120 |
+
metrics.log_derived(
|
| 121 |
+
"ppl", lambda meters: utils.get_perplexity(meters["nll_loss"].avg)
|
| 122 |
+
)
|
| 123 |
+
|
| 124 |
+
@staticmethod
|
| 125 |
+
def logging_outputs_can_be_summed() -> bool:
|
| 126 |
+
"""
|
| 127 |
+
Whether the logging outputs returned by `forward` can be summed
|
| 128 |
+
across workers prior to calling `reduce_metrics`. Setting this
|
| 129 |
+
to True will improves distributed training speed.
|
| 130 |
+
"""
|
| 131 |
+
return True
|
data/fairseq/fairseq/criterions/label_smoothed_cross_entropy_with_ctc.py
ADDED
|
@@ -0,0 +1,97 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) Facebook, Inc. and its affiliates.
|
| 2 |
+
#
|
| 3 |
+
# This source code is licensed under the MIT license found in the
|
| 4 |
+
# LICENSE file in the root directory of this source tree.
|
| 5 |
+
|
| 6 |
+
import math
|
| 7 |
+
from dataclasses import dataclass, field
|
| 8 |
+
|
| 9 |
+
import torch
|
| 10 |
+
import torch.nn.functional as F
|
| 11 |
+
|
| 12 |
+
from fairseq import utils
|
| 13 |
+
from fairseq.logging import metrics
|
| 14 |
+
from fairseq.criterions import register_criterion
|
| 15 |
+
from fairseq.criterions.label_smoothed_cross_entropy import (
|
| 16 |
+
LabelSmoothedCrossEntropyCriterion,
|
| 17 |
+
LabelSmoothedCrossEntropyCriterionConfig,
|
| 18 |
+
)
|
| 19 |
+
from fairseq.data.data_utils import lengths_to_mask
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
@dataclass
|
| 23 |
+
class LabelSmoothedCrossEntropyWithCtcCriterionConfig(
|
| 24 |
+
LabelSmoothedCrossEntropyCriterionConfig
|
| 25 |
+
):
|
| 26 |
+
ctc_weight: float = field(default=1.0, metadata={"help": "weight for CTC loss"})
|
| 27 |
+
|
| 28 |
+
|
| 29 |
+
@register_criterion(
|
| 30 |
+
"label_smoothed_cross_entropy_with_ctc",
|
| 31 |
+
dataclass=LabelSmoothedCrossEntropyWithCtcCriterionConfig,
|
| 32 |
+
)
|
| 33 |
+
class LabelSmoothedCrossEntropyWithCtcCriterion(LabelSmoothedCrossEntropyCriterion):
|
| 34 |
+
def __init__(
|
| 35 |
+
self,
|
| 36 |
+
task,
|
| 37 |
+
sentence_avg,
|
| 38 |
+
label_smoothing,
|
| 39 |
+
ignore_prefix_size,
|
| 40 |
+
report_accuracy,
|
| 41 |
+
ctc_weight,
|
| 42 |
+
):
|
| 43 |
+
super().__init__(
|
| 44 |
+
task, sentence_avg, label_smoothing, ignore_prefix_size, report_accuracy
|
| 45 |
+
)
|
| 46 |
+
self.ctc_weight = ctc_weight
|
| 47 |
+
|
| 48 |
+
def forward(self, model, sample, reduce=True):
|
| 49 |
+
net_output = model(**sample["net_input"])
|
| 50 |
+
loss, nll_loss = self.compute_loss(model, net_output, sample, reduce=reduce)
|
| 51 |
+
|
| 52 |
+
ctc_loss = torch.tensor(0.0).type_as(loss)
|
| 53 |
+
if self.ctc_weight > 0.0:
|
| 54 |
+
ctc_lprobs, ctc_lens = model.get_ctc_output(net_output, sample)
|
| 55 |
+
ctc_tgt, ctc_tgt_lens = model.get_ctc_target(sample)
|
| 56 |
+
ctc_tgt_mask = lengths_to_mask(ctc_tgt_lens)
|
| 57 |
+
ctc_tgt_flat = ctc_tgt.masked_select(ctc_tgt_mask)
|
| 58 |
+
reduction = "sum" if reduce else "none"
|
| 59 |
+
ctc_loss = (
|
| 60 |
+
F.ctc_loss(
|
| 61 |
+
ctc_lprobs,
|
| 62 |
+
ctc_tgt_flat,
|
| 63 |
+
ctc_lens,
|
| 64 |
+
ctc_tgt_lens,
|
| 65 |
+
reduction=reduction,
|
| 66 |
+
zero_infinity=True,
|
| 67 |
+
)
|
| 68 |
+
* self.ctc_weight
|
| 69 |
+
)
|
| 70 |
+
loss += ctc_loss
|
| 71 |
+
|
| 72 |
+
sample_size = (
|
| 73 |
+
sample["target"].size(0) if self.sentence_avg else sample["ntokens"]
|
| 74 |
+
)
|
| 75 |
+
logging_output = {
|
| 76 |
+
"loss": utils.item(loss.data),
|
| 77 |
+
"nll_loss": utils.item(nll_loss.data),
|
| 78 |
+
"ctc_loss": utils.item(ctc_loss.data),
|
| 79 |
+
"ntokens": sample["ntokens"],
|
| 80 |
+
"nsentences": sample["target"].size(0),
|
| 81 |
+
"sample_size": sample_size,
|
| 82 |
+
}
|
| 83 |
+
if self.report_accuracy:
|
| 84 |
+
n_correct, total = self.compute_accuracy(model, net_output, sample)
|
| 85 |
+
logging_output["n_correct"] = utils.item(n_correct.data)
|
| 86 |
+
logging_output["total"] = utils.item(total.data)
|
| 87 |
+
return loss, sample_size, logging_output
|
| 88 |
+
|
| 89 |
+
@classmethod
|
| 90 |
+
def reduce_metrics(cls, logging_outputs) -> None:
|
| 91 |
+
super().reduce_metrics(logging_outputs)
|
| 92 |
+
loss_sum = sum(log.get("ctc_loss", 0) for log in logging_outputs)
|
| 93 |
+
sample_size = sum(log.get("sample_size", 0) for log in logging_outputs)
|
| 94 |
+
|
| 95 |
+
metrics.log_scalar(
|
| 96 |
+
"ctc_loss", loss_sum / sample_size / math.log(2), sample_size, round=3
|
| 97 |
+
)
|
data/fairseq/fairseq/criterions/label_smoothed_cross_entropy_with_rdrop.py
ADDED
|
@@ -0,0 +1,177 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) Facebook, Inc. and its affiliates.
|
| 2 |
+
#
|
| 3 |
+
# This source code is licensed under the MIT license found in the
|
| 4 |
+
# LICENSE file in the root directory of this source tree.
|
| 5 |
+
|
| 6 |
+
import math
|
| 7 |
+
from dataclasses import dataclass, field
|
| 8 |
+
|
| 9 |
+
import torch
|
| 10 |
+
|
| 11 |
+
from fairseq import utils
|
| 12 |
+
from fairseq.logging import metrics
|
| 13 |
+
from fairseq.criterions import register_criterion
|
| 14 |
+
from fairseq.criterions.label_smoothed_cross_entropy import (
|
| 15 |
+
LabelSmoothedCrossEntropyCriterion,
|
| 16 |
+
LabelSmoothedCrossEntropyCriterionConfig,
|
| 17 |
+
label_smoothed_nll_loss,
|
| 18 |
+
)
|
| 19 |
+
|
| 20 |
+
|
| 21 |
+
@dataclass
|
| 22 |
+
class RdropLabelSmoothedCrossEntropyCriterionConfig(
|
| 23 |
+
LabelSmoothedCrossEntropyCriterionConfig
|
| 24 |
+
):
|
| 25 |
+
rdrop_alpha: float = field(
|
| 26 |
+
default=0.0,
|
| 27 |
+
metadata={"help": "alpha for r-drop, 0 means no r-drop"},
|
| 28 |
+
)
|
| 29 |
+
|
| 30 |
+
|
| 31 |
+
@register_criterion(
|
| 32 |
+
"label_smoothed_cross_entropy_with_rdrop",
|
| 33 |
+
dataclass=RdropLabelSmoothedCrossEntropyCriterionConfig,
|
| 34 |
+
)
|
| 35 |
+
class RdropLabelSmoothedCrossEntropyCriterion(LabelSmoothedCrossEntropyCriterion):
|
| 36 |
+
def __init__(
|
| 37 |
+
self,
|
| 38 |
+
task,
|
| 39 |
+
sentence_avg,
|
| 40 |
+
label_smoothing,
|
| 41 |
+
ignore_prefix_size=0,
|
| 42 |
+
report_accuracy=False,
|
| 43 |
+
rdrop_alpha=0.0,
|
| 44 |
+
):
|
| 45 |
+
super().__init__(
|
| 46 |
+
task,
|
| 47 |
+
sentence_avg,
|
| 48 |
+
label_smoothing,
|
| 49 |
+
ignore_prefix_size=ignore_prefix_size,
|
| 50 |
+
report_accuracy=report_accuracy,
|
| 51 |
+
)
|
| 52 |
+
self.sentence_avg = sentence_avg
|
| 53 |
+
self.eps = label_smoothing
|
| 54 |
+
self.ignore_prefix_size = ignore_prefix_size
|
| 55 |
+
self.report_accuracy = report_accuracy
|
| 56 |
+
self.rdrop_alpha = rdrop_alpha
|
| 57 |
+
|
| 58 |
+
def forward(self, model, sample, reduce=True, net_output=None):
|
| 59 |
+
"""Compute the loss for the given sample.
|
| 60 |
+
|
| 61 |
+
Returns a tuple with three elements:
|
| 62 |
+
1) the loss
|
| 63 |
+
2) the sample size, which is used as the denominator for the gradient
|
| 64 |
+
3) logging outputs to display while training
|
| 65 |
+
"""
|
| 66 |
+
if net_output is None:
|
| 67 |
+
if self.rdrop_alpha > 0 and sample["net_input"]["src_tokens"].size(
|
| 68 |
+
0
|
| 69 |
+
) == sample["target"].size(0):
|
| 70 |
+
sample = duplicate_input(sample)
|
| 71 |
+
net_output = model(**sample["net_input"])
|
| 72 |
+
loss, nll_loss, rdrop_kl_loss = self.compute_loss(
|
| 73 |
+
model, net_output, sample, reduce=reduce
|
| 74 |
+
)
|
| 75 |
+
sample_size = (
|
| 76 |
+
sample["target"].size(0) if self.sentence_avg else sample["ntokens"]
|
| 77 |
+
)
|
| 78 |
+
logging_output = {
|
| 79 |
+
"loss": loss.data,
|
| 80 |
+
"nll_loss": nll_loss.data,
|
| 81 |
+
"ntokens": sample["ntokens"],
|
| 82 |
+
"nsentences": sample["target"].size(0),
|
| 83 |
+
"sample_size": sample_size,
|
| 84 |
+
}
|
| 85 |
+
if self.report_accuracy:
|
| 86 |
+
n_correct, total = self.compute_accuracy(model, net_output, sample)
|
| 87 |
+
logging_output["n_correct"] = utils.item(n_correct.data)
|
| 88 |
+
logging_output["total"] = utils.item(total.data)
|
| 89 |
+
if self.rdrop_alpha > 0:
|
| 90 |
+
logging_output["rdrop_kl_loss"] = utils.item(rdrop_kl_loss.data)
|
| 91 |
+
return loss, sample_size, logging_output
|
| 92 |
+
|
| 93 |
+
def get_lprobs_and_target(self, model, net_output, sample):
|
| 94 |
+
lprobs = model.get_normalized_probs(net_output, log_probs=True)
|
| 95 |
+
target = model.get_targets(sample, net_output)
|
| 96 |
+
if self.rdrop_alpha > 0 or target.size(0) != lprobs.size(0):
|
| 97 |
+
target = torch.cat([target, target.clone()], dim=0)
|
| 98 |
+
|
| 99 |
+
if self.ignore_prefix_size > 0:
|
| 100 |
+
# lprobs: B x T x C
|
| 101 |
+
lprobs = lprobs[:, self.ignore_prefix_size :, :].contiguous()
|
| 102 |
+
target = target[:, self.ignore_prefix_size :].contiguous()
|
| 103 |
+
return lprobs.view(-1, lprobs.size(-1)), target.view(-1)
|
| 104 |
+
|
| 105 |
+
def compute_loss(self, model, net_output, sample, reduce=True):
|
| 106 |
+
lprobs, target = self.get_lprobs_and_target(model, net_output, sample)
|
| 107 |
+
loss, nll_loss = label_smoothed_nll_loss(
|
| 108 |
+
lprobs,
|
| 109 |
+
target,
|
| 110 |
+
self.eps,
|
| 111 |
+
ignore_index=self.padding_idx,
|
| 112 |
+
reduce=reduce,
|
| 113 |
+
)
|
| 114 |
+
|
| 115 |
+
if self.rdrop_alpha > 0:
|
| 116 |
+
pad_mask = target[: target.size(0) // 2].unsqueeze(-1).eq(self.padding_idx)
|
| 117 |
+
rdrop_kl_loss = compute_kl_loss(model, net_output, pad_mask)
|
| 118 |
+
loss += self.rdrop_alpha * rdrop_kl_loss
|
| 119 |
+
else:
|
| 120 |
+
rdrop_kl_loss = loss.new_zeros(1)
|
| 121 |
+
return loss, nll_loss, rdrop_kl_loss
|
| 122 |
+
|
| 123 |
+
@classmethod
|
| 124 |
+
def reduce_metrics(cls, logging_outputs) -> None:
|
| 125 |
+
"""Aggregate logging outputs from data parallel training."""
|
| 126 |
+
super().reduce_metrics(logging_outputs)
|
| 127 |
+
|
| 128 |
+
sample_size = sum(log.get("sample_size", 0) for log in logging_outputs)
|
| 129 |
+
|
| 130 |
+
rdrop_kl_loss = utils.item(
|
| 131 |
+
sum(log.get("rdrop_kl_loss", 0) for log in logging_outputs)
|
| 132 |
+
/ sample_size
|
| 133 |
+
/ math.log(2)
|
| 134 |
+
)
|
| 135 |
+
if rdrop_kl_loss > 0:
|
| 136 |
+
metrics.log_scalar("rdrop_kl_loss", rdrop_kl_loss)
|
| 137 |
+
|
| 138 |
+
|
| 139 |
+
def duplicate_input(sample):
|
| 140 |
+
if "net_input" in sample.keys():
|
| 141 |
+
sample_input = sample["net_input"]
|
| 142 |
+
else:
|
| 143 |
+
sample_input = sample
|
| 144 |
+
|
| 145 |
+
for k, v in sample_input.items():
|
| 146 |
+
if isinstance(v, torch.Tensor):
|
| 147 |
+
sample_input[k] = torch.cat([v, v.clone()], dim=0)
|
| 148 |
+
if "net_input" in sample.keys():
|
| 149 |
+
sample["net_input"] = sample_input
|
| 150 |
+
else:
|
| 151 |
+
sample = sample_input
|
| 152 |
+
return sample
|
| 153 |
+
|
| 154 |
+
|
| 155 |
+
def compute_kl_loss(model, net_output, pad_mask=None, reduce=True):
|
| 156 |
+
net_prob = model.get_normalized_probs(net_output, log_probs=True)
|
| 157 |
+
net_prob_tec = model.get_normalized_probs(net_output, log_probs=False)
|
| 158 |
+
|
| 159 |
+
net_prob = net_prob.view(-1, net_prob.size(-1))
|
| 160 |
+
net_prob_tec = net_prob_tec.view(-1, net_prob_tec.size(-1))
|
| 161 |
+
|
| 162 |
+
p, q = torch.split(net_prob, net_prob.size(0) // 2, dim=0)
|
| 163 |
+
p_tec, q_tec = torch.split(net_prob_tec, net_prob_tec.size(0) // 2, dim=0)
|
| 164 |
+
|
| 165 |
+
p_loss = torch.nn.functional.kl_div(p, q_tec, reduction="none")
|
| 166 |
+
q_loss = torch.nn.functional.kl_div(q, p_tec, reduction="none")
|
| 167 |
+
|
| 168 |
+
if pad_mask is not None:
|
| 169 |
+
p_loss.masked_fill_(pad_mask, 0.0)
|
| 170 |
+
q_loss.masked_fill_(pad_mask, 0.0)
|
| 171 |
+
|
| 172 |
+
if reduce:
|
| 173 |
+
p_loss = p_loss.sum()
|
| 174 |
+
q_loss = q_loss.sum()
|
| 175 |
+
|
| 176 |
+
loss = (p_loss + q_loss) / 2
|
| 177 |
+
return loss
|
data/fairseq/fairseq/criterions/legacy_masked_lm.py
ADDED
|
@@ -0,0 +1,178 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) Facebook, Inc. and its affiliates.
|
| 2 |
+
#
|
| 3 |
+
# This source code is licensed under the MIT license found in the
|
| 4 |
+
# LICENSE file in the root directory of this source tree.
|
| 5 |
+
|
| 6 |
+
import math
|
| 7 |
+
|
| 8 |
+
import torch
|
| 9 |
+
import torch.nn.functional as F
|
| 10 |
+
from fairseq import utils
|
| 11 |
+
from fairseq.logging import metrics
|
| 12 |
+
from fairseq.criterions import FairseqCriterion, register_criterion
|
| 13 |
+
|
| 14 |
+
|
| 15 |
+
def compute_cross_entropy_loss(logits, targets, ignore_index=-100):
|
| 16 |
+
"""
|
| 17 |
+
Function to compute the cross entropy loss. The default value of
|
| 18 |
+
ignore_index is the same as the default value for F.cross_entropy in
|
| 19 |
+
pytorch.
|
| 20 |
+
"""
|
| 21 |
+
assert logits.size(0) == targets.size(
|
| 22 |
+
-1
|
| 23 |
+
), "Logits and Targets tensor shapes don't match up"
|
| 24 |
+
|
| 25 |
+
loss = F.nll_loss(
|
| 26 |
+
F.log_softmax(logits, -1, dtype=torch.float32),
|
| 27 |
+
targets,
|
| 28 |
+
reduction="sum",
|
| 29 |
+
ignore_index=ignore_index,
|
| 30 |
+
)
|
| 31 |
+
return loss
|
| 32 |
+
|
| 33 |
+
|
| 34 |
+
@register_criterion("legacy_masked_lm_loss")
|
| 35 |
+
class LegacyMaskedLmLoss(FairseqCriterion):
|
| 36 |
+
"""
|
| 37 |
+
Implementation for the loss used in masked language model (MLM) training.
|
| 38 |
+
This optionally also computes the next sentence prediction (NSP) loss and
|
| 39 |
+
adds it to the overall loss based on the specified args. There are three
|
| 40 |
+
cases to consider:
|
| 41 |
+
1) Generic MLM training without NSP loss. In this case sentence_targets
|
| 42 |
+
and sentence_logits are both None.
|
| 43 |
+
2) BERT training without NSP loss. In this case sentence_targets is
|
| 44 |
+
not None but sentence_logits is None and we should not be computing
|
| 45 |
+
a sentence level loss.
|
| 46 |
+
3) BERT training with NSP loss. In this case both sentence_targets and
|
| 47 |
+
sentence_logits are not None and we should be computing a sentence
|
| 48 |
+
level loss. The weight of the sentence level loss is specified as
|
| 49 |
+
an argument.
|
| 50 |
+
"""
|
| 51 |
+
|
| 52 |
+
def __init__(self, task, masked_lm_only, nsp_loss_weight):
|
| 53 |
+
super().__init__(task)
|
| 54 |
+
self.masked_lm_only = masked_lm_only
|
| 55 |
+
self.nsp_loss_weight = nsp_loss_weight
|
| 56 |
+
|
| 57 |
+
@staticmethod
|
| 58 |
+
def add_args(parser):
|
| 59 |
+
"""Args for MaskedLM Loss"""
|
| 60 |
+
# Default for masked_lm_only is False so as to not break BERT training
|
| 61 |
+
parser.add_argument(
|
| 62 |
+
"--masked-lm-only",
|
| 63 |
+
default=False,
|
| 64 |
+
action="store_true",
|
| 65 |
+
help="compute MLM loss only",
|
| 66 |
+
)
|
| 67 |
+
parser.add_argument(
|
| 68 |
+
"--nsp-loss-weight",
|
| 69 |
+
default=1.0,
|
| 70 |
+
type=float,
|
| 71 |
+
help="weight for next sentence prediction" " loss (default 1)",
|
| 72 |
+
)
|
| 73 |
+
|
| 74 |
+
def forward(self, model, sample, reduce=True):
|
| 75 |
+
"""Compute the loss for the given sample.
|
| 76 |
+
Returns a tuple with three elements:
|
| 77 |
+
1) the loss
|
| 78 |
+
2) the sample size, which is used as the denominator for the gradient
|
| 79 |
+
3) logging outputs to display while training
|
| 80 |
+
"""
|
| 81 |
+
lm_logits, output_metadata = model(**sample["net_input"])
|
| 82 |
+
|
| 83 |
+
# reshape lm_logits from (N,T,C) to (N*T,C)
|
| 84 |
+
lm_logits = lm_logits.view(-1, lm_logits.size(-1))
|
| 85 |
+
lm_targets = sample["lm_target"].view(-1)
|
| 86 |
+
lm_loss = compute_cross_entropy_loss(lm_logits, lm_targets, self.padding_idx)
|
| 87 |
+
|
| 88 |
+
# compute the number of tokens for which loss is computed. This is used
|
| 89 |
+
# to normalize the loss
|
| 90 |
+
ntokens = utils.strip_pad(lm_targets, self.padding_idx).numel()
|
| 91 |
+
loss = lm_loss / ntokens
|
| 92 |
+
nsentences = sample["nsentences"]
|
| 93 |
+
# nsentences = 0
|
| 94 |
+
|
| 95 |
+
# Compute sentence loss if masked_lm_only is False
|
| 96 |
+
sentence_loss = None
|
| 97 |
+
if not self.masked_lm_only:
|
| 98 |
+
sentence_logits = output_metadata["sentence_logits"]
|
| 99 |
+
sentence_targets = sample["sentence_target"].view(-1)
|
| 100 |
+
# This needs to be recomputed due to some differences between
|
| 101 |
+
# TokenBlock and BlockPair dataset. This can be resolved with a
|
| 102 |
+
# refactor of BERTModel which we will do in the future.
|
| 103 |
+
# TODO: Remove this after refactor of BERTModel
|
| 104 |
+
nsentences = sentence_targets.size(0)
|
| 105 |
+
|
| 106 |
+
# Check for logits being none which can happen when remove_heads
|
| 107 |
+
# is set to true in the BERT model. Ideally we should set
|
| 108 |
+
# masked_lm_only to true in this case, but that requires some
|
| 109 |
+
# refactor in the BERT model.
|
| 110 |
+
if sentence_logits is not None:
|
| 111 |
+
sentence_loss = compute_cross_entropy_loss(
|
| 112 |
+
sentence_logits, sentence_targets
|
| 113 |
+
)
|
| 114 |
+
|
| 115 |
+
loss += self.nsp_loss_weight * (sentence_loss / nsentences)
|
| 116 |
+
|
| 117 |
+
# NOTE: as we are summing up per token mlm loss and per sentence nsp loss
|
| 118 |
+
# we don't need to use sample_size as denominator for the gradient
|
| 119 |
+
# here sample_size is just used for logging
|
| 120 |
+
sample_size = 1
|
| 121 |
+
logging_output = {
|
| 122 |
+
"loss": utils.item(loss.data) if reduce else loss.data,
|
| 123 |
+
"lm_loss": utils.item(lm_loss.data) if reduce else lm_loss.data,
|
| 124 |
+
# sentence loss is not always computed
|
| 125 |
+
"sentence_loss": (
|
| 126 |
+
(utils.item(sentence_loss.data) if reduce else sentence_loss.data)
|
| 127 |
+
if sentence_loss is not None
|
| 128 |
+
else 0.0
|
| 129 |
+
),
|
| 130 |
+
"ntokens": ntokens,
|
| 131 |
+
"nsentences": nsentences,
|
| 132 |
+
"sample_size": sample_size,
|
| 133 |
+
}
|
| 134 |
+
return loss, sample_size, logging_output
|
| 135 |
+
|
| 136 |
+
@staticmethod
|
| 137 |
+
def reduce_metrics(logging_outputs) -> None:
|
| 138 |
+
"""Aggregate logging outputs from data parallel training."""
|
| 139 |
+
lm_loss_sum = sum(log.get("lm_loss", 0) for log in logging_outputs)
|
| 140 |
+
sentence_loss_sum = sum(log.get("sentence_loss", 0) for log in logging_outputs)
|
| 141 |
+
ntokens = sum(log.get("ntokens", 0) for log in logging_outputs)
|
| 142 |
+
nsentences = sum(log.get("nsentences", 0) for log in logging_outputs)
|
| 143 |
+
sample_size = sum(log.get("sample_size", 0) for log in logging_outputs)
|
| 144 |
+
agg_loss = sum(log.get("loss", 0) for log in logging_outputs)
|
| 145 |
+
|
| 146 |
+
metrics.log_scalar(
|
| 147 |
+
"loss",
|
| 148 |
+
agg_loss / sample_size / math.log(2) if sample_size > 0 else 0.0,
|
| 149 |
+
sample_size,
|
| 150 |
+
round=3,
|
| 151 |
+
)
|
| 152 |
+
metrics.log_scalar(
|
| 153 |
+
"lm_loss",
|
| 154 |
+
lm_loss_sum / ntokens / math.log(2) if ntokens > 0 else 0.0,
|
| 155 |
+
ntokens,
|
| 156 |
+
round=3,
|
| 157 |
+
)
|
| 158 |
+
metrics.log_scalar(
|
| 159 |
+
"sentence_loss",
|
| 160 |
+
sentence_loss_sum / nsentences / math.log(2) if nsentences > 0 else 0.0,
|
| 161 |
+
nsentences,
|
| 162 |
+
round=3,
|
| 163 |
+
)
|
| 164 |
+
metrics.log_scalar(
|
| 165 |
+
"nll_loss",
|
| 166 |
+
lm_loss_sum / ntokens / math.log(2) if ntokens > 0 else 0.0,
|
| 167 |
+
ntokens,
|
| 168 |
+
round=3,
|
| 169 |
+
)
|
| 170 |
+
|
| 171 |
+
@staticmethod
|
| 172 |
+
def logging_outputs_can_be_summed() -> bool:
|
| 173 |
+
"""
|
| 174 |
+
Whether the logging outputs returned by `forward` can be summed
|
| 175 |
+
across workers prior to calling `reduce_metrics`. Setting this
|
| 176 |
+
to True will improves distributed training speed.
|
| 177 |
+
"""
|
| 178 |
+
return True
|
data/fairseq/fairseq/criterions/masked_lm.py
ADDED
|
@@ -0,0 +1,99 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) Facebook, Inc. and its affiliates.
|
| 2 |
+
#
|
| 3 |
+
# This source code is licensed under the MIT license found in the
|
| 4 |
+
# LICENSE file in the root directory of this source tree.
|
| 5 |
+
|
| 6 |
+
from dataclasses import dataclass
|
| 7 |
+
import math
|
| 8 |
+
from omegaconf import II
|
| 9 |
+
|
| 10 |
+
import torch
|
| 11 |
+
from fairseq import modules, utils
|
| 12 |
+
from fairseq.logging import metrics
|
| 13 |
+
from fairseq.criterions import FairseqCriterion, register_criterion
|
| 14 |
+
from fairseq.dataclass import FairseqDataclass
|
| 15 |
+
|
| 16 |
+
|
| 17 |
+
@dataclass
|
| 18 |
+
class MaskedLmConfig(FairseqDataclass):
|
| 19 |
+
tpu: bool = II("common.tpu")
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
@register_criterion("masked_lm", dataclass=MaskedLmConfig)
|
| 23 |
+
class MaskedLmLoss(FairseqCriterion):
|
| 24 |
+
"""
|
| 25 |
+
Implementation for the loss used in masked language model (MLM) training.
|
| 26 |
+
"""
|
| 27 |
+
|
| 28 |
+
def __init__(self, cfg: MaskedLmConfig, task):
|
| 29 |
+
super().__init__(task)
|
| 30 |
+
self.tpu = cfg.tpu
|
| 31 |
+
|
| 32 |
+
def forward(self, model, sample, reduce=True):
|
| 33 |
+
"""Compute the loss for the given sample.
|
| 34 |
+
|
| 35 |
+
Returns a tuple with three elements:
|
| 36 |
+
1) the loss
|
| 37 |
+
2) the sample size, which is used as the denominator for the gradient
|
| 38 |
+
3) logging outputs to display while training
|
| 39 |
+
"""
|
| 40 |
+
masked_tokens = sample["target"].ne(self.padding_idx)
|
| 41 |
+
sample_size = masked_tokens.int().sum()
|
| 42 |
+
|
| 43 |
+
# Rare: when all tokens are masked, project all tokens.
|
| 44 |
+
# We use torch.where to avoid device-to-host transfers,
|
| 45 |
+
# except on CPU where torch.where is not well supported
|
| 46 |
+
# (see github.com/pytorch/pytorch/issues/26247).
|
| 47 |
+
if self.tpu:
|
| 48 |
+
masked_tokens = None # always project all tokens on TPU
|
| 49 |
+
elif masked_tokens.device == torch.device("cpu"):
|
| 50 |
+
if not masked_tokens.any():
|
| 51 |
+
masked_tokens = None
|
| 52 |
+
else:
|
| 53 |
+
masked_tokens = torch.where(
|
| 54 |
+
masked_tokens.any(),
|
| 55 |
+
masked_tokens,
|
| 56 |
+
masked_tokens.new([True]),
|
| 57 |
+
)
|
| 58 |
+
|
| 59 |
+
logits = model(**sample["net_input"], masked_tokens=masked_tokens)[0]
|
| 60 |
+
targets = model.get_targets(sample, [logits])
|
| 61 |
+
if masked_tokens is not None:
|
| 62 |
+
targets = targets[masked_tokens]
|
| 63 |
+
|
| 64 |
+
loss = modules.cross_entropy(
|
| 65 |
+
logits.view(-1, logits.size(-1)),
|
| 66 |
+
targets.view(-1),
|
| 67 |
+
reduction="sum",
|
| 68 |
+
ignore_index=self.padding_idx,
|
| 69 |
+
)
|
| 70 |
+
|
| 71 |
+
logging_output = {
|
| 72 |
+
"loss": loss if self.tpu else loss.data,
|
| 73 |
+
"ntokens": sample["ntokens"],
|
| 74 |
+
"nsentences": sample["nsentences"],
|
| 75 |
+
"sample_size": sample_size,
|
| 76 |
+
}
|
| 77 |
+
return loss, sample_size, logging_output
|
| 78 |
+
|
| 79 |
+
@staticmethod
|
| 80 |
+
def reduce_metrics(logging_outputs) -> None:
|
| 81 |
+
"""Aggregate logging outputs from data parallel training."""
|
| 82 |
+
loss_sum = sum(log.get("loss", 0) for log in logging_outputs)
|
| 83 |
+
sample_size = sum(log.get("sample_size", 0) for log in logging_outputs)
|
| 84 |
+
|
| 85 |
+
metrics.log_scalar(
|
| 86 |
+
"loss", loss_sum / sample_size / math.log(2), sample_size, round=3
|
| 87 |
+
)
|
| 88 |
+
metrics.log_derived(
|
| 89 |
+
"ppl", lambda meters: utils.get_perplexity(meters["loss"].avg)
|
| 90 |
+
)
|
| 91 |
+
|
| 92 |
+
@staticmethod
|
| 93 |
+
def logging_outputs_can_be_summed() -> bool:
|
| 94 |
+
"""
|
| 95 |
+
Whether the logging outputs returned by `forward` can be summed
|
| 96 |
+
across workers prior to calling `reduce_metrics`. Setting this
|
| 97 |
+
to True will improves distributed training speed.
|
| 98 |
+
"""
|
| 99 |
+
return True
|
data/fairseq/fairseq/criterions/model_criterion.py
ADDED
|
@@ -0,0 +1,177 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) Facebook, Inc. and its affiliates.
|
| 2 |
+
#
|
| 3 |
+
# This source code is licensed under the MIT license found in the
|
| 4 |
+
# LICENSE file in the root directory of this source tree.
|
| 5 |
+
|
| 6 |
+
import logging
|
| 7 |
+
from dataclasses import dataclass, field
|
| 8 |
+
from typing import Dict, List
|
| 9 |
+
|
| 10 |
+
import torch
|
| 11 |
+
|
| 12 |
+
from fairseq import utils
|
| 13 |
+
from fairseq.logging import metrics
|
| 14 |
+
from fairseq.criterions import FairseqCriterion, register_criterion
|
| 15 |
+
from fairseq.dataclass import FairseqDataclass
|
| 16 |
+
from fairseq.logging.meters import safe_round
|
| 17 |
+
|
| 18 |
+
|
| 19 |
+
logger = logging.getLogger(__name__)
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
@dataclass
|
| 23 |
+
class ModelCriterionConfig(FairseqDataclass):
|
| 24 |
+
loss_weights: Dict[str, float] = field(
|
| 25 |
+
default_factory=dict,
|
| 26 |
+
metadata={"help": "weights for the loss terms"},
|
| 27 |
+
)
|
| 28 |
+
log_keys: List[str] = field(
|
| 29 |
+
default_factory=list,
|
| 30 |
+
metadata={"help": "additional output keys to log"},
|
| 31 |
+
)
|
| 32 |
+
can_sum: bool = True
|
| 33 |
+
|
| 34 |
+
|
| 35 |
+
@register_criterion("model", dataclass=ModelCriterionConfig)
|
| 36 |
+
class ModelCriterion(FairseqCriterion):
|
| 37 |
+
"""
|
| 38 |
+
This criterion relies on the model to supply losses.
|
| 39 |
+
The losses should be a dictionary of name -> scalar returned by
|
| 40 |
+
the model either by including it in the net_output dict or by
|
| 41 |
+
implementing a get_losses(net_output, sample) method. The final loss is
|
| 42 |
+
a scaled sum of all losses according to weights in loss_weights.
|
| 43 |
+
If no weights are provided, then all losses are scaled by 1.0.
|
| 44 |
+
|
| 45 |
+
The losses will be automatically logged. Additional keys from
|
| 46 |
+
net_output dict can be logged via the log_keys parameter.
|
| 47 |
+
"""
|
| 48 |
+
|
| 49 |
+
def __init__(self, task, loss_weights=None, log_keys=None, can_sum=True):
|
| 50 |
+
super().__init__(task)
|
| 51 |
+
self.loss_weights = loss_weights
|
| 52 |
+
self.log_keys = log_keys
|
| 53 |
+
self.can_sum = can_sum
|
| 54 |
+
|
| 55 |
+
def forward(self, model, sample, reduce=True):
|
| 56 |
+
net_output = model(**sample["net_input"])
|
| 57 |
+
|
| 58 |
+
scaled_losses = {}
|
| 59 |
+
|
| 60 |
+
if hasattr(model, "get_losses"):
|
| 61 |
+
losses = model.get_losses(net_output, sample)
|
| 62 |
+
elif isinstance(net_output, dict) and "losses" in net_output:
|
| 63 |
+
losses = net_output["losses"]
|
| 64 |
+
else:
|
| 65 |
+
raise Exception("Could not retrieve losses")
|
| 66 |
+
|
| 67 |
+
for lk, p in losses.items():
|
| 68 |
+
try:
|
| 69 |
+
coef = 1.0 if len(self.loss_weights) == 0 else self.loss_weights[lk]
|
| 70 |
+
except KeyError:
|
| 71 |
+
logger.error(
|
| 72 |
+
f"weight for loss {lk} is not in loss_weights ({self.loss_weights})"
|
| 73 |
+
)
|
| 74 |
+
raise
|
| 75 |
+
if coef != 0 and p is not None:
|
| 76 |
+
scaled_losses[lk] = coef * p.float().sum()
|
| 77 |
+
|
| 78 |
+
loss = sum(scaled_losses.values())
|
| 79 |
+
|
| 80 |
+
if "sample_size" in net_output:
|
| 81 |
+
sample_size = net_output["sample_size"]
|
| 82 |
+
else:
|
| 83 |
+
sample_size = loss.numel()
|
| 84 |
+
|
| 85 |
+
if reduce and loss.numel() > 1:
|
| 86 |
+
loss = loss.sum()
|
| 87 |
+
|
| 88 |
+
logging_output = {
|
| 89 |
+
"loss": loss.data,
|
| 90 |
+
"ntokens": sample_size,
|
| 91 |
+
"nsentences": sample["id"].numel(),
|
| 92 |
+
"sample_size": sample_size,
|
| 93 |
+
"_world_size": 1,
|
| 94 |
+
}
|
| 95 |
+
|
| 96 |
+
for lk in self.log_keys:
|
| 97 |
+
if lk in net_output and net_output[lk] is not None:
|
| 98 |
+
if not torch.is_tensor(net_output[lk]) or net_output[lk].numel() == 1:
|
| 99 |
+
logging_output[lk] = float(net_output[lk])
|
| 100 |
+
elif lk.startswith("_"):
|
| 101 |
+
logging_output[lk] = net_output[lk]
|
| 102 |
+
else:
|
| 103 |
+
for i, v in enumerate(net_output[lk]):
|
| 104 |
+
logging_output[f"{lk}_{i}"] = float(v)
|
| 105 |
+
|
| 106 |
+
if len(scaled_losses) > 1:
|
| 107 |
+
for lk, l in scaled_losses.items():
|
| 108 |
+
if l.numel() > 1:
|
| 109 |
+
l = l.sum()
|
| 110 |
+
logging_output[f"loss_{lk}"] = l.item()
|
| 111 |
+
|
| 112 |
+
if "logs" in net_output:
|
| 113 |
+
for lgw in net_output["logs"]:
|
| 114 |
+
logging_output[lgw] = net_output["logs"][lgw]
|
| 115 |
+
|
| 116 |
+
return loss, sample_size, logging_output
|
| 117 |
+
|
| 118 |
+
@staticmethod
|
| 119 |
+
def reduce_metrics(logging_outputs) -> None:
|
| 120 |
+
"""Aggregate logging outputs from data parallel training."""
|
| 121 |
+
loss_sum = utils.item(sum(log.get("loss", 0) for log in logging_outputs))
|
| 122 |
+
ntokens = utils.item(sum(log.get("ntokens", 0) for log in logging_outputs))
|
| 123 |
+
nsentences = utils.item(
|
| 124 |
+
sum(log.get("nsentences", 0) for log in logging_outputs)
|
| 125 |
+
)
|
| 126 |
+
sample_size = utils.item(
|
| 127 |
+
sum(log.get("sample_size", 0) for log in logging_outputs)
|
| 128 |
+
)
|
| 129 |
+
|
| 130 |
+
metrics.log_scalar("loss", loss_sum / sample_size, sample_size, round=3)
|
| 131 |
+
metrics.log_scalar("ntokens", ntokens)
|
| 132 |
+
metrics.log_scalar("nsentences", nsentences)
|
| 133 |
+
metrics.log_scalar("sample_size", sample_size)
|
| 134 |
+
|
| 135 |
+
builtin_keys = {
|
| 136 |
+
"loss",
|
| 137 |
+
"ntokens",
|
| 138 |
+
"nsentences",
|
| 139 |
+
"sample_size",
|
| 140 |
+
"_world_size",
|
| 141 |
+
}
|
| 142 |
+
|
| 143 |
+
world_size = utils.item(
|
| 144 |
+
sum(log.get("_world_size", 0) for log in logging_outputs)
|
| 145 |
+
)
|
| 146 |
+
|
| 147 |
+
for k in logging_outputs[0]:
|
| 148 |
+
if k not in builtin_keys and not k.startswith("_"):
|
| 149 |
+
val = sum(log.get(k, 0) for log in logging_outputs)
|
| 150 |
+
if k.startswith("loss_"):
|
| 151 |
+
metrics.log_scalar(k, val / sample_size, sample_size, round=3)
|
| 152 |
+
else:
|
| 153 |
+
metrics.log_scalar(k, val / world_size, round=3)
|
| 154 |
+
|
| 155 |
+
correct = sum(log.get("correct", 0) for log in logging_outputs)
|
| 156 |
+
total = sum(log.get("count", 0) for log in logging_outputs)
|
| 157 |
+
|
| 158 |
+
if total > 0:
|
| 159 |
+
metrics.log_scalar("_correct", correct)
|
| 160 |
+
metrics.log_scalar("_total", total)
|
| 161 |
+
|
| 162 |
+
metrics.log_derived(
|
| 163 |
+
"accuracy",
|
| 164 |
+
lambda meters: safe_round(
|
| 165 |
+
meters["_correct"].sum / meters["_total"].sum, 5
|
| 166 |
+
)
|
| 167 |
+
if meters["_total"].sum > 0
|
| 168 |
+
else float("nan"),
|
| 169 |
+
)
|
| 170 |
+
|
| 171 |
+
def logging_outputs_can_be_summed(self) -> bool:
|
| 172 |
+
"""
|
| 173 |
+
Whether the logging outputs returned by `forward` can be summed
|
| 174 |
+
across workers prior to calling `reduce_metrics`. Setting this
|
| 175 |
+
to True will improves distributed training speed.
|
| 176 |
+
"""
|
| 177 |
+
return self.can_sum
|
data/fairseq/fairseq/criterions/nat_loss.py
ADDED
|
@@ -0,0 +1,181 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) Facebook, Inc. and its affiliates.
|
| 2 |
+
#
|
| 3 |
+
# This source code is licensed under the MIT license found in the
|
| 4 |
+
# LICENSE file in the root directory of this source tree.
|
| 5 |
+
|
| 6 |
+
import math
|
| 7 |
+
|
| 8 |
+
import torch
|
| 9 |
+
import torch.nn.functional as F
|
| 10 |
+
from fairseq import utils
|
| 11 |
+
from fairseq.logging import metrics
|
| 12 |
+
from fairseq.criterions import FairseqCriterion, register_criterion
|
| 13 |
+
from fairseq.dataclass import FairseqDataclass
|
| 14 |
+
from torch import Tensor
|
| 15 |
+
|
| 16 |
+
from dataclasses import dataclass, field
|
| 17 |
+
|
| 18 |
+
|
| 19 |
+
@dataclass
|
| 20 |
+
class LabelSmoothedDualImitationCriterionConfig(FairseqDataclass):
|
| 21 |
+
label_smoothing: float = field(
|
| 22 |
+
default=0.0,
|
| 23 |
+
metadata={"help": "epsilon for label smoothing, 0 means no label smoothing"},
|
| 24 |
+
)
|
| 25 |
+
|
| 26 |
+
|
| 27 |
+
@register_criterion("nat_loss", dataclass=LabelSmoothedDualImitationCriterionConfig)
|
| 28 |
+
class LabelSmoothedDualImitationCriterion(FairseqCriterion):
|
| 29 |
+
def __init__(self, task, label_smoothing):
|
| 30 |
+
super().__init__(task)
|
| 31 |
+
self.label_smoothing = label_smoothing
|
| 32 |
+
|
| 33 |
+
def _compute_loss(
|
| 34 |
+
self, outputs, targets, masks=None, label_smoothing=0.0, name="loss", factor=1.0
|
| 35 |
+
):
|
| 36 |
+
"""
|
| 37 |
+
outputs: batch x len x d_model
|
| 38 |
+
targets: batch x len
|
| 39 |
+
masks: batch x len
|
| 40 |
+
|
| 41 |
+
policy_logprob: if there is some policy
|
| 42 |
+
depends on the likelihood score as rewards.
|
| 43 |
+
"""
|
| 44 |
+
|
| 45 |
+
def mean_ds(x: Tensor, dim=None) -> Tensor:
|
| 46 |
+
return (
|
| 47 |
+
x.float().mean().type_as(x)
|
| 48 |
+
if dim is None
|
| 49 |
+
else x.float().mean(dim).type_as(x)
|
| 50 |
+
)
|
| 51 |
+
|
| 52 |
+
if masks is not None:
|
| 53 |
+
outputs, targets = outputs[masks], targets[masks]
|
| 54 |
+
|
| 55 |
+
if masks is not None and not masks.any():
|
| 56 |
+
nll_loss = torch.tensor(0)
|
| 57 |
+
loss = nll_loss
|
| 58 |
+
else:
|
| 59 |
+
logits = F.log_softmax(outputs, dim=-1)
|
| 60 |
+
if targets.dim() == 1:
|
| 61 |
+
losses = F.nll_loss(logits, targets.to(logits.device), reduction="none")
|
| 62 |
+
|
| 63 |
+
else: # soft-labels
|
| 64 |
+
losses = F.kl_div(logits, targets.to(logits.device), reduction="none")
|
| 65 |
+
losses = losses.sum(-1)
|
| 66 |
+
|
| 67 |
+
nll_loss = mean_ds(losses)
|
| 68 |
+
if label_smoothing > 0:
|
| 69 |
+
loss = (
|
| 70 |
+
nll_loss * (1 - label_smoothing) - mean_ds(logits) * label_smoothing
|
| 71 |
+
)
|
| 72 |
+
else:
|
| 73 |
+
loss = nll_loss
|
| 74 |
+
|
| 75 |
+
loss = loss * factor
|
| 76 |
+
return {"name": name, "loss": loss, "nll_loss": nll_loss, "factor": factor}
|
| 77 |
+
|
| 78 |
+
def _custom_loss(self, loss, name="loss", factor=1.0):
|
| 79 |
+
return {"name": name, "loss": loss, "factor": factor}
|
| 80 |
+
|
| 81 |
+
def forward(self, model, sample, reduce=True):
|
| 82 |
+
"""Compute the loss for the given sample.
|
| 83 |
+
Returns a tuple with three elements:
|
| 84 |
+
1) the loss
|
| 85 |
+
2) the sample size, which is used as the denominator for the gradient
|
| 86 |
+
3) logging outputs to display while training
|
| 87 |
+
"""
|
| 88 |
+
nsentences, ntokens = sample["nsentences"], sample["ntokens"]
|
| 89 |
+
|
| 90 |
+
# B x T
|
| 91 |
+
src_tokens, src_lengths = (
|
| 92 |
+
sample["net_input"]["src_tokens"],
|
| 93 |
+
sample["net_input"]["src_lengths"],
|
| 94 |
+
)
|
| 95 |
+
tgt_tokens, prev_output_tokens = sample["target"], sample["prev_target"]
|
| 96 |
+
|
| 97 |
+
outputs = model(src_tokens, src_lengths, prev_output_tokens, tgt_tokens)
|
| 98 |
+
losses, nll_loss = [], []
|
| 99 |
+
|
| 100 |
+
for obj in outputs:
|
| 101 |
+
if outputs[obj].get("loss", None) is None:
|
| 102 |
+
_losses = self._compute_loss(
|
| 103 |
+
outputs[obj].get("out"),
|
| 104 |
+
outputs[obj].get("tgt"),
|
| 105 |
+
outputs[obj].get("mask", None),
|
| 106 |
+
outputs[obj].get("ls", 0.0),
|
| 107 |
+
name=obj + "-loss",
|
| 108 |
+
factor=outputs[obj].get("factor", 1.0),
|
| 109 |
+
)
|
| 110 |
+
else:
|
| 111 |
+
_losses = self._custom_loss(
|
| 112 |
+
outputs[obj].get("loss"),
|
| 113 |
+
name=obj + "-loss",
|
| 114 |
+
factor=outputs[obj].get("factor", 1.0),
|
| 115 |
+
)
|
| 116 |
+
|
| 117 |
+
losses += [_losses]
|
| 118 |
+
if outputs[obj].get("nll_loss", False):
|
| 119 |
+
nll_loss += [_losses.get("nll_loss", 0.0)]
|
| 120 |
+
|
| 121 |
+
loss = sum(l["loss"] for l in losses)
|
| 122 |
+
nll_loss = sum(l for l in nll_loss) if len(nll_loss) > 0 else loss.new_tensor(0)
|
| 123 |
+
|
| 124 |
+
# NOTE:
|
| 125 |
+
# we don't need to use sample_size as denominator for the gradient
|
| 126 |
+
# here sample_size is just used for logging
|
| 127 |
+
sample_size = 1
|
| 128 |
+
logging_output = {
|
| 129 |
+
"loss": loss.data,
|
| 130 |
+
"nll_loss": nll_loss.data,
|
| 131 |
+
"ntokens": ntokens,
|
| 132 |
+
"nsentences": nsentences,
|
| 133 |
+
"sample_size": sample_size,
|
| 134 |
+
}
|
| 135 |
+
|
| 136 |
+
for l in losses:
|
| 137 |
+
logging_output[l["name"]] = (
|
| 138 |
+
utils.item(l["loss"].data / l["factor"])
|
| 139 |
+
if reduce
|
| 140 |
+
else l[["loss"]].data / l["factor"]
|
| 141 |
+
)
|
| 142 |
+
|
| 143 |
+
return loss, sample_size, logging_output
|
| 144 |
+
|
| 145 |
+
@staticmethod
|
| 146 |
+
def reduce_metrics(logging_outputs) -> None:
|
| 147 |
+
"""Aggregate logging outputs from data parallel training."""
|
| 148 |
+
sample_size = utils.item(
|
| 149 |
+
sum(log.get("sample_size", 0) for log in logging_outputs)
|
| 150 |
+
)
|
| 151 |
+
loss = utils.item(sum(log.get("loss", 0) for log in logging_outputs))
|
| 152 |
+
nll_loss = utils.item(sum(log.get("nll_loss", 0) for log in logging_outputs))
|
| 153 |
+
|
| 154 |
+
metrics.log_scalar(
|
| 155 |
+
"loss", loss / sample_size / math.log(2), sample_size, round=3
|
| 156 |
+
)
|
| 157 |
+
metrics.log_scalar(
|
| 158 |
+
"nll_loss", nll_loss / sample_size / math.log(2), sample_size, round=3
|
| 159 |
+
)
|
| 160 |
+
metrics.log_derived(
|
| 161 |
+
"ppl", lambda meters: utils.get_perplexity(meters["loss"].avg)
|
| 162 |
+
)
|
| 163 |
+
|
| 164 |
+
for key in logging_outputs[0]:
|
| 165 |
+
if key[-5:] == "-loss":
|
| 166 |
+
val = sum(log.get(key, 0) for log in logging_outputs)
|
| 167 |
+
metrics.log_scalar(
|
| 168 |
+
key[:-5],
|
| 169 |
+
val / sample_size / math.log(2) if sample_size > 0 else 0.0,
|
| 170 |
+
sample_size,
|
| 171 |
+
round=3,
|
| 172 |
+
)
|
| 173 |
+
|
| 174 |
+
@staticmethod
|
| 175 |
+
def logging_outputs_can_be_summed() -> bool:
|
| 176 |
+
"""
|
| 177 |
+
Whether the logging outputs returned by `forward` can be summed
|
| 178 |
+
across workers prior to calling `reduce_metrics`. Setting this
|
| 179 |
+
to True will improves distributed training speed.
|
| 180 |
+
"""
|
| 181 |
+
return True
|
data/fairseq/fairseq/criterions/sentence_prediction.py
ADDED
|
@@ -0,0 +1,288 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) Facebook, Inc. and its affiliates.
|
| 2 |
+
#
|
| 3 |
+
# This source code is licensed under the MIT license found in the
|
| 4 |
+
# LICENSE file in the root directory of this source tree.
|
| 5 |
+
|
| 6 |
+
import math
|
| 7 |
+
from dataclasses import dataclass, field
|
| 8 |
+
from itertools import chain
|
| 9 |
+
|
| 10 |
+
import numpy as np
|
| 11 |
+
import torch
|
| 12 |
+
import torch.nn.functional as F
|
| 13 |
+
from sklearn.metrics import f1_score
|
| 14 |
+
from sklearn.metrics import matthews_corrcoef as _matthews_corrcoef
|
| 15 |
+
from scipy.stats import pearsonr, spearmanr
|
| 16 |
+
|
| 17 |
+
from fairseq.logging import metrics
|
| 18 |
+
from fairseq.criterions import FairseqCriterion, register_criterion
|
| 19 |
+
from fairseq.dataclass import FairseqDataclass
|
| 20 |
+
from fairseq.logging.meters import safe_round
|
| 21 |
+
|
| 22 |
+
|
| 23 |
+
def simple_accuracy(preds, labels):
|
| 24 |
+
return (preds == labels).mean()
|
| 25 |
+
|
| 26 |
+
|
| 27 |
+
def acc_and_f1(preds, labels):
|
| 28 |
+
acc = simple_accuracy(preds, labels)
|
| 29 |
+
f1 = f1_score(y_true=labels, y_pred=preds)
|
| 30 |
+
return {
|
| 31 |
+
"acc": acc,
|
| 32 |
+
"f1": f1,
|
| 33 |
+
"acc_and_f1": (acc + f1) / 2,
|
| 34 |
+
}
|
| 35 |
+
|
| 36 |
+
|
| 37 |
+
def pearson_and_spearman(preds, labels):
|
| 38 |
+
pearson_corr = pearsonr(preds, labels)[0]
|
| 39 |
+
spearman_corr = spearmanr(preds, labels)[0]
|
| 40 |
+
return {
|
| 41 |
+
"pearson": pearson_corr,
|
| 42 |
+
"spearmanr": spearman_corr,
|
| 43 |
+
"corr": (pearson_corr + spearman_corr) / 2,
|
| 44 |
+
}
|
| 45 |
+
|
| 46 |
+
|
| 47 |
+
def matthews_corrcoef(preds, labels):
|
| 48 |
+
# make it consistent with other metrics taking (preds, labels) as input
|
| 49 |
+
mcc = _matthews_corrcoef(labels, preds)
|
| 50 |
+
return mcc
|
| 51 |
+
|
| 52 |
+
|
| 53 |
+
@dataclass
|
| 54 |
+
class SentencePredictionConfig(FairseqDataclass):
|
| 55 |
+
classification_head_name: str = field(
|
| 56 |
+
default="sentence_classification_head",
|
| 57 |
+
metadata={"help": "name of the classification head to use"},
|
| 58 |
+
)
|
| 59 |
+
regression_target: bool = field(
|
| 60 |
+
default=False,
|
| 61 |
+
)
|
| 62 |
+
report_mcc: bool = False
|
| 63 |
+
report_acc_and_f1: bool = False
|
| 64 |
+
report_pearson_and_spearman: bool = False
|
| 65 |
+
|
| 66 |
+
|
| 67 |
+
@register_criterion("sentence_prediction", dataclass=SentencePredictionConfig)
|
| 68 |
+
class SentencePredictionCriterion(FairseqCriterion):
|
| 69 |
+
def __init__(self, cfg: SentencePredictionConfig, task):
|
| 70 |
+
super().__init__(task)
|
| 71 |
+
self.classification_head_name = cfg.classification_head_name
|
| 72 |
+
self.regression_target = cfg.regression_target
|
| 73 |
+
self.keep_pred_and_targ = (
|
| 74 |
+
cfg.report_mcc or cfg.report_acc_and_f1 or cfg.report_pearson_and_spearman
|
| 75 |
+
)
|
| 76 |
+
self.report_mcc = cfg.report_mcc
|
| 77 |
+
self.report_acc_and_f1 = cfg.report_acc_and_f1
|
| 78 |
+
self.report_pearson_and_spearman = cfg.report_pearson_and_spearman
|
| 79 |
+
self.label_dict = task.label_dictionary
|
| 80 |
+
|
| 81 |
+
def forward(self, model, sample, reduce=True):
|
| 82 |
+
"""Compute the loss for the given sample.
|
| 83 |
+
|
| 84 |
+
Returns a tuple with three elements:
|
| 85 |
+
1) the loss
|
| 86 |
+
2) the sample size, which is used as the denominator for the gradient
|
| 87 |
+
3) logging outputs to display while training
|
| 88 |
+
"""
|
| 89 |
+
assert (
|
| 90 |
+
hasattr(model, "classification_heads")
|
| 91 |
+
and self.classification_head_name in model.classification_heads
|
| 92 |
+
), "model must provide sentence classification head for --criterion=sentence_prediction"
|
| 93 |
+
|
| 94 |
+
logits, _ = model(
|
| 95 |
+
**sample["net_input"],
|
| 96 |
+
features_only=True,
|
| 97 |
+
classification_head_name=self.classification_head_name,
|
| 98 |
+
)
|
| 99 |
+
targets = model.get_targets(sample, [logits]).view(-1)
|
| 100 |
+
sample_size = targets.numel()
|
| 101 |
+
|
| 102 |
+
if not self.regression_target:
|
| 103 |
+
lprobs = F.log_softmax(logits, dim=-1, dtype=torch.float32)
|
| 104 |
+
task_loss = F.nll_loss(lprobs, targets, reduction="sum")
|
| 105 |
+
else:
|
| 106 |
+
logits = logits.view(-1).float()
|
| 107 |
+
targets = targets.float()
|
| 108 |
+
task_loss = F.mse_loss(logits, targets, reduction="sum")
|
| 109 |
+
|
| 110 |
+
logging_output = {}
|
| 111 |
+
loss = task_loss
|
| 112 |
+
# mha & ffn regularization update
|
| 113 |
+
if (
|
| 114 |
+
hasattr(model, "args")
|
| 115 |
+
and hasattr(model.args, "mha_reg_scale_factor")
|
| 116 |
+
and model.args.mha_reg_scale_factor != 0.0
|
| 117 |
+
):
|
| 118 |
+
mha_reg_loss = model._get_adaptive_head_loss()
|
| 119 |
+
loss += mha_reg_loss
|
| 120 |
+
logging_output.update({"mha_reg_loss": mha_reg_loss})
|
| 121 |
+
if (
|
| 122 |
+
hasattr(model, "args")
|
| 123 |
+
and hasattr(model.args, "ffn_reg_scale_factor")
|
| 124 |
+
and model.args.ffn_reg_scale_factor != 0.0
|
| 125 |
+
):
|
| 126 |
+
ffn_reg_loss = model._get_adaptive_ffn_loss()
|
| 127 |
+
loss += ffn_reg_loss
|
| 128 |
+
logging_output.update({"ffn_reg_loss": ffn_reg_loss})
|
| 129 |
+
|
| 130 |
+
logging_output.update(
|
| 131 |
+
{
|
| 132 |
+
"loss": loss.data,
|
| 133 |
+
"ntokens": sample["ntokens"],
|
| 134 |
+
"nsentences": sample_size,
|
| 135 |
+
"sample_size": sample_size,
|
| 136 |
+
}
|
| 137 |
+
)
|
| 138 |
+
if not self.regression_target:
|
| 139 |
+
preds = logits.argmax(dim=1)
|
| 140 |
+
logging_output["ncorrect"] = (preds == targets).sum()
|
| 141 |
+
if self.keep_pred_and_targ and not model.training:
|
| 142 |
+
if self.regression_target:
|
| 143 |
+
logging_output["pred"] = logits.detach().cpu().tolist()
|
| 144 |
+
logging_output["targ"] = targets.detach().cpu().tolist()
|
| 145 |
+
else:
|
| 146 |
+
# remove offset `self.label_dict.nspecial` from OffsetTokensDataset
|
| 147 |
+
preds = self.label_dict.string(preds + self.label_dict.nspecial).split()
|
| 148 |
+
targets = self.label_dict.string(
|
| 149 |
+
targets + self.label_dict.nspecial
|
| 150 |
+
).split()
|
| 151 |
+
logging_output["pred"] = list(map(int, preds))
|
| 152 |
+
logging_output["targ"] = list(map(int, targets))
|
| 153 |
+
|
| 154 |
+
if self.report_mcc:
|
| 155 |
+
logging_output["report_mcc"] = True
|
| 156 |
+
if self.report_acc_and_f1:
|
| 157 |
+
logging_output["report_acc_and_f1"] = True
|
| 158 |
+
if self.report_pearson_and_spearman:
|
| 159 |
+
logging_output["report_pearson_and_spearman"] = True
|
| 160 |
+
|
| 161 |
+
return loss, sample_size, logging_output
|
| 162 |
+
|
| 163 |
+
@staticmethod
|
| 164 |
+
def reduce_metrics(logging_outputs) -> None:
|
| 165 |
+
"""Aggregate logging outputs from data parallel training."""
|
| 166 |
+
loss_sum = sum(log.get("loss", 0) for log in logging_outputs)
|
| 167 |
+
ntokens = sum(log.get("ntokens", 0) for log in logging_outputs)
|
| 168 |
+
nsentences = sum(log.get("nsentences", 0) for log in logging_outputs)
|
| 169 |
+
sample_size = sum(log.get("sample_size", 0) for log in logging_outputs)
|
| 170 |
+
mha_reg_loss_sum = sum(log.get("mha_reg_loss", 0) for log in logging_outputs)
|
| 171 |
+
ffn_reg_loss_sum = sum(log.get("ffn_reg_loss", 0) for log in logging_outputs)
|
| 172 |
+
|
| 173 |
+
metrics.log_scalar(
|
| 174 |
+
"loss", loss_sum / sample_size / math.log(2), sample_size, round=3
|
| 175 |
+
)
|
| 176 |
+
if mha_reg_loss_sum:
|
| 177 |
+
metrics.log_scalar(
|
| 178 |
+
"mha_reg_loss",
|
| 179 |
+
mha_reg_loss_sum / sample_size / math.log(2),
|
| 180 |
+
sample_size,
|
| 181 |
+
round=3,
|
| 182 |
+
)
|
| 183 |
+
if ffn_reg_loss_sum:
|
| 184 |
+
metrics.log_scalar(
|
| 185 |
+
"ffn_reg_loss",
|
| 186 |
+
ffn_reg_loss_sum / sample_size / math.log(2),
|
| 187 |
+
sample_size,
|
| 188 |
+
round=3,
|
| 189 |
+
)
|
| 190 |
+
if sample_size != ntokens:
|
| 191 |
+
metrics.log_scalar(
|
| 192 |
+
"nll_loss", loss_sum / ntokens / math.log(2), ntokens, round=3
|
| 193 |
+
)
|
| 194 |
+
|
| 195 |
+
if len(logging_outputs) > 0 and "ncorrect" in logging_outputs[0]:
|
| 196 |
+
ncorrect = sum(log.get("ncorrect", 0) for log in logging_outputs)
|
| 197 |
+
metrics.log_scalar(
|
| 198 |
+
"accuracy", 100.0 * ncorrect / nsentences, nsentences, round=1
|
| 199 |
+
)
|
| 200 |
+
|
| 201 |
+
# Metrics used by GLUE
|
| 202 |
+
pred = np.array(
|
| 203 |
+
list(chain.from_iterable(log.get("pred", []) for log in logging_outputs))
|
| 204 |
+
)
|
| 205 |
+
targ = np.array(
|
| 206 |
+
list(chain.from_iterable(log.get("targ", []) for log in logging_outputs))
|
| 207 |
+
)
|
| 208 |
+
if len(pred):
|
| 209 |
+
metrics.log_concat_tensor("pred", torch.from_numpy(pred), dim=0)
|
| 210 |
+
metrics.log_concat_tensor("targ", torch.from_numpy(targ), dim=0)
|
| 211 |
+
if any("report_mcc" in log for log in logging_outputs):
|
| 212 |
+
metrics.log_derived(
|
| 213 |
+
"mcc",
|
| 214 |
+
lambda meters: safe_round(
|
| 215 |
+
matthews_corrcoef(
|
| 216 |
+
meters["pred"].tensor.numpy(),
|
| 217 |
+
meters["targ"].tensor.numpy(),
|
| 218 |
+
)
|
| 219 |
+
* 100,
|
| 220 |
+
1,
|
| 221 |
+
),
|
| 222 |
+
)
|
| 223 |
+
if any("report_acc_and_f1" in log for log in logging_outputs):
|
| 224 |
+
metrics.log_derived(
|
| 225 |
+
"acc_and_f1",
|
| 226 |
+
lambda meters: safe_round(
|
| 227 |
+
acc_and_f1(
|
| 228 |
+
meters["pred"].tensor.numpy(),
|
| 229 |
+
meters["targ"].tensor.numpy(),
|
| 230 |
+
)["acc_and_f1"]
|
| 231 |
+
* 100,
|
| 232 |
+
1,
|
| 233 |
+
),
|
| 234 |
+
)
|
| 235 |
+
metrics.log_derived(
|
| 236 |
+
"f1",
|
| 237 |
+
lambda meters: safe_round(
|
| 238 |
+
acc_and_f1(
|
| 239 |
+
meters["pred"].tensor.numpy(),
|
| 240 |
+
meters["targ"].tensor.numpy(),
|
| 241 |
+
)["f1"]
|
| 242 |
+
* 100,
|
| 243 |
+
1,
|
| 244 |
+
),
|
| 245 |
+
)
|
| 246 |
+
if any("report_pearson_and_spearman" in log for log in logging_outputs):
|
| 247 |
+
metrics.log_derived(
|
| 248 |
+
"pearson_and_spearman",
|
| 249 |
+
lambda meters: safe_round(
|
| 250 |
+
pearson_and_spearman(
|
| 251 |
+
meters["pred"].tensor.numpy(),
|
| 252 |
+
meters["targ"].tensor.numpy(),
|
| 253 |
+
)["corr"]
|
| 254 |
+
* 100,
|
| 255 |
+
1,
|
| 256 |
+
),
|
| 257 |
+
)
|
| 258 |
+
metrics.log_derived(
|
| 259 |
+
"pearson",
|
| 260 |
+
lambda meters: safe_round(
|
| 261 |
+
pearson_and_spearman(
|
| 262 |
+
meters["pred"].tensor.numpy(),
|
| 263 |
+
meters["targ"].tensor.numpy(),
|
| 264 |
+
)["pearson"]
|
| 265 |
+
* 100,
|
| 266 |
+
1,
|
| 267 |
+
),
|
| 268 |
+
)
|
| 269 |
+
metrics.log_derived(
|
| 270 |
+
"spearman",
|
| 271 |
+
lambda meters: safe_round(
|
| 272 |
+
pearson_and_spearman(
|
| 273 |
+
meters["pred"].tensor.numpy(),
|
| 274 |
+
meters["targ"].tensor.numpy(),
|
| 275 |
+
)["spearmanr"]
|
| 276 |
+
* 100,
|
| 277 |
+
1,
|
| 278 |
+
),
|
| 279 |
+
)
|
| 280 |
+
|
| 281 |
+
@staticmethod
|
| 282 |
+
def logging_outputs_can_be_summed() -> bool:
|
| 283 |
+
"""
|
| 284 |
+
Whether the logging outputs returned by `forward` can be summed
|
| 285 |
+
across workers prior to calling `reduce_metrics`. Setting this
|
| 286 |
+
to True will improves distributed training speed.
|
| 287 |
+
"""
|
| 288 |
+
return True
|
data/fairseq/fairseq/criterions/sentence_prediction_adapters.py
ADDED
|
@@ -0,0 +1,63 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) Facebook, Inc. and its affiliates.
|
| 2 |
+
#
|
| 3 |
+
# This source code is licensed under the MIT license found in the
|
| 4 |
+
# LICENSE file in the root directory of this source tree.
|
| 5 |
+
|
| 6 |
+
import torch
|
| 7 |
+
import torch.nn.functional as F
|
| 8 |
+
from fairseq.criterions import register_criterion
|
| 9 |
+
from fairseq.criterions.sentence_prediction import (
|
| 10 |
+
SentencePredictionCriterion,
|
| 11 |
+
SentencePredictionConfig,
|
| 12 |
+
)
|
| 13 |
+
|
| 14 |
+
|
| 15 |
+
@register_criterion("sentence_prediction_adapters", dataclass=SentencePredictionConfig)
|
| 16 |
+
class SentencePredictionCriterionAdapters(SentencePredictionCriterion):
|
| 17 |
+
def forward(self, model, sample, reduce=True):
|
| 18 |
+
"""Compute the loss for the given sample.
|
| 19 |
+
|
| 20 |
+
Returns a tuple with three elements:
|
| 21 |
+
1) the loss
|
| 22 |
+
2) the sample size, which is used as the denominator for the gradient
|
| 23 |
+
3) logging outputs to display while training
|
| 24 |
+
"""
|
| 25 |
+
assert (
|
| 26 |
+
hasattr(model, "classification_heads")
|
| 27 |
+
and self.classification_head_name in model.classification_heads
|
| 28 |
+
), "model must provide sentence classification head for --criterion=sentence_prediction"
|
| 29 |
+
|
| 30 |
+
if not hasattr(sample, "lang_id"):
|
| 31 |
+
# If no language ID is given, we fall back to English
|
| 32 |
+
lang_id = ["en_XX"] * sample["nsentences"]
|
| 33 |
+
else:
|
| 34 |
+
lang_id = sample["lang_id"]
|
| 35 |
+
|
| 36 |
+
logits, _ = model(
|
| 37 |
+
**sample["net_input"],
|
| 38 |
+
features_only=True,
|
| 39 |
+
classification_head_name=self.classification_head_name,
|
| 40 |
+
lang_id=lang_id,
|
| 41 |
+
)
|
| 42 |
+
targets = model.get_targets(sample, [logits]).view(-1)
|
| 43 |
+
sample_size = targets.numel()
|
| 44 |
+
|
| 45 |
+
if not self.regression_target:
|
| 46 |
+
lprobs = F.log_softmax(logits, dim=-1, dtype=torch.float32)
|
| 47 |
+
loss = F.nll_loss(lprobs, targets, reduction="sum")
|
| 48 |
+
else:
|
| 49 |
+
logits = logits.view(-1).float()
|
| 50 |
+
targets = targets.float()
|
| 51 |
+
loss = F.mse_loss(logits, targets, reduction="sum")
|
| 52 |
+
|
| 53 |
+
logging_output = {
|
| 54 |
+
"loss": loss.data,
|
| 55 |
+
"ntokens": sample["ntokens"],
|
| 56 |
+
"nsentences": sample_size,
|
| 57 |
+
"sample_size": sample_size,
|
| 58 |
+
}
|
| 59 |
+
if not self.regression_target:
|
| 60 |
+
preds = logits.argmax(dim=1)
|
| 61 |
+
logging_output["ncorrect"] = (preds == targets).sum()
|
| 62 |
+
|
| 63 |
+
return loss, sample_size, logging_output
|
data/fairseq/fairseq/criterions/sentence_ranking.py
ADDED
|
@@ -0,0 +1,121 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) Facebook, Inc. and its affiliates.
|
| 2 |
+
#
|
| 3 |
+
# This source code is licensed under the MIT license found in the
|
| 4 |
+
# LICENSE file in the root directory of this source tree.
|
| 5 |
+
|
| 6 |
+
import math
|
| 7 |
+
|
| 8 |
+
import torch
|
| 9 |
+
import torch.nn.functional as F
|
| 10 |
+
from fairseq import utils
|
| 11 |
+
from fairseq.logging import metrics
|
| 12 |
+
from fairseq.criterions import FairseqCriterion, register_criterion
|
| 13 |
+
|
| 14 |
+
|
| 15 |
+
@register_criterion("sentence_ranking")
|
| 16 |
+
class SentenceRankingCriterion(FairseqCriterion):
|
| 17 |
+
def __init__(self, task, ranking_head_name, save_predictions, num_classes):
|
| 18 |
+
super().__init__(task)
|
| 19 |
+
self.ranking_head_name = ranking_head_name
|
| 20 |
+
if save_predictions is not None:
|
| 21 |
+
self.prediction_h = open(save_predictions, "w")
|
| 22 |
+
else:
|
| 23 |
+
self.prediction_h = None
|
| 24 |
+
self.num_classes = num_classes
|
| 25 |
+
|
| 26 |
+
def __del__(self):
|
| 27 |
+
if self.prediction_h is not None:
|
| 28 |
+
self.prediction_h.close()
|
| 29 |
+
|
| 30 |
+
@staticmethod
|
| 31 |
+
def add_args(parser):
|
| 32 |
+
# fmt: off
|
| 33 |
+
parser.add_argument('--save-predictions', metavar='FILE',
|
| 34 |
+
help='file to save predictions to')
|
| 35 |
+
parser.add_argument('--ranking-head-name',
|
| 36 |
+
default='sentence_classification_head',
|
| 37 |
+
help='name of the ranking head to use')
|
| 38 |
+
# fmt: on
|
| 39 |
+
|
| 40 |
+
def forward(self, model, sample, reduce=True):
|
| 41 |
+
"""Compute ranking loss for the given sample.
|
| 42 |
+
|
| 43 |
+
Returns a tuple with three elements:
|
| 44 |
+
1) the loss
|
| 45 |
+
2) the sample size, which is used as the denominator for the gradient
|
| 46 |
+
3) logging outputs to display while training
|
| 47 |
+
"""
|
| 48 |
+
assert (
|
| 49 |
+
hasattr(model, "classification_heads")
|
| 50 |
+
and self.ranking_head_name in model.classification_heads
|
| 51 |
+
), "model must provide sentence ranking head for --criterion=sentence_ranking"
|
| 52 |
+
|
| 53 |
+
scores = []
|
| 54 |
+
for idx in range(self.num_classes):
|
| 55 |
+
score, _ = model(
|
| 56 |
+
**sample["net_input{idx}".format(idx=idx + 1)],
|
| 57 |
+
classification_head_name=self.ranking_head_name,
|
| 58 |
+
)
|
| 59 |
+
scores.append(score)
|
| 60 |
+
|
| 61 |
+
logits = torch.cat(scores, dim=1)
|
| 62 |
+
sample_size = logits.size(0)
|
| 63 |
+
|
| 64 |
+
if "target" in sample:
|
| 65 |
+
targets = model.get_targets(sample, [logits]).view(-1)
|
| 66 |
+
lprobs = F.log_softmax(logits, dim=-1, dtype=torch.float32)
|
| 67 |
+
loss = F.nll_loss(lprobs, targets, reduction="sum")
|
| 68 |
+
else:
|
| 69 |
+
targets = None
|
| 70 |
+
loss = torch.tensor(0.0, requires_grad=True)
|
| 71 |
+
|
| 72 |
+
if self.prediction_h is not None:
|
| 73 |
+
preds = logits.argmax(dim=1)
|
| 74 |
+
for i, (id, pred) in enumerate(zip(sample["id"].tolist(), preds.tolist())):
|
| 75 |
+
if targets is not None:
|
| 76 |
+
label = targets[i].item()
|
| 77 |
+
print("{}\t{}\t{}".format(id, pred, label), file=self.prediction_h)
|
| 78 |
+
else:
|
| 79 |
+
print("{}\t{}".format(id, pred), file=self.prediction_h)
|
| 80 |
+
|
| 81 |
+
logging_output = {
|
| 82 |
+
"loss": loss.data,
|
| 83 |
+
"ntokens": sample["ntokens"],
|
| 84 |
+
"nsentences": sample_size,
|
| 85 |
+
"sample_size": sample_size,
|
| 86 |
+
}
|
| 87 |
+
if targets is not None:
|
| 88 |
+
logging_output["ncorrect"] = (logits.argmax(dim=1) == targets).sum()
|
| 89 |
+
|
| 90 |
+
return loss, sample_size, logging_output
|
| 91 |
+
|
| 92 |
+
@staticmethod
|
| 93 |
+
def reduce_metrics(logging_outputs) -> None:
|
| 94 |
+
"""Aggregate logging outputs from data parallel training."""
|
| 95 |
+
loss_sum = sum(log.get("loss", 0) for log in logging_outputs)
|
| 96 |
+
ntokens = sum(log.get("ntokens", 0) for log in logging_outputs)
|
| 97 |
+
nsentences = sum(log.get("nsentences", 0) for log in logging_outputs)
|
| 98 |
+
sample_size = sum(log.get("sample_size", 0) for log in logging_outputs)
|
| 99 |
+
|
| 100 |
+
metrics.log_scalar(
|
| 101 |
+
"loss", loss_sum / sample_size / math.log(2), sample_size, round=3
|
| 102 |
+
)
|
| 103 |
+
if sample_size != ntokens:
|
| 104 |
+
metrics.log_scalar(
|
| 105 |
+
"nll_loss", loss_sum / ntokens / math.log(2), ntokens, round=3
|
| 106 |
+
)
|
| 107 |
+
|
| 108 |
+
if len(logging_outputs) > 0 and "ncorrect" in logging_outputs[0]:
|
| 109 |
+
ncorrect = sum(log.get("ncorrect", 0) for log in logging_outputs)
|
| 110 |
+
metrics.log_scalar(
|
| 111 |
+
"accuracy", 100.0 * ncorrect / nsentences, nsentences, round=1
|
| 112 |
+
)
|
| 113 |
+
|
| 114 |
+
@staticmethod
|
| 115 |
+
def logging_outputs_can_be_summed() -> bool:
|
| 116 |
+
"""
|
| 117 |
+
Whether the logging outputs returned by `forward` can be summed
|
| 118 |
+
across workers prior to calling `reduce_metrics`. Setting this
|
| 119 |
+
to True will improves distributed training speed.
|
| 120 |
+
"""
|
| 121 |
+
return True
|
data/fairseq/fairseq/criterions/speech_dlm_criterion.py
ADDED
|
@@ -0,0 +1,335 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) Facebook, Inc. and its affiliates.
|
| 2 |
+
#
|
| 3 |
+
# This source code is licensed under the MIT license found in the
|
| 4 |
+
# LICENSE file in the root directory of this source tree.
|
| 5 |
+
|
| 6 |
+
import math
|
| 7 |
+
from dataclasses import dataclass, field
|
| 8 |
+
from typing import Optional
|
| 9 |
+
|
| 10 |
+
import torch.nn.functional as F
|
| 11 |
+
from fairseq import metrics, utils
|
| 12 |
+
from fairseq.criterions import FairseqCriterion, register_criterion
|
| 13 |
+
from fairseq.dataclass import FairseqDataclass
|
| 14 |
+
from omegaconf import II
|
| 15 |
+
|
| 16 |
+
|
| 17 |
+
@dataclass
|
| 18 |
+
class SpeechDLMCriterionConfig(FairseqDataclass):
|
| 19 |
+
sentence_avg: bool = II("optimization.sentence_avg")
|
| 20 |
+
main_and_cross_weights: Optional[str] = field(
|
| 21 |
+
default="1,0",
|
| 22 |
+
metadata={
|
| 23 |
+
"help": "Comma-separated list of weights of Main-channel vs Cross-channel Prediction Losses"
|
| 24 |
+
"(default: 1,0)"
|
| 25 |
+
},
|
| 26 |
+
)
|
| 27 |
+
general_unit_loss_weight: float = field(
|
| 28 |
+
default=0,
|
| 29 |
+
metadata={
|
| 30 |
+
"help": "The weight of the General Prediction Loss (Next-step Unit Prediction Loss)"
|
| 31 |
+
"(default: 0)"
|
| 32 |
+
},
|
| 33 |
+
)
|
| 34 |
+
edge_unit_loss_weight: float = field(
|
| 35 |
+
default=1,
|
| 36 |
+
metadata={"help": "The weight of the Edge Unit Prediction Loss" "(default: 1)"},
|
| 37 |
+
)
|
| 38 |
+
duration_loss_weight: float = field(
|
| 39 |
+
default=1,
|
| 40 |
+
metadata={
|
| 41 |
+
"help": "The weight of the Edge Unit Duration Prediction Loss"
|
| 42 |
+
"(default: 1)"
|
| 43 |
+
},
|
| 44 |
+
)
|
| 45 |
+
|
| 46 |
+
|
| 47 |
+
@register_criterion("speech_dlm_criterion", dataclass=SpeechDLMCriterionConfig)
|
| 48 |
+
class SpeechDLMCriterion(FairseqCriterion):
|
| 49 |
+
"""Criteron for the SpeechDLM model as described in the paper:
|
| 50 |
+
https://arxiv.org/pdf/2203.16502.pdf
|
| 51 |
+
|
| 52 |
+
There are 3 possible losses depending on the targets of the model:
|
| 53 |
+
- general_unit_loss : The next unit prediction loss, corresponding to
|
| 54 |
+
'next' target
|
| 55 |
+
- edge_unit_loss : The edge unit prediction loss, corresponding to
|
| 56 |
+
'edge' target
|
| 57 |
+
- duration_loss : The duration prediction loss, corresponding to
|
| 58 |
+
'duration' target
|
| 59 |
+
"""
|
| 60 |
+
|
| 61 |
+
def __init__(
|
| 62 |
+
self,
|
| 63 |
+
task,
|
| 64 |
+
sentence_avg,
|
| 65 |
+
main_and_cross_weights,
|
| 66 |
+
general_unit_loss_weight,
|
| 67 |
+
edge_unit_loss_weight,
|
| 68 |
+
duration_loss_weight,
|
| 69 |
+
):
|
| 70 |
+
super().__init__(task)
|
| 71 |
+
self.sentence_avg = sentence_avg
|
| 72 |
+
|
| 73 |
+
self.channels = task.channels
|
| 74 |
+
self.targets = task.targets
|
| 75 |
+
self.delayed_duration_target = task.delayed_duration_target
|
| 76 |
+
|
| 77 |
+
self.main_channel_weight = float(main_and_cross_weights.split(",")[0])
|
| 78 |
+
self.cross_channel_weight = float(main_and_cross_weights.split(",")[1])
|
| 79 |
+
assert self.main_channel_weight >= 0 and self.cross_channel_weight >= 0
|
| 80 |
+
|
| 81 |
+
self.channel_weights = {
|
| 82 |
+
channel: weight
|
| 83 |
+
for channel, weight in zip(self.channels, task.channel_weights)
|
| 84 |
+
}
|
| 85 |
+
|
| 86 |
+
self.target_weights = {}
|
| 87 |
+
for t in self.targets:
|
| 88 |
+
if t == "next":
|
| 89 |
+
self.target_weights[t] = general_unit_loss_weight
|
| 90 |
+
assert (
|
| 91 |
+
general_unit_loss_weight > 0
|
| 92 |
+
), "Expect a positive --general-unit-loss-weight for next unit prediction"
|
| 93 |
+
elif t == "edge":
|
| 94 |
+
self.target_weights[t] = edge_unit_loss_weight
|
| 95 |
+
assert (
|
| 96 |
+
edge_unit_loss_weight > 0
|
| 97 |
+
), "Expect a positive --edge-unit-loss-weight for edge unit prediction"
|
| 98 |
+
elif t == "duration":
|
| 99 |
+
self.target_weights[t] = duration_loss_weight
|
| 100 |
+
assert (
|
| 101 |
+
duration_loss_weight > 0
|
| 102 |
+
), "Expect a positive --duration-loss-weight for duration prediction"
|
| 103 |
+
|
| 104 |
+
def forward(self, model, sample, reduce=True):
|
| 105 |
+
"""Compute the loss for the given sample.
|
| 106 |
+
|
| 107 |
+
Returns a tuple with three elements:
|
| 108 |
+
1) the loss
|
| 109 |
+
2) the sample size, which is used as the denominator for the gradient
|
| 110 |
+
3) logging outputs to display while training
|
| 111 |
+
"""
|
| 112 |
+
net_output = model(**sample["net_input"])
|
| 113 |
+
loss_dict, stats_dict = self.compute_loss(
|
| 114 |
+
model, net_output, sample, reduce=reduce
|
| 115 |
+
)
|
| 116 |
+
nsentences = sample["net_input"]["src_tokens"][self.channels[0]].size(0)
|
| 117 |
+
|
| 118 |
+
logging_output = {
|
| 119 |
+
"nsentences": nsentences,
|
| 120 |
+
}
|
| 121 |
+
logging_output["nsentences"] = nsentences
|
| 122 |
+
|
| 123 |
+
loss_all = {t: 0 for t in self.targets}
|
| 124 |
+
correct_all = {t: 0 for t in self.targets}
|
| 125 |
+
count_all = {t: 0 for t in self.targets}
|
| 126 |
+
ntokens_all = 0
|
| 127 |
+
sample_size_all = 0
|
| 128 |
+
for channel in loss_dict:
|
| 129 |
+
for pred_channel in loss_dict[channel]:
|
| 130 |
+
# Get ntokens & sample_size
|
| 131 |
+
ntokens = sample["net_input"]["src_tokens"][channel].numel()
|
| 132 |
+
sample_size = nsentences if self.sentence_avg else ntokens
|
| 133 |
+
prefix = "[{}-{}]".format(channel, pred_channel)
|
| 134 |
+
log_keys = {
|
| 135 |
+
"next": "general_token",
|
| 136 |
+
"edge": "edge_token",
|
| 137 |
+
"duration": "edge_duration",
|
| 138 |
+
}
|
| 139 |
+
|
| 140 |
+
# Log & Update the sizes
|
| 141 |
+
logging_output["{}ntokens".format(prefix)] = ntokens
|
| 142 |
+
logging_output["{}sample_size".format(prefix)] = sample_size
|
| 143 |
+
ntokens_all += ntokens
|
| 144 |
+
sample_size_all += sample_size
|
| 145 |
+
|
| 146 |
+
for t in self.targets:
|
| 147 |
+
log_key = log_keys[t]
|
| 148 |
+
loss = loss_dict[channel][pred_channel][t]
|
| 149 |
+
correct, count = stats_dict[channel][pred_channel][t]
|
| 150 |
+
|
| 151 |
+
# Log the statistics
|
| 152 |
+
logging_output["{}{}_loss".format(prefix, log_key)] = loss.data
|
| 153 |
+
logging_output["{}{}_correct".format(prefix, log_key)] = correct
|
| 154 |
+
logging_output["{}{}_count".format(prefix, log_key)] = count
|
| 155 |
+
|
| 156 |
+
# Scale the training loss by weights
|
| 157 |
+
target_loss = loss * self.channel_weights[channel]
|
| 158 |
+
if pred_channel == channel:
|
| 159 |
+
target_loss = target_loss * self.main_channel_weight
|
| 160 |
+
else:
|
| 161 |
+
target_loss = target_loss * self.cross_channel_weight
|
| 162 |
+
# Normalize the losses in the training by the number of edges
|
| 163 |
+
if t in ["edge", "duration"]:
|
| 164 |
+
target_loss = target_loss / count * sample_size
|
| 165 |
+
|
| 166 |
+
# Update the statistics
|
| 167 |
+
loss_all[t] += target_loss
|
| 168 |
+
correct_all[t] += correct
|
| 169 |
+
count_all[t] += count
|
| 170 |
+
|
| 171 |
+
# Logging the average statistics
|
| 172 |
+
logging_output["ntokens"] = ntokens_all
|
| 173 |
+
logging_output["sample_size"] = sample_size_all
|
| 174 |
+
for t in self.targets:
|
| 175 |
+
log_key = {
|
| 176 |
+
"next": "general_token",
|
| 177 |
+
"edge": "edge_token",
|
| 178 |
+
"duration": "edge_duration",
|
| 179 |
+
}[t]
|
| 180 |
+
logging_output["{}_loss".format(log_key)] = loss_all[t].data
|
| 181 |
+
logging_output["{}_correct".format(log_key)] = correct_all[t]
|
| 182 |
+
logging_output["{}_count".format(log_key)] = count_all[t]
|
| 183 |
+
|
| 184 |
+
# Define the training loss
|
| 185 |
+
training_loss = 0
|
| 186 |
+
for t in self.targets:
|
| 187 |
+
training_loss += loss_all[t] * self.target_weights[t]
|
| 188 |
+
logging_output["loss"] = training_loss.data
|
| 189 |
+
|
| 190 |
+
return training_loss, sample_size_all, logging_output
|
| 191 |
+
|
| 192 |
+
def compute_loss(self, model, net_output, sample, reduce=True):
|
| 193 |
+
# Get the model outputs and target
|
| 194 |
+
lprobs_dict = model.get_normalized_probs(net_output, log_probs=True)
|
| 195 |
+
target_dict = model.get_targets(sample, net_output)
|
| 196 |
+
|
| 197 |
+
# Init the dictionaries
|
| 198 |
+
loss_dict, stats_dict = {}, {}
|
| 199 |
+
|
| 200 |
+
for channel in lprobs_dict:
|
| 201 |
+
# Init the dictionaries
|
| 202 |
+
loss_dict[channel], stats_dict[channel] = {}, {}
|
| 203 |
+
|
| 204 |
+
for pred_channel in lprobs_dict[channel]:
|
| 205 |
+
# Init the dictionaries
|
| 206 |
+
loss_dict[channel][pred_channel] = {}
|
| 207 |
+
stats_dict[channel][pred_channel] = {}
|
| 208 |
+
|
| 209 |
+
# Get token & duration predictions
|
| 210 |
+
outputs = lprobs_dict[channel][pred_channel]
|
| 211 |
+
if not isinstance(outputs, dict):
|
| 212 |
+
token_lprobs = outputs
|
| 213 |
+
else:
|
| 214 |
+
token_lprobs = outputs["pred_token"]
|
| 215 |
+
dur_preds = outputs["pred_duration"]
|
| 216 |
+
dur_preds = dur_preds.view(-1)
|
| 217 |
+
token_lprobs = token_lprobs.view(-1, token_lprobs.size(-1))
|
| 218 |
+
token_preds = token_lprobs.argmax(dim=-1)
|
| 219 |
+
|
| 220 |
+
# Get edge indices
|
| 221 |
+
if "edge" in self.targets or "duration" in self.targets:
|
| 222 |
+
edge_indices = target_dict["edge_indices"][pred_channel]
|
| 223 |
+
|
| 224 |
+
# Compute loss and statistics
|
| 225 |
+
for t in self.targets:
|
| 226 |
+
if t in ["next", "edge"]:
|
| 227 |
+
if t == "next":
|
| 228 |
+
target = target_dict["next"][pred_channel].view(-1)
|
| 229 |
+
lprobs = token_lprobs
|
| 230 |
+
preds = token_preds
|
| 231 |
+
elif t == "edge":
|
| 232 |
+
target = target_dict["edge"][pred_channel]
|
| 233 |
+
lprobs = token_lprobs[edge_indices]
|
| 234 |
+
preds = token_preds[edge_indices]
|
| 235 |
+
|
| 236 |
+
loss = F.nll_loss(
|
| 237 |
+
lprobs,
|
| 238 |
+
target,
|
| 239 |
+
ignore_index=self.padding_idx,
|
| 240 |
+
reduction="sum" if reduce else "none",
|
| 241 |
+
)
|
| 242 |
+
elif t == "duration":
|
| 243 |
+
target = target_dict["duration"][pred_channel]
|
| 244 |
+
if self.delayed_duration_target:
|
| 245 |
+
duration_indices = edge_indices + 1
|
| 246 |
+
if duration_indices[-1] == len(dur_preds):
|
| 247 |
+
duration_indices = duration_indices[:-1]
|
| 248 |
+
target = target[:-1]
|
| 249 |
+
else:
|
| 250 |
+
duration_indices = edge_indices
|
| 251 |
+
preds = dur_preds[duration_indices]
|
| 252 |
+
|
| 253 |
+
loss = F.l1_loss(
|
| 254 |
+
preds,
|
| 255 |
+
target,
|
| 256 |
+
reduction="sum" if reduce else "none",
|
| 257 |
+
)
|
| 258 |
+
preds = preds.round()
|
| 259 |
+
|
| 260 |
+
correct = (preds == target).sum().float().cpu().item()
|
| 261 |
+
count = float(target.size(0))
|
| 262 |
+
|
| 263 |
+
loss_dict[channel][pred_channel][t] = loss
|
| 264 |
+
stats_dict[channel][pred_channel][t] = (correct, count)
|
| 265 |
+
|
| 266 |
+
return loss_dict, stats_dict
|
| 267 |
+
|
| 268 |
+
@staticmethod
|
| 269 |
+
def reduce_metrics(logging_outputs) -> None:
|
| 270 |
+
"""Aggregate logging outputs from data parallel training."""
|
| 271 |
+
logging_keys = next(iter(logging_outputs)).keys()
|
| 272 |
+
channels = [item[:-7] for item in logging_keys if item.endswith("ntokens")]
|
| 273 |
+
target_prefixes = set(
|
| 274 |
+
[
|
| 275 |
+
item[:-5].split("]")[-1]
|
| 276 |
+
for item in logging_keys
|
| 277 |
+
if item.endswith("_loss")
|
| 278 |
+
]
|
| 279 |
+
)
|
| 280 |
+
for channel_prefix in channels:
|
| 281 |
+
for target_prefix in target_prefixes:
|
| 282 |
+
prefix = "{}{}".format(channel_prefix, target_prefix)
|
| 283 |
+
count_sum = sum(
|
| 284 |
+
log.get("{}_count".format(prefix), 0) for log in logging_outputs
|
| 285 |
+
)
|
| 286 |
+
correct_sum = sum(
|
| 287 |
+
log.get("{}_correct".format(prefix), 0) for log in logging_outputs
|
| 288 |
+
)
|
| 289 |
+
loss_sum = sum(
|
| 290 |
+
log.get("{}_loss".format(prefix), 0) for log in logging_outputs
|
| 291 |
+
)
|
| 292 |
+
|
| 293 |
+
if "duration" not in target_prefix:
|
| 294 |
+
# we divide by log(2) to convert the loss from base e to base 2
|
| 295 |
+
metrics.log_scalar(
|
| 296 |
+
"{}_loss".format(prefix),
|
| 297 |
+
loss_sum / count_sum / math.log(2),
|
| 298 |
+
count_sum,
|
| 299 |
+
round=3,
|
| 300 |
+
)
|
| 301 |
+
metrics.log_derived(
|
| 302 |
+
"{}_ppl".format(prefix),
|
| 303 |
+
lambda meters, prefix=prefix: utils.get_perplexity(
|
| 304 |
+
meters["{}_loss".format(prefix)].avg
|
| 305 |
+
),
|
| 306 |
+
)
|
| 307 |
+
else:
|
| 308 |
+
# for duration we don't need to divide by log(2)
|
| 309 |
+
metrics.log_scalar(
|
| 310 |
+
"{}_loss".format(prefix),
|
| 311 |
+
loss_sum / count_sum,
|
| 312 |
+
count_sum,
|
| 313 |
+
round=3,
|
| 314 |
+
)
|
| 315 |
+
|
| 316 |
+
accuracy = 100 * correct_sum / count_sum
|
| 317 |
+
metrics.log_scalar("{}_pred_acc".format(prefix), accuracy, round=3)
|
| 318 |
+
|
| 319 |
+
# Logging training loss
|
| 320 |
+
sample_size = sum(log.get("sample_size", 0) for log in logging_outputs)
|
| 321 |
+
loss_sum = sum(log.get("loss", 0) for log in logging_outputs)
|
| 322 |
+
|
| 323 |
+
# we divide by log(2) to convert the loss from base e to base 2
|
| 324 |
+
metrics.log_scalar(
|
| 325 |
+
"loss", loss_sum / sample_size / math.log(2), sample_size, round=3
|
| 326 |
+
)
|
| 327 |
+
|
| 328 |
+
@staticmethod
|
| 329 |
+
def logging_outputs_can_be_summed() -> bool:
|
| 330 |
+
"""
|
| 331 |
+
Whether the logging outputs returned by `forward` can be summed
|
| 332 |
+
across workers prior to calling `reduce_metrics`. Setting this
|
| 333 |
+
to True will improves distributed training speed.
|
| 334 |
+
"""
|
| 335 |
+
return True
|
data/fairseq/fairseq/criterions/speech_to_speech_criterion.py
ADDED
|
@@ -0,0 +1,517 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) Facebook, Inc. and its affiliates.
|
| 2 |
+
#
|
| 3 |
+
# This source code is licensed under the MIT license found in the
|
| 4 |
+
# LICENSE file in the root directory of this source tree.
|
| 5 |
+
|
| 6 |
+
import logging
|
| 7 |
+
import math
|
| 8 |
+
from collections import OrderedDict
|
| 9 |
+
|
| 10 |
+
import torch
|
| 11 |
+
|
| 12 |
+
from fairseq import utils
|
| 13 |
+
from fairseq.logging import metrics
|
| 14 |
+
from fairseq.criterions import register_criterion
|
| 15 |
+
from fairseq.criterions.ctc import CtcCriterion
|
| 16 |
+
from fairseq.criterions.label_smoothed_cross_entropy_with_rdrop import (
|
| 17 |
+
RdropLabelSmoothedCrossEntropyCriterion,
|
| 18 |
+
RdropLabelSmoothedCrossEntropyCriterionConfig,
|
| 19 |
+
duplicate_input,
|
| 20 |
+
)
|
| 21 |
+
from fairseq.criterions.tacotron2_loss import (
|
| 22 |
+
Tacotron2Criterion,
|
| 23 |
+
Tacotron2CriterionConfig,
|
| 24 |
+
)
|
| 25 |
+
|
| 26 |
+
logger = logging.getLogger(__name__)
|
| 27 |
+
|
| 28 |
+
|
| 29 |
+
class MultitaskCriterion:
|
| 30 |
+
def __init__(self, multitask_tasks, rdrop_alpha=0.0):
|
| 31 |
+
self.rdrop_alpha = rdrop_alpha
|
| 32 |
+
self.rdrop_alpha_mtl = rdrop_alpha
|
| 33 |
+
|
| 34 |
+
self.multitask_criterion = OrderedDict()
|
| 35 |
+
self.multitask_loss_weight = OrderedDict()
|
| 36 |
+
for task_name, task_obj in multitask_tasks.items():
|
| 37 |
+
if task_obj.args.get_loss_weight(0) == 0:
|
| 38 |
+
logger.info(f"Skip {task_name} loss criterion")
|
| 39 |
+
continue
|
| 40 |
+
|
| 41 |
+
rdrop_alpha_task = task_obj.args.rdrop_alpha
|
| 42 |
+
if rdrop_alpha_task is None:
|
| 43 |
+
rdrop_alpha_task = rdrop_alpha
|
| 44 |
+
self.rdrop_alpha_mtl = rdrop_alpha_task
|
| 45 |
+
logger.info(f"rdrop_alpha is set to {rdrop_alpha_task} for {task_name}")
|
| 46 |
+
|
| 47 |
+
if task_obj.args.decoder_type == "ctc":
|
| 48 |
+
self.multitask_criterion[task_name] = CtcCriterion(
|
| 49 |
+
task_obj.args.criterion_cfg,
|
| 50 |
+
task_obj,
|
| 51 |
+
rdrop_alpha=rdrop_alpha_task,
|
| 52 |
+
)
|
| 53 |
+
else:
|
| 54 |
+
self.multitask_criterion[
|
| 55 |
+
task_name
|
| 56 |
+
] = RdropLabelSmoothedCrossEntropyCriterion(
|
| 57 |
+
task_obj,
|
| 58 |
+
task_obj.args.criterion_cfg.sentence_avg,
|
| 59 |
+
label_smoothing=task_obj.args.criterion_cfg.label_smoothing,
|
| 60 |
+
rdrop_alpha=rdrop_alpha_task,
|
| 61 |
+
)
|
| 62 |
+
|
| 63 |
+
def set_multitask_loss_weight(self, task_name, weight=0.0):
|
| 64 |
+
self.multitask_loss_weight[task_name] = weight
|
| 65 |
+
|
| 66 |
+
def get_multitask_loss(self, model, sample, model_out):
|
| 67 |
+
logging_output = {}
|
| 68 |
+
loss = 0.0
|
| 69 |
+
for task_name, task_criterion in self.multitask_criterion.items():
|
| 70 |
+
layer_id = task_criterion.task.args.input_layer
|
| 71 |
+
if isinstance(task_criterion, CtcCriterion):
|
| 72 |
+
if task_criterion.task.args.input_from == "encoder":
|
| 73 |
+
if len(model_out["encoder_padding_mask"]) > 0:
|
| 74 |
+
non_padding_mask = ~model_out["encoder_padding_mask"][0]
|
| 75 |
+
input_lengths = non_padding_mask.long().sum(-1)
|
| 76 |
+
else:
|
| 77 |
+
out = model_out["encoder_states"][layer_id]
|
| 78 |
+
input_lengths = out.new_full(
|
| 79 |
+
(out.shape[1],), out.shape[0]
|
| 80 |
+
).long()
|
| 81 |
+
|
| 82 |
+
task_sample = {
|
| 83 |
+
"net_input": {
|
| 84 |
+
"src_tokens": model_out["encoder_states"][
|
| 85 |
+
layer_id
|
| 86 |
+
], # check batch idx
|
| 87 |
+
"src_lengths": input_lengths,
|
| 88 |
+
},
|
| 89 |
+
"id": sample["id"],
|
| 90 |
+
}
|
| 91 |
+
else:
|
| 92 |
+
task_sample = {
|
| 93 |
+
"net_input": {
|
| 94 |
+
"src_tokens": model_out["inner_states"][layer_id],
|
| 95 |
+
"src_lengths": sample["target_lengths"],
|
| 96 |
+
},
|
| 97 |
+
"id": sample["id"],
|
| 98 |
+
}
|
| 99 |
+
else:
|
| 100 |
+
task_sample = {
|
| 101 |
+
"net_input": {
|
| 102 |
+
"src_tokens": sample["multitask"][task_name]["net_input"][
|
| 103 |
+
"prev_output_tokens"
|
| 104 |
+
],
|
| 105 |
+
"encoder_out": {
|
| 106 |
+
"encoder_out": [model_out["encoder_states"][layer_id]],
|
| 107 |
+
"encoder_padding_mask": model_out["encoder_padding_mask"],
|
| 108 |
+
},
|
| 109 |
+
}
|
| 110 |
+
}
|
| 111 |
+
|
| 112 |
+
for key in ["target", "target_lengths", "ntokens"]:
|
| 113 |
+
task_sample[key] = sample["multitask"][task_name][key]
|
| 114 |
+
|
| 115 |
+
if task_name == getattr(model, "mt_task_name", None):
|
| 116 |
+
decoder_out = model_out["mt_decoder_out"]
|
| 117 |
+
else:
|
| 118 |
+
decoder_out = None
|
| 119 |
+
task_loss, task_sample_size, task_logging_output = task_criterion(
|
| 120 |
+
model.multitask_decoders[task_name], task_sample, net_output=decoder_out
|
| 121 |
+
)
|
| 122 |
+
|
| 123 |
+
loss = loss + self.multitask_loss_weight[task_name] * task_loss
|
| 124 |
+
task_logging_output["loss_weight"] = self.multitask_loss_weight[task_name]
|
| 125 |
+
logging_output[task_name] = task_logging_output
|
| 126 |
+
return loss, logging_output
|
| 127 |
+
|
| 128 |
+
@classmethod
|
| 129 |
+
def reduce_metrics(cls, logging_outputs) -> None:
|
| 130 |
+
for task_name in logging_outputs[0]["multitask"].keys():
|
| 131 |
+
# different criterion may return different logging
|
| 132 |
+
# currently only reduce on loss, the most common one
|
| 133 |
+
# ideally the way that losses are reduced should also depend on the task type
|
| 134 |
+
loss_sum = sum(
|
| 135 |
+
log["multitask"][task_name].get("loss", 0) for log in logging_outputs
|
| 136 |
+
)
|
| 137 |
+
sample_size = sum(
|
| 138 |
+
log["multitask"][task_name].get("sample_size", 0)
|
| 139 |
+
for log in logging_outputs
|
| 140 |
+
)
|
| 141 |
+
|
| 142 |
+
metrics.log_scalar(
|
| 143 |
+
f"multitask_{task_name}_loss",
|
| 144 |
+
loss_sum / sample_size / math.log(2),
|
| 145 |
+
sample_size,
|
| 146 |
+
round=3,
|
| 147 |
+
)
|
| 148 |
+
|
| 149 |
+
loss_weight = logging_outputs[0]["multitask"][task_name].get(
|
| 150 |
+
"loss_weight", 0
|
| 151 |
+
)
|
| 152 |
+
metrics.log_scalar(
|
| 153 |
+
f"multitask_{task_name}_loss_weight",
|
| 154 |
+
loss_weight,
|
| 155 |
+
weight=0,
|
| 156 |
+
priority=250,
|
| 157 |
+
)
|
| 158 |
+
|
| 159 |
+
|
| 160 |
+
@register_criterion(
|
| 161 |
+
"speech_to_unit", dataclass=RdropLabelSmoothedCrossEntropyCriterionConfig
|
| 162 |
+
)
|
| 163 |
+
class SpeechToUnitMultitaskTaskCriterion(
|
| 164 |
+
RdropLabelSmoothedCrossEntropyCriterion, MultitaskCriterion
|
| 165 |
+
):
|
| 166 |
+
def __init__(
|
| 167 |
+
self,
|
| 168 |
+
task,
|
| 169 |
+
sentence_avg,
|
| 170 |
+
label_smoothing,
|
| 171 |
+
ignore_prefix_size=0,
|
| 172 |
+
report_accuracy=False,
|
| 173 |
+
rdrop_alpha=0.0,
|
| 174 |
+
):
|
| 175 |
+
super().__init__(
|
| 176 |
+
task,
|
| 177 |
+
sentence_avg,
|
| 178 |
+
label_smoothing,
|
| 179 |
+
ignore_prefix_size,
|
| 180 |
+
report_accuracy,
|
| 181 |
+
rdrop_alpha,
|
| 182 |
+
)
|
| 183 |
+
MultitaskCriterion.__init__(self, task.multitask_tasks, rdrop_alpha)
|
| 184 |
+
|
| 185 |
+
def forward(self, model, sample, reduce=True):
|
| 186 |
+
net_input_concat = {
|
| 187 |
+
"src_tokens": sample["net_input"]["src_tokens"],
|
| 188 |
+
"src_lengths": sample["net_input"]["src_lengths"],
|
| 189 |
+
"prev_output_tokens": sample["net_input"]["prev_output_tokens"],
|
| 190 |
+
"tgt_speaker": sample["net_input"].get("tgt_speaker", None),
|
| 191 |
+
"return_all_hiddens": True,
|
| 192 |
+
}
|
| 193 |
+
|
| 194 |
+
if self.rdrop_alpha > 0 or self.rdrop_alpha_mtl > 0:
|
| 195 |
+
net_input_concat = duplicate_input(net_input_concat)
|
| 196 |
+
|
| 197 |
+
net_output, extra = model(**net_input_concat)
|
| 198 |
+
loss, nll_loss, rdrop_kl_loss = self.compute_loss(
|
| 199 |
+
model, [net_output], sample, reduce=reduce
|
| 200 |
+
)
|
| 201 |
+
sample_size = (
|
| 202 |
+
sample["target"].size(0) if self.sentence_avg else sample["ntokens"]
|
| 203 |
+
)
|
| 204 |
+
logging_output = {
|
| 205 |
+
"loss": loss.data,
|
| 206 |
+
"nll_loss": nll_loss.data,
|
| 207 |
+
"ntokens": sample["ntokens"],
|
| 208 |
+
"nsentences": sample["target"].size(0),
|
| 209 |
+
"sample_size": sample_size,
|
| 210 |
+
}
|
| 211 |
+
if self.report_accuracy:
|
| 212 |
+
n_correct, total = self.compute_accuracy(model, [net_output], sample)
|
| 213 |
+
logging_output["n_correct"] = utils.item(n_correct.data)
|
| 214 |
+
logging_output["total"] = utils.item(total.data)
|
| 215 |
+
if self.rdrop_alpha > 0:
|
| 216 |
+
logging_output["rdrop_kl_loss"] = utils.item(rdrop_kl_loss.data)
|
| 217 |
+
|
| 218 |
+
if len(self.multitask_criterion) == 0:
|
| 219 |
+
return loss, sample_size, logging_output
|
| 220 |
+
|
| 221 |
+
# multitask
|
| 222 |
+
multitask_loss, multitask_log = self.get_multitask_loss(model, sample, extra)
|
| 223 |
+
loss += multitask_loss
|
| 224 |
+
logging_output["multitask"] = multitask_log
|
| 225 |
+
|
| 226 |
+
return loss, sample_size, logging_output
|
| 227 |
+
|
| 228 |
+
@classmethod
|
| 229 |
+
def reduce_metrics(cls, logging_outputs) -> None:
|
| 230 |
+
super().reduce_metrics(logging_outputs)
|
| 231 |
+
|
| 232 |
+
# inference metrics
|
| 233 |
+
if "targ_frames" in logging_outputs[0]:
|
| 234 |
+
n = sum(log.get("norm_frames", 0) for log in logging_outputs)
|
| 235 |
+
for key, new_key in [
|
| 236 |
+
("mcd_loss", "mcd_loss"),
|
| 237 |
+
("pred_frames", "pred_ratio"),
|
| 238 |
+
("nins", "ins_rate"),
|
| 239 |
+
("ndel", "del_rate"),
|
| 240 |
+
]:
|
| 241 |
+
val = sum(log.get(key, 0) for log in logging_outputs)
|
| 242 |
+
metrics.log_scalar(new_key, val / n, n, round=3)
|
| 243 |
+
|
| 244 |
+
if "multitask" not in logging_outputs[0]:
|
| 245 |
+
return
|
| 246 |
+
|
| 247 |
+
MultitaskCriterion.reduce_metrics(logging_outputs)
|
| 248 |
+
|
| 249 |
+
@staticmethod
|
| 250 |
+
def logging_outputs_can_be_summed() -> bool:
|
| 251 |
+
"""
|
| 252 |
+
Whether the logging outputs returned by `forward` can be summed
|
| 253 |
+
across workers prior to calling `reduce_metrics`. Setting this
|
| 254 |
+
to True will improves distributed training speed.
|
| 255 |
+
"""
|
| 256 |
+
return False
|
| 257 |
+
|
| 258 |
+
|
| 259 |
+
@register_criterion(
|
| 260 |
+
"speech_to_unit_2pass", dataclass=RdropLabelSmoothedCrossEntropyCriterionConfig
|
| 261 |
+
)
|
| 262 |
+
class SpeechToUnit2passMultitaskTaskCriterion(SpeechToUnitMultitaskTaskCriterion):
|
| 263 |
+
def __init__(
|
| 264 |
+
self,
|
| 265 |
+
task,
|
| 266 |
+
sentence_avg,
|
| 267 |
+
label_smoothing,
|
| 268 |
+
ignore_prefix_size=0,
|
| 269 |
+
report_accuracy=False,
|
| 270 |
+
rdrop_alpha=0.0,
|
| 271 |
+
):
|
| 272 |
+
super().__init__(
|
| 273 |
+
task,
|
| 274 |
+
sentence_avg,
|
| 275 |
+
label_smoothing,
|
| 276 |
+
ignore_prefix_size,
|
| 277 |
+
report_accuracy,
|
| 278 |
+
rdrop_alpha,
|
| 279 |
+
)
|
| 280 |
+
|
| 281 |
+
def forward(self, model, sample, reduce=True):
|
| 282 |
+
net_input_concat = {
|
| 283 |
+
"src_tokens": sample["net_input"]["src_tokens"],
|
| 284 |
+
"src_lengths": sample["net_input"]["src_lengths"],
|
| 285 |
+
"prev_output_tokens": sample["net_input"]["prev_output_tokens"],
|
| 286 |
+
"prev_output_tokens_mt": sample["multitask"][model.mt_task_name][
|
| 287 |
+
"net_input"
|
| 288 |
+
]["prev_output_tokens"],
|
| 289 |
+
"tgt_speaker": sample["net_input"].get("tgt_speaker", None),
|
| 290 |
+
"return_all_hiddens": True,
|
| 291 |
+
}
|
| 292 |
+
if getattr(model, "asr_task_name", None) is not None:
|
| 293 |
+
net_input_concat["prev_output_tokens_asr"] = sample["multitask"][
|
| 294 |
+
model.asr_task_name
|
| 295 |
+
]["net_input"]["prev_output_tokens"]
|
| 296 |
+
|
| 297 |
+
if self.rdrop_alpha > 0 or self.rdrop_alpha_mtl > 0:
|
| 298 |
+
net_input_concat = duplicate_input(net_input_concat)
|
| 299 |
+
|
| 300 |
+
net_output, extra = model(**net_input_concat)
|
| 301 |
+
loss, nll_loss, rdrop_kl_loss = self.compute_loss(
|
| 302 |
+
model, [net_output], sample, reduce=reduce
|
| 303 |
+
)
|
| 304 |
+
|
| 305 |
+
sample_size = (
|
| 306 |
+
sample["target"].size(0) if self.sentence_avg else sample["ntokens"]
|
| 307 |
+
)
|
| 308 |
+
logging_output = {
|
| 309 |
+
"loss": loss.data,
|
| 310 |
+
"nll_loss": nll_loss.data,
|
| 311 |
+
"ntokens": sample["ntokens"],
|
| 312 |
+
"nsentences": sample["target"].size(0),
|
| 313 |
+
"sample_size": sample_size,
|
| 314 |
+
}
|
| 315 |
+
if self.report_accuracy:
|
| 316 |
+
n_correct, total = self.compute_accuracy(model, [net_output], sample)
|
| 317 |
+
logging_output["n_correct"] = utils.item(n_correct.data)
|
| 318 |
+
logging_output["total"] = utils.item(total.data)
|
| 319 |
+
if self.rdrop_alpha > 0:
|
| 320 |
+
logging_output["rdrop_kl_loss"] = utils.item(rdrop_kl_loss.data)
|
| 321 |
+
|
| 322 |
+
if len(self.multitask_criterion) == 0:
|
| 323 |
+
return loss, sample_size, logging_output
|
| 324 |
+
|
| 325 |
+
# multitask
|
| 326 |
+
multitask_loss, multitask_log = self.get_multitask_loss(model, sample, extra)
|
| 327 |
+
loss += multitask_loss
|
| 328 |
+
logging_output["multitask"] = multitask_log
|
| 329 |
+
|
| 330 |
+
return loss, sample_size, logging_output
|
| 331 |
+
|
| 332 |
+
|
| 333 |
+
@register_criterion("speech_to_spectrogram", dataclass=Tacotron2CriterionConfig)
|
| 334 |
+
class SpeechToSpectrogramMultitaskTaskCriterion(Tacotron2Criterion, MultitaskCriterion):
|
| 335 |
+
def __init__(
|
| 336 |
+
self,
|
| 337 |
+
task,
|
| 338 |
+
sentence_avg,
|
| 339 |
+
use_guided_attention_loss,
|
| 340 |
+
guided_attention_loss_sigma,
|
| 341 |
+
bce_pos_weight,
|
| 342 |
+
ctc_weight,
|
| 343 |
+
):
|
| 344 |
+
super().__init__(
|
| 345 |
+
task,
|
| 346 |
+
sentence_avg,
|
| 347 |
+
use_guided_attention_loss,
|
| 348 |
+
guided_attention_loss_sigma,
|
| 349 |
+
bce_pos_weight,
|
| 350 |
+
ctc_weight,
|
| 351 |
+
)
|
| 352 |
+
MultitaskCriterion.__init__(self, task.multitask_tasks)
|
| 353 |
+
|
| 354 |
+
def forward(self, model, sample, reduction="mean"):
|
| 355 |
+
bsz, max_len, _ = sample["target"].size()
|
| 356 |
+
feat_tgt = sample["target"]
|
| 357 |
+
feat_len = sample["target_lengths"].view(bsz, 1).expand(-1, max_len)
|
| 358 |
+
eos_tgt = torch.arange(max_len).to(sample["target"].device)
|
| 359 |
+
eos_tgt = eos_tgt.view(1, max_len).expand(bsz, -1)
|
| 360 |
+
eos_tgt = (eos_tgt == (feat_len - 1)).float()
|
| 361 |
+
|
| 362 |
+
feat_out, eos_out, extra = model(
|
| 363 |
+
src_tokens=sample["net_input"]["src_tokens"],
|
| 364 |
+
src_lengths=sample["net_input"]["src_lengths"],
|
| 365 |
+
prev_output_tokens=sample["net_input"]["prev_output_tokens"],
|
| 366 |
+
tgt_speaker=sample["net_input"]["tgt_speaker"],
|
| 367 |
+
target_lengths=sample["target_lengths"],
|
| 368 |
+
return_all_hiddens=True,
|
| 369 |
+
)
|
| 370 |
+
|
| 371 |
+
l1_loss, mse_loss, eos_loss = self.compute_loss(
|
| 372 |
+
extra["feature_out"],
|
| 373 |
+
feat_out,
|
| 374 |
+
eos_out,
|
| 375 |
+
feat_tgt,
|
| 376 |
+
eos_tgt,
|
| 377 |
+
sample["target_lengths"],
|
| 378 |
+
reduction,
|
| 379 |
+
)
|
| 380 |
+
attn_loss = torch.tensor(0.0).type_as(l1_loss)
|
| 381 |
+
if self.guided_attn is not None:
|
| 382 |
+
attn_loss = self.guided_attn(
|
| 383 |
+
extra["attn"],
|
| 384 |
+
sample["net_input"]["src_lengths"],
|
| 385 |
+
sample["target_lengths"],
|
| 386 |
+
reduction,
|
| 387 |
+
)
|
| 388 |
+
loss = (
|
| 389 |
+
l1_loss + mse_loss + eos_loss + attn_loss
|
| 390 |
+
) # do not include ctc loss as there's no text target
|
| 391 |
+
|
| 392 |
+
sample_size = sample["nsentences"] if self.sentence_avg else sample["ntokens"]
|
| 393 |
+
logging_output = {
|
| 394 |
+
"loss": utils.item(loss.data),
|
| 395 |
+
"ntokens": sample["ntokens"],
|
| 396 |
+
"nsentences": sample["nsentences"],
|
| 397 |
+
"sample_size": sample_size,
|
| 398 |
+
"l1_loss": utils.item(l1_loss.data),
|
| 399 |
+
"mse_loss": utils.item(mse_loss.data),
|
| 400 |
+
"eos_loss": utils.item(eos_loss.data),
|
| 401 |
+
"attn_loss": utils.item(attn_loss.data),
|
| 402 |
+
}
|
| 403 |
+
|
| 404 |
+
if len(self.multitask_criterion) == 0:
|
| 405 |
+
return loss, sample_size, logging_output
|
| 406 |
+
|
| 407 |
+
# multitask
|
| 408 |
+
multitask_loss, multitask_log = self.get_multitask_loss(model, sample, extra)
|
| 409 |
+
loss += multitask_loss
|
| 410 |
+
logging_output["multitask"] = multitask_log
|
| 411 |
+
return loss, sample_size, logging_output
|
| 412 |
+
|
| 413 |
+
@classmethod
|
| 414 |
+
def reduce_metrics(cls, logging_outputs) -> None:
|
| 415 |
+
super().reduce_metrics(logging_outputs)
|
| 416 |
+
|
| 417 |
+
# inference metrics
|
| 418 |
+
if "targ_frames" in logging_outputs[0]:
|
| 419 |
+
n = sum(log.get("norm_frames", 0) for log in logging_outputs)
|
| 420 |
+
for key, new_key in [
|
| 421 |
+
("mcd_loss", "mcd_loss"),
|
| 422 |
+
("pred_frames", "pred_ratio"),
|
| 423 |
+
("nins", "ins_rate"),
|
| 424 |
+
("ndel", "del_rate"),
|
| 425 |
+
]:
|
| 426 |
+
val = sum(log.get(key, 0) for log in logging_outputs)
|
| 427 |
+
metrics.log_scalar(new_key, val / n, n, round=3)
|
| 428 |
+
|
| 429 |
+
if "multitask" not in logging_outputs[0]:
|
| 430 |
+
return
|
| 431 |
+
|
| 432 |
+
MultitaskCriterion.reduce_metrics(logging_outputs)
|
| 433 |
+
|
| 434 |
+
|
| 435 |
+
@register_criterion("speech_to_spectrogram_2pass", dataclass=Tacotron2CriterionConfig)
|
| 436 |
+
class SpeechToSpectrogram2passMultitaskTaskCriterion(
|
| 437 |
+
SpeechToSpectrogramMultitaskTaskCriterion
|
| 438 |
+
):
|
| 439 |
+
def __init__(
|
| 440 |
+
self,
|
| 441 |
+
task,
|
| 442 |
+
sentence_avg,
|
| 443 |
+
use_guided_attention_loss,
|
| 444 |
+
guided_attention_loss_sigma,
|
| 445 |
+
bce_pos_weight,
|
| 446 |
+
ctc_weight,
|
| 447 |
+
):
|
| 448 |
+
super().__init__(
|
| 449 |
+
task,
|
| 450 |
+
sentence_avg,
|
| 451 |
+
use_guided_attention_loss,
|
| 452 |
+
guided_attention_loss_sigma,
|
| 453 |
+
bce_pos_weight,
|
| 454 |
+
ctc_weight,
|
| 455 |
+
)
|
| 456 |
+
|
| 457 |
+
def forward(self, model, sample, reduction="mean"):
|
| 458 |
+
bsz, max_len, _ = sample["target"].size()
|
| 459 |
+
feat_tgt = sample["target"]
|
| 460 |
+
feat_len = sample["target_lengths"].view(bsz, 1).expand(-1, max_len)
|
| 461 |
+
eos_tgt = torch.arange(max_len).to(sample["target"].device)
|
| 462 |
+
eos_tgt = eos_tgt.view(1, max_len).expand(bsz, -1)
|
| 463 |
+
eos_tgt = (eos_tgt == (feat_len - 1)).float()
|
| 464 |
+
|
| 465 |
+
feat_out, eos_out, extra = model(
|
| 466 |
+
src_tokens=sample["net_input"]["src_tokens"],
|
| 467 |
+
src_lengths=sample["net_input"]["src_lengths"],
|
| 468 |
+
prev_output_tokens=sample["net_input"]["prev_output_tokens"],
|
| 469 |
+
prev_output_tokens_mt=sample["multitask"][model.mt_task_name]["net_input"][
|
| 470 |
+
"prev_output_tokens"
|
| 471 |
+
],
|
| 472 |
+
tgt_speaker=sample["net_input"]["tgt_speaker"],
|
| 473 |
+
target_lengths=sample["target_lengths"],
|
| 474 |
+
return_all_hiddens=True,
|
| 475 |
+
)
|
| 476 |
+
|
| 477 |
+
l1_loss, mse_loss, eos_loss = self.compute_loss(
|
| 478 |
+
extra["feature_out"],
|
| 479 |
+
feat_out,
|
| 480 |
+
eos_out,
|
| 481 |
+
feat_tgt,
|
| 482 |
+
eos_tgt,
|
| 483 |
+
sample["target_lengths"],
|
| 484 |
+
reduction,
|
| 485 |
+
)
|
| 486 |
+
attn_loss = torch.tensor(0.0).type_as(l1_loss)
|
| 487 |
+
if self.guided_attn is not None:
|
| 488 |
+
attn_loss = self.guided_attn(
|
| 489 |
+
extra["attn"],
|
| 490 |
+
sample["net_input"]["src_lengths"],
|
| 491 |
+
sample["target_lengths"],
|
| 492 |
+
reduction,
|
| 493 |
+
)
|
| 494 |
+
loss = (
|
| 495 |
+
l1_loss + mse_loss + eos_loss + attn_loss
|
| 496 |
+
) # do not include ctc loss as there's no text target
|
| 497 |
+
|
| 498 |
+
sample_size = sample["nsentences"] if self.sentence_avg else sample["ntokens"]
|
| 499 |
+
logging_output = {
|
| 500 |
+
"loss": utils.item(loss.data),
|
| 501 |
+
"ntokens": sample["ntokens"],
|
| 502 |
+
"nsentences": sample["nsentences"],
|
| 503 |
+
"sample_size": sample_size,
|
| 504 |
+
"l1_loss": utils.item(l1_loss.data),
|
| 505 |
+
"mse_loss": utils.item(mse_loss.data),
|
| 506 |
+
"eos_loss": utils.item(eos_loss.data),
|
| 507 |
+
"attn_loss": utils.item(attn_loss.data),
|
| 508 |
+
}
|
| 509 |
+
|
| 510 |
+
if len(self.multitask_criterion) == 0:
|
| 511 |
+
return loss, sample_size, logging_output
|
| 512 |
+
|
| 513 |
+
# multitask
|
| 514 |
+
multitask_loss, multitask_log = self.get_multitask_loss(model, sample, extra)
|
| 515 |
+
loss += multitask_loss
|
| 516 |
+
logging_output["multitask"] = multitask_log
|
| 517 |
+
return loss, sample_size, logging_output
|
data/fairseq/fairseq/criterions/speech_ulm_criterion.py
ADDED
|
@@ -0,0 +1,126 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) Facebook, Inc. and its affiliates.
|
| 2 |
+
#
|
| 3 |
+
# This source code is licensed under the MIT license found in the
|
| 4 |
+
# LICENSE file in the root directory of this source tree.
|
| 5 |
+
|
| 6 |
+
import torch
|
| 7 |
+
from dataclasses import dataclass, field
|
| 8 |
+
|
| 9 |
+
import torch.nn.functional as F
|
| 10 |
+
from fairseq.logging import metrics
|
| 11 |
+
from fairseq.tasks import FairseqTask
|
| 12 |
+
from fairseq.criterions import FairseqCriterion, register_criterion
|
| 13 |
+
from fairseq.dataclass import FairseqDataclass
|
| 14 |
+
from omegaconf import II
|
| 15 |
+
|
| 16 |
+
|
| 17 |
+
@dataclass
|
| 18 |
+
class SpeechUnitLmCriterionConfig(FairseqDataclass):
|
| 19 |
+
sentence_avg: bool = II("optimization.sentence_avg")
|
| 20 |
+
loss_weights: str = field(
|
| 21 |
+
default="1.;0.0;0.0",
|
| 22 |
+
metadata={
|
| 23 |
+
"help": "Weights of the losses that correspond to token, duration, and F0 streams"
|
| 24 |
+
},
|
| 25 |
+
)
|
| 26 |
+
discrete_duration: bool = II("task.discrete_duration")
|
| 27 |
+
discrete_f0: bool = II("task.discrete_f0")
|
| 28 |
+
|
| 29 |
+
|
| 30 |
+
def mae_loss(pred, targ, mask, reduce=True):
|
| 31 |
+
if pred.ndim == 3:
|
| 32 |
+
pred = pred.squeeze(2)
|
| 33 |
+
else:
|
| 34 |
+
assert pred.ndim == 2
|
| 35 |
+
loss = (pred.float() - targ.float()).abs() * (~mask).float()
|
| 36 |
+
loss = loss.sum() if reduce else loss.view(-1)
|
| 37 |
+
return loss
|
| 38 |
+
|
| 39 |
+
|
| 40 |
+
def nll_loss(pred, targ, mask, reduce=True):
|
| 41 |
+
lprob = F.log_softmax(pred, dim=-1)
|
| 42 |
+
loss = F.nll_loss(lprob.view(-1, lprob.size(-1)), targ.view(-1), reduction="none")
|
| 43 |
+
loss = loss * (~mask).float().view(-1)
|
| 44 |
+
loss = loss.sum() if reduce else loss.view(-1)
|
| 45 |
+
return loss
|
| 46 |
+
|
| 47 |
+
|
| 48 |
+
@register_criterion("speech_unit_lm_criterion", dataclass=SpeechUnitLmCriterionConfig)
|
| 49 |
+
class SpeechUnitLmCriterion(FairseqCriterion):
|
| 50 |
+
def __init__(self, cfg: SpeechUnitLmCriterionConfig, task: FairseqTask):
|
| 51 |
+
super().__init__(task)
|
| 52 |
+
self.sentence_avg = cfg.sentence_avg
|
| 53 |
+
self.weights = torch.tensor([float(w) for w in cfg.loss_weights.split(";")])
|
| 54 |
+
assert self.weights.size(0) == 3
|
| 55 |
+
assert (self.weights >= 0.0).all()
|
| 56 |
+
|
| 57 |
+
self.dur_loss_fn = nll_loss if cfg.discrete_duration else mae_loss
|
| 58 |
+
self.f0_loss_fn = nll_loss if cfg.discrete_f0 else mae_loss
|
| 59 |
+
|
| 60 |
+
def forward(self, model, sample, reduce=True):
|
| 61 |
+
"""Compute the loss for the given sample.
|
| 62 |
+
|
| 63 |
+
Returns a tuple with three elements:
|
| 64 |
+
1) the loss
|
| 65 |
+
2) the sample size, which is used as the denominator for the gradient
|
| 66 |
+
3) logging outputs to display while training
|
| 67 |
+
"""
|
| 68 |
+
net_output = model(**sample["net_input"])
|
| 69 |
+
|
| 70 |
+
token_loss = nll_loss(
|
| 71 |
+
net_output["token"], sample["target"], sample["mask"], reduce
|
| 72 |
+
)
|
| 73 |
+
dur_loss = self.dur_loss_fn(
|
| 74 |
+
net_output["duration"],
|
| 75 |
+
sample["dur_target"],
|
| 76 |
+
sample["dur_mask"],
|
| 77 |
+
reduce,
|
| 78 |
+
)
|
| 79 |
+
f0_loss = self.f0_loss_fn(
|
| 80 |
+
net_output["f0"],
|
| 81 |
+
sample["f0_target"],
|
| 82 |
+
sample["f0_mask"],
|
| 83 |
+
reduce,
|
| 84 |
+
)
|
| 85 |
+
loss = self.weights.to(token_loss.device) * torch.stack(
|
| 86 |
+
[token_loss, dur_loss, f0_loss], dim=-1
|
| 87 |
+
)
|
| 88 |
+
loss = loss.sum() if reduce else loss.sum(-1)
|
| 89 |
+
|
| 90 |
+
sample_size = (
|
| 91 |
+
sample["target"].size(0) if self.sentence_avg else sample["ntokens"]
|
| 92 |
+
)
|
| 93 |
+
logging_output = {
|
| 94 |
+
"loss": loss.detach().sum().item(),
|
| 95 |
+
"token_loss": token_loss.detach().sum().item(),
|
| 96 |
+
"dur_loss": dur_loss.detach().sum().item(),
|
| 97 |
+
"f0_loss": f0_loss.detach().sum().item(),
|
| 98 |
+
"ntokens": sample["ntokens"],
|
| 99 |
+
"nsentences": sample["target"].size(0),
|
| 100 |
+
"sample_size": sample_size,
|
| 101 |
+
}
|
| 102 |
+
return loss, sample_size, logging_output
|
| 103 |
+
|
| 104 |
+
@staticmethod
|
| 105 |
+
def reduce_metrics(logging_outputs) -> None:
|
| 106 |
+
"""Aggregate logging outputs from data parallel training."""
|
| 107 |
+
loss_sum = sum(log.get("loss", 0) for log in logging_outputs)
|
| 108 |
+
token_loss_sum = sum(log.get("token_loss", 0) for log in logging_outputs)
|
| 109 |
+
dur_loss_sum = sum(log.get("dur_loss", 0) for log in logging_outputs)
|
| 110 |
+
f0_loss_sum = sum(log.get("f0_loss", 0) for log in logging_outputs)
|
| 111 |
+
|
| 112 |
+
sample_size = sum(log.get("sample_size", 0) for log in logging_outputs)
|
| 113 |
+
|
| 114 |
+
metrics.log_scalar("loss", loss_sum / sample_size, sample_size, round=3)
|
| 115 |
+
|
| 116 |
+
metrics.log_scalar(
|
| 117 |
+
"token_loss", token_loss_sum / sample_size, sample_size, round=3
|
| 118 |
+
)
|
| 119 |
+
|
| 120 |
+
metrics.log_scalar("dur_loss", dur_loss_sum / sample_size, sample_size, round=3)
|
| 121 |
+
|
| 122 |
+
metrics.log_scalar("f0_loss", f0_loss_sum / sample_size, sample_size, round=3)
|
| 123 |
+
|
| 124 |
+
@staticmethod
|
| 125 |
+
def logging_outputs_can_be_summed() -> bool:
|
| 126 |
+
return True
|
data/fairseq/fairseq/criterions/tacotron2_loss.py
ADDED
|
@@ -0,0 +1,227 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) 2017-present, Facebook, Inc.
|
| 2 |
+
# All rights reserved.
|
| 3 |
+
#
|
| 4 |
+
# This source code is licensed under the license found in the LICENSE file in
|
| 5 |
+
# the root directory of this source tree. An additional grant of patent rights
|
| 6 |
+
# can be found in the PATENTS file in the same directory.
|
| 7 |
+
|
| 8 |
+
import logging
|
| 9 |
+
from dataclasses import dataclass, field
|
| 10 |
+
from functools import lru_cache
|
| 11 |
+
from typing import Any, Dict, List
|
| 12 |
+
|
| 13 |
+
import torch
|
| 14 |
+
import torch.nn.functional as F
|
| 15 |
+
from omegaconf import II
|
| 16 |
+
|
| 17 |
+
from fairseq import utils
|
| 18 |
+
from fairseq.logging import metrics
|
| 19 |
+
from fairseq.criterions import FairseqCriterion, register_criterion
|
| 20 |
+
from fairseq.data.data_utils import lengths_to_mask
|
| 21 |
+
from fairseq.dataclass import FairseqDataclass
|
| 22 |
+
|
| 23 |
+
logger = logging.getLogger(__name__)
|
| 24 |
+
|
| 25 |
+
|
| 26 |
+
@dataclass
|
| 27 |
+
class Tacotron2CriterionConfig(FairseqDataclass):
|
| 28 |
+
bce_pos_weight: float = field(
|
| 29 |
+
default=1.0,
|
| 30 |
+
metadata={"help": "weight of positive examples for BCE loss"},
|
| 31 |
+
)
|
| 32 |
+
use_guided_attention_loss: bool = field(
|
| 33 |
+
default=False,
|
| 34 |
+
metadata={"help": "use guided attention loss"},
|
| 35 |
+
)
|
| 36 |
+
guided_attention_loss_sigma: float = field(
|
| 37 |
+
default=0.4,
|
| 38 |
+
metadata={"help": "weight of positive examples for BCE loss"},
|
| 39 |
+
)
|
| 40 |
+
ctc_weight: float = field(default=0.0, metadata={"help": "weight for CTC loss"})
|
| 41 |
+
sentence_avg: bool = II("optimization.sentence_avg")
|
| 42 |
+
|
| 43 |
+
|
| 44 |
+
class GuidedAttentionLoss(torch.nn.Module):
|
| 45 |
+
"""
|
| 46 |
+
Efficiently Trainable Text-to-Speech System Based on Deep Convolutional
|
| 47 |
+
Networks with Guided Attention (https://arxiv.org/abs/1710.08969)
|
| 48 |
+
"""
|
| 49 |
+
|
| 50 |
+
def __init__(self, sigma):
|
| 51 |
+
super().__init__()
|
| 52 |
+
self.sigma = sigma
|
| 53 |
+
|
| 54 |
+
@staticmethod
|
| 55 |
+
@lru_cache(maxsize=8)
|
| 56 |
+
def _get_weight(s_len, t_len, sigma):
|
| 57 |
+
grid_x, grid_y = torch.meshgrid(torch.arange(t_len), torch.arange(s_len))
|
| 58 |
+
grid_x = grid_x.to(s_len.device)
|
| 59 |
+
grid_y = grid_y.to(s_len.device)
|
| 60 |
+
w = (grid_y.float() / s_len - grid_x.float() / t_len) ** 2
|
| 61 |
+
return 1.0 - torch.exp(-w / (2 * (sigma**2)))
|
| 62 |
+
|
| 63 |
+
def _get_weights(self, src_lens, tgt_lens):
|
| 64 |
+
bsz, max_s_len, max_t_len = len(src_lens), max(src_lens), max(tgt_lens)
|
| 65 |
+
weights = torch.zeros((bsz, max_t_len, max_s_len))
|
| 66 |
+
for i, (s_len, t_len) in enumerate(zip(src_lens, tgt_lens)):
|
| 67 |
+
weights[i, :t_len, :s_len] = self._get_weight(s_len, t_len, self.sigma)
|
| 68 |
+
return weights
|
| 69 |
+
|
| 70 |
+
@staticmethod
|
| 71 |
+
def _get_masks(src_lens, tgt_lens):
|
| 72 |
+
in_masks = lengths_to_mask(src_lens)
|
| 73 |
+
out_masks = lengths_to_mask(tgt_lens)
|
| 74 |
+
return out_masks.unsqueeze(2) & in_masks.unsqueeze(1)
|
| 75 |
+
|
| 76 |
+
def forward(self, attn, src_lens, tgt_lens, reduction="mean"):
|
| 77 |
+
weights = self._get_weights(src_lens, tgt_lens).to(attn.device)
|
| 78 |
+
masks = self._get_masks(src_lens, tgt_lens).to(attn.device)
|
| 79 |
+
loss = (weights * attn.transpose(1, 2)).masked_select(masks)
|
| 80 |
+
loss = torch.sum(loss) if reduction == "sum" else torch.mean(loss)
|
| 81 |
+
return loss
|
| 82 |
+
|
| 83 |
+
|
| 84 |
+
@register_criterion("tacotron2", dataclass=Tacotron2CriterionConfig)
|
| 85 |
+
class Tacotron2Criterion(FairseqCriterion):
|
| 86 |
+
def __init__(
|
| 87 |
+
self,
|
| 88 |
+
task,
|
| 89 |
+
sentence_avg,
|
| 90 |
+
use_guided_attention_loss,
|
| 91 |
+
guided_attention_loss_sigma,
|
| 92 |
+
bce_pos_weight,
|
| 93 |
+
ctc_weight,
|
| 94 |
+
):
|
| 95 |
+
super().__init__(task)
|
| 96 |
+
self.sentence_avg = sentence_avg
|
| 97 |
+
self.bce_pos_weight = bce_pos_weight
|
| 98 |
+
|
| 99 |
+
self.guided_attn = None
|
| 100 |
+
if use_guided_attention_loss:
|
| 101 |
+
self.guided_attn = GuidedAttentionLoss(guided_attention_loss_sigma)
|
| 102 |
+
self.ctc_weight = ctc_weight
|
| 103 |
+
|
| 104 |
+
def forward(self, model, sample, reduction="mean"):
|
| 105 |
+
bsz, max_len, _ = sample["target"].size()
|
| 106 |
+
feat_tgt = sample["target"]
|
| 107 |
+
feat_len = sample["target_lengths"].view(bsz, 1).expand(-1, max_len)
|
| 108 |
+
eos_tgt = torch.arange(max_len).to(sample["target"].device)
|
| 109 |
+
eos_tgt = eos_tgt.view(1, max_len).expand(bsz, -1)
|
| 110 |
+
eos_tgt = (eos_tgt == (feat_len - 1)).float()
|
| 111 |
+
src_tokens = sample["net_input"]["src_tokens"]
|
| 112 |
+
src_lens = sample["net_input"]["src_lengths"]
|
| 113 |
+
tgt_lens = sample["target_lengths"]
|
| 114 |
+
|
| 115 |
+
feat_out, eos_out, extra = model(
|
| 116 |
+
src_tokens=src_tokens,
|
| 117 |
+
src_lengths=src_lens,
|
| 118 |
+
prev_output_tokens=sample["net_input"]["prev_output_tokens"],
|
| 119 |
+
incremental_state=None,
|
| 120 |
+
target_lengths=tgt_lens,
|
| 121 |
+
speaker=sample["speaker"],
|
| 122 |
+
)
|
| 123 |
+
|
| 124 |
+
l1_loss, mse_loss, eos_loss = self.compute_loss(
|
| 125 |
+
extra["feature_out"],
|
| 126 |
+
feat_out,
|
| 127 |
+
eos_out,
|
| 128 |
+
feat_tgt,
|
| 129 |
+
eos_tgt,
|
| 130 |
+
tgt_lens,
|
| 131 |
+
reduction,
|
| 132 |
+
)
|
| 133 |
+
attn_loss = torch.tensor(0.0).type_as(l1_loss)
|
| 134 |
+
if self.guided_attn is not None:
|
| 135 |
+
attn_loss = self.guided_attn(extra["attn"], src_lens, tgt_lens, reduction)
|
| 136 |
+
ctc_loss = torch.tensor(0.0).type_as(l1_loss)
|
| 137 |
+
if self.ctc_weight > 0.0:
|
| 138 |
+
net_output = (feat_out, eos_out, extra)
|
| 139 |
+
lprobs = model.get_normalized_probs(net_output, log_probs=True)
|
| 140 |
+
lprobs = lprobs.transpose(0, 1) # T x B x C
|
| 141 |
+
src_mask = lengths_to_mask(src_lens)
|
| 142 |
+
src_tokens_flat = src_tokens.masked_select(src_mask)
|
| 143 |
+
ctc_loss = (
|
| 144 |
+
F.ctc_loss(
|
| 145 |
+
lprobs,
|
| 146 |
+
src_tokens_flat,
|
| 147 |
+
tgt_lens,
|
| 148 |
+
src_lens,
|
| 149 |
+
reduction=reduction,
|
| 150 |
+
zero_infinity=True,
|
| 151 |
+
)
|
| 152 |
+
* self.ctc_weight
|
| 153 |
+
)
|
| 154 |
+
loss = l1_loss + mse_loss + eos_loss + attn_loss + ctc_loss
|
| 155 |
+
|
| 156 |
+
sample_size = sample["nsentences"] if self.sentence_avg else sample["ntokens"]
|
| 157 |
+
logging_output = {
|
| 158 |
+
"loss": utils.item(loss.data),
|
| 159 |
+
"ntokens": sample["ntokens"],
|
| 160 |
+
"nsentences": sample["nsentences"],
|
| 161 |
+
"sample_size": sample_size,
|
| 162 |
+
"l1_loss": utils.item(l1_loss.data),
|
| 163 |
+
"mse_loss": utils.item(mse_loss.data),
|
| 164 |
+
"eos_loss": utils.item(eos_loss.data),
|
| 165 |
+
"attn_loss": utils.item(attn_loss.data),
|
| 166 |
+
"ctc_loss": utils.item(ctc_loss.data),
|
| 167 |
+
}
|
| 168 |
+
return loss, sample_size, logging_output
|
| 169 |
+
|
| 170 |
+
def compute_loss(
|
| 171 |
+
self,
|
| 172 |
+
feat_out,
|
| 173 |
+
feat_out_post,
|
| 174 |
+
eos_out,
|
| 175 |
+
feat_tgt,
|
| 176 |
+
eos_tgt,
|
| 177 |
+
tgt_lens,
|
| 178 |
+
reduction="mean",
|
| 179 |
+
):
|
| 180 |
+
mask = lengths_to_mask(tgt_lens)
|
| 181 |
+
_eos_out = eos_out[mask].squeeze()
|
| 182 |
+
_eos_tgt = eos_tgt[mask]
|
| 183 |
+
_feat_tgt = feat_tgt[mask]
|
| 184 |
+
_feat_out = feat_out[mask]
|
| 185 |
+
_feat_out_post = feat_out_post[mask]
|
| 186 |
+
|
| 187 |
+
l1_loss = F.l1_loss(_feat_out, _feat_tgt, reduction=reduction) + F.l1_loss(
|
| 188 |
+
_feat_out_post, _feat_tgt, reduction=reduction
|
| 189 |
+
)
|
| 190 |
+
mse_loss = F.mse_loss(_feat_out, _feat_tgt, reduction=reduction) + F.mse_loss(
|
| 191 |
+
_feat_out_post, _feat_tgt, reduction=reduction
|
| 192 |
+
)
|
| 193 |
+
eos_loss = F.binary_cross_entropy_with_logits(
|
| 194 |
+
_eos_out,
|
| 195 |
+
_eos_tgt,
|
| 196 |
+
pos_weight=torch.tensor(self.bce_pos_weight),
|
| 197 |
+
reduction=reduction,
|
| 198 |
+
)
|
| 199 |
+
return l1_loss, mse_loss, eos_loss
|
| 200 |
+
|
| 201 |
+
@classmethod
|
| 202 |
+
def reduce_metrics(cls, logging_outputs: List[Dict[str, Any]]) -> None:
|
| 203 |
+
ns = [log.get("sample_size", 0) for log in logging_outputs]
|
| 204 |
+
ntot = sum(ns)
|
| 205 |
+
ws = [n / (ntot + 1e-8) for n in ns]
|
| 206 |
+
for key in ["loss", "l1_loss", "mse_loss", "eos_loss", "attn_loss", "ctc_loss"]:
|
| 207 |
+
vals = [log.get(key, 0) for log in logging_outputs]
|
| 208 |
+
val = sum(val * w for val, w in zip(vals, ws))
|
| 209 |
+
metrics.log_scalar(key, val, ntot, round=3)
|
| 210 |
+
metrics.log_scalar("sample_size", ntot, len(logging_outputs))
|
| 211 |
+
|
| 212 |
+
# inference metrics
|
| 213 |
+
if "targ_frames" not in logging_outputs[0]:
|
| 214 |
+
return
|
| 215 |
+
n = sum(log.get("targ_frames", 0) for log in logging_outputs)
|
| 216 |
+
for key, new_key in [
|
| 217 |
+
("mcd_loss", "mcd_loss"),
|
| 218 |
+
("pred_frames", "pred_ratio"),
|
| 219 |
+
("nins", "ins_rate"),
|
| 220 |
+
("ndel", "del_rate"),
|
| 221 |
+
]:
|
| 222 |
+
val = sum(log.get(key, 0) for log in logging_outputs)
|
| 223 |
+
metrics.log_scalar(new_key, val / n, n, round=3)
|
| 224 |
+
|
| 225 |
+
@staticmethod
|
| 226 |
+
def logging_outputs_can_be_summed() -> bool:
|
| 227 |
+
return False
|
data/fairseq/fairseq/criterions/wav2vec_criterion.py
ADDED
|
@@ -0,0 +1,231 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) Facebook, Inc. and its affiliates.
|
| 2 |
+
#
|
| 3 |
+
# This source code is licensed under the MIT license found in the
|
| 4 |
+
# LICENSE file in the root directory of this source tree.
|
| 5 |
+
|
| 6 |
+
import math
|
| 7 |
+
from dataclasses import dataclass, field
|
| 8 |
+
from typing import List, Optional
|
| 9 |
+
|
| 10 |
+
import torch
|
| 11 |
+
import torch.nn.functional as F
|
| 12 |
+
from fairseq import utils
|
| 13 |
+
from fairseq.logging import metrics
|
| 14 |
+
from fairseq.criterions import FairseqCriterion, register_criterion
|
| 15 |
+
from fairseq.dataclass import FairseqDataclass
|
| 16 |
+
from fairseq.logging.meters import safe_round
|
| 17 |
+
from fairseq.utils import is_xla_tensor
|
| 18 |
+
|
| 19 |
+
|
| 20 |
+
@dataclass
|
| 21 |
+
class Wav2VecCriterionConfig(FairseqDataclass):
|
| 22 |
+
infonce: bool = field(
|
| 23 |
+
default=False,
|
| 24 |
+
metadata={
|
| 25 |
+
"help": "if set, uses cross entropy instead of binary cross entropy (i.e. InfoNCE loss)"
|
| 26 |
+
},
|
| 27 |
+
)
|
| 28 |
+
loss_weights: Optional[List[float]] = field(
|
| 29 |
+
default=None,
|
| 30 |
+
metadata={"help": "weights for additional loss terms (not first one)"},
|
| 31 |
+
)
|
| 32 |
+
log_keys: List[str] = field(
|
| 33 |
+
default_factory=lambda: [],
|
| 34 |
+
metadata={"help": "output keys to log"},
|
| 35 |
+
)
|
| 36 |
+
|
| 37 |
+
|
| 38 |
+
@register_criterion("wav2vec", dataclass=Wav2VecCriterionConfig)
|
| 39 |
+
class Wav2vecCriterion(FairseqCriterion):
|
| 40 |
+
def __init__(self, task, infonce=False, loss_weights=None, log_keys=None):
|
| 41 |
+
super().__init__(task)
|
| 42 |
+
self.infonce = infonce
|
| 43 |
+
self.loss_weights = loss_weights
|
| 44 |
+
self.log_keys = [] if log_keys is None else log_keys
|
| 45 |
+
|
| 46 |
+
def forward(self, model, sample, reduce=True):
|
| 47 |
+
"""Compute the loss for the given sample.
|
| 48 |
+
|
| 49 |
+
Returns a tuple with three elements:
|
| 50 |
+
1) the loss
|
| 51 |
+
2) the sample size, which is used as the denominator for the gradient
|
| 52 |
+
3) logging outputs to display while training
|
| 53 |
+
"""
|
| 54 |
+
net_output = model(**sample["net_input"])
|
| 55 |
+
logits = model.get_logits(net_output).float()
|
| 56 |
+
target = model.get_targets(sample, net_output)
|
| 57 |
+
self.xla = is_xla_tensor(logits)
|
| 58 |
+
|
| 59 |
+
# XXX: handle weights on xla.
|
| 60 |
+
weights = None
|
| 61 |
+
if hasattr(model, "get_target_weights") and not self.infonce:
|
| 62 |
+
weights = model.get_target_weights(target, net_output)
|
| 63 |
+
if torch.is_tensor(weights):
|
| 64 |
+
weights = weights.float()
|
| 65 |
+
|
| 66 |
+
losses = []
|
| 67 |
+
|
| 68 |
+
reduction = "none" if ((not reduce) or self.xla) else "sum"
|
| 69 |
+
if self.infonce:
|
| 70 |
+
loss = F.cross_entropy(logits, target, reduction=reduction)
|
| 71 |
+
else:
|
| 72 |
+
loss = F.binary_cross_entropy_with_logits(
|
| 73 |
+
logits, target.float(), weights, reduction=reduction
|
| 74 |
+
)
|
| 75 |
+
|
| 76 |
+
if self.xla:
|
| 77 |
+
# tpu-comment: since dynamic shapes lead to recompilations on xla,
|
| 78 |
+
# we don't shrink tensors using mask_indices.
|
| 79 |
+
# Instead, we use mask indices to adjust loss.
|
| 80 |
+
mi = (
|
| 81 |
+
sample["net_input"]["mask_indices"]
|
| 82 |
+
.transpose(0, 1) # logits are transposed in `model.get_logits`
|
| 83 |
+
.reshape(logits.size(0))
|
| 84 |
+
)
|
| 85 |
+
loss = (loss * mi).sum() if reduce else (loss * mi)
|
| 86 |
+
|
| 87 |
+
if "sample_size" in sample:
|
| 88 |
+
sample_size = sample["sample_size"]
|
| 89 |
+
elif "mask_indices" in sample["net_input"]:
|
| 90 |
+
sample_size = sample["net_input"]["mask_indices"].sum()
|
| 91 |
+
else:
|
| 92 |
+
sample_size = target.numel() if self.infonce else target.long().sum().item()
|
| 93 |
+
losses.append(loss.detach().clone())
|
| 94 |
+
|
| 95 |
+
if self.loss_weights is not None:
|
| 96 |
+
assert hasattr(model, "get_extra_losses")
|
| 97 |
+
extra_losses = model.get_extra_losses(net_output)
|
| 98 |
+
if torch.is_tensor(extra_losses):
|
| 99 |
+
extra_losses = [extra_losses]
|
| 100 |
+
if len(self.loss_weights) == 1 and len(extra_losses) != 1:
|
| 101 |
+
self.loss_weights = [self.loss_weights[0]] * len(extra_losses)
|
| 102 |
+
assert len(extra_losses) == len(
|
| 103 |
+
self.loss_weights
|
| 104 |
+
), f"{len(extra_losses)}, {len(self.loss_weights)}"
|
| 105 |
+
for p, coef in zip(extra_losses, self.loss_weights):
|
| 106 |
+
if coef != 0 and p is not None:
|
| 107 |
+
p = coef * p.float() * sample_size
|
| 108 |
+
loss += p
|
| 109 |
+
losses.append(p)
|
| 110 |
+
|
| 111 |
+
logging_output = {
|
| 112 |
+
"loss": loss.item() if (reduce and not self.xla) else loss.detach(),
|
| 113 |
+
"ntokens": sample_size,
|
| 114 |
+
"nsentences": sample["id"].numel(),
|
| 115 |
+
"sample_size": sample_size,
|
| 116 |
+
}
|
| 117 |
+
|
| 118 |
+
for lk in self.log_keys:
|
| 119 |
+
# Only store "logits" and "target" for computing MAP and MAUC
|
| 120 |
+
# during validation
|
| 121 |
+
if lk == "logits":
|
| 122 |
+
if not self.training:
|
| 123 |
+
logging_output["logits"] = logits.cpu().numpy()
|
| 124 |
+
elif lk == "target":
|
| 125 |
+
if not self.training:
|
| 126 |
+
# If the targets have been mixed with the predictions of
|
| 127 |
+
# teacher models, find the original targets
|
| 128 |
+
if hasattr(model, "get_original_targets"):
|
| 129 |
+
original_target = model.get_original_targets(sample, net_output)
|
| 130 |
+
else:
|
| 131 |
+
original_target = target
|
| 132 |
+
logging_output["target"] = original_target.cpu().numpy()
|
| 133 |
+
elif lk in net_output:
|
| 134 |
+
value = net_output[lk]
|
| 135 |
+
if not is_xla_tensor(value):
|
| 136 |
+
value = float(value)
|
| 137 |
+
logging_output[lk] = value
|
| 138 |
+
|
| 139 |
+
if len(losses) > 1:
|
| 140 |
+
for i, l in enumerate(losses):
|
| 141 |
+
logging_output[f"loss_{i}"] = l.item() if not self.xla else l.detach()
|
| 142 |
+
|
| 143 |
+
if self.infonce:
|
| 144 |
+
with torch.no_grad():
|
| 145 |
+
if logits.numel() == 0:
|
| 146 |
+
corr = 0
|
| 147 |
+
count = 0
|
| 148 |
+
else:
|
| 149 |
+
assert logits.dim() > 1, logits.shape
|
| 150 |
+
max = logits.argmax(-1) == 0
|
| 151 |
+
min = logits.argmin(-1) == 0
|
| 152 |
+
if is_xla_tensor(logits):
|
| 153 |
+
max, min = max * mi, min * mi
|
| 154 |
+
both = max & min
|
| 155 |
+
corr = max.long().sum() - both.long().sum()
|
| 156 |
+
count = mi.sum()
|
| 157 |
+
else:
|
| 158 |
+
both = max & min
|
| 159 |
+
corr = max.long().sum().item() - both.long().sum().item()
|
| 160 |
+
count = float(max.numel())
|
| 161 |
+
|
| 162 |
+
logging_output["correct"] = corr
|
| 163 |
+
logging_output["count"] = count
|
| 164 |
+
|
| 165 |
+
return loss, sample_size, logging_output
|
| 166 |
+
|
| 167 |
+
@staticmethod
|
| 168 |
+
def reduce_metrics(logging_outputs) -> None:
|
| 169 |
+
"""Aggregate logging outputs from data parallel training."""
|
| 170 |
+
loss_sum = utils.item(sum(log.get("loss", 0) for log in logging_outputs))
|
| 171 |
+
ntokens = utils.item(sum(log.get("ntokens", 0) for log in logging_outputs))
|
| 172 |
+
nsentences = utils.item(
|
| 173 |
+
sum(log.get("nsentences", 0) for log in logging_outputs)
|
| 174 |
+
)
|
| 175 |
+
sample_size = utils.item(
|
| 176 |
+
sum(log.get("sample_size", 0) for log in logging_outputs)
|
| 177 |
+
)
|
| 178 |
+
|
| 179 |
+
metrics.log_scalar(
|
| 180 |
+
"loss", loss_sum / (sample_size or 1) / math.log(2), sample_size, round=3
|
| 181 |
+
)
|
| 182 |
+
metrics.log_scalar("ntokens", ntokens)
|
| 183 |
+
metrics.log_scalar("nsentences", nsentences)
|
| 184 |
+
|
| 185 |
+
correct = sum(log.get("correct", 0) for log in logging_outputs)
|
| 186 |
+
metrics.log_scalar("_correct", correct)
|
| 187 |
+
|
| 188 |
+
total = sum(log.get("count", 0) for log in logging_outputs)
|
| 189 |
+
metrics.log_scalar("_total", total)
|
| 190 |
+
|
| 191 |
+
if total > 0:
|
| 192 |
+
metrics.log_derived(
|
| 193 |
+
"accuracy",
|
| 194 |
+
lambda meters: safe_round(
|
| 195 |
+
meters["_correct"].sum / meters["_total"].sum, 5
|
| 196 |
+
)
|
| 197 |
+
if meters["_total"].sum > 0
|
| 198 |
+
else float("nan"),
|
| 199 |
+
)
|
| 200 |
+
|
| 201 |
+
builtin_keys = {
|
| 202 |
+
"loss",
|
| 203 |
+
"ntokens",
|
| 204 |
+
"nsentences",
|
| 205 |
+
"sample_size",
|
| 206 |
+
"correct",
|
| 207 |
+
"count",
|
| 208 |
+
}
|
| 209 |
+
|
| 210 |
+
for k in logging_outputs[0]:
|
| 211 |
+
if k not in builtin_keys:
|
| 212 |
+
val = sum(log.get(k, 0) for log in logging_outputs)
|
| 213 |
+
if k.startswith("loss"):
|
| 214 |
+
metrics.log_scalar(
|
| 215 |
+
k, val / (sample_size or 1) / math.log(2), sample_size, round=3
|
| 216 |
+
)
|
| 217 |
+
else:
|
| 218 |
+
metrics.log_scalar(k, val / len(logging_outputs), round=3)
|
| 219 |
+
|
| 220 |
+
# FIXME: revert when gather based xla reduction is implemented
|
| 221 |
+
# @staticmethod
|
| 222 |
+
# def logging_outputs_can_be_summed() -> bool:
|
| 223 |
+
def logging_outputs_can_be_summed(self) -> bool:
|
| 224 |
+
"""
|
| 225 |
+
Whether the logging outputs returned by `forward` can be summed
|
| 226 |
+
across workers prior to calling `reduce_metrics`. Setting this
|
| 227 |
+
to True will improves distributed training speed.
|
| 228 |
+
"""
|
| 229 |
+
# XXX: Gather based reduction not implemented for xla yet.
|
| 230 |
+
# So we fall to sum based reduction for xla.
|
| 231 |
+
return self.xla
|