sonithoiningombam commited on
Commit
d6424aa
·
verified ·
1 Parent(s): 12f413a

Add files using upload-large-folder tool

Browse files
This view is limited to 50 files because it contains too many changes.   See raw diff
Files changed (50) hide show
  1. data/fairseq/fairseq/criterions/__init__.py +36 -0
  2. data/fairseq/fairseq/criterions/__pycache__/__init__.cpython-38.pyc +0 -0
  3. data/fairseq/fairseq/criterions/__pycache__/adaptive_loss.cpython-38.pyc +0 -0
  4. data/fairseq/fairseq/criterions/__pycache__/composite_loss.cpython-38.pyc +0 -0
  5. data/fairseq/fairseq/criterions/__pycache__/cross_entropy.cpython-38.pyc +0 -0
  6. data/fairseq/fairseq/criterions/__pycache__/ctc.cpython-38.pyc +0 -0
  7. data/fairseq/fairseq/criterions/__pycache__/fairseq_criterion.cpython-38.pyc +0 -0
  8. data/fairseq/fairseq/criterions/__pycache__/fastspeech2_loss.cpython-38.pyc +0 -0
  9. data/fairseq/fairseq/criterions/__pycache__/hubert_criterion.cpython-38.pyc +0 -0
  10. data/fairseq/fairseq/criterions/__pycache__/label_smoothed_cross_entropy.cpython-38.pyc +0 -0
  11. data/fairseq/fairseq/criterions/__pycache__/label_smoothed_cross_entropy_latency_augmented.cpython-38.pyc +0 -0
  12. data/fairseq/fairseq/criterions/__pycache__/label_smoothed_cross_entropy_with_alignment.cpython-38.pyc +0 -0
  13. data/fairseq/fairseq/criterions/__pycache__/label_smoothed_cross_entropy_with_ctc.cpython-38.pyc +0 -0
  14. data/fairseq/fairseq/criterions/__pycache__/label_smoothed_cross_entropy_with_rdrop.cpython-38.pyc +0 -0
  15. data/fairseq/fairseq/criterions/__pycache__/legacy_masked_lm.cpython-38.pyc +0 -0
  16. data/fairseq/fairseq/criterions/__pycache__/masked_lm.cpython-38.pyc +0 -0
  17. data/fairseq/fairseq/criterions/__pycache__/model_criterion.cpython-38.pyc +0 -0
  18. data/fairseq/fairseq/criterions/__pycache__/nat_loss.cpython-38.pyc +0 -0
  19. data/fairseq/fairseq/criterions/__pycache__/sentence_prediction.cpython-38.pyc +0 -0
  20. data/fairseq/fairseq/criterions/__pycache__/sentence_prediction_adapters.cpython-38.pyc +0 -0
  21. data/fairseq/fairseq/criterions/__pycache__/sentence_ranking.cpython-38.pyc +0 -0
  22. data/fairseq/fairseq/criterions/__pycache__/speech_dlm_criterion.cpython-38.pyc +0 -0
  23. data/fairseq/fairseq/criterions/__pycache__/speech_to_speech_criterion.cpython-38.pyc +0 -0
  24. data/fairseq/fairseq/criterions/__pycache__/speech_ulm_criterion.cpython-38.pyc +0 -0
  25. data/fairseq/fairseq/criterions/__pycache__/tacotron2_loss.cpython-38.pyc +0 -0
  26. data/fairseq/fairseq/criterions/__pycache__/wav2vec_criterion.cpython-38.pyc +0 -0
  27. data/fairseq/fairseq/criterions/adaptive_loss.py +124 -0
  28. data/fairseq/fairseq/criterions/composite_loss.py +100 -0
  29. data/fairseq/fairseq/criterions/cross_entropy.py +91 -0
  30. data/fairseq/fairseq/criterions/ctc.py +325 -0
  31. data/fairseq/fairseq/criterions/fairseq_criterion.py +121 -0
  32. data/fairseq/fairseq/criterions/fastspeech2_loss.py +137 -0
  33. data/fairseq/fairseq/criterions/hubert_criterion.py +195 -0
  34. data/fairseq/fairseq/criterions/label_smoothed_cross_entropy.py +168 -0
  35. data/fairseq/fairseq/criterions/label_smoothed_cross_entropy_latency_augmented.py +221 -0
  36. data/fairseq/fairseq/criterions/label_smoothed_cross_entropy_with_alignment.py +131 -0
  37. data/fairseq/fairseq/criterions/label_smoothed_cross_entropy_with_ctc.py +97 -0
  38. data/fairseq/fairseq/criterions/label_smoothed_cross_entropy_with_rdrop.py +177 -0
  39. data/fairseq/fairseq/criterions/legacy_masked_lm.py +178 -0
  40. data/fairseq/fairseq/criterions/masked_lm.py +99 -0
  41. data/fairseq/fairseq/criterions/model_criterion.py +177 -0
  42. data/fairseq/fairseq/criterions/nat_loss.py +181 -0
  43. data/fairseq/fairseq/criterions/sentence_prediction.py +288 -0
  44. data/fairseq/fairseq/criterions/sentence_prediction_adapters.py +63 -0
  45. data/fairseq/fairseq/criterions/sentence_ranking.py +121 -0
  46. data/fairseq/fairseq/criterions/speech_dlm_criterion.py +335 -0
  47. data/fairseq/fairseq/criterions/speech_to_speech_criterion.py +517 -0
  48. data/fairseq/fairseq/criterions/speech_ulm_criterion.py +126 -0
  49. data/fairseq/fairseq/criterions/tacotron2_loss.py +227 -0
  50. data/fairseq/fairseq/criterions/wav2vec_criterion.py +231 -0
data/fairseq/fairseq/criterions/__init__.py ADDED
@@ -0,0 +1,36 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright (c) Facebook, Inc. and its affiliates.
2
+ #
3
+ # This source code is licensed under the MIT license found in the
4
+ # LICENSE file in the root directory of this source tree.
5
+ """isort:skip_file"""
6
+
7
+ import importlib
8
+ import os
9
+
10
+ from fairseq import registry
11
+ from fairseq.criterions.fairseq_criterion import ( # noqa
12
+ FairseqCriterion,
13
+ LegacyFairseqCriterion,
14
+ )
15
+ from omegaconf import DictConfig
16
+
17
+
18
+ (
19
+ build_criterion_,
20
+ register_criterion,
21
+ CRITERION_REGISTRY,
22
+ CRITERION_DATACLASS_REGISTRY,
23
+ ) = registry.setup_registry(
24
+ "--criterion", base_class=FairseqCriterion, default="cross_entropy"
25
+ )
26
+
27
+
28
+ def build_criterion(cfg: DictConfig, task, from_checkpoint=False):
29
+ return build_criterion_(cfg, task, from_checkpoint=from_checkpoint)
30
+
31
+
32
+ # automatically import any Python files in the criterions/ directory
33
+ for file in sorted(os.listdir(os.path.dirname(__file__))):
34
+ if file.endswith(".py") and not file.startswith("_"):
35
+ file_name = file[: file.find(".py")]
36
+ importlib.import_module("fairseq.criterions." + file_name)
data/fairseq/fairseq/criterions/__pycache__/__init__.cpython-38.pyc ADDED
Binary file (1.02 kB). View file
 
data/fairseq/fairseq/criterions/__pycache__/adaptive_loss.cpython-38.pyc ADDED
Binary file (4.81 kB). View file
 
data/fairseq/fairseq/criterions/__pycache__/composite_loss.cpython-38.pyc ADDED
Binary file (4.45 kB). View file
 
data/fairseq/fairseq/criterions/__pycache__/cross_entropy.cpython-38.pyc ADDED
Binary file (3.92 kB). View file
 
data/fairseq/fairseq/criterions/__pycache__/ctc.cpython-38.pyc ADDED
Binary file (8.68 kB). View file
 
data/fairseq/fairseq/criterions/__pycache__/fairseq_criterion.cpython-38.pyc ADDED
Binary file (4.49 kB). View file
 
data/fairseq/fairseq/criterions/__pycache__/fastspeech2_loss.cpython-38.pyc ADDED
Binary file (4.73 kB). View file
 
data/fairseq/fairseq/criterions/__pycache__/hubert_criterion.cpython-38.pyc ADDED
Binary file (6.85 kB). View file
 
data/fairseq/fairseq/criterions/__pycache__/label_smoothed_cross_entropy.cpython-38.pyc ADDED
Binary file (6.36 kB). View file
 
data/fairseq/fairseq/criterions/__pycache__/label_smoothed_cross_entropy_latency_augmented.cpython-38.pyc ADDED
Binary file (6 kB). View file
 
data/fairseq/fairseq/criterions/__pycache__/label_smoothed_cross_entropy_with_alignment.cpython-38.pyc ADDED
Binary file (4.85 kB). View file
 
data/fairseq/fairseq/criterions/__pycache__/label_smoothed_cross_entropy_with_ctc.cpython-38.pyc ADDED
Binary file (3.31 kB). View file
 
data/fairseq/fairseq/criterions/__pycache__/label_smoothed_cross_entropy_with_rdrop.cpython-38.pyc ADDED
Binary file (5.26 kB). View file
 
data/fairseq/fairseq/criterions/__pycache__/legacy_masked_lm.cpython-38.pyc ADDED
Binary file (5.64 kB). View file
 
data/fairseq/fairseq/criterions/__pycache__/masked_lm.cpython-38.pyc ADDED
Binary file (3.5 kB). View file
 
data/fairseq/fairseq/criterions/__pycache__/model_criterion.cpython-38.pyc ADDED
Binary file (5.84 kB). View file
 
data/fairseq/fairseq/criterions/__pycache__/nat_loss.cpython-38.pyc ADDED
Binary file (6.11 kB). View file
 
data/fairseq/fairseq/criterions/__pycache__/sentence_prediction.cpython-38.pyc ADDED
Binary file (8.7 kB). View file
 
data/fairseq/fairseq/criterions/__pycache__/sentence_prediction_adapters.cpython-38.pyc ADDED
Binary file (1.95 kB). View file
 
data/fairseq/fairseq/criterions/__pycache__/sentence_ranking.cpython-38.pyc ADDED
Binary file (4.69 kB). View file
 
data/fairseq/fairseq/criterions/__pycache__/speech_dlm_criterion.cpython-38.pyc ADDED
Binary file (9.17 kB). View file
 
data/fairseq/fairseq/criterions/__pycache__/speech_to_speech_criterion.cpython-38.pyc ADDED
Binary file (11.7 kB). View file
 
data/fairseq/fairseq/criterions/__pycache__/speech_ulm_criterion.cpython-38.pyc ADDED
Binary file (5.02 kB). View file
 
data/fairseq/fairseq/criterions/__pycache__/tacotron2_loss.cpython-38.pyc ADDED
Binary file (7.43 kB). View file
 
data/fairseq/fairseq/criterions/__pycache__/wav2vec_criterion.cpython-38.pyc ADDED
Binary file (6.61 kB). View file
 
data/fairseq/fairseq/criterions/adaptive_loss.py ADDED
@@ -0,0 +1,124 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright (c) Facebook, Inc. and its affiliates.
2
+ #
3
+ # This source code is licensed under the MIT license found in the
4
+ # LICENSE file in the root directory of this source tree.
5
+
6
+ import math
7
+ from dataclasses import dataclass
8
+
9
+ import torch.nn.functional as F
10
+ from fairseq import utils
11
+ from fairseq.logging import metrics
12
+ from fairseq.criterions import FairseqCriterion, register_criterion
13
+ from fairseq.dataclass import FairseqDataclass
14
+ from fairseq.dataclass.constants import DDP_BACKEND_CHOICES
15
+ from omegaconf import II
16
+
17
+
18
+ @dataclass
19
+ class AdaptiveLossConfig(FairseqDataclass):
20
+ sentence_avg: bool = II("optimization.sentence_avg")
21
+ ddp_backend: DDP_BACKEND_CHOICES = II("distributed_training.ddp_backend")
22
+
23
+
24
+ @register_criterion("adaptive_loss", dataclass=AdaptiveLossConfig)
25
+ class AdaptiveLoss(FairseqCriterion):
26
+ """This is an implementation of the loss function accompanying the adaptive softmax approximation for
27
+ graphical processing units (GPU), described in the paper "Efficient softmax approximation for GPUs"
28
+ (http://arxiv.org/abs/1609.04309)."""
29
+
30
+ def __init__(self, task, sentence_avg):
31
+ super().__init__(task)
32
+ self.sentence_avg = sentence_avg
33
+
34
+ @classmethod
35
+ def build_criterion(cls, cfg: AdaptiveLossConfig, task):
36
+ if cfg.ddp_backend in {"c10d", "pytorch_ddp"}:
37
+ raise Exception(
38
+ "AdaptiveLoss is not compatible with the PyTorch "
39
+ "version of DistributedDataParallel. Please use "
40
+ "`--ddp-backend=legacy_ddp` instead."
41
+ )
42
+ return cls(task, cfg.sentence_avg)
43
+
44
+ def forward(self, model, sample, reduce=True):
45
+ """Compute the loss for the given sample.
46
+
47
+ Returns a tuple with three elements:
48
+ 1) the loss
49
+ 2) the sample size, which is used as the denominator for the gradient
50
+ 3) logging outputs to display while training
51
+ """
52
+
53
+ assert (
54
+ hasattr(model.decoder, "adaptive_softmax")
55
+ and model.decoder.adaptive_softmax is not None
56
+ )
57
+ adaptive_softmax = model.decoder.adaptive_softmax
58
+
59
+ net_output = model(**sample["net_input"])
60
+ orig_target = model.get_targets(sample, net_output)
61
+
62
+ nsentences = orig_target.size(0)
63
+ orig_target = orig_target.view(-1)
64
+
65
+ bsz = orig_target.size(0)
66
+
67
+ logits, target = adaptive_softmax(net_output[0], orig_target)
68
+ assert len(target) == len(logits)
69
+
70
+ loss = net_output[0].new(1 if reduce else bsz).zero_()
71
+
72
+ for i in range(len(target)):
73
+ if target[i] is not None:
74
+ assert target[i].min() >= 0 and target[i].max() <= logits[i].size(1)
75
+ loss += F.cross_entropy(
76
+ logits[i],
77
+ target[i],
78
+ ignore_index=self.padding_idx,
79
+ reduction="sum" if reduce else "none",
80
+ )
81
+
82
+ orig = utils.strip_pad(orig_target, self.padding_idx)
83
+ ntokens = orig.numel()
84
+ sample_size = sample["target"].size(0) if self.sentence_avg else ntokens
85
+ logging_output = {
86
+ "loss": loss.data,
87
+ "ntokens": ntokens,
88
+ "nsentences": nsentences,
89
+ "sample_size": sample_size,
90
+ }
91
+ return loss, sample_size, logging_output
92
+
93
+ @staticmethod
94
+ def reduce_metrics(logging_outputs) -> None:
95
+ """Aggregate logging outputs from data parallel training."""
96
+ loss_sum = utils.item(sum(log.get("loss", 0) for log in logging_outputs))
97
+ ntokens = utils.item(sum(log.get("ntokens", 0) for log in logging_outputs))
98
+ sample_size = utils.item(
99
+ sum(log.get("sample_size", 0) for log in logging_outputs)
100
+ )
101
+
102
+ metrics.log_scalar(
103
+ "loss", loss_sum / sample_size / math.log(2), sample_size, round=3
104
+ )
105
+ if sample_size != ntokens:
106
+ metrics.log_scalar(
107
+ "nll_loss", loss_sum / ntokens / math.log(2), ntokens, round=3
108
+ )
109
+ metrics.log_derived(
110
+ "ppl", lambda meters: utils.get_perplexity(meters["nll_loss"].avg)
111
+ )
112
+ else:
113
+ metrics.log_derived(
114
+ "ppl", lambda meters: utils.get_perplexity(meters["loss"].avg)
115
+ )
116
+
117
+ @staticmethod
118
+ def logging_outputs_can_be_summed() -> bool:
119
+ """
120
+ Whether the logging outputs returned by `forward` can be summed
121
+ across workers prior to calling `reduce_metrics`. Setting this
122
+ to True will improves distributed training speed.
123
+ """
124
+ return True
data/fairseq/fairseq/criterions/composite_loss.py ADDED
@@ -0,0 +1,100 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright (c) Facebook, Inc. and its affiliates.
2
+ #
3
+ # This source code is licensed under the MIT license found in the
4
+ # LICENSE file in the root directory of this source tree.
5
+
6
+ from fairseq import utils
7
+ from fairseq.criterions import LegacyFairseqCriterion, register_criterion
8
+ from torch import nn
9
+
10
+
11
+ @register_criterion("composite_loss")
12
+ class CompositeLoss(LegacyFairseqCriterion):
13
+ """This is a composite loss that, given a list of model outputs and a list of targets,
14
+ computes an average of losses for each output-target pair"""
15
+
16
+ def __init__(self, args, task):
17
+ super().__init__(args, task)
18
+ self.underlying_criterion = args.underlying_criterion
19
+
20
+ @staticmethod
21
+ def add_args(parser):
22
+ """Add criterion-specific arguments to the parser."""
23
+ # fmt: off
24
+ parser.add_argument('--underlying-criterion', type=str, metavar='VAL', required=True,
25
+ help='underlying criterion to use for the composite loss')
26
+ # fmt: on
27
+
28
+ @staticmethod
29
+ def build_underlying_criterion(args, task):
30
+ saved_criterion = args.criterion
31
+ args.criterion = args.underlying_criterion
32
+ assert saved_criterion != args.underlying_criterion
33
+ underlying_criterion = task.build_criterion(args)
34
+ args.criterion = saved_criterion
35
+ return underlying_criterion
36
+
37
+ @classmethod
38
+ def build_criterion(cls, args, task):
39
+ underlying_criterion = CompositeLoss.build_underlying_criterion(args, task)
40
+
41
+ class FakeModel(nn.Module):
42
+ def __init__(self, model, net_out, target):
43
+ super().__init__()
44
+ self.model = model
45
+ self.net_out = net_out
46
+ self.target = target
47
+
48
+ def forward(self, **unused):
49
+ return self.net_out
50
+
51
+ def get_normalized_probs(self, net_output, log_probs, sample=None):
52
+ return self.model.get_normalized_probs(
53
+ net_output, log_probs, sample=sample
54
+ )
55
+
56
+ def get_targets(self, *unused):
57
+ return self.target
58
+
59
+ @property
60
+ def decoder(self):
61
+ return self.model.decoder
62
+
63
+ class _CompositeLoss(LegacyFairseqCriterion):
64
+ def __init__(self, args, task, underlying_criterion):
65
+ super().__init__(args, task)
66
+ self.underlying_criterion = underlying_criterion
67
+
68
+ def forward(self, model, sample, reduce=True):
69
+ net_outputs = model(**sample["net_input"])
70
+ targets = sample["target"]
71
+
72
+ bsz = targets[0].size(0)
73
+ loss = net_outputs[0][0].new(1 if reduce else bsz).float().zero_()
74
+
75
+ sample_size = 0
76
+ logging_output = {}
77
+ for o, t in zip(net_outputs[0], targets):
78
+ m = FakeModel(model, (o, net_outputs[1]), t)
79
+ sample["target"] = t
80
+ l, ss, logging_output = self.underlying_criterion(m, sample, reduce)
81
+ loss += l
82
+ sample_size += ss
83
+
84
+ loss.div_(len(targets))
85
+ sample_size /= len(targets)
86
+
87
+ logging_output["loss"] = utils.item(loss.data) if reduce else loss.data
88
+ return loss, sample_size, logging_output
89
+
90
+ @staticmethod
91
+ def aggregate_logging_outputs(logging_outputs):
92
+ return underlying_criterion.__class__.aggregate_logging_outputs(
93
+ logging_outputs
94
+ )
95
+
96
+ @staticmethod
97
+ def reduce_metrics(logging_outputs) -> None:
98
+ underlying_criterion.__class__.reduce_metrics(logging_outputs)
99
+
100
+ return _CompositeLoss(args, task, underlying_criterion)
data/fairseq/fairseq/criterions/cross_entropy.py ADDED
@@ -0,0 +1,91 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright (c) Facebook, Inc. and its affiliates.
2
+ #
3
+ # This source code is licensed under the MIT license found in the
4
+ # LICENSE file in the root directory of this source tree.
5
+
6
+ import math
7
+ from dataclasses import dataclass
8
+
9
+ import torch.nn.functional as F
10
+ from fairseq import utils
11
+ from fairseq.logging import metrics
12
+ from fairseq.criterions import FairseqCriterion, register_criterion
13
+ from fairseq.dataclass import FairseqDataclass
14
+ from omegaconf import II
15
+
16
+
17
+ @dataclass
18
+ class CrossEntropyCriterionConfig(FairseqDataclass):
19
+ sentence_avg: bool = II("optimization.sentence_avg")
20
+
21
+
22
+ @register_criterion("cross_entropy", dataclass=CrossEntropyCriterionConfig)
23
+ class CrossEntropyCriterion(FairseqCriterion):
24
+ def __init__(self, task, sentence_avg):
25
+ super().__init__(task)
26
+ self.sentence_avg = sentence_avg
27
+
28
+ def forward(self, model, sample, reduce=True):
29
+ """Compute the loss for the given sample.
30
+
31
+ Returns a tuple with three elements:
32
+ 1) the loss
33
+ 2) the sample size, which is used as the denominator for the gradient
34
+ 3) logging outputs to display while training
35
+ """
36
+ net_output = model(**sample["net_input"])
37
+ loss, _ = self.compute_loss(model, net_output, sample, reduce=reduce)
38
+ sample_size = (
39
+ sample["target"].size(0) if self.sentence_avg else sample["ntokens"]
40
+ )
41
+ logging_output = {
42
+ "loss": loss.data,
43
+ "ntokens": sample["ntokens"],
44
+ "nsentences": sample["target"].size(0),
45
+ "sample_size": sample_size,
46
+ }
47
+ return loss, sample_size, logging_output
48
+
49
+ def compute_loss(self, model, net_output, sample, reduce=True):
50
+ lprobs = model.get_normalized_probs(net_output, log_probs=True)
51
+ lprobs = lprobs.view(-1, lprobs.size(-1))
52
+ target = model.get_targets(sample, net_output).view(-1)
53
+ loss = F.nll_loss(
54
+ lprobs,
55
+ target,
56
+ ignore_index=self.padding_idx,
57
+ reduction="sum" if reduce else "none",
58
+ )
59
+ return loss, loss
60
+
61
+ @staticmethod
62
+ def reduce_metrics(logging_outputs) -> None:
63
+ """Aggregate logging outputs from data parallel training."""
64
+ loss_sum = sum(log.get("loss", 0) for log in logging_outputs)
65
+ ntokens = sum(log.get("ntokens", 0) for log in logging_outputs)
66
+ sample_size = sum(log.get("sample_size", 0) for log in logging_outputs)
67
+
68
+ # we divide by log(2) to convert the loss from base e to base 2
69
+ metrics.log_scalar(
70
+ "loss", loss_sum / sample_size / math.log(2), sample_size, round=3
71
+ )
72
+ if sample_size != ntokens:
73
+ metrics.log_scalar(
74
+ "nll_loss", loss_sum / ntokens / math.log(2), ntokens, round=3
75
+ )
76
+ metrics.log_derived(
77
+ "ppl", lambda meters: utils.get_perplexity(meters["nll_loss"].avg)
78
+ )
79
+ else:
80
+ metrics.log_derived(
81
+ "ppl", lambda meters: utils.get_perplexity(meters["loss"].avg)
82
+ )
83
+
84
+ @staticmethod
85
+ def logging_outputs_can_be_summed() -> bool:
86
+ """
87
+ Whether the logging outputs returned by `forward` can be summed
88
+ across workers prior to calling `reduce_metrics`. Setting this
89
+ to True will improves distributed training speed.
90
+ """
91
+ return True
data/fairseq/fairseq/criterions/ctc.py ADDED
@@ -0,0 +1,325 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # All rights reserved.
2
+ #
3
+ # This source code is licensed under the license found in the LICENSE file in
4
+ # the root directory of this source tree. An additional grant of patent rights
5
+ # can be found in the PATENTS file in the same directory.
6
+
7
+ import math
8
+ from argparse import Namespace
9
+ from dataclasses import dataclass, field
10
+ from omegaconf import II
11
+ from typing import Optional
12
+
13
+ import torch
14
+ import torch.nn.functional as F
15
+
16
+ from fairseq import utils
17
+ from fairseq.logging import metrics
18
+ from fairseq.criterions import FairseqCriterion, register_criterion
19
+ from fairseq.dataclass import FairseqDataclass
20
+ from fairseq.data.data_utils import post_process
21
+ from fairseq.tasks import FairseqTask
22
+ from fairseq.logging.meters import safe_round
23
+
24
+
25
+ @dataclass
26
+ class CtcCriterionConfig(FairseqDataclass):
27
+ zero_infinity: bool = field(
28
+ default=False,
29
+ metadata={"help": "zero inf loss when source length <= target length"},
30
+ )
31
+ sentence_avg: bool = II("optimization.sentence_avg")
32
+ post_process: str = field(
33
+ default="letter",
34
+ metadata={
35
+ "help": "how to post process predictions into words. can be letter, "
36
+ "wordpiece, BPE symbols, etc. "
37
+ "See fairseq.data.data_utils.post_process() for full list of options"
38
+ },
39
+ )
40
+ wer_kenlm_model: Optional[str] = field(
41
+ default=None,
42
+ metadata={
43
+ "help": "if this is provided, use kenlm to compute wer (along with other wer_* args)"
44
+ },
45
+ )
46
+ wer_lexicon: Optional[str] = field(
47
+ default=None,
48
+ metadata={"help": "lexicon to use with wer_kenlm_model"},
49
+ )
50
+ wer_lm_weight: float = field(
51
+ default=2.0,
52
+ metadata={"help": "lm weight to use with wer_kenlm_model"},
53
+ )
54
+ wer_word_score: float = field(
55
+ default=-1.0,
56
+ metadata={"help": "lm word score to use with wer_kenlm_model"},
57
+ )
58
+ wer_sil_weight: float = field(
59
+ default=0,
60
+ metadata={"help": "lm word score to use with wer_kenlm_model"},
61
+ )
62
+
63
+ wer_args: Optional[str] = field(
64
+ default=None,
65
+ metadata={
66
+ "help": "DEPRECATED: tuple of (wer_kenlm_model, wer_lexicon, wer_lm_weight, wer_word_score)"
67
+ },
68
+ )
69
+
70
+
71
+ @register_criterion("ctc", dataclass=CtcCriterionConfig)
72
+ class CtcCriterion(FairseqCriterion):
73
+ def __init__(
74
+ self, cfg: CtcCriterionConfig, task: FairseqTask, rdrop_alpha: int = 0.0
75
+ ):
76
+ super().__init__(task)
77
+ self.blank_idx = (
78
+ task.target_dictionary.index(task.blank_symbol)
79
+ if hasattr(task, "blank_symbol")
80
+ else 0
81
+ )
82
+ self.pad_idx = task.target_dictionary.pad()
83
+ self.eos_idx = task.target_dictionary.eos()
84
+ self.post_process = cfg.post_process
85
+
86
+ self.rdrop_alpha = rdrop_alpha
87
+
88
+ if cfg.wer_args is not None:
89
+ (
90
+ cfg.wer_kenlm_model,
91
+ cfg.wer_lexicon,
92
+ cfg.wer_lm_weight,
93
+ cfg.wer_word_score,
94
+ ) = eval(cfg.wer_args)
95
+
96
+ if cfg.wer_kenlm_model is not None and cfg.wer_kenlm_model != "":
97
+ from examples.speech_recognition.w2l_decoder import W2lKenLMDecoder
98
+
99
+ dec_args = Namespace()
100
+ dec_args.nbest = 1
101
+ dec_args.criterion = "ctc"
102
+ dec_args.kenlm_model = cfg.wer_kenlm_model
103
+ dec_args.lexicon = cfg.wer_lexicon
104
+ dec_args.beam = 50
105
+ dec_args.beam_size_token = min(50, len(task.target_dictionary))
106
+ dec_args.beam_threshold = min(50, len(task.target_dictionary))
107
+ dec_args.lm_weight = cfg.wer_lm_weight
108
+ dec_args.word_score = cfg.wer_word_score
109
+ dec_args.sil_weight = cfg.wer_sil_weight
110
+ dec_args.unk_weight = -math.inf
111
+ dec_args.sil_weight = 0
112
+
113
+ self.w2l_decoder = W2lKenLMDecoder(dec_args, task.target_dictionary)
114
+ else:
115
+ self.w2l_decoder = None
116
+
117
+ self.zero_infinity = cfg.zero_infinity
118
+ self.sentence_avg = cfg.sentence_avg
119
+
120
+ def forward(self, model, sample, reduce=True, **kwargs):
121
+ net_output = model(**sample["net_input"])
122
+ lprobs = model.get_normalized_probs(
123
+ net_output, log_probs=True
124
+ ).contiguous() # (T, B, C) from the encoder
125
+
126
+ # CTC loss is calculated over duplicated inputs
127
+ # sample is already duplicated for R-Drop
128
+ if self.rdrop_alpha > 0:
129
+ for k, v in sample.items():
130
+ if k in ["target", "target_lengths"]:
131
+ sample[k] = torch.cat([v, v.clone()], dim=0)
132
+ elif k == "net_input":
133
+ if sample[k]["src_tokens"].size(1) != sample[k]["src_lengths"].size(
134
+ 0
135
+ ):
136
+ # for decoder CTC loss
137
+ sample[k]["src_lengths"] = torch.cat(
138
+ [
139
+ sample[k]["src_lengths"],
140
+ sample[k]["src_lengths"].clone(),
141
+ ],
142
+ dim=0,
143
+ )
144
+
145
+ if "src_lengths" in sample["net_input"]:
146
+ input_lengths = sample["net_input"]["src_lengths"]
147
+ else:
148
+ if net_output["padding_mask"] is not None:
149
+ non_padding_mask = ~net_output["padding_mask"]
150
+ input_lengths = non_padding_mask.long().sum(-1)
151
+ else:
152
+ input_lengths = lprobs.new_full(
153
+ (lprobs.size(1),), lprobs.size(0), dtype=torch.long
154
+ )
155
+
156
+ pad_mask = (sample["target"] != self.pad_idx) & (
157
+ sample["target"] != self.eos_idx
158
+ )
159
+ targets_flat = sample["target"].masked_select(pad_mask)
160
+ if "target_lengths" in sample:
161
+ target_lengths = sample["target_lengths"]
162
+ else:
163
+ target_lengths = pad_mask.sum(-1)
164
+
165
+ with torch.backends.cudnn.flags(enabled=False):
166
+ loss = F.ctc_loss(
167
+ lprobs,
168
+ targets_flat,
169
+ input_lengths,
170
+ target_lengths,
171
+ blank=self.blank_idx,
172
+ reduction="sum",
173
+ zero_infinity=self.zero_infinity,
174
+ )
175
+
176
+ ntokens = (
177
+ sample["ntokens"] if "ntokens" in sample else target_lengths.sum().item()
178
+ )
179
+
180
+ sample_size = sample["target"].size(0) if self.sentence_avg else ntokens
181
+ logging_output = {
182
+ "loss": utils.item(loss.data), # * sample['ntokens'],
183
+ "ntokens": ntokens,
184
+ "nsentences": sample["id"].numel(),
185
+ "sample_size": sample_size,
186
+ }
187
+
188
+ if not model.training:
189
+ import editdistance
190
+
191
+ with torch.no_grad():
192
+ lprobs_t = lprobs.transpose(0, 1).float().contiguous().cpu()
193
+
194
+ c_err = 0
195
+ c_len = 0
196
+ w_errs = 0
197
+ w_len = 0
198
+ wv_errs = 0
199
+ for lp, t, inp_l in zip(
200
+ lprobs_t,
201
+ sample["target_label"]
202
+ if "target_label" in sample
203
+ else sample["target"],
204
+ input_lengths,
205
+ ):
206
+ lp = lp[:inp_l].unsqueeze(0)
207
+
208
+ decoded = None
209
+ if self.w2l_decoder is not None:
210
+ decoded = self.w2l_decoder.decode(lp)
211
+ if len(decoded) < 1:
212
+ decoded = None
213
+ else:
214
+ decoded = decoded[0]
215
+ if len(decoded) < 1:
216
+ decoded = None
217
+ else:
218
+ decoded = decoded[0]
219
+
220
+ p = (t != self.task.target_dictionary.pad()) & (
221
+ t != self.task.target_dictionary.eos()
222
+ )
223
+ targ = t[p]
224
+ targ_units = self.task.target_dictionary.string(targ)
225
+ targ_units_arr = targ.tolist()
226
+
227
+ toks = lp.argmax(dim=-1).unique_consecutive()
228
+ pred_units_arr = toks[toks != self.blank_idx].tolist()
229
+
230
+ c_err += editdistance.eval(pred_units_arr, targ_units_arr)
231
+ c_len += len(targ_units_arr)
232
+
233
+ targ_words = post_process(targ_units, self.post_process).split()
234
+
235
+ pred_units = self.task.target_dictionary.string(pred_units_arr)
236
+ pred_words_raw = post_process(pred_units, self.post_process).split()
237
+
238
+ if decoded is not None and "words" in decoded:
239
+ pred_words = decoded["words"]
240
+ w_errs += editdistance.eval(pred_words, targ_words)
241
+ wv_errs += editdistance.eval(pred_words_raw, targ_words)
242
+ else:
243
+ dist = editdistance.eval(pred_words_raw, targ_words)
244
+ w_errs += dist
245
+ wv_errs += dist
246
+
247
+ w_len += len(targ_words)
248
+
249
+ logging_output["wv_errors"] = wv_errs
250
+ logging_output["w_errors"] = w_errs
251
+ logging_output["w_total"] = w_len
252
+ logging_output["c_errors"] = c_err
253
+ logging_output["c_total"] = c_len
254
+
255
+ return loss, sample_size, logging_output
256
+
257
+ @staticmethod
258
+ def reduce_metrics(logging_outputs) -> None:
259
+ """Aggregate logging outputs from data parallel training."""
260
+
261
+ loss_sum = utils.item(sum(log.get("loss", 0) for log in logging_outputs))
262
+ ntokens = utils.item(sum(log.get("ntokens", 0) for log in logging_outputs))
263
+ nsentences = utils.item(
264
+ sum(log.get("nsentences", 0) for log in logging_outputs)
265
+ )
266
+ sample_size = utils.item(
267
+ sum(log.get("sample_size", 0) for log in logging_outputs)
268
+ )
269
+
270
+ metrics.log_scalar(
271
+ "loss", loss_sum / sample_size / math.log(2), sample_size, round=3
272
+ )
273
+ metrics.log_scalar("ntokens", ntokens)
274
+ metrics.log_scalar("nsentences", nsentences)
275
+ if sample_size != ntokens:
276
+ metrics.log_scalar(
277
+ "nll_loss", loss_sum / ntokens / math.log(2), ntokens, round=3
278
+ )
279
+
280
+ c_errors = sum(log.get("c_errors", 0) for log in logging_outputs)
281
+ metrics.log_scalar("_c_errors", c_errors)
282
+ c_total = sum(log.get("c_total", 0) for log in logging_outputs)
283
+ metrics.log_scalar("_c_total", c_total)
284
+ w_errors = sum(log.get("w_errors", 0) for log in logging_outputs)
285
+ metrics.log_scalar("_w_errors", w_errors)
286
+ wv_errors = sum(log.get("wv_errors", 0) for log in logging_outputs)
287
+ metrics.log_scalar("_wv_errors", wv_errors)
288
+ w_total = sum(log.get("w_total", 0) for log in logging_outputs)
289
+ metrics.log_scalar("_w_total", w_total)
290
+
291
+ if c_total > 0:
292
+ metrics.log_derived(
293
+ "uer",
294
+ lambda meters: safe_round(
295
+ meters["_c_errors"].sum * 100.0 / meters["_c_total"].sum, 3
296
+ )
297
+ if meters["_c_total"].sum > 0
298
+ else float("nan"),
299
+ )
300
+ if w_total > 0:
301
+ metrics.log_derived(
302
+ "wer",
303
+ lambda meters: safe_round(
304
+ meters["_w_errors"].sum * 100.0 / meters["_w_total"].sum, 3
305
+ )
306
+ if meters["_w_total"].sum > 0
307
+ else float("nan"),
308
+ )
309
+ metrics.log_derived(
310
+ "raw_wer",
311
+ lambda meters: safe_round(
312
+ meters["_wv_errors"].sum * 100.0 / meters["_w_total"].sum, 3
313
+ )
314
+ if meters["_w_total"].sum > 0
315
+ else float("nan"),
316
+ )
317
+
318
+ @staticmethod
319
+ def logging_outputs_can_be_summed() -> bool:
320
+ """
321
+ Whether the logging outputs returned by `forward` can be summed
322
+ across workers prior to calling `reduce_metrics`. Setting this
323
+ to True will improves distributed training speed.
324
+ """
325
+ return True
data/fairseq/fairseq/criterions/fairseq_criterion.py ADDED
@@ -0,0 +1,121 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright (c) Facebook, Inc. and its affiliates.
2
+ #
3
+ # This source code is licensed under the MIT license found in the
4
+ # LICENSE file in the root directory of this source tree.
5
+
6
+ import inspect
7
+ from typing import Any, Dict, List
8
+
9
+ from fairseq import utils
10
+ from fairseq.logging import metrics
11
+ from fairseq.dataclass import FairseqDataclass
12
+ from fairseq.dataclass.utils import gen_parser_from_dataclass
13
+ from torch.nn.modules.loss import _Loss
14
+
15
+
16
+ class FairseqCriterion(_Loss):
17
+ def __init__(self, task):
18
+ super().__init__()
19
+ self.task = task
20
+ if hasattr(task, "target_dictionary"):
21
+ tgt_dict = task.target_dictionary
22
+ self.padding_idx = tgt_dict.pad() if tgt_dict is not None else -100
23
+
24
+ @classmethod
25
+ def add_args(cls, parser):
26
+ """Add criterion-specific arguments to the parser."""
27
+ dc = getattr(cls, "__dataclass", None)
28
+ if dc is not None:
29
+ gen_parser_from_dataclass(parser, dc())
30
+
31
+ @classmethod
32
+ def build_criterion(cls, cfg: FairseqDataclass, task):
33
+ """Construct a criterion from command-line args."""
34
+ # arguments in the __init__.
35
+ init_args = {}
36
+ for p in inspect.signature(cls).parameters.values():
37
+ if (
38
+ p.kind == p.POSITIONAL_ONLY
39
+ or p.kind == p.VAR_POSITIONAL
40
+ or p.kind == p.VAR_KEYWORD
41
+ ):
42
+ # we haven't implemented inference for these argument types,
43
+ # but PRs welcome :)
44
+ raise NotImplementedError("{} not supported".format(p.kind))
45
+
46
+ assert p.kind in {p.POSITIONAL_OR_KEYWORD, p.KEYWORD_ONLY}
47
+
48
+ if p.name == "task":
49
+ init_args["task"] = task
50
+ elif p.name == "cfg":
51
+ init_args["cfg"] = cfg
52
+ elif hasattr(cfg, p.name):
53
+ init_args[p.name] = getattr(cfg, p.name)
54
+ elif p.default != p.empty:
55
+ pass # we'll use the default value
56
+ else:
57
+ raise NotImplementedError(
58
+ "Unable to infer Criterion arguments, please implement "
59
+ "{}.build_criterion".format(cls.__name__)
60
+ )
61
+ return cls(**init_args)
62
+
63
+ def forward(self, model, sample, reduce=True):
64
+ """Compute the loss for the given sample.
65
+
66
+ Returns a tuple with three elements:
67
+ 1) the loss
68
+ 2) the sample size, which is used as the denominator for the gradient
69
+ 3) logging outputs to display while training
70
+ """
71
+ raise NotImplementedError
72
+
73
+ @staticmethod
74
+ def aggregate_logging_outputs(
75
+ logging_outputs: List[Dict[str, Any]]
76
+ ) -> Dict[str, Any]:
77
+ """Aggregate logging outputs from data parallel training."""
78
+ utils.deprecation_warning(
79
+ "The aggregate_logging_outputs API is deprecated. "
80
+ "Please use the reduce_metrics API instead."
81
+ )
82
+ raise NotImplementedError
83
+
84
+ @classmethod
85
+ def reduce_metrics(cls, logging_outputs: List[Dict[str, Any]]) -> None:
86
+ """Aggregate logging outputs from data parallel training."""
87
+ utils.deprecation_warning(
88
+ "Criterions should implement the reduce_metrics API. "
89
+ "Falling back to deprecated aggregate_logging_outputs API."
90
+ )
91
+ agg_logging_outputs = cls.aggregate_logging_outputs(logging_outputs)
92
+ for k, v in agg_logging_outputs.items():
93
+ if k in {"nsentences", "ntokens", "sample_size"}:
94
+ continue
95
+ metrics.log_scalar(k, v)
96
+
97
+ @staticmethod
98
+ def logging_outputs_can_be_summed() -> bool:
99
+ """
100
+ Whether the logging outputs returned by `forward` can be summed
101
+ across workers prior to calling `reduce_metrics`. Setting this
102
+ to True will improves distributed training speed.
103
+ """
104
+ return False
105
+
106
+
107
+ class LegacyFairseqCriterion(FairseqCriterion):
108
+ def __init__(self, args, task):
109
+ super().__init__(task=task)
110
+ self.args = args
111
+
112
+ utils.deprecation_warning(
113
+ "Criterions should take explicit arguments instead of an "
114
+ "argparse.Namespace object, please update your criterion by "
115
+ "extending FairseqCriterion instead of LegacyFairseqCriterion."
116
+ )
117
+
118
+ @classmethod
119
+ def build_criterion(cls, args, task):
120
+ """Construct a criterion from command-line args."""
121
+ return cls(args, task)
data/fairseq/fairseq/criterions/fastspeech2_loss.py ADDED
@@ -0,0 +1,137 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright (c) 2017-present, Facebook, Inc.
2
+ # All rights reserved.
3
+ #
4
+ # This source code is licensed under the license found in the LICENSE file in
5
+ # the root directory of this source tree. An additional grant of patent rights
6
+ # can be found in the PATENTS file in the same directory.
7
+
8
+ from typing import List, Dict, Any
9
+ from dataclasses import dataclass, field
10
+
11
+ import torch
12
+ import torch.nn.functional as F
13
+
14
+ from fairseq import utils
15
+ from fairseq.logging import metrics
16
+ from fairseq.criterions import FairseqCriterion, register_criterion
17
+ from fairseq.dataclass import FairseqDataclass
18
+ from fairseq.data.data_utils import lengths_to_mask
19
+ from fairseq.models.fairseq_model import FairseqEncoderModel
20
+
21
+
22
+ @dataclass
23
+ class FastSpeech2CriterionConfig(FairseqDataclass):
24
+ ctc_weight: float = field(default=0.0, metadata={"help": "weight for CTC loss"})
25
+
26
+
27
+ @register_criterion("fastspeech2", dataclass=FastSpeech2CriterionConfig)
28
+ class FastSpeech2Loss(FairseqCriterion):
29
+ def __init__(self, task, ctc_weight):
30
+ super().__init__(task)
31
+ self.ctc_weight = ctc_weight
32
+
33
+ def forward(self, model: FairseqEncoderModel, sample, reduction="mean"):
34
+ src_tokens = sample["net_input"]["src_tokens"]
35
+ src_lens = sample["net_input"]["src_lengths"]
36
+ tgt_lens = sample["target_lengths"]
37
+ _feat_out, _feat_out_post, _, log_dur_out, pitch_out, energy_out = model(
38
+ src_tokens=src_tokens,
39
+ src_lengths=src_lens,
40
+ prev_output_tokens=sample["net_input"]["prev_output_tokens"],
41
+ incremental_state=None,
42
+ target_lengths=tgt_lens,
43
+ speaker=sample["speaker"],
44
+ durations=sample["durations"],
45
+ pitches=sample["pitches"],
46
+ energies=sample["energies"],
47
+ )
48
+
49
+ src_mask = lengths_to_mask(sample["net_input"]["src_lengths"])
50
+ tgt_mask = lengths_to_mask(sample["target_lengths"])
51
+
52
+ pitches, energies = sample["pitches"], sample["energies"]
53
+ pitch_out, pitches = pitch_out[src_mask], pitches[src_mask]
54
+ energy_out, energies = energy_out[src_mask], energies[src_mask]
55
+
56
+ feat_out, feat = _feat_out[tgt_mask], sample["target"][tgt_mask]
57
+ l1_loss = F.l1_loss(feat_out, feat, reduction=reduction)
58
+ if _feat_out_post is not None:
59
+ l1_loss += F.l1_loss(_feat_out_post[tgt_mask], feat, reduction=reduction)
60
+
61
+ pitch_loss = F.mse_loss(pitch_out, pitches, reduction=reduction)
62
+ energy_loss = F.mse_loss(energy_out, energies, reduction=reduction)
63
+
64
+ log_dur_out = log_dur_out[src_mask]
65
+ dur = sample["durations"].float()
66
+ dur = dur.half() if log_dur_out.type().endswith(".HalfTensor") else dur
67
+ log_dur = torch.log(dur + 1)[src_mask]
68
+ dur_loss = F.mse_loss(log_dur_out, log_dur, reduction=reduction)
69
+
70
+ ctc_loss = torch.tensor(0.0).type_as(l1_loss)
71
+ if self.ctc_weight > 0.0:
72
+ lprobs = model.get_normalized_probs((_feat_out,), log_probs=True)
73
+ lprobs = lprobs.transpose(0, 1) # T x B x C
74
+ src_mask = lengths_to_mask(src_lens)
75
+ src_tokens_flat = src_tokens.masked_select(src_mask)
76
+ ctc_loss = (
77
+ F.ctc_loss(
78
+ lprobs,
79
+ src_tokens_flat,
80
+ tgt_lens,
81
+ src_lens,
82
+ reduction=reduction,
83
+ zero_infinity=True,
84
+ )
85
+ * self.ctc_weight
86
+ )
87
+
88
+ loss = l1_loss + dur_loss + pitch_loss + energy_loss + ctc_loss
89
+
90
+ sample_size = sample["nsentences"]
91
+ logging_output = {
92
+ "loss": utils.item(loss.data),
93
+ "ntokens": sample["ntokens"],
94
+ "nsentences": sample["nsentences"],
95
+ "sample_size": sample_size,
96
+ "l1_loss": utils.item(l1_loss.data),
97
+ "dur_loss": utils.item(dur_loss.data),
98
+ "pitch_loss": utils.item(pitch_loss.data),
99
+ "energy_loss": utils.item(energy_loss.data),
100
+ "ctc_loss": utils.item(ctc_loss.data),
101
+ }
102
+ return loss, sample_size, logging_output
103
+
104
+ @classmethod
105
+ def reduce_metrics(cls, logging_outputs: List[Dict[str, Any]]) -> None:
106
+ ns = [log.get("sample_size", 0) for log in logging_outputs]
107
+ ntot = sum(ns)
108
+ ws = [n / (ntot + 1e-8) for n in ns]
109
+ for key in [
110
+ "loss",
111
+ "l1_loss",
112
+ "dur_loss",
113
+ "pitch_loss",
114
+ "energy_loss",
115
+ "ctc_loss",
116
+ ]:
117
+ vals = [log.get(key, 0) for log in logging_outputs]
118
+ val = sum(val * w for val, w in zip(vals, ws))
119
+ metrics.log_scalar(key, val, ntot, round=3)
120
+ metrics.log_scalar("sample_size", ntot, len(logging_outputs))
121
+
122
+ # inference metrics
123
+ if "targ_frames" not in logging_outputs[0]:
124
+ return
125
+ n = sum(log.get("targ_frames", 0) for log in logging_outputs)
126
+ for key, new_key in [
127
+ ("mcd_loss", "mcd_loss"),
128
+ ("pred_frames", "pred_ratio"),
129
+ ("nins", "ins_rate"),
130
+ ("ndel", "del_rate"),
131
+ ]:
132
+ val = sum(log.get(key, 0) for log in logging_outputs)
133
+ metrics.log_scalar(new_key, val / n, n, round=3)
134
+
135
+ @staticmethod
136
+ def logging_outputs_can_be_summed() -> bool:
137
+ return False
data/fairseq/fairseq/criterions/hubert_criterion.py ADDED
@@ -0,0 +1,195 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright (c) Facebook, Inc. and its affiliates.
2
+ #
3
+ # This source code is licensed under the MIT license found in the
4
+ # LICENSE file in the root directory of this source tree.
5
+
6
+ import math
7
+ import re
8
+ from dataclasses import dataclass, field
9
+ from typing import List, Optional
10
+
11
+ import torch
12
+ import torch.nn.functional as F
13
+ from fairseq import utils
14
+ from fairseq.logging import metrics
15
+ from fairseq.criterions import FairseqCriterion, register_criterion
16
+ from fairseq.dataclass import FairseqDataclass
17
+
18
+
19
+ @dataclass
20
+ class HubertCriterionConfig(FairseqDataclass):
21
+ pred_masked_weight: float = field(
22
+ default=1.0,
23
+ metadata={"help": "weight for predictive loss for masked frames"},
24
+ )
25
+ pred_nomask_weight: float = field(
26
+ default=0.0,
27
+ metadata={"help": "weight for predictive loss for unmasked frames"},
28
+ )
29
+ loss_weights: Optional[List[float]] = field(
30
+ default=None,
31
+ metadata={"help": "weights for additional loss terms (not first one)"},
32
+ )
33
+ log_keys: List[str] = field(
34
+ default_factory=lambda: [],
35
+ metadata={"help": "output keys to log"},
36
+ )
37
+
38
+
39
+ @register_criterion("hubert", dataclass=HubertCriterionConfig)
40
+ class HubertCriterion(FairseqCriterion):
41
+ def __init__(
42
+ self,
43
+ task,
44
+ pred_masked_weight,
45
+ pred_nomask_weight,
46
+ loss_weights=None,
47
+ log_keys=None,
48
+ ):
49
+ super().__init__(task)
50
+ self.pred_masked_weight = pred_masked_weight
51
+ self.pred_nomask_weight = pred_nomask_weight
52
+ self.loss_weights = loss_weights
53
+ self.log_keys = [] if log_keys is None else log_keys
54
+
55
+ def forward(self, model, sample, reduce=True, log_pred=False):
56
+ """Compute the loss for the given sample.
57
+ Returns a tuple with three elements:
58
+ 1) the loss
59
+ 2) the sample size, which is used as the denominator for the gradient
60
+ 3) logging outputs to display while training
61
+ """
62
+ net_output = model(target_list=sample["target_list"], **sample["net_input"])
63
+ loss = 0.0
64
+ sample_size = 0
65
+ logging_output = {}
66
+ reduction = "sum" if reduce else "none"
67
+
68
+ loss_m_list = []
69
+ logp_m_list = model.get_logits(net_output, True)
70
+ targ_m_list = model.get_targets(net_output, True)
71
+ assert self.pred_masked_weight == 0 or len(logp_m_list) > 0
72
+ for i, (logp_m, targ_m) in enumerate(zip(logp_m_list, targ_m_list)):
73
+ loss_m = F.cross_entropy(logp_m, targ_m, reduction=reduction)
74
+ loss_m_list.append(loss_m)
75
+ logging_output[f"loss_m_{i}"] = loss_m.detach().item()
76
+ if self.pred_masked_weight > 0:
77
+ loss += self.pred_masked_weight * sum(loss_m_list)
78
+ sample_size += targ_m_list[0].numel()
79
+
80
+ loss_u_list = []
81
+ logp_u_list = model.get_logits(net_output, False)
82
+ targ_u_list = model.get_targets(net_output, False)
83
+ assert self.pred_nomask_weight == 0 or len(logp_u_list) > 0
84
+ for i, (logp_u, targ_u) in enumerate(zip(logp_u_list, targ_u_list)):
85
+ loss_u = F.cross_entropy(logp_u, targ_u, reduction=reduction)
86
+ loss_u_list.append(loss_u)
87
+ logging_output[f"loss_u_{i}"] = loss_u.detach().item()
88
+ if self.pred_nomask_weight > 0:
89
+ loss += self.pred_nomask_weight * sum(loss_u_list)
90
+ sample_size += targ_u_list[0].numel()
91
+
92
+ if self.loss_weights is not None:
93
+ assert hasattr(model, "get_extra_losses")
94
+ extra_losses, names = model.get_extra_losses(net_output)
95
+ if torch.is_tensor(extra_losses):
96
+ extra_losses = [extra_losses]
97
+ names = [names]
98
+ if len(self.loss_weights) == 1 and len(extra_losses) != 1:
99
+ self.loss_weights = [self.loss_weights[0]] * len(extra_losses)
100
+ assert len(extra_losses) == len(
101
+ self.loss_weights
102
+ ), f"{len(extra_losses)}, {len(self.loss_weights)}"
103
+ for p, n, coef in zip(extra_losses, names, self.loss_weights):
104
+ if coef != 0 and p is not None:
105
+ p = coef * p.float() * sample_size
106
+ loss += p
107
+ logging_output[f"loss_{n}"] = p.item()
108
+
109
+ logging_output = {
110
+ "loss": loss.item() if reduce else loss,
111
+ "ntokens": sample_size,
112
+ "nsentences": sample["id"].numel(),
113
+ "sample_size": sample_size,
114
+ **logging_output,
115
+ }
116
+
117
+ for lk in self.log_keys:
118
+ if lk in net_output:
119
+ logging_output[lk] = float((net_output[lk]))
120
+
121
+ def compute_correct(logits):
122
+ if logits.numel() == 0:
123
+ return 0, 0
124
+ else:
125
+ assert logits.dim() > 1, logits.shape
126
+ max = logits.argmax(-1) == 0
127
+ min = logits.argmin(-1) == 0
128
+ both = max & min
129
+ corr = max.long().sum().item() - both.long().sum().item()
130
+ count = max.numel()
131
+ return corr, count
132
+
133
+ with torch.no_grad():
134
+ for i, logp_m in enumerate(logp_m_list):
135
+ corr_m, count_m = compute_correct(logp_m)
136
+ logging_output[f"correct_m_{i}"] = corr_m
137
+ logging_output[f"count_m_{i}"] = count_m
138
+
139
+ for i, logp_u in enumerate(logp_u_list):
140
+ corr_u, count_u = compute_correct(logp_u)
141
+ logging_output[f"correct_u_{i}"] = corr_u
142
+ logging_output[f"count_u_{i}"] = count_u
143
+
144
+ return loss, sample_size, logging_output
145
+
146
+ @staticmethod
147
+ def reduce_metrics(logging_outputs) -> None:
148
+ """Aggregate logging outputs from data parallel training (copied from normal cross entropy)."""
149
+ loss_sum = sum(log.get("loss", 0) for log in logging_outputs)
150
+ ntokens = sum(log.get("ntokens", 0) for log in logging_outputs)
151
+ sample_size = sum(log.get("sample_size", 0) for log in logging_outputs)
152
+
153
+ metrics.log_scalar(
154
+ "loss", loss_sum / sample_size / math.log(2), sample_size, round=3
155
+ )
156
+ if sample_size != ntokens:
157
+ metrics.log_scalar(
158
+ "nll_loss", loss_sum / ntokens / math.log(2), ntokens, round=3
159
+ )
160
+ metrics.log_derived(
161
+ "ppl", lambda meters: utils.get_perplexity(meters["nll_loss"].avg)
162
+ )
163
+ else:
164
+ metrics.log_derived(
165
+ "ppl", lambda meters: utils.get_perplexity(meters["loss"].avg)
166
+ )
167
+
168
+ counts = {}
169
+ for lk in logging_outputs[0].keys():
170
+ if lk.startswith("count_"):
171
+ val = sum(log[lk] for log in logging_outputs)
172
+ metrics.log_scalar(lk, val)
173
+ counts[lk] = val
174
+
175
+ for lk in logging_outputs[0].keys():
176
+ if lk.startswith("loss_"):
177
+ val = sum(log[lk] for log in logging_outputs)
178
+ metrics.log_scalar(lk, val / sample_size / math.log(2), round=3)
179
+ elif lk.startswith("correct_"):
180
+ val = sum(log[lk] for log in logging_outputs)
181
+ metrics.log_scalar(lk, val / counts[re.sub("correct", "count", lk)])
182
+
183
+ @staticmethod
184
+ def aggregate_logging_outputs(logging_outputs):
185
+ """Aggregate logging outputs from data parallel training."""
186
+ raise NotImplementedError()
187
+
188
+ @staticmethod
189
+ def logging_outputs_can_be_summed() -> bool:
190
+ """
191
+ Whether the logging outputs returned by `forward` can be summed
192
+ across workers prior to calling `reduce_metrics`. Setting this
193
+ to True will improves distributed training speed.
194
+ """
195
+ return False
data/fairseq/fairseq/criterions/label_smoothed_cross_entropy.py ADDED
@@ -0,0 +1,168 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright (c) Facebook, Inc. and its affiliates.
2
+ #
3
+ # This source code is licensed under the MIT license found in the
4
+ # LICENSE file in the root directory of this source tree.
5
+
6
+ import math
7
+ from dataclasses import dataclass, field
8
+
9
+ import torch
10
+ from fairseq import utils
11
+ from fairseq.logging import metrics
12
+ from fairseq.criterions import FairseqCriterion, register_criterion
13
+ from fairseq.dataclass import FairseqDataclass
14
+ from omegaconf import II
15
+
16
+
17
+ @dataclass
18
+ class LabelSmoothedCrossEntropyCriterionConfig(FairseqDataclass):
19
+ label_smoothing: float = field(
20
+ default=0.0,
21
+ metadata={"help": "epsilon for label smoothing, 0 means no label smoothing"},
22
+ )
23
+ report_accuracy: bool = field(
24
+ default=False,
25
+ metadata={"help": "report accuracy metric"},
26
+ )
27
+ ignore_prefix_size: int = field(
28
+ default=0,
29
+ metadata={"help": "Ignore first N tokens"},
30
+ )
31
+ sentence_avg: bool = II("optimization.sentence_avg")
32
+
33
+
34
+ def label_smoothed_nll_loss(lprobs, target, epsilon, ignore_index=None, reduce=True):
35
+ if target.dim() == lprobs.dim() - 1:
36
+ target = target.unsqueeze(-1)
37
+ nll_loss = -lprobs.gather(dim=-1, index=target)
38
+ smooth_loss = -lprobs.sum(dim=-1, keepdim=True)
39
+ if ignore_index is not None:
40
+ pad_mask = target.eq(ignore_index)
41
+ nll_loss.masked_fill_(pad_mask, 0.0)
42
+ smooth_loss.masked_fill_(pad_mask, 0.0)
43
+ else:
44
+ nll_loss = nll_loss.squeeze(-1)
45
+ smooth_loss = smooth_loss.squeeze(-1)
46
+ if reduce:
47
+ nll_loss = nll_loss.sum()
48
+ smooth_loss = smooth_loss.sum()
49
+ eps_i = epsilon / (lprobs.size(-1) - 1)
50
+ loss = (1.0 - epsilon - eps_i) * nll_loss + eps_i * smooth_loss
51
+ return loss, nll_loss
52
+
53
+
54
+ @register_criterion(
55
+ "label_smoothed_cross_entropy", dataclass=LabelSmoothedCrossEntropyCriterionConfig
56
+ )
57
+ class LabelSmoothedCrossEntropyCriterion(FairseqCriterion):
58
+ def __init__(
59
+ self,
60
+ task,
61
+ sentence_avg,
62
+ label_smoothing,
63
+ ignore_prefix_size=0,
64
+ report_accuracy=False,
65
+ ):
66
+ super().__init__(task)
67
+ self.sentence_avg = sentence_avg
68
+ self.eps = label_smoothing
69
+ self.ignore_prefix_size = ignore_prefix_size
70
+ self.report_accuracy = report_accuracy
71
+
72
+ def forward(self, model, sample, reduce=True):
73
+ """Compute the loss for the given sample.
74
+
75
+ Returns a tuple with three elements:
76
+ 1) the loss
77
+ 2) the sample size, which is used as the denominator for the gradient
78
+ 3) logging outputs to display while training
79
+ """
80
+ net_output = model(**sample["net_input"])
81
+ loss, nll_loss = self.compute_loss(model, net_output, sample, reduce=reduce)
82
+ sample_size = (
83
+ sample["target"].size(0) if self.sentence_avg else sample["ntokens"]
84
+ )
85
+ logging_output = {
86
+ "loss": loss.data,
87
+ "nll_loss": nll_loss.data,
88
+ "ntokens": sample["ntokens"],
89
+ "nsentences": sample["target"].size(0),
90
+ "sample_size": sample_size,
91
+ }
92
+ if self.report_accuracy:
93
+ n_correct, total = self.compute_accuracy(model, net_output, sample)
94
+ logging_output["n_correct"] = utils.item(n_correct.data)
95
+ logging_output["total"] = utils.item(total.data)
96
+ return loss, sample_size, logging_output
97
+
98
+ def get_lprobs_and_target(self, model, net_output, sample):
99
+ lprobs = model.get_normalized_probs(net_output, log_probs=True)
100
+ target = model.get_targets(sample, net_output)
101
+ if self.ignore_prefix_size > 0:
102
+ # lprobs: B x T x C
103
+ lprobs = lprobs[:, self.ignore_prefix_size :, :].contiguous()
104
+ target = target[:, self.ignore_prefix_size :].contiguous()
105
+ return lprobs.view(-1, lprobs.size(-1)), target.view(-1)
106
+
107
+ def compute_loss(self, model, net_output, sample, reduce=True):
108
+ lprobs, target = self.get_lprobs_and_target(model, net_output, sample)
109
+ loss, nll_loss = label_smoothed_nll_loss(
110
+ lprobs,
111
+ target,
112
+ self.eps,
113
+ ignore_index=self.padding_idx,
114
+ reduce=reduce,
115
+ )
116
+ return loss, nll_loss
117
+
118
+ def compute_accuracy(self, model, net_output, sample):
119
+ lprobs, target = self.get_lprobs_and_target(model, net_output, sample)
120
+ mask = target.ne(self.padding_idx)
121
+ n_correct = torch.sum(
122
+ lprobs.argmax(1).masked_select(mask).eq(target.masked_select(mask))
123
+ )
124
+ total = torch.sum(mask)
125
+ return n_correct, total
126
+
127
+ @classmethod
128
+ def reduce_metrics(cls, logging_outputs) -> None:
129
+ """Aggregate logging outputs from data parallel training."""
130
+ loss_sum = sum(log.get("loss", 0) for log in logging_outputs)
131
+ nll_loss_sum = sum(log.get("nll_loss", 0) for log in logging_outputs)
132
+ ntokens = sum(log.get("ntokens", 0) for log in logging_outputs)
133
+ sample_size = sum(log.get("sample_size", 0) for log in logging_outputs)
134
+
135
+ metrics.log_scalar(
136
+ "loss", loss_sum / sample_size / math.log(2), sample_size, round=3
137
+ )
138
+ metrics.log_scalar(
139
+ "nll_loss", nll_loss_sum / ntokens / math.log(2), ntokens, round=3
140
+ )
141
+ metrics.log_derived(
142
+ "ppl", lambda meters: utils.get_perplexity(meters["nll_loss"].avg)
143
+ )
144
+
145
+ total = utils.item(sum(log.get("total", 0) for log in logging_outputs))
146
+ if total > 0:
147
+ metrics.log_scalar("total", total)
148
+ n_correct = utils.item(
149
+ sum(log.get("n_correct", 0) for log in logging_outputs)
150
+ )
151
+ metrics.log_scalar("n_correct", n_correct)
152
+ metrics.log_derived(
153
+ "accuracy",
154
+ lambda meters: round(
155
+ meters["n_correct"].sum * 100.0 / meters["total"].sum, 3
156
+ )
157
+ if meters["total"].sum > 0
158
+ else float("nan"),
159
+ )
160
+
161
+ @staticmethod
162
+ def logging_outputs_can_be_summed() -> bool:
163
+ """
164
+ Whether the logging outputs returned by `forward` can be summed
165
+ across workers prior to calling `reduce_metrics`. Setting this
166
+ to True will improves distributed training speed.
167
+ """
168
+ return True
data/fairseq/fairseq/criterions/label_smoothed_cross_entropy_latency_augmented.py ADDED
@@ -0,0 +1,221 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright (c) Facebook, Inc. and its affiliates.
2
+ #
3
+ # This source code is licensed under the MIT license found in the
4
+ # LICENSE file in the root directory of this source tree.
5
+
6
+ from dataclasses import dataclass, field
7
+ import torch
8
+ from fairseq import utils
9
+ from fairseq.logging import metrics
10
+ from fairseq.criterions import register_criterion
11
+ from fairseq.criterions.label_smoothed_cross_entropy import (
12
+ LabelSmoothedCrossEntropyCriterion,
13
+ LabelSmoothedCrossEntropyCriterionConfig,
14
+ )
15
+
16
+ try:
17
+ from simuleval.metrics.latency import (
18
+ AverageLagging,
19
+ AverageProportion,
20
+ DifferentiableAverageLagging,
21
+ )
22
+
23
+ LATENCY_METRICS = {
24
+ "average_lagging": AverageLagging,
25
+ "average_proportion": AverageProportion,
26
+ "differentiable_average_lagging": DifferentiableAverageLagging,
27
+ }
28
+ except ImportError:
29
+ LATENCY_METRICS = None
30
+
31
+
32
+ @dataclass
33
+ class LabelSmoothedCrossEntropyCriterionLatencyAugmentConfig(
34
+ LabelSmoothedCrossEntropyCriterionConfig
35
+ ):
36
+ latency_avg_weight: float = field(
37
+ default=0.0,
38
+ metadata={"help": "weight fot average latency loss."},
39
+ )
40
+ latency_var_weight: float = field(
41
+ default=0.0,
42
+ metadata={"help": "weight fot variance latency loss."},
43
+ )
44
+ latency_avg_type: str = field(
45
+ default="differentiable_average_lagging",
46
+ metadata={"help": "latency type for average loss"},
47
+ )
48
+ latency_var_type: str = field(
49
+ default="variance_delay",
50
+ metadata={"help": "latency typ for variance loss"},
51
+ )
52
+ latency_gather_method: str = field(
53
+ default="weighted_average",
54
+ metadata={"help": "method to gather latency loss for all heads"},
55
+ )
56
+ latency_update_after: int = field(
57
+ default=0,
58
+ metadata={"help": "Add latency loss after certain steps"},
59
+ )
60
+
61
+
62
+ @register_criterion(
63
+ "latency_augmented_label_smoothed_cross_entropy",
64
+ dataclass=LabelSmoothedCrossEntropyCriterionLatencyAugmentConfig,
65
+ )
66
+ class LatencyAugmentedLabelSmoothedCrossEntropyCriterion(
67
+ LabelSmoothedCrossEntropyCriterion
68
+ ):
69
+ def __init__(
70
+ self,
71
+ task,
72
+ sentence_avg,
73
+ label_smoothing,
74
+ ignore_prefix_size,
75
+ report_accuracy,
76
+ latency_avg_weight,
77
+ latency_var_weight,
78
+ latency_avg_type,
79
+ latency_var_type,
80
+ latency_gather_method,
81
+ latency_update_after,
82
+ ):
83
+ super().__init__(
84
+ task, sentence_avg, label_smoothing, ignore_prefix_size, report_accuracy
85
+ )
86
+ assert LATENCY_METRICS is not None, "Please make sure SimulEval is installed."
87
+
88
+ self.latency_avg_weight = latency_avg_weight
89
+ self.latency_var_weight = latency_var_weight
90
+ self.latency_avg_type = latency_avg_type
91
+ self.latency_var_type = latency_var_type
92
+ self.latency_gather_method = latency_gather_method
93
+ self.latency_update_after = latency_update_after
94
+
95
+ def forward(self, model, sample, reduce=True):
96
+ net_output = model(**sample["net_input"])
97
+ # 1. Compute cross entropy loss
98
+ loss, nll_loss = self.compute_loss(model, net_output, sample, reduce=reduce)
99
+
100
+ # 2. Compute cross latency loss
101
+ latency_loss, expected_latency, expected_delays_var = self.compute_latency_loss(
102
+ model, sample, net_output
103
+ )
104
+
105
+ if self.latency_update_after > 0:
106
+ num_updates = getattr(model.decoder, "num_updates", None)
107
+ assert (
108
+ num_updates is not None
109
+ ), "model.decoder doesn't have attribute 'num_updates'"
110
+ if num_updates <= self.latency_update_after:
111
+ latency_loss = 0
112
+
113
+ loss += latency_loss
114
+
115
+ sample_size = (
116
+ sample["target"].size(0) if self.sentence_avg else sample["ntokens"]
117
+ )
118
+
119
+ logging_output = {
120
+ "loss": loss.data,
121
+ "nll_loss": nll_loss.data,
122
+ "ntokens": sample["ntokens"],
123
+ "nsentences": sample["target"].size(0),
124
+ "sample_size": sample_size,
125
+ "latency": expected_latency,
126
+ "delays_var": expected_delays_var,
127
+ "latency_loss": latency_loss,
128
+ }
129
+
130
+ if self.report_accuracy:
131
+ n_correct, total = self.compute_accuracy(model, net_output, sample)
132
+ logging_output["n_correct"] = utils.item(n_correct.data)
133
+ logging_output["total"] = utils.item(total.data)
134
+ return loss, sample_size, logging_output
135
+
136
+ def compute_latency_loss(self, model, sample, net_output):
137
+ assert (
138
+ net_output[-1].encoder_padding_mask is None
139
+ or not net_output[-1].encoder_padding_mask[:, 0].any()
140
+ ), "Only right padding on source is supported."
141
+ # 1. Obtain the expected alignment
142
+ alpha_list = [item["alpha"] for item in net_output[1].attn_list]
143
+ num_layers = len(alpha_list)
144
+ bsz, num_heads, tgt_len, src_len = alpha_list[0].size()
145
+
146
+ # bsz * num_layers * num_heads, tgt_len, src_len
147
+ alpha_all = torch.cat(alpha_list, dim=1).view(-1, tgt_len, src_len)
148
+
149
+ # 2 compute expected delays
150
+ # bsz * num_heads * num_layers, tgt_len, src_len for MMA
151
+ steps = (
152
+ torch.arange(1, 1 + src_len)
153
+ .unsqueeze(0)
154
+ .unsqueeze(1)
155
+ .expand_as(alpha_all)
156
+ .type_as(alpha_all)
157
+ )
158
+
159
+ expected_delays = torch.sum(steps * alpha_all, dim=-1)
160
+
161
+ target_padding_mask = (
162
+ model.get_targets(sample, net_output)
163
+ .eq(self.padding_idx)
164
+ .unsqueeze(1)
165
+ .expand(bsz, num_layers * num_heads, tgt_len)
166
+ .contiguous()
167
+ .view(-1, tgt_len)
168
+ )
169
+
170
+ src_lengths = (
171
+ sample["net_input"]["src_lengths"]
172
+ .unsqueeze(1)
173
+ .expand(bsz, num_layers * num_heads)
174
+ .contiguous()
175
+ .view(-1)
176
+ )
177
+ expected_latency = LATENCY_METRICS[self.latency_avg_type](
178
+ expected_delays, src_lengths, None, target_padding_mask=target_padding_mask
179
+ )
180
+
181
+ # 2.1 average expected latency of heads
182
+ # bsz, num_layers * num_heads
183
+ expected_latency = expected_latency.view(bsz, -1)
184
+ if self.latency_gather_method == "average":
185
+ # bsz * tgt_len
186
+ expected_latency = expected_delays.mean(dim=1)
187
+ elif self.latency_gather_method == "weighted_average":
188
+ weights = torch.nn.functional.softmax(expected_latency, dim=1)
189
+ expected_latency = torch.sum(expected_latency * weights, dim=1)
190
+ elif self.latency_gather_method == "max":
191
+ expected_latency = expected_latency.max(dim=1)[0]
192
+ else:
193
+ raise NotImplementedError
194
+
195
+ expected_latency = expected_latency.sum()
196
+ avg_loss = self.latency_avg_weight * expected_latency
197
+
198
+ # 2.2 variance of expected delays
199
+ expected_delays_var = (
200
+ expected_delays.view(bsz, -1, tgt_len).var(dim=1).mean(dim=1)
201
+ )
202
+ expected_delays_var = expected_delays_var.sum()
203
+ var_loss = self.latency_avg_weight * expected_delays_var
204
+
205
+ # 3. Final loss
206
+ latency_loss = avg_loss + var_loss
207
+
208
+ return latency_loss, expected_latency, expected_delays_var
209
+
210
+ @classmethod
211
+ def reduce_metrics(cls, logging_outputs) -> None:
212
+ super().reduce_metrics(logging_outputs)
213
+ latency = sum(log.get("latency", 0) for log in logging_outputs)
214
+ delays_var = sum(log.get("delays_var", 0) for log in logging_outputs)
215
+ latency_loss = sum(log.get("latency_loss", 0) for log in logging_outputs)
216
+ nsentences = sum(log.get("nsentences", 0) for log in logging_outputs)
217
+ metrics.log_scalar("latency", latency.float() / nsentences, nsentences, round=3)
218
+ metrics.log_scalar("delays_var", delays_var / nsentences, nsentences, round=3)
219
+ metrics.log_scalar(
220
+ "latency_loss", latency_loss / nsentences, nsentences, round=3
221
+ )
data/fairseq/fairseq/criterions/label_smoothed_cross_entropy_with_alignment.py ADDED
@@ -0,0 +1,131 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright (c) Facebook, Inc. and its affiliates.
2
+ #
3
+ # This source code is licensed under the MIT license found in the
4
+ # LICENSE file in the root directory of this source tree.
5
+
6
+ import math
7
+
8
+ from fairseq import utils
9
+ from fairseq.logging import metrics
10
+ from fairseq.criterions import register_criterion
11
+
12
+ from .label_smoothed_cross_entropy import (
13
+ LabelSmoothedCrossEntropyCriterion,
14
+ LabelSmoothedCrossEntropyCriterionConfig,
15
+ )
16
+
17
+ from dataclasses import dataclass, field
18
+
19
+
20
+ @dataclass
21
+ class LabelSmoothedCrossEntropyCriterionWithAlignmentConfig(
22
+ LabelSmoothedCrossEntropyCriterionConfig
23
+ ):
24
+ alignment_lambda: float = field(
25
+ default=0.05, metadata={"help": "weight for the alignment loss"}
26
+ )
27
+
28
+
29
+ @register_criterion(
30
+ "label_smoothed_cross_entropy_with_alignment",
31
+ dataclass=LabelSmoothedCrossEntropyCriterionWithAlignmentConfig,
32
+ )
33
+ class LabelSmoothedCrossEntropyCriterionWithAlignment(
34
+ LabelSmoothedCrossEntropyCriterion
35
+ ):
36
+ def __init__(self, task, sentence_avg, label_smoothing, alignment_lambda):
37
+ super().__init__(task, sentence_avg, label_smoothing)
38
+ self.alignment_lambda = alignment_lambda
39
+
40
+ def forward(self, model, sample, reduce=True):
41
+ """Compute the loss for the given sample.
42
+
43
+ Returns a tuple with three elements:
44
+ 1) the loss
45
+ 2) the sample size, which is used as the denominator for the gradient
46
+ 3) logging outputs to display while training
47
+ """
48
+ net_output = model(**sample["net_input"])
49
+ loss, nll_loss = self.compute_loss(model, net_output, sample, reduce=reduce)
50
+ sample_size = (
51
+ sample["target"].size(0) if self.sentence_avg else sample["ntokens"]
52
+ )
53
+ logging_output = {
54
+ "loss": utils.item(loss.data) if reduce else loss.data,
55
+ "nll_loss": utils.item(nll_loss.data) if reduce else nll_loss.data,
56
+ "ntokens": sample["ntokens"],
57
+ "nsentences": sample["target"].size(0),
58
+ "sample_size": sample_size,
59
+ }
60
+
61
+ alignment_loss = None
62
+
63
+ # Compute alignment loss only for training set and non dummy batches.
64
+ if "alignments" in sample and sample["alignments"] is not None:
65
+ alignment_loss = self.compute_alignment_loss(sample, net_output)
66
+
67
+ if alignment_loss is not None:
68
+ logging_output["alignment_loss"] = utils.item(alignment_loss.data)
69
+ loss += self.alignment_lambda * alignment_loss
70
+
71
+ return loss, sample_size, logging_output
72
+
73
+ def compute_alignment_loss(self, sample, net_output):
74
+ attn_prob = net_output[1]["attn"][0]
75
+ bsz, tgt_sz, src_sz = attn_prob.shape
76
+ attn = attn_prob.view(bsz * tgt_sz, src_sz)
77
+
78
+ align = sample["alignments"]
79
+ align_weights = sample["align_weights"].float()
80
+
81
+ if len(align) > 0:
82
+ # Alignment loss computation. align (shape [:, 2]) contains the src-tgt index pairs corresponding to
83
+ # the alignments. align_weights (shape [:]) contains the 1 / frequency of a tgt index for normalizing.
84
+ loss = -(
85
+ (attn[align[:, 1][:, None], align[:, 0][:, None]]).log()
86
+ * align_weights[:, None]
87
+ ).sum()
88
+ else:
89
+ return None
90
+
91
+ return loss
92
+
93
+ @staticmethod
94
+ def reduce_metrics(logging_outputs) -> None:
95
+ """Aggregate logging outputs from data parallel training."""
96
+ loss_sum = utils.item(sum(log.get("loss", 0) for log in logging_outputs))
97
+ nll_loss_sum = utils.item(
98
+ sum(log.get("nll_loss", 0) for log in logging_outputs)
99
+ )
100
+ alignment_loss_sum = utils.item(
101
+ sum(log.get("alignment_loss", 0) for log in logging_outputs)
102
+ )
103
+ ntokens = utils.item(sum(log.get("ntokens", 0) for log in logging_outputs))
104
+ sample_size = utils.item(
105
+ sum(log.get("sample_size", 0) for log in logging_outputs)
106
+ )
107
+
108
+ metrics.log_scalar(
109
+ "loss", loss_sum / sample_size / math.log(2), sample_size, round=3
110
+ )
111
+ metrics.log_scalar(
112
+ "nll_loss", nll_loss_sum / ntokens / math.log(2), ntokens, round=3
113
+ )
114
+ metrics.log_scalar(
115
+ "alignment_loss",
116
+ alignment_loss_sum / sample_size / math.log(2),
117
+ sample_size,
118
+ round=3,
119
+ )
120
+ metrics.log_derived(
121
+ "ppl", lambda meters: utils.get_perplexity(meters["nll_loss"].avg)
122
+ )
123
+
124
+ @staticmethod
125
+ def logging_outputs_can_be_summed() -> bool:
126
+ """
127
+ Whether the logging outputs returned by `forward` can be summed
128
+ across workers prior to calling `reduce_metrics`. Setting this
129
+ to True will improves distributed training speed.
130
+ """
131
+ return True
data/fairseq/fairseq/criterions/label_smoothed_cross_entropy_with_ctc.py ADDED
@@ -0,0 +1,97 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright (c) Facebook, Inc. and its affiliates.
2
+ #
3
+ # This source code is licensed under the MIT license found in the
4
+ # LICENSE file in the root directory of this source tree.
5
+
6
+ import math
7
+ from dataclasses import dataclass, field
8
+
9
+ import torch
10
+ import torch.nn.functional as F
11
+
12
+ from fairseq import utils
13
+ from fairseq.logging import metrics
14
+ from fairseq.criterions import register_criterion
15
+ from fairseq.criterions.label_smoothed_cross_entropy import (
16
+ LabelSmoothedCrossEntropyCriterion,
17
+ LabelSmoothedCrossEntropyCriterionConfig,
18
+ )
19
+ from fairseq.data.data_utils import lengths_to_mask
20
+
21
+
22
+ @dataclass
23
+ class LabelSmoothedCrossEntropyWithCtcCriterionConfig(
24
+ LabelSmoothedCrossEntropyCriterionConfig
25
+ ):
26
+ ctc_weight: float = field(default=1.0, metadata={"help": "weight for CTC loss"})
27
+
28
+
29
+ @register_criterion(
30
+ "label_smoothed_cross_entropy_with_ctc",
31
+ dataclass=LabelSmoothedCrossEntropyWithCtcCriterionConfig,
32
+ )
33
+ class LabelSmoothedCrossEntropyWithCtcCriterion(LabelSmoothedCrossEntropyCriterion):
34
+ def __init__(
35
+ self,
36
+ task,
37
+ sentence_avg,
38
+ label_smoothing,
39
+ ignore_prefix_size,
40
+ report_accuracy,
41
+ ctc_weight,
42
+ ):
43
+ super().__init__(
44
+ task, sentence_avg, label_smoothing, ignore_prefix_size, report_accuracy
45
+ )
46
+ self.ctc_weight = ctc_weight
47
+
48
+ def forward(self, model, sample, reduce=True):
49
+ net_output = model(**sample["net_input"])
50
+ loss, nll_loss = self.compute_loss(model, net_output, sample, reduce=reduce)
51
+
52
+ ctc_loss = torch.tensor(0.0).type_as(loss)
53
+ if self.ctc_weight > 0.0:
54
+ ctc_lprobs, ctc_lens = model.get_ctc_output(net_output, sample)
55
+ ctc_tgt, ctc_tgt_lens = model.get_ctc_target(sample)
56
+ ctc_tgt_mask = lengths_to_mask(ctc_tgt_lens)
57
+ ctc_tgt_flat = ctc_tgt.masked_select(ctc_tgt_mask)
58
+ reduction = "sum" if reduce else "none"
59
+ ctc_loss = (
60
+ F.ctc_loss(
61
+ ctc_lprobs,
62
+ ctc_tgt_flat,
63
+ ctc_lens,
64
+ ctc_tgt_lens,
65
+ reduction=reduction,
66
+ zero_infinity=True,
67
+ )
68
+ * self.ctc_weight
69
+ )
70
+ loss += ctc_loss
71
+
72
+ sample_size = (
73
+ sample["target"].size(0) if self.sentence_avg else sample["ntokens"]
74
+ )
75
+ logging_output = {
76
+ "loss": utils.item(loss.data),
77
+ "nll_loss": utils.item(nll_loss.data),
78
+ "ctc_loss": utils.item(ctc_loss.data),
79
+ "ntokens": sample["ntokens"],
80
+ "nsentences": sample["target"].size(0),
81
+ "sample_size": sample_size,
82
+ }
83
+ if self.report_accuracy:
84
+ n_correct, total = self.compute_accuracy(model, net_output, sample)
85
+ logging_output["n_correct"] = utils.item(n_correct.data)
86
+ logging_output["total"] = utils.item(total.data)
87
+ return loss, sample_size, logging_output
88
+
89
+ @classmethod
90
+ def reduce_metrics(cls, logging_outputs) -> None:
91
+ super().reduce_metrics(logging_outputs)
92
+ loss_sum = sum(log.get("ctc_loss", 0) for log in logging_outputs)
93
+ sample_size = sum(log.get("sample_size", 0) for log in logging_outputs)
94
+
95
+ metrics.log_scalar(
96
+ "ctc_loss", loss_sum / sample_size / math.log(2), sample_size, round=3
97
+ )
data/fairseq/fairseq/criterions/label_smoothed_cross_entropy_with_rdrop.py ADDED
@@ -0,0 +1,177 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright (c) Facebook, Inc. and its affiliates.
2
+ #
3
+ # This source code is licensed under the MIT license found in the
4
+ # LICENSE file in the root directory of this source tree.
5
+
6
+ import math
7
+ from dataclasses import dataclass, field
8
+
9
+ import torch
10
+
11
+ from fairseq import utils
12
+ from fairseq.logging import metrics
13
+ from fairseq.criterions import register_criterion
14
+ from fairseq.criterions.label_smoothed_cross_entropy import (
15
+ LabelSmoothedCrossEntropyCriterion,
16
+ LabelSmoothedCrossEntropyCriterionConfig,
17
+ label_smoothed_nll_loss,
18
+ )
19
+
20
+
21
+ @dataclass
22
+ class RdropLabelSmoothedCrossEntropyCriterionConfig(
23
+ LabelSmoothedCrossEntropyCriterionConfig
24
+ ):
25
+ rdrop_alpha: float = field(
26
+ default=0.0,
27
+ metadata={"help": "alpha for r-drop, 0 means no r-drop"},
28
+ )
29
+
30
+
31
+ @register_criterion(
32
+ "label_smoothed_cross_entropy_with_rdrop",
33
+ dataclass=RdropLabelSmoothedCrossEntropyCriterionConfig,
34
+ )
35
+ class RdropLabelSmoothedCrossEntropyCriterion(LabelSmoothedCrossEntropyCriterion):
36
+ def __init__(
37
+ self,
38
+ task,
39
+ sentence_avg,
40
+ label_smoothing,
41
+ ignore_prefix_size=0,
42
+ report_accuracy=False,
43
+ rdrop_alpha=0.0,
44
+ ):
45
+ super().__init__(
46
+ task,
47
+ sentence_avg,
48
+ label_smoothing,
49
+ ignore_prefix_size=ignore_prefix_size,
50
+ report_accuracy=report_accuracy,
51
+ )
52
+ self.sentence_avg = sentence_avg
53
+ self.eps = label_smoothing
54
+ self.ignore_prefix_size = ignore_prefix_size
55
+ self.report_accuracy = report_accuracy
56
+ self.rdrop_alpha = rdrop_alpha
57
+
58
+ def forward(self, model, sample, reduce=True, net_output=None):
59
+ """Compute the loss for the given sample.
60
+
61
+ Returns a tuple with three elements:
62
+ 1) the loss
63
+ 2) the sample size, which is used as the denominator for the gradient
64
+ 3) logging outputs to display while training
65
+ """
66
+ if net_output is None:
67
+ if self.rdrop_alpha > 0 and sample["net_input"]["src_tokens"].size(
68
+ 0
69
+ ) == sample["target"].size(0):
70
+ sample = duplicate_input(sample)
71
+ net_output = model(**sample["net_input"])
72
+ loss, nll_loss, rdrop_kl_loss = self.compute_loss(
73
+ model, net_output, sample, reduce=reduce
74
+ )
75
+ sample_size = (
76
+ sample["target"].size(0) if self.sentence_avg else sample["ntokens"]
77
+ )
78
+ logging_output = {
79
+ "loss": loss.data,
80
+ "nll_loss": nll_loss.data,
81
+ "ntokens": sample["ntokens"],
82
+ "nsentences": sample["target"].size(0),
83
+ "sample_size": sample_size,
84
+ }
85
+ if self.report_accuracy:
86
+ n_correct, total = self.compute_accuracy(model, net_output, sample)
87
+ logging_output["n_correct"] = utils.item(n_correct.data)
88
+ logging_output["total"] = utils.item(total.data)
89
+ if self.rdrop_alpha > 0:
90
+ logging_output["rdrop_kl_loss"] = utils.item(rdrop_kl_loss.data)
91
+ return loss, sample_size, logging_output
92
+
93
+ def get_lprobs_and_target(self, model, net_output, sample):
94
+ lprobs = model.get_normalized_probs(net_output, log_probs=True)
95
+ target = model.get_targets(sample, net_output)
96
+ if self.rdrop_alpha > 0 or target.size(0) != lprobs.size(0):
97
+ target = torch.cat([target, target.clone()], dim=0)
98
+
99
+ if self.ignore_prefix_size > 0:
100
+ # lprobs: B x T x C
101
+ lprobs = lprobs[:, self.ignore_prefix_size :, :].contiguous()
102
+ target = target[:, self.ignore_prefix_size :].contiguous()
103
+ return lprobs.view(-1, lprobs.size(-1)), target.view(-1)
104
+
105
+ def compute_loss(self, model, net_output, sample, reduce=True):
106
+ lprobs, target = self.get_lprobs_and_target(model, net_output, sample)
107
+ loss, nll_loss = label_smoothed_nll_loss(
108
+ lprobs,
109
+ target,
110
+ self.eps,
111
+ ignore_index=self.padding_idx,
112
+ reduce=reduce,
113
+ )
114
+
115
+ if self.rdrop_alpha > 0:
116
+ pad_mask = target[: target.size(0) // 2].unsqueeze(-1).eq(self.padding_idx)
117
+ rdrop_kl_loss = compute_kl_loss(model, net_output, pad_mask)
118
+ loss += self.rdrop_alpha * rdrop_kl_loss
119
+ else:
120
+ rdrop_kl_loss = loss.new_zeros(1)
121
+ return loss, nll_loss, rdrop_kl_loss
122
+
123
+ @classmethod
124
+ def reduce_metrics(cls, logging_outputs) -> None:
125
+ """Aggregate logging outputs from data parallel training."""
126
+ super().reduce_metrics(logging_outputs)
127
+
128
+ sample_size = sum(log.get("sample_size", 0) for log in logging_outputs)
129
+
130
+ rdrop_kl_loss = utils.item(
131
+ sum(log.get("rdrop_kl_loss", 0) for log in logging_outputs)
132
+ / sample_size
133
+ / math.log(2)
134
+ )
135
+ if rdrop_kl_loss > 0:
136
+ metrics.log_scalar("rdrop_kl_loss", rdrop_kl_loss)
137
+
138
+
139
+ def duplicate_input(sample):
140
+ if "net_input" in sample.keys():
141
+ sample_input = sample["net_input"]
142
+ else:
143
+ sample_input = sample
144
+
145
+ for k, v in sample_input.items():
146
+ if isinstance(v, torch.Tensor):
147
+ sample_input[k] = torch.cat([v, v.clone()], dim=0)
148
+ if "net_input" in sample.keys():
149
+ sample["net_input"] = sample_input
150
+ else:
151
+ sample = sample_input
152
+ return sample
153
+
154
+
155
+ def compute_kl_loss(model, net_output, pad_mask=None, reduce=True):
156
+ net_prob = model.get_normalized_probs(net_output, log_probs=True)
157
+ net_prob_tec = model.get_normalized_probs(net_output, log_probs=False)
158
+
159
+ net_prob = net_prob.view(-1, net_prob.size(-1))
160
+ net_prob_tec = net_prob_tec.view(-1, net_prob_tec.size(-1))
161
+
162
+ p, q = torch.split(net_prob, net_prob.size(0) // 2, dim=0)
163
+ p_tec, q_tec = torch.split(net_prob_tec, net_prob_tec.size(0) // 2, dim=0)
164
+
165
+ p_loss = torch.nn.functional.kl_div(p, q_tec, reduction="none")
166
+ q_loss = torch.nn.functional.kl_div(q, p_tec, reduction="none")
167
+
168
+ if pad_mask is not None:
169
+ p_loss.masked_fill_(pad_mask, 0.0)
170
+ q_loss.masked_fill_(pad_mask, 0.0)
171
+
172
+ if reduce:
173
+ p_loss = p_loss.sum()
174
+ q_loss = q_loss.sum()
175
+
176
+ loss = (p_loss + q_loss) / 2
177
+ return loss
data/fairseq/fairseq/criterions/legacy_masked_lm.py ADDED
@@ -0,0 +1,178 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright (c) Facebook, Inc. and its affiliates.
2
+ #
3
+ # This source code is licensed under the MIT license found in the
4
+ # LICENSE file in the root directory of this source tree.
5
+
6
+ import math
7
+
8
+ import torch
9
+ import torch.nn.functional as F
10
+ from fairseq import utils
11
+ from fairseq.logging import metrics
12
+ from fairseq.criterions import FairseqCriterion, register_criterion
13
+
14
+
15
+ def compute_cross_entropy_loss(logits, targets, ignore_index=-100):
16
+ """
17
+ Function to compute the cross entropy loss. The default value of
18
+ ignore_index is the same as the default value for F.cross_entropy in
19
+ pytorch.
20
+ """
21
+ assert logits.size(0) == targets.size(
22
+ -1
23
+ ), "Logits and Targets tensor shapes don't match up"
24
+
25
+ loss = F.nll_loss(
26
+ F.log_softmax(logits, -1, dtype=torch.float32),
27
+ targets,
28
+ reduction="sum",
29
+ ignore_index=ignore_index,
30
+ )
31
+ return loss
32
+
33
+
34
+ @register_criterion("legacy_masked_lm_loss")
35
+ class LegacyMaskedLmLoss(FairseqCriterion):
36
+ """
37
+ Implementation for the loss used in masked language model (MLM) training.
38
+ This optionally also computes the next sentence prediction (NSP) loss and
39
+ adds it to the overall loss based on the specified args. There are three
40
+ cases to consider:
41
+ 1) Generic MLM training without NSP loss. In this case sentence_targets
42
+ and sentence_logits are both None.
43
+ 2) BERT training without NSP loss. In this case sentence_targets is
44
+ not None but sentence_logits is None and we should not be computing
45
+ a sentence level loss.
46
+ 3) BERT training with NSP loss. In this case both sentence_targets and
47
+ sentence_logits are not None and we should be computing a sentence
48
+ level loss. The weight of the sentence level loss is specified as
49
+ an argument.
50
+ """
51
+
52
+ def __init__(self, task, masked_lm_only, nsp_loss_weight):
53
+ super().__init__(task)
54
+ self.masked_lm_only = masked_lm_only
55
+ self.nsp_loss_weight = nsp_loss_weight
56
+
57
+ @staticmethod
58
+ def add_args(parser):
59
+ """Args for MaskedLM Loss"""
60
+ # Default for masked_lm_only is False so as to not break BERT training
61
+ parser.add_argument(
62
+ "--masked-lm-only",
63
+ default=False,
64
+ action="store_true",
65
+ help="compute MLM loss only",
66
+ )
67
+ parser.add_argument(
68
+ "--nsp-loss-weight",
69
+ default=1.0,
70
+ type=float,
71
+ help="weight for next sentence prediction" " loss (default 1)",
72
+ )
73
+
74
+ def forward(self, model, sample, reduce=True):
75
+ """Compute the loss for the given sample.
76
+ Returns a tuple with three elements:
77
+ 1) the loss
78
+ 2) the sample size, which is used as the denominator for the gradient
79
+ 3) logging outputs to display while training
80
+ """
81
+ lm_logits, output_metadata = model(**sample["net_input"])
82
+
83
+ # reshape lm_logits from (N,T,C) to (N*T,C)
84
+ lm_logits = lm_logits.view(-1, lm_logits.size(-1))
85
+ lm_targets = sample["lm_target"].view(-1)
86
+ lm_loss = compute_cross_entropy_loss(lm_logits, lm_targets, self.padding_idx)
87
+
88
+ # compute the number of tokens for which loss is computed. This is used
89
+ # to normalize the loss
90
+ ntokens = utils.strip_pad(lm_targets, self.padding_idx).numel()
91
+ loss = lm_loss / ntokens
92
+ nsentences = sample["nsentences"]
93
+ # nsentences = 0
94
+
95
+ # Compute sentence loss if masked_lm_only is False
96
+ sentence_loss = None
97
+ if not self.masked_lm_only:
98
+ sentence_logits = output_metadata["sentence_logits"]
99
+ sentence_targets = sample["sentence_target"].view(-1)
100
+ # This needs to be recomputed due to some differences between
101
+ # TokenBlock and BlockPair dataset. This can be resolved with a
102
+ # refactor of BERTModel which we will do in the future.
103
+ # TODO: Remove this after refactor of BERTModel
104
+ nsentences = sentence_targets.size(0)
105
+
106
+ # Check for logits being none which can happen when remove_heads
107
+ # is set to true in the BERT model. Ideally we should set
108
+ # masked_lm_only to true in this case, but that requires some
109
+ # refactor in the BERT model.
110
+ if sentence_logits is not None:
111
+ sentence_loss = compute_cross_entropy_loss(
112
+ sentence_logits, sentence_targets
113
+ )
114
+
115
+ loss += self.nsp_loss_weight * (sentence_loss / nsentences)
116
+
117
+ # NOTE: as we are summing up per token mlm loss and per sentence nsp loss
118
+ # we don't need to use sample_size as denominator for the gradient
119
+ # here sample_size is just used for logging
120
+ sample_size = 1
121
+ logging_output = {
122
+ "loss": utils.item(loss.data) if reduce else loss.data,
123
+ "lm_loss": utils.item(lm_loss.data) if reduce else lm_loss.data,
124
+ # sentence loss is not always computed
125
+ "sentence_loss": (
126
+ (utils.item(sentence_loss.data) if reduce else sentence_loss.data)
127
+ if sentence_loss is not None
128
+ else 0.0
129
+ ),
130
+ "ntokens": ntokens,
131
+ "nsentences": nsentences,
132
+ "sample_size": sample_size,
133
+ }
134
+ return loss, sample_size, logging_output
135
+
136
+ @staticmethod
137
+ def reduce_metrics(logging_outputs) -> None:
138
+ """Aggregate logging outputs from data parallel training."""
139
+ lm_loss_sum = sum(log.get("lm_loss", 0) for log in logging_outputs)
140
+ sentence_loss_sum = sum(log.get("sentence_loss", 0) for log in logging_outputs)
141
+ ntokens = sum(log.get("ntokens", 0) for log in logging_outputs)
142
+ nsentences = sum(log.get("nsentences", 0) for log in logging_outputs)
143
+ sample_size = sum(log.get("sample_size", 0) for log in logging_outputs)
144
+ agg_loss = sum(log.get("loss", 0) for log in logging_outputs)
145
+
146
+ metrics.log_scalar(
147
+ "loss",
148
+ agg_loss / sample_size / math.log(2) if sample_size > 0 else 0.0,
149
+ sample_size,
150
+ round=3,
151
+ )
152
+ metrics.log_scalar(
153
+ "lm_loss",
154
+ lm_loss_sum / ntokens / math.log(2) if ntokens > 0 else 0.0,
155
+ ntokens,
156
+ round=3,
157
+ )
158
+ metrics.log_scalar(
159
+ "sentence_loss",
160
+ sentence_loss_sum / nsentences / math.log(2) if nsentences > 0 else 0.0,
161
+ nsentences,
162
+ round=3,
163
+ )
164
+ metrics.log_scalar(
165
+ "nll_loss",
166
+ lm_loss_sum / ntokens / math.log(2) if ntokens > 0 else 0.0,
167
+ ntokens,
168
+ round=3,
169
+ )
170
+
171
+ @staticmethod
172
+ def logging_outputs_can_be_summed() -> bool:
173
+ """
174
+ Whether the logging outputs returned by `forward` can be summed
175
+ across workers prior to calling `reduce_metrics`. Setting this
176
+ to True will improves distributed training speed.
177
+ """
178
+ return True
data/fairseq/fairseq/criterions/masked_lm.py ADDED
@@ -0,0 +1,99 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright (c) Facebook, Inc. and its affiliates.
2
+ #
3
+ # This source code is licensed under the MIT license found in the
4
+ # LICENSE file in the root directory of this source tree.
5
+
6
+ from dataclasses import dataclass
7
+ import math
8
+ from omegaconf import II
9
+
10
+ import torch
11
+ from fairseq import modules, utils
12
+ from fairseq.logging import metrics
13
+ from fairseq.criterions import FairseqCriterion, register_criterion
14
+ from fairseq.dataclass import FairseqDataclass
15
+
16
+
17
+ @dataclass
18
+ class MaskedLmConfig(FairseqDataclass):
19
+ tpu: bool = II("common.tpu")
20
+
21
+
22
+ @register_criterion("masked_lm", dataclass=MaskedLmConfig)
23
+ class MaskedLmLoss(FairseqCriterion):
24
+ """
25
+ Implementation for the loss used in masked language model (MLM) training.
26
+ """
27
+
28
+ def __init__(self, cfg: MaskedLmConfig, task):
29
+ super().__init__(task)
30
+ self.tpu = cfg.tpu
31
+
32
+ def forward(self, model, sample, reduce=True):
33
+ """Compute the loss for the given sample.
34
+
35
+ Returns a tuple with three elements:
36
+ 1) the loss
37
+ 2) the sample size, which is used as the denominator for the gradient
38
+ 3) logging outputs to display while training
39
+ """
40
+ masked_tokens = sample["target"].ne(self.padding_idx)
41
+ sample_size = masked_tokens.int().sum()
42
+
43
+ # Rare: when all tokens are masked, project all tokens.
44
+ # We use torch.where to avoid device-to-host transfers,
45
+ # except on CPU where torch.where is not well supported
46
+ # (see github.com/pytorch/pytorch/issues/26247).
47
+ if self.tpu:
48
+ masked_tokens = None # always project all tokens on TPU
49
+ elif masked_tokens.device == torch.device("cpu"):
50
+ if not masked_tokens.any():
51
+ masked_tokens = None
52
+ else:
53
+ masked_tokens = torch.where(
54
+ masked_tokens.any(),
55
+ masked_tokens,
56
+ masked_tokens.new([True]),
57
+ )
58
+
59
+ logits = model(**sample["net_input"], masked_tokens=masked_tokens)[0]
60
+ targets = model.get_targets(sample, [logits])
61
+ if masked_tokens is not None:
62
+ targets = targets[masked_tokens]
63
+
64
+ loss = modules.cross_entropy(
65
+ logits.view(-1, logits.size(-1)),
66
+ targets.view(-1),
67
+ reduction="sum",
68
+ ignore_index=self.padding_idx,
69
+ )
70
+
71
+ logging_output = {
72
+ "loss": loss if self.tpu else loss.data,
73
+ "ntokens": sample["ntokens"],
74
+ "nsentences": sample["nsentences"],
75
+ "sample_size": sample_size,
76
+ }
77
+ return loss, sample_size, logging_output
78
+
79
+ @staticmethod
80
+ def reduce_metrics(logging_outputs) -> None:
81
+ """Aggregate logging outputs from data parallel training."""
82
+ loss_sum = sum(log.get("loss", 0) for log in logging_outputs)
83
+ sample_size = sum(log.get("sample_size", 0) for log in logging_outputs)
84
+
85
+ metrics.log_scalar(
86
+ "loss", loss_sum / sample_size / math.log(2), sample_size, round=3
87
+ )
88
+ metrics.log_derived(
89
+ "ppl", lambda meters: utils.get_perplexity(meters["loss"].avg)
90
+ )
91
+
92
+ @staticmethod
93
+ def logging_outputs_can_be_summed() -> bool:
94
+ """
95
+ Whether the logging outputs returned by `forward` can be summed
96
+ across workers prior to calling `reduce_metrics`. Setting this
97
+ to True will improves distributed training speed.
98
+ """
99
+ return True
data/fairseq/fairseq/criterions/model_criterion.py ADDED
@@ -0,0 +1,177 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright (c) Facebook, Inc. and its affiliates.
2
+ #
3
+ # This source code is licensed under the MIT license found in the
4
+ # LICENSE file in the root directory of this source tree.
5
+
6
+ import logging
7
+ from dataclasses import dataclass, field
8
+ from typing import Dict, List
9
+
10
+ import torch
11
+
12
+ from fairseq import utils
13
+ from fairseq.logging import metrics
14
+ from fairseq.criterions import FairseqCriterion, register_criterion
15
+ from fairseq.dataclass import FairseqDataclass
16
+ from fairseq.logging.meters import safe_round
17
+
18
+
19
+ logger = logging.getLogger(__name__)
20
+
21
+
22
+ @dataclass
23
+ class ModelCriterionConfig(FairseqDataclass):
24
+ loss_weights: Dict[str, float] = field(
25
+ default_factory=dict,
26
+ metadata={"help": "weights for the loss terms"},
27
+ )
28
+ log_keys: List[str] = field(
29
+ default_factory=list,
30
+ metadata={"help": "additional output keys to log"},
31
+ )
32
+ can_sum: bool = True
33
+
34
+
35
+ @register_criterion("model", dataclass=ModelCriterionConfig)
36
+ class ModelCriterion(FairseqCriterion):
37
+ """
38
+ This criterion relies on the model to supply losses.
39
+ The losses should be a dictionary of name -> scalar returned by
40
+ the model either by including it in the net_output dict or by
41
+ implementing a get_losses(net_output, sample) method. The final loss is
42
+ a scaled sum of all losses according to weights in loss_weights.
43
+ If no weights are provided, then all losses are scaled by 1.0.
44
+
45
+ The losses will be automatically logged. Additional keys from
46
+ net_output dict can be logged via the log_keys parameter.
47
+ """
48
+
49
+ def __init__(self, task, loss_weights=None, log_keys=None, can_sum=True):
50
+ super().__init__(task)
51
+ self.loss_weights = loss_weights
52
+ self.log_keys = log_keys
53
+ self.can_sum = can_sum
54
+
55
+ def forward(self, model, sample, reduce=True):
56
+ net_output = model(**sample["net_input"])
57
+
58
+ scaled_losses = {}
59
+
60
+ if hasattr(model, "get_losses"):
61
+ losses = model.get_losses(net_output, sample)
62
+ elif isinstance(net_output, dict) and "losses" in net_output:
63
+ losses = net_output["losses"]
64
+ else:
65
+ raise Exception("Could not retrieve losses")
66
+
67
+ for lk, p in losses.items():
68
+ try:
69
+ coef = 1.0 if len(self.loss_weights) == 0 else self.loss_weights[lk]
70
+ except KeyError:
71
+ logger.error(
72
+ f"weight for loss {lk} is not in loss_weights ({self.loss_weights})"
73
+ )
74
+ raise
75
+ if coef != 0 and p is not None:
76
+ scaled_losses[lk] = coef * p.float().sum()
77
+
78
+ loss = sum(scaled_losses.values())
79
+
80
+ if "sample_size" in net_output:
81
+ sample_size = net_output["sample_size"]
82
+ else:
83
+ sample_size = loss.numel()
84
+
85
+ if reduce and loss.numel() > 1:
86
+ loss = loss.sum()
87
+
88
+ logging_output = {
89
+ "loss": loss.data,
90
+ "ntokens": sample_size,
91
+ "nsentences": sample["id"].numel(),
92
+ "sample_size": sample_size,
93
+ "_world_size": 1,
94
+ }
95
+
96
+ for lk in self.log_keys:
97
+ if lk in net_output and net_output[lk] is not None:
98
+ if not torch.is_tensor(net_output[lk]) or net_output[lk].numel() == 1:
99
+ logging_output[lk] = float(net_output[lk])
100
+ elif lk.startswith("_"):
101
+ logging_output[lk] = net_output[lk]
102
+ else:
103
+ for i, v in enumerate(net_output[lk]):
104
+ logging_output[f"{lk}_{i}"] = float(v)
105
+
106
+ if len(scaled_losses) > 1:
107
+ for lk, l in scaled_losses.items():
108
+ if l.numel() > 1:
109
+ l = l.sum()
110
+ logging_output[f"loss_{lk}"] = l.item()
111
+
112
+ if "logs" in net_output:
113
+ for lgw in net_output["logs"]:
114
+ logging_output[lgw] = net_output["logs"][lgw]
115
+
116
+ return loss, sample_size, logging_output
117
+
118
+ @staticmethod
119
+ def reduce_metrics(logging_outputs) -> None:
120
+ """Aggregate logging outputs from data parallel training."""
121
+ loss_sum = utils.item(sum(log.get("loss", 0) for log in logging_outputs))
122
+ ntokens = utils.item(sum(log.get("ntokens", 0) for log in logging_outputs))
123
+ nsentences = utils.item(
124
+ sum(log.get("nsentences", 0) for log in logging_outputs)
125
+ )
126
+ sample_size = utils.item(
127
+ sum(log.get("sample_size", 0) for log in logging_outputs)
128
+ )
129
+
130
+ metrics.log_scalar("loss", loss_sum / sample_size, sample_size, round=3)
131
+ metrics.log_scalar("ntokens", ntokens)
132
+ metrics.log_scalar("nsentences", nsentences)
133
+ metrics.log_scalar("sample_size", sample_size)
134
+
135
+ builtin_keys = {
136
+ "loss",
137
+ "ntokens",
138
+ "nsentences",
139
+ "sample_size",
140
+ "_world_size",
141
+ }
142
+
143
+ world_size = utils.item(
144
+ sum(log.get("_world_size", 0) for log in logging_outputs)
145
+ )
146
+
147
+ for k in logging_outputs[0]:
148
+ if k not in builtin_keys and not k.startswith("_"):
149
+ val = sum(log.get(k, 0) for log in logging_outputs)
150
+ if k.startswith("loss_"):
151
+ metrics.log_scalar(k, val / sample_size, sample_size, round=3)
152
+ else:
153
+ metrics.log_scalar(k, val / world_size, round=3)
154
+
155
+ correct = sum(log.get("correct", 0) for log in logging_outputs)
156
+ total = sum(log.get("count", 0) for log in logging_outputs)
157
+
158
+ if total > 0:
159
+ metrics.log_scalar("_correct", correct)
160
+ metrics.log_scalar("_total", total)
161
+
162
+ metrics.log_derived(
163
+ "accuracy",
164
+ lambda meters: safe_round(
165
+ meters["_correct"].sum / meters["_total"].sum, 5
166
+ )
167
+ if meters["_total"].sum > 0
168
+ else float("nan"),
169
+ )
170
+
171
+ def logging_outputs_can_be_summed(self) -> bool:
172
+ """
173
+ Whether the logging outputs returned by `forward` can be summed
174
+ across workers prior to calling `reduce_metrics`. Setting this
175
+ to True will improves distributed training speed.
176
+ """
177
+ return self.can_sum
data/fairseq/fairseq/criterions/nat_loss.py ADDED
@@ -0,0 +1,181 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright (c) Facebook, Inc. and its affiliates.
2
+ #
3
+ # This source code is licensed under the MIT license found in the
4
+ # LICENSE file in the root directory of this source tree.
5
+
6
+ import math
7
+
8
+ import torch
9
+ import torch.nn.functional as F
10
+ from fairseq import utils
11
+ from fairseq.logging import metrics
12
+ from fairseq.criterions import FairseqCriterion, register_criterion
13
+ from fairseq.dataclass import FairseqDataclass
14
+ from torch import Tensor
15
+
16
+ from dataclasses import dataclass, field
17
+
18
+
19
+ @dataclass
20
+ class LabelSmoothedDualImitationCriterionConfig(FairseqDataclass):
21
+ label_smoothing: float = field(
22
+ default=0.0,
23
+ metadata={"help": "epsilon for label smoothing, 0 means no label smoothing"},
24
+ )
25
+
26
+
27
+ @register_criterion("nat_loss", dataclass=LabelSmoothedDualImitationCriterionConfig)
28
+ class LabelSmoothedDualImitationCriterion(FairseqCriterion):
29
+ def __init__(self, task, label_smoothing):
30
+ super().__init__(task)
31
+ self.label_smoothing = label_smoothing
32
+
33
+ def _compute_loss(
34
+ self, outputs, targets, masks=None, label_smoothing=0.0, name="loss", factor=1.0
35
+ ):
36
+ """
37
+ outputs: batch x len x d_model
38
+ targets: batch x len
39
+ masks: batch x len
40
+
41
+ policy_logprob: if there is some policy
42
+ depends on the likelihood score as rewards.
43
+ """
44
+
45
+ def mean_ds(x: Tensor, dim=None) -> Tensor:
46
+ return (
47
+ x.float().mean().type_as(x)
48
+ if dim is None
49
+ else x.float().mean(dim).type_as(x)
50
+ )
51
+
52
+ if masks is not None:
53
+ outputs, targets = outputs[masks], targets[masks]
54
+
55
+ if masks is not None and not masks.any():
56
+ nll_loss = torch.tensor(0)
57
+ loss = nll_loss
58
+ else:
59
+ logits = F.log_softmax(outputs, dim=-1)
60
+ if targets.dim() == 1:
61
+ losses = F.nll_loss(logits, targets.to(logits.device), reduction="none")
62
+
63
+ else: # soft-labels
64
+ losses = F.kl_div(logits, targets.to(logits.device), reduction="none")
65
+ losses = losses.sum(-1)
66
+
67
+ nll_loss = mean_ds(losses)
68
+ if label_smoothing > 0:
69
+ loss = (
70
+ nll_loss * (1 - label_smoothing) - mean_ds(logits) * label_smoothing
71
+ )
72
+ else:
73
+ loss = nll_loss
74
+
75
+ loss = loss * factor
76
+ return {"name": name, "loss": loss, "nll_loss": nll_loss, "factor": factor}
77
+
78
+ def _custom_loss(self, loss, name="loss", factor=1.0):
79
+ return {"name": name, "loss": loss, "factor": factor}
80
+
81
+ def forward(self, model, sample, reduce=True):
82
+ """Compute the loss for the given sample.
83
+ Returns a tuple with three elements:
84
+ 1) the loss
85
+ 2) the sample size, which is used as the denominator for the gradient
86
+ 3) logging outputs to display while training
87
+ """
88
+ nsentences, ntokens = sample["nsentences"], sample["ntokens"]
89
+
90
+ # B x T
91
+ src_tokens, src_lengths = (
92
+ sample["net_input"]["src_tokens"],
93
+ sample["net_input"]["src_lengths"],
94
+ )
95
+ tgt_tokens, prev_output_tokens = sample["target"], sample["prev_target"]
96
+
97
+ outputs = model(src_tokens, src_lengths, prev_output_tokens, tgt_tokens)
98
+ losses, nll_loss = [], []
99
+
100
+ for obj in outputs:
101
+ if outputs[obj].get("loss", None) is None:
102
+ _losses = self._compute_loss(
103
+ outputs[obj].get("out"),
104
+ outputs[obj].get("tgt"),
105
+ outputs[obj].get("mask", None),
106
+ outputs[obj].get("ls", 0.0),
107
+ name=obj + "-loss",
108
+ factor=outputs[obj].get("factor", 1.0),
109
+ )
110
+ else:
111
+ _losses = self._custom_loss(
112
+ outputs[obj].get("loss"),
113
+ name=obj + "-loss",
114
+ factor=outputs[obj].get("factor", 1.0),
115
+ )
116
+
117
+ losses += [_losses]
118
+ if outputs[obj].get("nll_loss", False):
119
+ nll_loss += [_losses.get("nll_loss", 0.0)]
120
+
121
+ loss = sum(l["loss"] for l in losses)
122
+ nll_loss = sum(l for l in nll_loss) if len(nll_loss) > 0 else loss.new_tensor(0)
123
+
124
+ # NOTE:
125
+ # we don't need to use sample_size as denominator for the gradient
126
+ # here sample_size is just used for logging
127
+ sample_size = 1
128
+ logging_output = {
129
+ "loss": loss.data,
130
+ "nll_loss": nll_loss.data,
131
+ "ntokens": ntokens,
132
+ "nsentences": nsentences,
133
+ "sample_size": sample_size,
134
+ }
135
+
136
+ for l in losses:
137
+ logging_output[l["name"]] = (
138
+ utils.item(l["loss"].data / l["factor"])
139
+ if reduce
140
+ else l[["loss"]].data / l["factor"]
141
+ )
142
+
143
+ return loss, sample_size, logging_output
144
+
145
+ @staticmethod
146
+ def reduce_metrics(logging_outputs) -> None:
147
+ """Aggregate logging outputs from data parallel training."""
148
+ sample_size = utils.item(
149
+ sum(log.get("sample_size", 0) for log in logging_outputs)
150
+ )
151
+ loss = utils.item(sum(log.get("loss", 0) for log in logging_outputs))
152
+ nll_loss = utils.item(sum(log.get("nll_loss", 0) for log in logging_outputs))
153
+
154
+ metrics.log_scalar(
155
+ "loss", loss / sample_size / math.log(2), sample_size, round=3
156
+ )
157
+ metrics.log_scalar(
158
+ "nll_loss", nll_loss / sample_size / math.log(2), sample_size, round=3
159
+ )
160
+ metrics.log_derived(
161
+ "ppl", lambda meters: utils.get_perplexity(meters["loss"].avg)
162
+ )
163
+
164
+ for key in logging_outputs[0]:
165
+ if key[-5:] == "-loss":
166
+ val = sum(log.get(key, 0) for log in logging_outputs)
167
+ metrics.log_scalar(
168
+ key[:-5],
169
+ val / sample_size / math.log(2) if sample_size > 0 else 0.0,
170
+ sample_size,
171
+ round=3,
172
+ )
173
+
174
+ @staticmethod
175
+ def logging_outputs_can_be_summed() -> bool:
176
+ """
177
+ Whether the logging outputs returned by `forward` can be summed
178
+ across workers prior to calling `reduce_metrics`. Setting this
179
+ to True will improves distributed training speed.
180
+ """
181
+ return True
data/fairseq/fairseq/criterions/sentence_prediction.py ADDED
@@ -0,0 +1,288 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright (c) Facebook, Inc. and its affiliates.
2
+ #
3
+ # This source code is licensed under the MIT license found in the
4
+ # LICENSE file in the root directory of this source tree.
5
+
6
+ import math
7
+ from dataclasses import dataclass, field
8
+ from itertools import chain
9
+
10
+ import numpy as np
11
+ import torch
12
+ import torch.nn.functional as F
13
+ from sklearn.metrics import f1_score
14
+ from sklearn.metrics import matthews_corrcoef as _matthews_corrcoef
15
+ from scipy.stats import pearsonr, spearmanr
16
+
17
+ from fairseq.logging import metrics
18
+ from fairseq.criterions import FairseqCriterion, register_criterion
19
+ from fairseq.dataclass import FairseqDataclass
20
+ from fairseq.logging.meters import safe_round
21
+
22
+
23
+ def simple_accuracy(preds, labels):
24
+ return (preds == labels).mean()
25
+
26
+
27
+ def acc_and_f1(preds, labels):
28
+ acc = simple_accuracy(preds, labels)
29
+ f1 = f1_score(y_true=labels, y_pred=preds)
30
+ return {
31
+ "acc": acc,
32
+ "f1": f1,
33
+ "acc_and_f1": (acc + f1) / 2,
34
+ }
35
+
36
+
37
+ def pearson_and_spearman(preds, labels):
38
+ pearson_corr = pearsonr(preds, labels)[0]
39
+ spearman_corr = spearmanr(preds, labels)[0]
40
+ return {
41
+ "pearson": pearson_corr,
42
+ "spearmanr": spearman_corr,
43
+ "corr": (pearson_corr + spearman_corr) / 2,
44
+ }
45
+
46
+
47
+ def matthews_corrcoef(preds, labels):
48
+ # make it consistent with other metrics taking (preds, labels) as input
49
+ mcc = _matthews_corrcoef(labels, preds)
50
+ return mcc
51
+
52
+
53
+ @dataclass
54
+ class SentencePredictionConfig(FairseqDataclass):
55
+ classification_head_name: str = field(
56
+ default="sentence_classification_head",
57
+ metadata={"help": "name of the classification head to use"},
58
+ )
59
+ regression_target: bool = field(
60
+ default=False,
61
+ )
62
+ report_mcc: bool = False
63
+ report_acc_and_f1: bool = False
64
+ report_pearson_and_spearman: bool = False
65
+
66
+
67
+ @register_criterion("sentence_prediction", dataclass=SentencePredictionConfig)
68
+ class SentencePredictionCriterion(FairseqCriterion):
69
+ def __init__(self, cfg: SentencePredictionConfig, task):
70
+ super().__init__(task)
71
+ self.classification_head_name = cfg.classification_head_name
72
+ self.regression_target = cfg.regression_target
73
+ self.keep_pred_and_targ = (
74
+ cfg.report_mcc or cfg.report_acc_and_f1 or cfg.report_pearson_and_spearman
75
+ )
76
+ self.report_mcc = cfg.report_mcc
77
+ self.report_acc_and_f1 = cfg.report_acc_and_f1
78
+ self.report_pearson_and_spearman = cfg.report_pearson_and_spearman
79
+ self.label_dict = task.label_dictionary
80
+
81
+ def forward(self, model, sample, reduce=True):
82
+ """Compute the loss for the given sample.
83
+
84
+ Returns a tuple with three elements:
85
+ 1) the loss
86
+ 2) the sample size, which is used as the denominator for the gradient
87
+ 3) logging outputs to display while training
88
+ """
89
+ assert (
90
+ hasattr(model, "classification_heads")
91
+ and self.classification_head_name in model.classification_heads
92
+ ), "model must provide sentence classification head for --criterion=sentence_prediction"
93
+
94
+ logits, _ = model(
95
+ **sample["net_input"],
96
+ features_only=True,
97
+ classification_head_name=self.classification_head_name,
98
+ )
99
+ targets = model.get_targets(sample, [logits]).view(-1)
100
+ sample_size = targets.numel()
101
+
102
+ if not self.regression_target:
103
+ lprobs = F.log_softmax(logits, dim=-1, dtype=torch.float32)
104
+ task_loss = F.nll_loss(lprobs, targets, reduction="sum")
105
+ else:
106
+ logits = logits.view(-1).float()
107
+ targets = targets.float()
108
+ task_loss = F.mse_loss(logits, targets, reduction="sum")
109
+
110
+ logging_output = {}
111
+ loss = task_loss
112
+ # mha & ffn regularization update
113
+ if (
114
+ hasattr(model, "args")
115
+ and hasattr(model.args, "mha_reg_scale_factor")
116
+ and model.args.mha_reg_scale_factor != 0.0
117
+ ):
118
+ mha_reg_loss = model._get_adaptive_head_loss()
119
+ loss += mha_reg_loss
120
+ logging_output.update({"mha_reg_loss": mha_reg_loss})
121
+ if (
122
+ hasattr(model, "args")
123
+ and hasattr(model.args, "ffn_reg_scale_factor")
124
+ and model.args.ffn_reg_scale_factor != 0.0
125
+ ):
126
+ ffn_reg_loss = model._get_adaptive_ffn_loss()
127
+ loss += ffn_reg_loss
128
+ logging_output.update({"ffn_reg_loss": ffn_reg_loss})
129
+
130
+ logging_output.update(
131
+ {
132
+ "loss": loss.data,
133
+ "ntokens": sample["ntokens"],
134
+ "nsentences": sample_size,
135
+ "sample_size": sample_size,
136
+ }
137
+ )
138
+ if not self.regression_target:
139
+ preds = logits.argmax(dim=1)
140
+ logging_output["ncorrect"] = (preds == targets).sum()
141
+ if self.keep_pred_and_targ and not model.training:
142
+ if self.regression_target:
143
+ logging_output["pred"] = logits.detach().cpu().tolist()
144
+ logging_output["targ"] = targets.detach().cpu().tolist()
145
+ else:
146
+ # remove offset `self.label_dict.nspecial` from OffsetTokensDataset
147
+ preds = self.label_dict.string(preds + self.label_dict.nspecial).split()
148
+ targets = self.label_dict.string(
149
+ targets + self.label_dict.nspecial
150
+ ).split()
151
+ logging_output["pred"] = list(map(int, preds))
152
+ logging_output["targ"] = list(map(int, targets))
153
+
154
+ if self.report_mcc:
155
+ logging_output["report_mcc"] = True
156
+ if self.report_acc_and_f1:
157
+ logging_output["report_acc_and_f1"] = True
158
+ if self.report_pearson_and_spearman:
159
+ logging_output["report_pearson_and_spearman"] = True
160
+
161
+ return loss, sample_size, logging_output
162
+
163
+ @staticmethod
164
+ def reduce_metrics(logging_outputs) -> None:
165
+ """Aggregate logging outputs from data parallel training."""
166
+ loss_sum = sum(log.get("loss", 0) for log in logging_outputs)
167
+ ntokens = sum(log.get("ntokens", 0) for log in logging_outputs)
168
+ nsentences = sum(log.get("nsentences", 0) for log in logging_outputs)
169
+ sample_size = sum(log.get("sample_size", 0) for log in logging_outputs)
170
+ mha_reg_loss_sum = sum(log.get("mha_reg_loss", 0) for log in logging_outputs)
171
+ ffn_reg_loss_sum = sum(log.get("ffn_reg_loss", 0) for log in logging_outputs)
172
+
173
+ metrics.log_scalar(
174
+ "loss", loss_sum / sample_size / math.log(2), sample_size, round=3
175
+ )
176
+ if mha_reg_loss_sum:
177
+ metrics.log_scalar(
178
+ "mha_reg_loss",
179
+ mha_reg_loss_sum / sample_size / math.log(2),
180
+ sample_size,
181
+ round=3,
182
+ )
183
+ if ffn_reg_loss_sum:
184
+ metrics.log_scalar(
185
+ "ffn_reg_loss",
186
+ ffn_reg_loss_sum / sample_size / math.log(2),
187
+ sample_size,
188
+ round=3,
189
+ )
190
+ if sample_size != ntokens:
191
+ metrics.log_scalar(
192
+ "nll_loss", loss_sum / ntokens / math.log(2), ntokens, round=3
193
+ )
194
+
195
+ if len(logging_outputs) > 0 and "ncorrect" in logging_outputs[0]:
196
+ ncorrect = sum(log.get("ncorrect", 0) for log in logging_outputs)
197
+ metrics.log_scalar(
198
+ "accuracy", 100.0 * ncorrect / nsentences, nsentences, round=1
199
+ )
200
+
201
+ # Metrics used by GLUE
202
+ pred = np.array(
203
+ list(chain.from_iterable(log.get("pred", []) for log in logging_outputs))
204
+ )
205
+ targ = np.array(
206
+ list(chain.from_iterable(log.get("targ", []) for log in logging_outputs))
207
+ )
208
+ if len(pred):
209
+ metrics.log_concat_tensor("pred", torch.from_numpy(pred), dim=0)
210
+ metrics.log_concat_tensor("targ", torch.from_numpy(targ), dim=0)
211
+ if any("report_mcc" in log for log in logging_outputs):
212
+ metrics.log_derived(
213
+ "mcc",
214
+ lambda meters: safe_round(
215
+ matthews_corrcoef(
216
+ meters["pred"].tensor.numpy(),
217
+ meters["targ"].tensor.numpy(),
218
+ )
219
+ * 100,
220
+ 1,
221
+ ),
222
+ )
223
+ if any("report_acc_and_f1" in log for log in logging_outputs):
224
+ metrics.log_derived(
225
+ "acc_and_f1",
226
+ lambda meters: safe_round(
227
+ acc_and_f1(
228
+ meters["pred"].tensor.numpy(),
229
+ meters["targ"].tensor.numpy(),
230
+ )["acc_and_f1"]
231
+ * 100,
232
+ 1,
233
+ ),
234
+ )
235
+ metrics.log_derived(
236
+ "f1",
237
+ lambda meters: safe_round(
238
+ acc_and_f1(
239
+ meters["pred"].tensor.numpy(),
240
+ meters["targ"].tensor.numpy(),
241
+ )["f1"]
242
+ * 100,
243
+ 1,
244
+ ),
245
+ )
246
+ if any("report_pearson_and_spearman" in log for log in logging_outputs):
247
+ metrics.log_derived(
248
+ "pearson_and_spearman",
249
+ lambda meters: safe_round(
250
+ pearson_and_spearman(
251
+ meters["pred"].tensor.numpy(),
252
+ meters["targ"].tensor.numpy(),
253
+ )["corr"]
254
+ * 100,
255
+ 1,
256
+ ),
257
+ )
258
+ metrics.log_derived(
259
+ "pearson",
260
+ lambda meters: safe_round(
261
+ pearson_and_spearman(
262
+ meters["pred"].tensor.numpy(),
263
+ meters["targ"].tensor.numpy(),
264
+ )["pearson"]
265
+ * 100,
266
+ 1,
267
+ ),
268
+ )
269
+ metrics.log_derived(
270
+ "spearman",
271
+ lambda meters: safe_round(
272
+ pearson_and_spearman(
273
+ meters["pred"].tensor.numpy(),
274
+ meters["targ"].tensor.numpy(),
275
+ )["spearmanr"]
276
+ * 100,
277
+ 1,
278
+ ),
279
+ )
280
+
281
+ @staticmethod
282
+ def logging_outputs_can_be_summed() -> bool:
283
+ """
284
+ Whether the logging outputs returned by `forward` can be summed
285
+ across workers prior to calling `reduce_metrics`. Setting this
286
+ to True will improves distributed training speed.
287
+ """
288
+ return True
data/fairseq/fairseq/criterions/sentence_prediction_adapters.py ADDED
@@ -0,0 +1,63 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright (c) Facebook, Inc. and its affiliates.
2
+ #
3
+ # This source code is licensed under the MIT license found in the
4
+ # LICENSE file in the root directory of this source tree.
5
+
6
+ import torch
7
+ import torch.nn.functional as F
8
+ from fairseq.criterions import register_criterion
9
+ from fairseq.criterions.sentence_prediction import (
10
+ SentencePredictionCriterion,
11
+ SentencePredictionConfig,
12
+ )
13
+
14
+
15
+ @register_criterion("sentence_prediction_adapters", dataclass=SentencePredictionConfig)
16
+ class SentencePredictionCriterionAdapters(SentencePredictionCriterion):
17
+ def forward(self, model, sample, reduce=True):
18
+ """Compute the loss for the given sample.
19
+
20
+ Returns a tuple with three elements:
21
+ 1) the loss
22
+ 2) the sample size, which is used as the denominator for the gradient
23
+ 3) logging outputs to display while training
24
+ """
25
+ assert (
26
+ hasattr(model, "classification_heads")
27
+ and self.classification_head_name in model.classification_heads
28
+ ), "model must provide sentence classification head for --criterion=sentence_prediction"
29
+
30
+ if not hasattr(sample, "lang_id"):
31
+ # If no language ID is given, we fall back to English
32
+ lang_id = ["en_XX"] * sample["nsentences"]
33
+ else:
34
+ lang_id = sample["lang_id"]
35
+
36
+ logits, _ = model(
37
+ **sample["net_input"],
38
+ features_only=True,
39
+ classification_head_name=self.classification_head_name,
40
+ lang_id=lang_id,
41
+ )
42
+ targets = model.get_targets(sample, [logits]).view(-1)
43
+ sample_size = targets.numel()
44
+
45
+ if not self.regression_target:
46
+ lprobs = F.log_softmax(logits, dim=-1, dtype=torch.float32)
47
+ loss = F.nll_loss(lprobs, targets, reduction="sum")
48
+ else:
49
+ logits = logits.view(-1).float()
50
+ targets = targets.float()
51
+ loss = F.mse_loss(logits, targets, reduction="sum")
52
+
53
+ logging_output = {
54
+ "loss": loss.data,
55
+ "ntokens": sample["ntokens"],
56
+ "nsentences": sample_size,
57
+ "sample_size": sample_size,
58
+ }
59
+ if not self.regression_target:
60
+ preds = logits.argmax(dim=1)
61
+ logging_output["ncorrect"] = (preds == targets).sum()
62
+
63
+ return loss, sample_size, logging_output
data/fairseq/fairseq/criterions/sentence_ranking.py ADDED
@@ -0,0 +1,121 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright (c) Facebook, Inc. and its affiliates.
2
+ #
3
+ # This source code is licensed under the MIT license found in the
4
+ # LICENSE file in the root directory of this source tree.
5
+
6
+ import math
7
+
8
+ import torch
9
+ import torch.nn.functional as F
10
+ from fairseq import utils
11
+ from fairseq.logging import metrics
12
+ from fairseq.criterions import FairseqCriterion, register_criterion
13
+
14
+
15
+ @register_criterion("sentence_ranking")
16
+ class SentenceRankingCriterion(FairseqCriterion):
17
+ def __init__(self, task, ranking_head_name, save_predictions, num_classes):
18
+ super().__init__(task)
19
+ self.ranking_head_name = ranking_head_name
20
+ if save_predictions is not None:
21
+ self.prediction_h = open(save_predictions, "w")
22
+ else:
23
+ self.prediction_h = None
24
+ self.num_classes = num_classes
25
+
26
+ def __del__(self):
27
+ if self.prediction_h is not None:
28
+ self.prediction_h.close()
29
+
30
+ @staticmethod
31
+ def add_args(parser):
32
+ # fmt: off
33
+ parser.add_argument('--save-predictions', metavar='FILE',
34
+ help='file to save predictions to')
35
+ parser.add_argument('--ranking-head-name',
36
+ default='sentence_classification_head',
37
+ help='name of the ranking head to use')
38
+ # fmt: on
39
+
40
+ def forward(self, model, sample, reduce=True):
41
+ """Compute ranking loss for the given sample.
42
+
43
+ Returns a tuple with three elements:
44
+ 1) the loss
45
+ 2) the sample size, which is used as the denominator for the gradient
46
+ 3) logging outputs to display while training
47
+ """
48
+ assert (
49
+ hasattr(model, "classification_heads")
50
+ and self.ranking_head_name in model.classification_heads
51
+ ), "model must provide sentence ranking head for --criterion=sentence_ranking"
52
+
53
+ scores = []
54
+ for idx in range(self.num_classes):
55
+ score, _ = model(
56
+ **sample["net_input{idx}".format(idx=idx + 1)],
57
+ classification_head_name=self.ranking_head_name,
58
+ )
59
+ scores.append(score)
60
+
61
+ logits = torch.cat(scores, dim=1)
62
+ sample_size = logits.size(0)
63
+
64
+ if "target" in sample:
65
+ targets = model.get_targets(sample, [logits]).view(-1)
66
+ lprobs = F.log_softmax(logits, dim=-1, dtype=torch.float32)
67
+ loss = F.nll_loss(lprobs, targets, reduction="sum")
68
+ else:
69
+ targets = None
70
+ loss = torch.tensor(0.0, requires_grad=True)
71
+
72
+ if self.prediction_h is not None:
73
+ preds = logits.argmax(dim=1)
74
+ for i, (id, pred) in enumerate(zip(sample["id"].tolist(), preds.tolist())):
75
+ if targets is not None:
76
+ label = targets[i].item()
77
+ print("{}\t{}\t{}".format(id, pred, label), file=self.prediction_h)
78
+ else:
79
+ print("{}\t{}".format(id, pred), file=self.prediction_h)
80
+
81
+ logging_output = {
82
+ "loss": loss.data,
83
+ "ntokens": sample["ntokens"],
84
+ "nsentences": sample_size,
85
+ "sample_size": sample_size,
86
+ }
87
+ if targets is not None:
88
+ logging_output["ncorrect"] = (logits.argmax(dim=1) == targets).sum()
89
+
90
+ return loss, sample_size, logging_output
91
+
92
+ @staticmethod
93
+ def reduce_metrics(logging_outputs) -> None:
94
+ """Aggregate logging outputs from data parallel training."""
95
+ loss_sum = sum(log.get("loss", 0) for log in logging_outputs)
96
+ ntokens = sum(log.get("ntokens", 0) for log in logging_outputs)
97
+ nsentences = sum(log.get("nsentences", 0) for log in logging_outputs)
98
+ sample_size = sum(log.get("sample_size", 0) for log in logging_outputs)
99
+
100
+ metrics.log_scalar(
101
+ "loss", loss_sum / sample_size / math.log(2), sample_size, round=3
102
+ )
103
+ if sample_size != ntokens:
104
+ metrics.log_scalar(
105
+ "nll_loss", loss_sum / ntokens / math.log(2), ntokens, round=3
106
+ )
107
+
108
+ if len(logging_outputs) > 0 and "ncorrect" in logging_outputs[0]:
109
+ ncorrect = sum(log.get("ncorrect", 0) for log in logging_outputs)
110
+ metrics.log_scalar(
111
+ "accuracy", 100.0 * ncorrect / nsentences, nsentences, round=1
112
+ )
113
+
114
+ @staticmethod
115
+ def logging_outputs_can_be_summed() -> bool:
116
+ """
117
+ Whether the logging outputs returned by `forward` can be summed
118
+ across workers prior to calling `reduce_metrics`. Setting this
119
+ to True will improves distributed training speed.
120
+ """
121
+ return True
data/fairseq/fairseq/criterions/speech_dlm_criterion.py ADDED
@@ -0,0 +1,335 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright (c) Facebook, Inc. and its affiliates.
2
+ #
3
+ # This source code is licensed under the MIT license found in the
4
+ # LICENSE file in the root directory of this source tree.
5
+
6
+ import math
7
+ from dataclasses import dataclass, field
8
+ from typing import Optional
9
+
10
+ import torch.nn.functional as F
11
+ from fairseq import metrics, utils
12
+ from fairseq.criterions import FairseqCriterion, register_criterion
13
+ from fairseq.dataclass import FairseqDataclass
14
+ from omegaconf import II
15
+
16
+
17
+ @dataclass
18
+ class SpeechDLMCriterionConfig(FairseqDataclass):
19
+ sentence_avg: bool = II("optimization.sentence_avg")
20
+ main_and_cross_weights: Optional[str] = field(
21
+ default="1,0",
22
+ metadata={
23
+ "help": "Comma-separated list of weights of Main-channel vs Cross-channel Prediction Losses"
24
+ "(default: 1,0)"
25
+ },
26
+ )
27
+ general_unit_loss_weight: float = field(
28
+ default=0,
29
+ metadata={
30
+ "help": "The weight of the General Prediction Loss (Next-step Unit Prediction Loss)"
31
+ "(default: 0)"
32
+ },
33
+ )
34
+ edge_unit_loss_weight: float = field(
35
+ default=1,
36
+ metadata={"help": "The weight of the Edge Unit Prediction Loss" "(default: 1)"},
37
+ )
38
+ duration_loss_weight: float = field(
39
+ default=1,
40
+ metadata={
41
+ "help": "The weight of the Edge Unit Duration Prediction Loss"
42
+ "(default: 1)"
43
+ },
44
+ )
45
+
46
+
47
+ @register_criterion("speech_dlm_criterion", dataclass=SpeechDLMCriterionConfig)
48
+ class SpeechDLMCriterion(FairseqCriterion):
49
+ """Criteron for the SpeechDLM model as described in the paper:
50
+ https://arxiv.org/pdf/2203.16502.pdf
51
+
52
+ There are 3 possible losses depending on the targets of the model:
53
+ - general_unit_loss : The next unit prediction loss, corresponding to
54
+ 'next' target
55
+ - edge_unit_loss : The edge unit prediction loss, corresponding to
56
+ 'edge' target
57
+ - duration_loss : The duration prediction loss, corresponding to
58
+ 'duration' target
59
+ """
60
+
61
+ def __init__(
62
+ self,
63
+ task,
64
+ sentence_avg,
65
+ main_and_cross_weights,
66
+ general_unit_loss_weight,
67
+ edge_unit_loss_weight,
68
+ duration_loss_weight,
69
+ ):
70
+ super().__init__(task)
71
+ self.sentence_avg = sentence_avg
72
+
73
+ self.channels = task.channels
74
+ self.targets = task.targets
75
+ self.delayed_duration_target = task.delayed_duration_target
76
+
77
+ self.main_channel_weight = float(main_and_cross_weights.split(",")[0])
78
+ self.cross_channel_weight = float(main_and_cross_weights.split(",")[1])
79
+ assert self.main_channel_weight >= 0 and self.cross_channel_weight >= 0
80
+
81
+ self.channel_weights = {
82
+ channel: weight
83
+ for channel, weight in zip(self.channels, task.channel_weights)
84
+ }
85
+
86
+ self.target_weights = {}
87
+ for t in self.targets:
88
+ if t == "next":
89
+ self.target_weights[t] = general_unit_loss_weight
90
+ assert (
91
+ general_unit_loss_weight > 0
92
+ ), "Expect a positive --general-unit-loss-weight for next unit prediction"
93
+ elif t == "edge":
94
+ self.target_weights[t] = edge_unit_loss_weight
95
+ assert (
96
+ edge_unit_loss_weight > 0
97
+ ), "Expect a positive --edge-unit-loss-weight for edge unit prediction"
98
+ elif t == "duration":
99
+ self.target_weights[t] = duration_loss_weight
100
+ assert (
101
+ duration_loss_weight > 0
102
+ ), "Expect a positive --duration-loss-weight for duration prediction"
103
+
104
+ def forward(self, model, sample, reduce=True):
105
+ """Compute the loss for the given sample.
106
+
107
+ Returns a tuple with three elements:
108
+ 1) the loss
109
+ 2) the sample size, which is used as the denominator for the gradient
110
+ 3) logging outputs to display while training
111
+ """
112
+ net_output = model(**sample["net_input"])
113
+ loss_dict, stats_dict = self.compute_loss(
114
+ model, net_output, sample, reduce=reduce
115
+ )
116
+ nsentences = sample["net_input"]["src_tokens"][self.channels[0]].size(0)
117
+
118
+ logging_output = {
119
+ "nsentences": nsentences,
120
+ }
121
+ logging_output["nsentences"] = nsentences
122
+
123
+ loss_all = {t: 0 for t in self.targets}
124
+ correct_all = {t: 0 for t in self.targets}
125
+ count_all = {t: 0 for t in self.targets}
126
+ ntokens_all = 0
127
+ sample_size_all = 0
128
+ for channel in loss_dict:
129
+ for pred_channel in loss_dict[channel]:
130
+ # Get ntokens & sample_size
131
+ ntokens = sample["net_input"]["src_tokens"][channel].numel()
132
+ sample_size = nsentences if self.sentence_avg else ntokens
133
+ prefix = "[{}-{}]".format(channel, pred_channel)
134
+ log_keys = {
135
+ "next": "general_token",
136
+ "edge": "edge_token",
137
+ "duration": "edge_duration",
138
+ }
139
+
140
+ # Log & Update the sizes
141
+ logging_output["{}ntokens".format(prefix)] = ntokens
142
+ logging_output["{}sample_size".format(prefix)] = sample_size
143
+ ntokens_all += ntokens
144
+ sample_size_all += sample_size
145
+
146
+ for t in self.targets:
147
+ log_key = log_keys[t]
148
+ loss = loss_dict[channel][pred_channel][t]
149
+ correct, count = stats_dict[channel][pred_channel][t]
150
+
151
+ # Log the statistics
152
+ logging_output["{}{}_loss".format(prefix, log_key)] = loss.data
153
+ logging_output["{}{}_correct".format(prefix, log_key)] = correct
154
+ logging_output["{}{}_count".format(prefix, log_key)] = count
155
+
156
+ # Scale the training loss by weights
157
+ target_loss = loss * self.channel_weights[channel]
158
+ if pred_channel == channel:
159
+ target_loss = target_loss * self.main_channel_weight
160
+ else:
161
+ target_loss = target_loss * self.cross_channel_weight
162
+ # Normalize the losses in the training by the number of edges
163
+ if t in ["edge", "duration"]:
164
+ target_loss = target_loss / count * sample_size
165
+
166
+ # Update the statistics
167
+ loss_all[t] += target_loss
168
+ correct_all[t] += correct
169
+ count_all[t] += count
170
+
171
+ # Logging the average statistics
172
+ logging_output["ntokens"] = ntokens_all
173
+ logging_output["sample_size"] = sample_size_all
174
+ for t in self.targets:
175
+ log_key = {
176
+ "next": "general_token",
177
+ "edge": "edge_token",
178
+ "duration": "edge_duration",
179
+ }[t]
180
+ logging_output["{}_loss".format(log_key)] = loss_all[t].data
181
+ logging_output["{}_correct".format(log_key)] = correct_all[t]
182
+ logging_output["{}_count".format(log_key)] = count_all[t]
183
+
184
+ # Define the training loss
185
+ training_loss = 0
186
+ for t in self.targets:
187
+ training_loss += loss_all[t] * self.target_weights[t]
188
+ logging_output["loss"] = training_loss.data
189
+
190
+ return training_loss, sample_size_all, logging_output
191
+
192
+ def compute_loss(self, model, net_output, sample, reduce=True):
193
+ # Get the model outputs and target
194
+ lprobs_dict = model.get_normalized_probs(net_output, log_probs=True)
195
+ target_dict = model.get_targets(sample, net_output)
196
+
197
+ # Init the dictionaries
198
+ loss_dict, stats_dict = {}, {}
199
+
200
+ for channel in lprobs_dict:
201
+ # Init the dictionaries
202
+ loss_dict[channel], stats_dict[channel] = {}, {}
203
+
204
+ for pred_channel in lprobs_dict[channel]:
205
+ # Init the dictionaries
206
+ loss_dict[channel][pred_channel] = {}
207
+ stats_dict[channel][pred_channel] = {}
208
+
209
+ # Get token & duration predictions
210
+ outputs = lprobs_dict[channel][pred_channel]
211
+ if not isinstance(outputs, dict):
212
+ token_lprobs = outputs
213
+ else:
214
+ token_lprobs = outputs["pred_token"]
215
+ dur_preds = outputs["pred_duration"]
216
+ dur_preds = dur_preds.view(-1)
217
+ token_lprobs = token_lprobs.view(-1, token_lprobs.size(-1))
218
+ token_preds = token_lprobs.argmax(dim=-1)
219
+
220
+ # Get edge indices
221
+ if "edge" in self.targets or "duration" in self.targets:
222
+ edge_indices = target_dict["edge_indices"][pred_channel]
223
+
224
+ # Compute loss and statistics
225
+ for t in self.targets:
226
+ if t in ["next", "edge"]:
227
+ if t == "next":
228
+ target = target_dict["next"][pred_channel].view(-1)
229
+ lprobs = token_lprobs
230
+ preds = token_preds
231
+ elif t == "edge":
232
+ target = target_dict["edge"][pred_channel]
233
+ lprobs = token_lprobs[edge_indices]
234
+ preds = token_preds[edge_indices]
235
+
236
+ loss = F.nll_loss(
237
+ lprobs,
238
+ target,
239
+ ignore_index=self.padding_idx,
240
+ reduction="sum" if reduce else "none",
241
+ )
242
+ elif t == "duration":
243
+ target = target_dict["duration"][pred_channel]
244
+ if self.delayed_duration_target:
245
+ duration_indices = edge_indices + 1
246
+ if duration_indices[-1] == len(dur_preds):
247
+ duration_indices = duration_indices[:-1]
248
+ target = target[:-1]
249
+ else:
250
+ duration_indices = edge_indices
251
+ preds = dur_preds[duration_indices]
252
+
253
+ loss = F.l1_loss(
254
+ preds,
255
+ target,
256
+ reduction="sum" if reduce else "none",
257
+ )
258
+ preds = preds.round()
259
+
260
+ correct = (preds == target).sum().float().cpu().item()
261
+ count = float(target.size(0))
262
+
263
+ loss_dict[channel][pred_channel][t] = loss
264
+ stats_dict[channel][pred_channel][t] = (correct, count)
265
+
266
+ return loss_dict, stats_dict
267
+
268
+ @staticmethod
269
+ def reduce_metrics(logging_outputs) -> None:
270
+ """Aggregate logging outputs from data parallel training."""
271
+ logging_keys = next(iter(logging_outputs)).keys()
272
+ channels = [item[:-7] for item in logging_keys if item.endswith("ntokens")]
273
+ target_prefixes = set(
274
+ [
275
+ item[:-5].split("]")[-1]
276
+ for item in logging_keys
277
+ if item.endswith("_loss")
278
+ ]
279
+ )
280
+ for channel_prefix in channels:
281
+ for target_prefix in target_prefixes:
282
+ prefix = "{}{}".format(channel_prefix, target_prefix)
283
+ count_sum = sum(
284
+ log.get("{}_count".format(prefix), 0) for log in logging_outputs
285
+ )
286
+ correct_sum = sum(
287
+ log.get("{}_correct".format(prefix), 0) for log in logging_outputs
288
+ )
289
+ loss_sum = sum(
290
+ log.get("{}_loss".format(prefix), 0) for log in logging_outputs
291
+ )
292
+
293
+ if "duration" not in target_prefix:
294
+ # we divide by log(2) to convert the loss from base e to base 2
295
+ metrics.log_scalar(
296
+ "{}_loss".format(prefix),
297
+ loss_sum / count_sum / math.log(2),
298
+ count_sum,
299
+ round=3,
300
+ )
301
+ metrics.log_derived(
302
+ "{}_ppl".format(prefix),
303
+ lambda meters, prefix=prefix: utils.get_perplexity(
304
+ meters["{}_loss".format(prefix)].avg
305
+ ),
306
+ )
307
+ else:
308
+ # for duration we don't need to divide by log(2)
309
+ metrics.log_scalar(
310
+ "{}_loss".format(prefix),
311
+ loss_sum / count_sum,
312
+ count_sum,
313
+ round=3,
314
+ )
315
+
316
+ accuracy = 100 * correct_sum / count_sum
317
+ metrics.log_scalar("{}_pred_acc".format(prefix), accuracy, round=3)
318
+
319
+ # Logging training loss
320
+ sample_size = sum(log.get("sample_size", 0) for log in logging_outputs)
321
+ loss_sum = sum(log.get("loss", 0) for log in logging_outputs)
322
+
323
+ # we divide by log(2) to convert the loss from base e to base 2
324
+ metrics.log_scalar(
325
+ "loss", loss_sum / sample_size / math.log(2), sample_size, round=3
326
+ )
327
+
328
+ @staticmethod
329
+ def logging_outputs_can_be_summed() -> bool:
330
+ """
331
+ Whether the logging outputs returned by `forward` can be summed
332
+ across workers prior to calling `reduce_metrics`. Setting this
333
+ to True will improves distributed training speed.
334
+ """
335
+ return True
data/fairseq/fairseq/criterions/speech_to_speech_criterion.py ADDED
@@ -0,0 +1,517 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright (c) Facebook, Inc. and its affiliates.
2
+ #
3
+ # This source code is licensed under the MIT license found in the
4
+ # LICENSE file in the root directory of this source tree.
5
+
6
+ import logging
7
+ import math
8
+ from collections import OrderedDict
9
+
10
+ import torch
11
+
12
+ from fairseq import utils
13
+ from fairseq.logging import metrics
14
+ from fairseq.criterions import register_criterion
15
+ from fairseq.criterions.ctc import CtcCriterion
16
+ from fairseq.criterions.label_smoothed_cross_entropy_with_rdrop import (
17
+ RdropLabelSmoothedCrossEntropyCriterion,
18
+ RdropLabelSmoothedCrossEntropyCriterionConfig,
19
+ duplicate_input,
20
+ )
21
+ from fairseq.criterions.tacotron2_loss import (
22
+ Tacotron2Criterion,
23
+ Tacotron2CriterionConfig,
24
+ )
25
+
26
+ logger = logging.getLogger(__name__)
27
+
28
+
29
+ class MultitaskCriterion:
30
+ def __init__(self, multitask_tasks, rdrop_alpha=0.0):
31
+ self.rdrop_alpha = rdrop_alpha
32
+ self.rdrop_alpha_mtl = rdrop_alpha
33
+
34
+ self.multitask_criterion = OrderedDict()
35
+ self.multitask_loss_weight = OrderedDict()
36
+ for task_name, task_obj in multitask_tasks.items():
37
+ if task_obj.args.get_loss_weight(0) == 0:
38
+ logger.info(f"Skip {task_name} loss criterion")
39
+ continue
40
+
41
+ rdrop_alpha_task = task_obj.args.rdrop_alpha
42
+ if rdrop_alpha_task is None:
43
+ rdrop_alpha_task = rdrop_alpha
44
+ self.rdrop_alpha_mtl = rdrop_alpha_task
45
+ logger.info(f"rdrop_alpha is set to {rdrop_alpha_task} for {task_name}")
46
+
47
+ if task_obj.args.decoder_type == "ctc":
48
+ self.multitask_criterion[task_name] = CtcCriterion(
49
+ task_obj.args.criterion_cfg,
50
+ task_obj,
51
+ rdrop_alpha=rdrop_alpha_task,
52
+ )
53
+ else:
54
+ self.multitask_criterion[
55
+ task_name
56
+ ] = RdropLabelSmoothedCrossEntropyCriterion(
57
+ task_obj,
58
+ task_obj.args.criterion_cfg.sentence_avg,
59
+ label_smoothing=task_obj.args.criterion_cfg.label_smoothing,
60
+ rdrop_alpha=rdrop_alpha_task,
61
+ )
62
+
63
+ def set_multitask_loss_weight(self, task_name, weight=0.0):
64
+ self.multitask_loss_weight[task_name] = weight
65
+
66
+ def get_multitask_loss(self, model, sample, model_out):
67
+ logging_output = {}
68
+ loss = 0.0
69
+ for task_name, task_criterion in self.multitask_criterion.items():
70
+ layer_id = task_criterion.task.args.input_layer
71
+ if isinstance(task_criterion, CtcCriterion):
72
+ if task_criterion.task.args.input_from == "encoder":
73
+ if len(model_out["encoder_padding_mask"]) > 0:
74
+ non_padding_mask = ~model_out["encoder_padding_mask"][0]
75
+ input_lengths = non_padding_mask.long().sum(-1)
76
+ else:
77
+ out = model_out["encoder_states"][layer_id]
78
+ input_lengths = out.new_full(
79
+ (out.shape[1],), out.shape[0]
80
+ ).long()
81
+
82
+ task_sample = {
83
+ "net_input": {
84
+ "src_tokens": model_out["encoder_states"][
85
+ layer_id
86
+ ], # check batch idx
87
+ "src_lengths": input_lengths,
88
+ },
89
+ "id": sample["id"],
90
+ }
91
+ else:
92
+ task_sample = {
93
+ "net_input": {
94
+ "src_tokens": model_out["inner_states"][layer_id],
95
+ "src_lengths": sample["target_lengths"],
96
+ },
97
+ "id": sample["id"],
98
+ }
99
+ else:
100
+ task_sample = {
101
+ "net_input": {
102
+ "src_tokens": sample["multitask"][task_name]["net_input"][
103
+ "prev_output_tokens"
104
+ ],
105
+ "encoder_out": {
106
+ "encoder_out": [model_out["encoder_states"][layer_id]],
107
+ "encoder_padding_mask": model_out["encoder_padding_mask"],
108
+ },
109
+ }
110
+ }
111
+
112
+ for key in ["target", "target_lengths", "ntokens"]:
113
+ task_sample[key] = sample["multitask"][task_name][key]
114
+
115
+ if task_name == getattr(model, "mt_task_name", None):
116
+ decoder_out = model_out["mt_decoder_out"]
117
+ else:
118
+ decoder_out = None
119
+ task_loss, task_sample_size, task_logging_output = task_criterion(
120
+ model.multitask_decoders[task_name], task_sample, net_output=decoder_out
121
+ )
122
+
123
+ loss = loss + self.multitask_loss_weight[task_name] * task_loss
124
+ task_logging_output["loss_weight"] = self.multitask_loss_weight[task_name]
125
+ logging_output[task_name] = task_logging_output
126
+ return loss, logging_output
127
+
128
+ @classmethod
129
+ def reduce_metrics(cls, logging_outputs) -> None:
130
+ for task_name in logging_outputs[0]["multitask"].keys():
131
+ # different criterion may return different logging
132
+ # currently only reduce on loss, the most common one
133
+ # ideally the way that losses are reduced should also depend on the task type
134
+ loss_sum = sum(
135
+ log["multitask"][task_name].get("loss", 0) for log in logging_outputs
136
+ )
137
+ sample_size = sum(
138
+ log["multitask"][task_name].get("sample_size", 0)
139
+ for log in logging_outputs
140
+ )
141
+
142
+ metrics.log_scalar(
143
+ f"multitask_{task_name}_loss",
144
+ loss_sum / sample_size / math.log(2),
145
+ sample_size,
146
+ round=3,
147
+ )
148
+
149
+ loss_weight = logging_outputs[0]["multitask"][task_name].get(
150
+ "loss_weight", 0
151
+ )
152
+ metrics.log_scalar(
153
+ f"multitask_{task_name}_loss_weight",
154
+ loss_weight,
155
+ weight=0,
156
+ priority=250,
157
+ )
158
+
159
+
160
+ @register_criterion(
161
+ "speech_to_unit", dataclass=RdropLabelSmoothedCrossEntropyCriterionConfig
162
+ )
163
+ class SpeechToUnitMultitaskTaskCriterion(
164
+ RdropLabelSmoothedCrossEntropyCriterion, MultitaskCriterion
165
+ ):
166
+ def __init__(
167
+ self,
168
+ task,
169
+ sentence_avg,
170
+ label_smoothing,
171
+ ignore_prefix_size=0,
172
+ report_accuracy=False,
173
+ rdrop_alpha=0.0,
174
+ ):
175
+ super().__init__(
176
+ task,
177
+ sentence_avg,
178
+ label_smoothing,
179
+ ignore_prefix_size,
180
+ report_accuracy,
181
+ rdrop_alpha,
182
+ )
183
+ MultitaskCriterion.__init__(self, task.multitask_tasks, rdrop_alpha)
184
+
185
+ def forward(self, model, sample, reduce=True):
186
+ net_input_concat = {
187
+ "src_tokens": sample["net_input"]["src_tokens"],
188
+ "src_lengths": sample["net_input"]["src_lengths"],
189
+ "prev_output_tokens": sample["net_input"]["prev_output_tokens"],
190
+ "tgt_speaker": sample["net_input"].get("tgt_speaker", None),
191
+ "return_all_hiddens": True,
192
+ }
193
+
194
+ if self.rdrop_alpha > 0 or self.rdrop_alpha_mtl > 0:
195
+ net_input_concat = duplicate_input(net_input_concat)
196
+
197
+ net_output, extra = model(**net_input_concat)
198
+ loss, nll_loss, rdrop_kl_loss = self.compute_loss(
199
+ model, [net_output], sample, reduce=reduce
200
+ )
201
+ sample_size = (
202
+ sample["target"].size(0) if self.sentence_avg else sample["ntokens"]
203
+ )
204
+ logging_output = {
205
+ "loss": loss.data,
206
+ "nll_loss": nll_loss.data,
207
+ "ntokens": sample["ntokens"],
208
+ "nsentences": sample["target"].size(0),
209
+ "sample_size": sample_size,
210
+ }
211
+ if self.report_accuracy:
212
+ n_correct, total = self.compute_accuracy(model, [net_output], sample)
213
+ logging_output["n_correct"] = utils.item(n_correct.data)
214
+ logging_output["total"] = utils.item(total.data)
215
+ if self.rdrop_alpha > 0:
216
+ logging_output["rdrop_kl_loss"] = utils.item(rdrop_kl_loss.data)
217
+
218
+ if len(self.multitask_criterion) == 0:
219
+ return loss, sample_size, logging_output
220
+
221
+ # multitask
222
+ multitask_loss, multitask_log = self.get_multitask_loss(model, sample, extra)
223
+ loss += multitask_loss
224
+ logging_output["multitask"] = multitask_log
225
+
226
+ return loss, sample_size, logging_output
227
+
228
+ @classmethod
229
+ def reduce_metrics(cls, logging_outputs) -> None:
230
+ super().reduce_metrics(logging_outputs)
231
+
232
+ # inference metrics
233
+ if "targ_frames" in logging_outputs[0]:
234
+ n = sum(log.get("norm_frames", 0) for log in logging_outputs)
235
+ for key, new_key in [
236
+ ("mcd_loss", "mcd_loss"),
237
+ ("pred_frames", "pred_ratio"),
238
+ ("nins", "ins_rate"),
239
+ ("ndel", "del_rate"),
240
+ ]:
241
+ val = sum(log.get(key, 0) for log in logging_outputs)
242
+ metrics.log_scalar(new_key, val / n, n, round=3)
243
+
244
+ if "multitask" not in logging_outputs[0]:
245
+ return
246
+
247
+ MultitaskCriterion.reduce_metrics(logging_outputs)
248
+
249
+ @staticmethod
250
+ def logging_outputs_can_be_summed() -> bool:
251
+ """
252
+ Whether the logging outputs returned by `forward` can be summed
253
+ across workers prior to calling `reduce_metrics`. Setting this
254
+ to True will improves distributed training speed.
255
+ """
256
+ return False
257
+
258
+
259
+ @register_criterion(
260
+ "speech_to_unit_2pass", dataclass=RdropLabelSmoothedCrossEntropyCriterionConfig
261
+ )
262
+ class SpeechToUnit2passMultitaskTaskCriterion(SpeechToUnitMultitaskTaskCriterion):
263
+ def __init__(
264
+ self,
265
+ task,
266
+ sentence_avg,
267
+ label_smoothing,
268
+ ignore_prefix_size=0,
269
+ report_accuracy=False,
270
+ rdrop_alpha=0.0,
271
+ ):
272
+ super().__init__(
273
+ task,
274
+ sentence_avg,
275
+ label_smoothing,
276
+ ignore_prefix_size,
277
+ report_accuracy,
278
+ rdrop_alpha,
279
+ )
280
+
281
+ def forward(self, model, sample, reduce=True):
282
+ net_input_concat = {
283
+ "src_tokens": sample["net_input"]["src_tokens"],
284
+ "src_lengths": sample["net_input"]["src_lengths"],
285
+ "prev_output_tokens": sample["net_input"]["prev_output_tokens"],
286
+ "prev_output_tokens_mt": sample["multitask"][model.mt_task_name][
287
+ "net_input"
288
+ ]["prev_output_tokens"],
289
+ "tgt_speaker": sample["net_input"].get("tgt_speaker", None),
290
+ "return_all_hiddens": True,
291
+ }
292
+ if getattr(model, "asr_task_name", None) is not None:
293
+ net_input_concat["prev_output_tokens_asr"] = sample["multitask"][
294
+ model.asr_task_name
295
+ ]["net_input"]["prev_output_tokens"]
296
+
297
+ if self.rdrop_alpha > 0 or self.rdrop_alpha_mtl > 0:
298
+ net_input_concat = duplicate_input(net_input_concat)
299
+
300
+ net_output, extra = model(**net_input_concat)
301
+ loss, nll_loss, rdrop_kl_loss = self.compute_loss(
302
+ model, [net_output], sample, reduce=reduce
303
+ )
304
+
305
+ sample_size = (
306
+ sample["target"].size(0) if self.sentence_avg else sample["ntokens"]
307
+ )
308
+ logging_output = {
309
+ "loss": loss.data,
310
+ "nll_loss": nll_loss.data,
311
+ "ntokens": sample["ntokens"],
312
+ "nsentences": sample["target"].size(0),
313
+ "sample_size": sample_size,
314
+ }
315
+ if self.report_accuracy:
316
+ n_correct, total = self.compute_accuracy(model, [net_output], sample)
317
+ logging_output["n_correct"] = utils.item(n_correct.data)
318
+ logging_output["total"] = utils.item(total.data)
319
+ if self.rdrop_alpha > 0:
320
+ logging_output["rdrop_kl_loss"] = utils.item(rdrop_kl_loss.data)
321
+
322
+ if len(self.multitask_criterion) == 0:
323
+ return loss, sample_size, logging_output
324
+
325
+ # multitask
326
+ multitask_loss, multitask_log = self.get_multitask_loss(model, sample, extra)
327
+ loss += multitask_loss
328
+ logging_output["multitask"] = multitask_log
329
+
330
+ return loss, sample_size, logging_output
331
+
332
+
333
+ @register_criterion("speech_to_spectrogram", dataclass=Tacotron2CriterionConfig)
334
+ class SpeechToSpectrogramMultitaskTaskCriterion(Tacotron2Criterion, MultitaskCriterion):
335
+ def __init__(
336
+ self,
337
+ task,
338
+ sentence_avg,
339
+ use_guided_attention_loss,
340
+ guided_attention_loss_sigma,
341
+ bce_pos_weight,
342
+ ctc_weight,
343
+ ):
344
+ super().__init__(
345
+ task,
346
+ sentence_avg,
347
+ use_guided_attention_loss,
348
+ guided_attention_loss_sigma,
349
+ bce_pos_weight,
350
+ ctc_weight,
351
+ )
352
+ MultitaskCriterion.__init__(self, task.multitask_tasks)
353
+
354
+ def forward(self, model, sample, reduction="mean"):
355
+ bsz, max_len, _ = sample["target"].size()
356
+ feat_tgt = sample["target"]
357
+ feat_len = sample["target_lengths"].view(bsz, 1).expand(-1, max_len)
358
+ eos_tgt = torch.arange(max_len).to(sample["target"].device)
359
+ eos_tgt = eos_tgt.view(1, max_len).expand(bsz, -1)
360
+ eos_tgt = (eos_tgt == (feat_len - 1)).float()
361
+
362
+ feat_out, eos_out, extra = model(
363
+ src_tokens=sample["net_input"]["src_tokens"],
364
+ src_lengths=sample["net_input"]["src_lengths"],
365
+ prev_output_tokens=sample["net_input"]["prev_output_tokens"],
366
+ tgt_speaker=sample["net_input"]["tgt_speaker"],
367
+ target_lengths=sample["target_lengths"],
368
+ return_all_hiddens=True,
369
+ )
370
+
371
+ l1_loss, mse_loss, eos_loss = self.compute_loss(
372
+ extra["feature_out"],
373
+ feat_out,
374
+ eos_out,
375
+ feat_tgt,
376
+ eos_tgt,
377
+ sample["target_lengths"],
378
+ reduction,
379
+ )
380
+ attn_loss = torch.tensor(0.0).type_as(l1_loss)
381
+ if self.guided_attn is not None:
382
+ attn_loss = self.guided_attn(
383
+ extra["attn"],
384
+ sample["net_input"]["src_lengths"],
385
+ sample["target_lengths"],
386
+ reduction,
387
+ )
388
+ loss = (
389
+ l1_loss + mse_loss + eos_loss + attn_loss
390
+ ) # do not include ctc loss as there's no text target
391
+
392
+ sample_size = sample["nsentences"] if self.sentence_avg else sample["ntokens"]
393
+ logging_output = {
394
+ "loss": utils.item(loss.data),
395
+ "ntokens": sample["ntokens"],
396
+ "nsentences": sample["nsentences"],
397
+ "sample_size": sample_size,
398
+ "l1_loss": utils.item(l1_loss.data),
399
+ "mse_loss": utils.item(mse_loss.data),
400
+ "eos_loss": utils.item(eos_loss.data),
401
+ "attn_loss": utils.item(attn_loss.data),
402
+ }
403
+
404
+ if len(self.multitask_criterion) == 0:
405
+ return loss, sample_size, logging_output
406
+
407
+ # multitask
408
+ multitask_loss, multitask_log = self.get_multitask_loss(model, sample, extra)
409
+ loss += multitask_loss
410
+ logging_output["multitask"] = multitask_log
411
+ return loss, sample_size, logging_output
412
+
413
+ @classmethod
414
+ def reduce_metrics(cls, logging_outputs) -> None:
415
+ super().reduce_metrics(logging_outputs)
416
+
417
+ # inference metrics
418
+ if "targ_frames" in logging_outputs[0]:
419
+ n = sum(log.get("norm_frames", 0) for log in logging_outputs)
420
+ for key, new_key in [
421
+ ("mcd_loss", "mcd_loss"),
422
+ ("pred_frames", "pred_ratio"),
423
+ ("nins", "ins_rate"),
424
+ ("ndel", "del_rate"),
425
+ ]:
426
+ val = sum(log.get(key, 0) for log in logging_outputs)
427
+ metrics.log_scalar(new_key, val / n, n, round=3)
428
+
429
+ if "multitask" not in logging_outputs[0]:
430
+ return
431
+
432
+ MultitaskCriterion.reduce_metrics(logging_outputs)
433
+
434
+
435
+ @register_criterion("speech_to_spectrogram_2pass", dataclass=Tacotron2CriterionConfig)
436
+ class SpeechToSpectrogram2passMultitaskTaskCriterion(
437
+ SpeechToSpectrogramMultitaskTaskCriterion
438
+ ):
439
+ def __init__(
440
+ self,
441
+ task,
442
+ sentence_avg,
443
+ use_guided_attention_loss,
444
+ guided_attention_loss_sigma,
445
+ bce_pos_weight,
446
+ ctc_weight,
447
+ ):
448
+ super().__init__(
449
+ task,
450
+ sentence_avg,
451
+ use_guided_attention_loss,
452
+ guided_attention_loss_sigma,
453
+ bce_pos_weight,
454
+ ctc_weight,
455
+ )
456
+
457
+ def forward(self, model, sample, reduction="mean"):
458
+ bsz, max_len, _ = sample["target"].size()
459
+ feat_tgt = sample["target"]
460
+ feat_len = sample["target_lengths"].view(bsz, 1).expand(-1, max_len)
461
+ eos_tgt = torch.arange(max_len).to(sample["target"].device)
462
+ eos_tgt = eos_tgt.view(1, max_len).expand(bsz, -1)
463
+ eos_tgt = (eos_tgt == (feat_len - 1)).float()
464
+
465
+ feat_out, eos_out, extra = model(
466
+ src_tokens=sample["net_input"]["src_tokens"],
467
+ src_lengths=sample["net_input"]["src_lengths"],
468
+ prev_output_tokens=sample["net_input"]["prev_output_tokens"],
469
+ prev_output_tokens_mt=sample["multitask"][model.mt_task_name]["net_input"][
470
+ "prev_output_tokens"
471
+ ],
472
+ tgt_speaker=sample["net_input"]["tgt_speaker"],
473
+ target_lengths=sample["target_lengths"],
474
+ return_all_hiddens=True,
475
+ )
476
+
477
+ l1_loss, mse_loss, eos_loss = self.compute_loss(
478
+ extra["feature_out"],
479
+ feat_out,
480
+ eos_out,
481
+ feat_tgt,
482
+ eos_tgt,
483
+ sample["target_lengths"],
484
+ reduction,
485
+ )
486
+ attn_loss = torch.tensor(0.0).type_as(l1_loss)
487
+ if self.guided_attn is not None:
488
+ attn_loss = self.guided_attn(
489
+ extra["attn"],
490
+ sample["net_input"]["src_lengths"],
491
+ sample["target_lengths"],
492
+ reduction,
493
+ )
494
+ loss = (
495
+ l1_loss + mse_loss + eos_loss + attn_loss
496
+ ) # do not include ctc loss as there's no text target
497
+
498
+ sample_size = sample["nsentences"] if self.sentence_avg else sample["ntokens"]
499
+ logging_output = {
500
+ "loss": utils.item(loss.data),
501
+ "ntokens": sample["ntokens"],
502
+ "nsentences": sample["nsentences"],
503
+ "sample_size": sample_size,
504
+ "l1_loss": utils.item(l1_loss.data),
505
+ "mse_loss": utils.item(mse_loss.data),
506
+ "eos_loss": utils.item(eos_loss.data),
507
+ "attn_loss": utils.item(attn_loss.data),
508
+ }
509
+
510
+ if len(self.multitask_criterion) == 0:
511
+ return loss, sample_size, logging_output
512
+
513
+ # multitask
514
+ multitask_loss, multitask_log = self.get_multitask_loss(model, sample, extra)
515
+ loss += multitask_loss
516
+ logging_output["multitask"] = multitask_log
517
+ return loss, sample_size, logging_output
data/fairseq/fairseq/criterions/speech_ulm_criterion.py ADDED
@@ -0,0 +1,126 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright (c) Facebook, Inc. and its affiliates.
2
+ #
3
+ # This source code is licensed under the MIT license found in the
4
+ # LICENSE file in the root directory of this source tree.
5
+
6
+ import torch
7
+ from dataclasses import dataclass, field
8
+
9
+ import torch.nn.functional as F
10
+ from fairseq.logging import metrics
11
+ from fairseq.tasks import FairseqTask
12
+ from fairseq.criterions import FairseqCriterion, register_criterion
13
+ from fairseq.dataclass import FairseqDataclass
14
+ from omegaconf import II
15
+
16
+
17
+ @dataclass
18
+ class SpeechUnitLmCriterionConfig(FairseqDataclass):
19
+ sentence_avg: bool = II("optimization.sentence_avg")
20
+ loss_weights: str = field(
21
+ default="1.;0.0;0.0",
22
+ metadata={
23
+ "help": "Weights of the losses that correspond to token, duration, and F0 streams"
24
+ },
25
+ )
26
+ discrete_duration: bool = II("task.discrete_duration")
27
+ discrete_f0: bool = II("task.discrete_f0")
28
+
29
+
30
+ def mae_loss(pred, targ, mask, reduce=True):
31
+ if pred.ndim == 3:
32
+ pred = pred.squeeze(2)
33
+ else:
34
+ assert pred.ndim == 2
35
+ loss = (pred.float() - targ.float()).abs() * (~mask).float()
36
+ loss = loss.sum() if reduce else loss.view(-1)
37
+ return loss
38
+
39
+
40
+ def nll_loss(pred, targ, mask, reduce=True):
41
+ lprob = F.log_softmax(pred, dim=-1)
42
+ loss = F.nll_loss(lprob.view(-1, lprob.size(-1)), targ.view(-1), reduction="none")
43
+ loss = loss * (~mask).float().view(-1)
44
+ loss = loss.sum() if reduce else loss.view(-1)
45
+ return loss
46
+
47
+
48
+ @register_criterion("speech_unit_lm_criterion", dataclass=SpeechUnitLmCriterionConfig)
49
+ class SpeechUnitLmCriterion(FairseqCriterion):
50
+ def __init__(self, cfg: SpeechUnitLmCriterionConfig, task: FairseqTask):
51
+ super().__init__(task)
52
+ self.sentence_avg = cfg.sentence_avg
53
+ self.weights = torch.tensor([float(w) for w in cfg.loss_weights.split(";")])
54
+ assert self.weights.size(0) == 3
55
+ assert (self.weights >= 0.0).all()
56
+
57
+ self.dur_loss_fn = nll_loss if cfg.discrete_duration else mae_loss
58
+ self.f0_loss_fn = nll_loss if cfg.discrete_f0 else mae_loss
59
+
60
+ def forward(self, model, sample, reduce=True):
61
+ """Compute the loss for the given sample.
62
+
63
+ Returns a tuple with three elements:
64
+ 1) the loss
65
+ 2) the sample size, which is used as the denominator for the gradient
66
+ 3) logging outputs to display while training
67
+ """
68
+ net_output = model(**sample["net_input"])
69
+
70
+ token_loss = nll_loss(
71
+ net_output["token"], sample["target"], sample["mask"], reduce
72
+ )
73
+ dur_loss = self.dur_loss_fn(
74
+ net_output["duration"],
75
+ sample["dur_target"],
76
+ sample["dur_mask"],
77
+ reduce,
78
+ )
79
+ f0_loss = self.f0_loss_fn(
80
+ net_output["f0"],
81
+ sample["f0_target"],
82
+ sample["f0_mask"],
83
+ reduce,
84
+ )
85
+ loss = self.weights.to(token_loss.device) * torch.stack(
86
+ [token_loss, dur_loss, f0_loss], dim=-1
87
+ )
88
+ loss = loss.sum() if reduce else loss.sum(-1)
89
+
90
+ sample_size = (
91
+ sample["target"].size(0) if self.sentence_avg else sample["ntokens"]
92
+ )
93
+ logging_output = {
94
+ "loss": loss.detach().sum().item(),
95
+ "token_loss": token_loss.detach().sum().item(),
96
+ "dur_loss": dur_loss.detach().sum().item(),
97
+ "f0_loss": f0_loss.detach().sum().item(),
98
+ "ntokens": sample["ntokens"],
99
+ "nsentences": sample["target"].size(0),
100
+ "sample_size": sample_size,
101
+ }
102
+ return loss, sample_size, logging_output
103
+
104
+ @staticmethod
105
+ def reduce_metrics(logging_outputs) -> None:
106
+ """Aggregate logging outputs from data parallel training."""
107
+ loss_sum = sum(log.get("loss", 0) for log in logging_outputs)
108
+ token_loss_sum = sum(log.get("token_loss", 0) for log in logging_outputs)
109
+ dur_loss_sum = sum(log.get("dur_loss", 0) for log in logging_outputs)
110
+ f0_loss_sum = sum(log.get("f0_loss", 0) for log in logging_outputs)
111
+
112
+ sample_size = sum(log.get("sample_size", 0) for log in logging_outputs)
113
+
114
+ metrics.log_scalar("loss", loss_sum / sample_size, sample_size, round=3)
115
+
116
+ metrics.log_scalar(
117
+ "token_loss", token_loss_sum / sample_size, sample_size, round=3
118
+ )
119
+
120
+ metrics.log_scalar("dur_loss", dur_loss_sum / sample_size, sample_size, round=3)
121
+
122
+ metrics.log_scalar("f0_loss", f0_loss_sum / sample_size, sample_size, round=3)
123
+
124
+ @staticmethod
125
+ def logging_outputs_can_be_summed() -> bool:
126
+ return True
data/fairseq/fairseq/criterions/tacotron2_loss.py ADDED
@@ -0,0 +1,227 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright (c) 2017-present, Facebook, Inc.
2
+ # All rights reserved.
3
+ #
4
+ # This source code is licensed under the license found in the LICENSE file in
5
+ # the root directory of this source tree. An additional grant of patent rights
6
+ # can be found in the PATENTS file in the same directory.
7
+
8
+ import logging
9
+ from dataclasses import dataclass, field
10
+ from functools import lru_cache
11
+ from typing import Any, Dict, List
12
+
13
+ import torch
14
+ import torch.nn.functional as F
15
+ from omegaconf import II
16
+
17
+ from fairseq import utils
18
+ from fairseq.logging import metrics
19
+ from fairseq.criterions import FairseqCriterion, register_criterion
20
+ from fairseq.data.data_utils import lengths_to_mask
21
+ from fairseq.dataclass import FairseqDataclass
22
+
23
+ logger = logging.getLogger(__name__)
24
+
25
+
26
+ @dataclass
27
+ class Tacotron2CriterionConfig(FairseqDataclass):
28
+ bce_pos_weight: float = field(
29
+ default=1.0,
30
+ metadata={"help": "weight of positive examples for BCE loss"},
31
+ )
32
+ use_guided_attention_loss: bool = field(
33
+ default=False,
34
+ metadata={"help": "use guided attention loss"},
35
+ )
36
+ guided_attention_loss_sigma: float = field(
37
+ default=0.4,
38
+ metadata={"help": "weight of positive examples for BCE loss"},
39
+ )
40
+ ctc_weight: float = field(default=0.0, metadata={"help": "weight for CTC loss"})
41
+ sentence_avg: bool = II("optimization.sentence_avg")
42
+
43
+
44
+ class GuidedAttentionLoss(torch.nn.Module):
45
+ """
46
+ Efficiently Trainable Text-to-Speech System Based on Deep Convolutional
47
+ Networks with Guided Attention (https://arxiv.org/abs/1710.08969)
48
+ """
49
+
50
+ def __init__(self, sigma):
51
+ super().__init__()
52
+ self.sigma = sigma
53
+
54
+ @staticmethod
55
+ @lru_cache(maxsize=8)
56
+ def _get_weight(s_len, t_len, sigma):
57
+ grid_x, grid_y = torch.meshgrid(torch.arange(t_len), torch.arange(s_len))
58
+ grid_x = grid_x.to(s_len.device)
59
+ grid_y = grid_y.to(s_len.device)
60
+ w = (grid_y.float() / s_len - grid_x.float() / t_len) ** 2
61
+ return 1.0 - torch.exp(-w / (2 * (sigma**2)))
62
+
63
+ def _get_weights(self, src_lens, tgt_lens):
64
+ bsz, max_s_len, max_t_len = len(src_lens), max(src_lens), max(tgt_lens)
65
+ weights = torch.zeros((bsz, max_t_len, max_s_len))
66
+ for i, (s_len, t_len) in enumerate(zip(src_lens, tgt_lens)):
67
+ weights[i, :t_len, :s_len] = self._get_weight(s_len, t_len, self.sigma)
68
+ return weights
69
+
70
+ @staticmethod
71
+ def _get_masks(src_lens, tgt_lens):
72
+ in_masks = lengths_to_mask(src_lens)
73
+ out_masks = lengths_to_mask(tgt_lens)
74
+ return out_masks.unsqueeze(2) & in_masks.unsqueeze(1)
75
+
76
+ def forward(self, attn, src_lens, tgt_lens, reduction="mean"):
77
+ weights = self._get_weights(src_lens, tgt_lens).to(attn.device)
78
+ masks = self._get_masks(src_lens, tgt_lens).to(attn.device)
79
+ loss = (weights * attn.transpose(1, 2)).masked_select(masks)
80
+ loss = torch.sum(loss) if reduction == "sum" else torch.mean(loss)
81
+ return loss
82
+
83
+
84
+ @register_criterion("tacotron2", dataclass=Tacotron2CriterionConfig)
85
+ class Tacotron2Criterion(FairseqCriterion):
86
+ def __init__(
87
+ self,
88
+ task,
89
+ sentence_avg,
90
+ use_guided_attention_loss,
91
+ guided_attention_loss_sigma,
92
+ bce_pos_weight,
93
+ ctc_weight,
94
+ ):
95
+ super().__init__(task)
96
+ self.sentence_avg = sentence_avg
97
+ self.bce_pos_weight = bce_pos_weight
98
+
99
+ self.guided_attn = None
100
+ if use_guided_attention_loss:
101
+ self.guided_attn = GuidedAttentionLoss(guided_attention_loss_sigma)
102
+ self.ctc_weight = ctc_weight
103
+
104
+ def forward(self, model, sample, reduction="mean"):
105
+ bsz, max_len, _ = sample["target"].size()
106
+ feat_tgt = sample["target"]
107
+ feat_len = sample["target_lengths"].view(bsz, 1).expand(-1, max_len)
108
+ eos_tgt = torch.arange(max_len).to(sample["target"].device)
109
+ eos_tgt = eos_tgt.view(1, max_len).expand(bsz, -1)
110
+ eos_tgt = (eos_tgt == (feat_len - 1)).float()
111
+ src_tokens = sample["net_input"]["src_tokens"]
112
+ src_lens = sample["net_input"]["src_lengths"]
113
+ tgt_lens = sample["target_lengths"]
114
+
115
+ feat_out, eos_out, extra = model(
116
+ src_tokens=src_tokens,
117
+ src_lengths=src_lens,
118
+ prev_output_tokens=sample["net_input"]["prev_output_tokens"],
119
+ incremental_state=None,
120
+ target_lengths=tgt_lens,
121
+ speaker=sample["speaker"],
122
+ )
123
+
124
+ l1_loss, mse_loss, eos_loss = self.compute_loss(
125
+ extra["feature_out"],
126
+ feat_out,
127
+ eos_out,
128
+ feat_tgt,
129
+ eos_tgt,
130
+ tgt_lens,
131
+ reduction,
132
+ )
133
+ attn_loss = torch.tensor(0.0).type_as(l1_loss)
134
+ if self.guided_attn is not None:
135
+ attn_loss = self.guided_attn(extra["attn"], src_lens, tgt_lens, reduction)
136
+ ctc_loss = torch.tensor(0.0).type_as(l1_loss)
137
+ if self.ctc_weight > 0.0:
138
+ net_output = (feat_out, eos_out, extra)
139
+ lprobs = model.get_normalized_probs(net_output, log_probs=True)
140
+ lprobs = lprobs.transpose(0, 1) # T x B x C
141
+ src_mask = lengths_to_mask(src_lens)
142
+ src_tokens_flat = src_tokens.masked_select(src_mask)
143
+ ctc_loss = (
144
+ F.ctc_loss(
145
+ lprobs,
146
+ src_tokens_flat,
147
+ tgt_lens,
148
+ src_lens,
149
+ reduction=reduction,
150
+ zero_infinity=True,
151
+ )
152
+ * self.ctc_weight
153
+ )
154
+ loss = l1_loss + mse_loss + eos_loss + attn_loss + ctc_loss
155
+
156
+ sample_size = sample["nsentences"] if self.sentence_avg else sample["ntokens"]
157
+ logging_output = {
158
+ "loss": utils.item(loss.data),
159
+ "ntokens": sample["ntokens"],
160
+ "nsentences": sample["nsentences"],
161
+ "sample_size": sample_size,
162
+ "l1_loss": utils.item(l1_loss.data),
163
+ "mse_loss": utils.item(mse_loss.data),
164
+ "eos_loss": utils.item(eos_loss.data),
165
+ "attn_loss": utils.item(attn_loss.data),
166
+ "ctc_loss": utils.item(ctc_loss.data),
167
+ }
168
+ return loss, sample_size, logging_output
169
+
170
+ def compute_loss(
171
+ self,
172
+ feat_out,
173
+ feat_out_post,
174
+ eos_out,
175
+ feat_tgt,
176
+ eos_tgt,
177
+ tgt_lens,
178
+ reduction="mean",
179
+ ):
180
+ mask = lengths_to_mask(tgt_lens)
181
+ _eos_out = eos_out[mask].squeeze()
182
+ _eos_tgt = eos_tgt[mask]
183
+ _feat_tgt = feat_tgt[mask]
184
+ _feat_out = feat_out[mask]
185
+ _feat_out_post = feat_out_post[mask]
186
+
187
+ l1_loss = F.l1_loss(_feat_out, _feat_tgt, reduction=reduction) + F.l1_loss(
188
+ _feat_out_post, _feat_tgt, reduction=reduction
189
+ )
190
+ mse_loss = F.mse_loss(_feat_out, _feat_tgt, reduction=reduction) + F.mse_loss(
191
+ _feat_out_post, _feat_tgt, reduction=reduction
192
+ )
193
+ eos_loss = F.binary_cross_entropy_with_logits(
194
+ _eos_out,
195
+ _eos_tgt,
196
+ pos_weight=torch.tensor(self.bce_pos_weight),
197
+ reduction=reduction,
198
+ )
199
+ return l1_loss, mse_loss, eos_loss
200
+
201
+ @classmethod
202
+ def reduce_metrics(cls, logging_outputs: List[Dict[str, Any]]) -> None:
203
+ ns = [log.get("sample_size", 0) for log in logging_outputs]
204
+ ntot = sum(ns)
205
+ ws = [n / (ntot + 1e-8) for n in ns]
206
+ for key in ["loss", "l1_loss", "mse_loss", "eos_loss", "attn_loss", "ctc_loss"]:
207
+ vals = [log.get(key, 0) for log in logging_outputs]
208
+ val = sum(val * w for val, w in zip(vals, ws))
209
+ metrics.log_scalar(key, val, ntot, round=3)
210
+ metrics.log_scalar("sample_size", ntot, len(logging_outputs))
211
+
212
+ # inference metrics
213
+ if "targ_frames" not in logging_outputs[0]:
214
+ return
215
+ n = sum(log.get("targ_frames", 0) for log in logging_outputs)
216
+ for key, new_key in [
217
+ ("mcd_loss", "mcd_loss"),
218
+ ("pred_frames", "pred_ratio"),
219
+ ("nins", "ins_rate"),
220
+ ("ndel", "del_rate"),
221
+ ]:
222
+ val = sum(log.get(key, 0) for log in logging_outputs)
223
+ metrics.log_scalar(new_key, val / n, n, round=3)
224
+
225
+ @staticmethod
226
+ def logging_outputs_can_be_summed() -> bool:
227
+ return False
data/fairseq/fairseq/criterions/wav2vec_criterion.py ADDED
@@ -0,0 +1,231 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright (c) Facebook, Inc. and its affiliates.
2
+ #
3
+ # This source code is licensed under the MIT license found in the
4
+ # LICENSE file in the root directory of this source tree.
5
+
6
+ import math
7
+ from dataclasses import dataclass, field
8
+ from typing import List, Optional
9
+
10
+ import torch
11
+ import torch.nn.functional as F
12
+ from fairseq import utils
13
+ from fairseq.logging import metrics
14
+ from fairseq.criterions import FairseqCriterion, register_criterion
15
+ from fairseq.dataclass import FairseqDataclass
16
+ from fairseq.logging.meters import safe_round
17
+ from fairseq.utils import is_xla_tensor
18
+
19
+
20
+ @dataclass
21
+ class Wav2VecCriterionConfig(FairseqDataclass):
22
+ infonce: bool = field(
23
+ default=False,
24
+ metadata={
25
+ "help": "if set, uses cross entropy instead of binary cross entropy (i.e. InfoNCE loss)"
26
+ },
27
+ )
28
+ loss_weights: Optional[List[float]] = field(
29
+ default=None,
30
+ metadata={"help": "weights for additional loss terms (not first one)"},
31
+ )
32
+ log_keys: List[str] = field(
33
+ default_factory=lambda: [],
34
+ metadata={"help": "output keys to log"},
35
+ )
36
+
37
+
38
+ @register_criterion("wav2vec", dataclass=Wav2VecCriterionConfig)
39
+ class Wav2vecCriterion(FairseqCriterion):
40
+ def __init__(self, task, infonce=False, loss_weights=None, log_keys=None):
41
+ super().__init__(task)
42
+ self.infonce = infonce
43
+ self.loss_weights = loss_weights
44
+ self.log_keys = [] if log_keys is None else log_keys
45
+
46
+ def forward(self, model, sample, reduce=True):
47
+ """Compute the loss for the given sample.
48
+
49
+ Returns a tuple with three elements:
50
+ 1) the loss
51
+ 2) the sample size, which is used as the denominator for the gradient
52
+ 3) logging outputs to display while training
53
+ """
54
+ net_output = model(**sample["net_input"])
55
+ logits = model.get_logits(net_output).float()
56
+ target = model.get_targets(sample, net_output)
57
+ self.xla = is_xla_tensor(logits)
58
+
59
+ # XXX: handle weights on xla.
60
+ weights = None
61
+ if hasattr(model, "get_target_weights") and not self.infonce:
62
+ weights = model.get_target_weights(target, net_output)
63
+ if torch.is_tensor(weights):
64
+ weights = weights.float()
65
+
66
+ losses = []
67
+
68
+ reduction = "none" if ((not reduce) or self.xla) else "sum"
69
+ if self.infonce:
70
+ loss = F.cross_entropy(logits, target, reduction=reduction)
71
+ else:
72
+ loss = F.binary_cross_entropy_with_logits(
73
+ logits, target.float(), weights, reduction=reduction
74
+ )
75
+
76
+ if self.xla:
77
+ # tpu-comment: since dynamic shapes lead to recompilations on xla,
78
+ # we don't shrink tensors using mask_indices.
79
+ # Instead, we use mask indices to adjust loss.
80
+ mi = (
81
+ sample["net_input"]["mask_indices"]
82
+ .transpose(0, 1) # logits are transposed in `model.get_logits`
83
+ .reshape(logits.size(0))
84
+ )
85
+ loss = (loss * mi).sum() if reduce else (loss * mi)
86
+
87
+ if "sample_size" in sample:
88
+ sample_size = sample["sample_size"]
89
+ elif "mask_indices" in sample["net_input"]:
90
+ sample_size = sample["net_input"]["mask_indices"].sum()
91
+ else:
92
+ sample_size = target.numel() if self.infonce else target.long().sum().item()
93
+ losses.append(loss.detach().clone())
94
+
95
+ if self.loss_weights is not None:
96
+ assert hasattr(model, "get_extra_losses")
97
+ extra_losses = model.get_extra_losses(net_output)
98
+ if torch.is_tensor(extra_losses):
99
+ extra_losses = [extra_losses]
100
+ if len(self.loss_weights) == 1 and len(extra_losses) != 1:
101
+ self.loss_weights = [self.loss_weights[0]] * len(extra_losses)
102
+ assert len(extra_losses) == len(
103
+ self.loss_weights
104
+ ), f"{len(extra_losses)}, {len(self.loss_weights)}"
105
+ for p, coef in zip(extra_losses, self.loss_weights):
106
+ if coef != 0 and p is not None:
107
+ p = coef * p.float() * sample_size
108
+ loss += p
109
+ losses.append(p)
110
+
111
+ logging_output = {
112
+ "loss": loss.item() if (reduce and not self.xla) else loss.detach(),
113
+ "ntokens": sample_size,
114
+ "nsentences": sample["id"].numel(),
115
+ "sample_size": sample_size,
116
+ }
117
+
118
+ for lk in self.log_keys:
119
+ # Only store "logits" and "target" for computing MAP and MAUC
120
+ # during validation
121
+ if lk == "logits":
122
+ if not self.training:
123
+ logging_output["logits"] = logits.cpu().numpy()
124
+ elif lk == "target":
125
+ if not self.training:
126
+ # If the targets have been mixed with the predictions of
127
+ # teacher models, find the original targets
128
+ if hasattr(model, "get_original_targets"):
129
+ original_target = model.get_original_targets(sample, net_output)
130
+ else:
131
+ original_target = target
132
+ logging_output["target"] = original_target.cpu().numpy()
133
+ elif lk in net_output:
134
+ value = net_output[lk]
135
+ if not is_xla_tensor(value):
136
+ value = float(value)
137
+ logging_output[lk] = value
138
+
139
+ if len(losses) > 1:
140
+ for i, l in enumerate(losses):
141
+ logging_output[f"loss_{i}"] = l.item() if not self.xla else l.detach()
142
+
143
+ if self.infonce:
144
+ with torch.no_grad():
145
+ if logits.numel() == 0:
146
+ corr = 0
147
+ count = 0
148
+ else:
149
+ assert logits.dim() > 1, logits.shape
150
+ max = logits.argmax(-1) == 0
151
+ min = logits.argmin(-1) == 0
152
+ if is_xla_tensor(logits):
153
+ max, min = max * mi, min * mi
154
+ both = max & min
155
+ corr = max.long().sum() - both.long().sum()
156
+ count = mi.sum()
157
+ else:
158
+ both = max & min
159
+ corr = max.long().sum().item() - both.long().sum().item()
160
+ count = float(max.numel())
161
+
162
+ logging_output["correct"] = corr
163
+ logging_output["count"] = count
164
+
165
+ return loss, sample_size, logging_output
166
+
167
+ @staticmethod
168
+ def reduce_metrics(logging_outputs) -> None:
169
+ """Aggregate logging outputs from data parallel training."""
170
+ loss_sum = utils.item(sum(log.get("loss", 0) for log in logging_outputs))
171
+ ntokens = utils.item(sum(log.get("ntokens", 0) for log in logging_outputs))
172
+ nsentences = utils.item(
173
+ sum(log.get("nsentences", 0) for log in logging_outputs)
174
+ )
175
+ sample_size = utils.item(
176
+ sum(log.get("sample_size", 0) for log in logging_outputs)
177
+ )
178
+
179
+ metrics.log_scalar(
180
+ "loss", loss_sum / (sample_size or 1) / math.log(2), sample_size, round=3
181
+ )
182
+ metrics.log_scalar("ntokens", ntokens)
183
+ metrics.log_scalar("nsentences", nsentences)
184
+
185
+ correct = sum(log.get("correct", 0) for log in logging_outputs)
186
+ metrics.log_scalar("_correct", correct)
187
+
188
+ total = sum(log.get("count", 0) for log in logging_outputs)
189
+ metrics.log_scalar("_total", total)
190
+
191
+ if total > 0:
192
+ metrics.log_derived(
193
+ "accuracy",
194
+ lambda meters: safe_round(
195
+ meters["_correct"].sum / meters["_total"].sum, 5
196
+ )
197
+ if meters["_total"].sum > 0
198
+ else float("nan"),
199
+ )
200
+
201
+ builtin_keys = {
202
+ "loss",
203
+ "ntokens",
204
+ "nsentences",
205
+ "sample_size",
206
+ "correct",
207
+ "count",
208
+ }
209
+
210
+ for k in logging_outputs[0]:
211
+ if k not in builtin_keys:
212
+ val = sum(log.get(k, 0) for log in logging_outputs)
213
+ if k.startswith("loss"):
214
+ metrics.log_scalar(
215
+ k, val / (sample_size or 1) / math.log(2), sample_size, round=3
216
+ )
217
+ else:
218
+ metrics.log_scalar(k, val / len(logging_outputs), round=3)
219
+
220
+ # FIXME: revert when gather based xla reduction is implemented
221
+ # @staticmethod
222
+ # def logging_outputs_can_be_summed() -> bool:
223
+ def logging_outputs_can_be_summed(self) -> bool:
224
+ """
225
+ Whether the logging outputs returned by `forward` can be summed
226
+ across workers prior to calling `reduce_metrics`. Setting this
227
+ to True will improves distributed training speed.
228
+ """
229
+ # XXX: Gather based reduction not implemented for xla yet.
230
+ # So we fall to sum based reduction for xla.
231
+ return self.xla